diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 00000000000..5db9e83d08c --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,21 @@ +--- +Checks: > + -*, + bugprone-sizeof-expression, + bugprone-suspicious-string-compare, + bugprone-use-after-move, + modernize-use-auto, + modernize-use-equals-default, + modernize-use-override, + modernize-use-using, + performance-for-range-copy, + readability-container-contains, + readability-const-return-type, + readability-container-size-empty, + readability-delete-null-pointer, + readability-duplicate-include, + readability-inconsistent-ifelse-braces +CheckOptions: + - key: modernize-use-auto.MinTypeNameLength + value: '11' +... diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 00000000000..7b55968c90d --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,24 @@ +name: 'Close stale pull requests' +on: + schedule: + - cron: '15 20 * * *' # 20:15 UTC + +permissions: + issues: write + pull-requests: write + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0 + with: + stale-pr-message: > + This PR is stale because it has been open 150 days with no activity. + Remove stale label or add a comment to prevent automatic closure in 14 days. + days-before-pr-stale: 150 + days-before-close: 14 + days-before-issue-stale: 5475 # 15 years, basically disabled for issues + # throttled to 20 per day + operations-per-run: 20 + ascending: true diff --git a/.github/workflows/windows-boost-test.yml b/.github/workflows/windows-boost-test.yml index 8bf7f551a05..e19b8d5e59c 100644 --- a/.github/workflows/windows-boost-test.yml +++ b/.github/workflows/windows-boost-test.yml @@ -24,7 +24,7 @@ jobs: run: | vcpkg install boost-filesystem:x64-windows boost-iostreams:x64-windows boost-serialization:x64-windows boost-system:x64-windows boost-program-options:x64-windows boost-url:x64-windows boost-context:x64-windows lz4:x64-windows shell: cmd - + - name: Configure and Build FoundationDB (without Swift) run: | mkdir build diff --git a/.gitignore b/.gitignore index b608ae140fd..fae11ee2e97 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ cmake-build-*/ *.g.S *.g.asm *.pom +fdbserver/include/fdbserver/FDBRocksDBVersion.h bindings/java/pom*.xml bindings/java/src*/main/overview.html bindings/java/src*/main/com/apple/foundationdb/NetworkOptions.java @@ -113,4 +114,5 @@ temp/ # AI assistants *.aider* -*.fdq \ No newline at end of file +*.fdq +.claude/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 01314db22ce..33bd957d068 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -314,14 +314,11 @@ if(NOT WIN32) if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when # x-compiling. add_subdirectory(fdbmonitor) + add_subdirectory(fdbkubernetesmonitor) endif() endif() add_subdirectory(fdbbackup) add_subdirectory(tests) -if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when - # x-compiling. - add_subdirectory(flowbench EXCLUDE_FROM_ALL) -endif() if(WITH_PYTHON AND WITH_C_BINDING) if(NOT FOUNDATIONDB_CROSS_COMPILING) # FIXME(swift): make this work when # x-compiling. diff --git a/README.md b/README.md index 3a74c410803..dfa32e585f2 100755 --- a/README.md +++ b/README.md @@ -140,17 +140,18 @@ Building FoundationDB requires at least 8GB of memory. More memory is needed whe ### macOS -The build under macOS will work the same way as on Linux. [Homebrew](https://brew.sh/) can be used to install the `boost` library and the `ninja` build tool. +The build under macOS will work the same way as on Linux. [Homebrew](https://brew.sh/) can be used to install the `boost` library and the `ninja` build tool. Be carefull, curent main branch use boost 1.86, do install this version or just let cmake download one. Also, if swift binding is not interest, use -DBUILD_SWIFT_BINDING=OFF. ```sh -cmake -G Ninja +cmake -G Ninja -B +cd ninja ``` To generate an installable package, ```sh -/packaging/osx/buildpkg.sh . +/packaging/osx/buildpkg.sh ``` ### Windows @@ -185,6 +186,10 @@ CMake will not produce a `compile_commands.json` by default; you must pass `-DCM Note that if the building is done inside the `foundationdb/build` Docker image, the resulting paths will still be incorrect and require manual fixing. One will wish to re-run `cmake` with `-DCMAKE_EXPORT_COMPILE_COMMANDS=OFF` to prevent it from reverting the manual changes. +### Code Formatting and Static Analysis + +`clang-format` and `clang-tidy` run as part of CI on every pull request. See the [clang-format](https://apple.github.io/foundationdb/clang-format.html) and [clang-tidy](https://apple.github.io/foundationdb/clang-tidy.html) guides for how to run them locally before pushing. + ### Using IDEs CMake provides built-in support for several popular IDEs. However, most FoundationDB files are written in the `flow` language, which is an extension of the C++ programming language, for coroutine support (Note that when FoundationDB was being developed, C++20 was not available). The `flow` language will be transpiled into C++ code using `actorcompiler`, while preventing most IDEs from recognizing `flow`-specific syntax. diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index a6405a1ccf9..422bbe6e0a4 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -211,7 +211,7 @@ if(NOT WIN32) target_link_libraries(fdb_c_client_memory_test PRIVATE fdb_c Threads::Threads) target_include_directories(fdb_c_api_tester_impl PUBLIC ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR}/foundationdb/ ${CMAKE_SOURCE_DIR}/flow/include ${CMAKE_BINARY_DIR}/flow/include) - target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11_target Threads::Threads fmt::fmt boost_target) + target_link_libraries(fdb_c_api_tester_impl PRIVATE fdb_cpp toml11::toml11 Threads::Threads fmt::fmt boost_target) if(NOT CMAKE_CXX_COMPILER_ID MATCHES "Clang") target_link_libraries(fdb_c_api_tester_impl PRIVATE stdc++fs) endif() @@ -230,6 +230,9 @@ if(NOT WIN32) # do not set RPATH for mako set_property(TARGET mako PROPERTY SKIP_BUILD_RPATH TRUE) target_link_libraries(mako PRIVATE fdb_c fdbclient fmt::fmt Threads::Threads fdb_cpp boost_target rapidjson) + if(NOT OPEN_FOR_IDE) + strip_debug_symbols(mako) + endif() if(NOT OPEN_FOR_IDE) # Make sure that fdb_c.h is compatible with c90 diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index bdb2f90f84b..d904e50f760 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -86,7 +86,7 @@ extern "C" DLLEXPORT fdb_bool_t fdb_error_predicate(int predicate_test, fdb_erro code == error_code_commit_proxy_memory_limit_exceeded || code == error_code_transaction_throttled_hot_shard || code == error_code_batch_transaction_throttled || code == error_code_process_behind || code == error_code_tag_throttled || - code == error_code_proxy_tag_throttled || code == error_code_transaction_rejected_range_locked; + code == error_code_transaction_rejected_range_locked; } return false; } @@ -222,7 +222,7 @@ class CAPICallback final : public ThreadCallback { extern "C" DLLEXPORT fdb_error_t fdb_future_set_callback(FDBFuture* f, void (*callbackf)(FDBFuture*, void*), void* userdata) { - CAPICallback* cb = new CAPICallback(callbackf, f, userdata); + auto* cb = new CAPICallback(callbackf, f, userdata); int ignore; CATCH_AND_RETURN(TSAVB(f)->callOrSetAsCallback(cb, ignore, 0);); } diff --git a/bindings/c/test/apitester/TesterApiWorkload.cpp b/bindings/c/test/apitester/TesterApiWorkload.cpp index 7a607da357f..b295aeb7a84 100644 --- a/bindings/c/test/apitester/TesterApiWorkload.cpp +++ b/bindings/c/test/apitester/TesterApiWorkload.cpp @@ -171,7 +171,7 @@ fdb::KeyRange ApiWorkload::randomNonEmptyKeyRange() { } std::optional ApiWorkload::randomTenant() { - if (tenants.size() > 0) { + if (!tenants.empty()) { return Random::get().randomInt(0, tenants.size() - 1); } else { return {}; @@ -244,7 +244,7 @@ void ApiWorkload::populateTenantData(TTaskFct cont, std::optional tenantId) } void ApiWorkload::createTenantsIfNecessary(TTaskFct cont) { - if (tenants.size() > 0) { + if (!tenants.empty()) { ASSERT(false); } else { schedule(cont); @@ -252,7 +252,7 @@ void ApiWorkload::createTenantsIfNecessary(TTaskFct cont) { } void ApiWorkload::populateData(TTaskFct cont) { - if (tenants.size() > 0) { + if (!tenants.empty()) { populateTenantData(cont, std::make_optional(0)); } else { populateTenantData(cont, {}); diff --git a/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp b/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp index 3743bbab882..fa619e55536 100644 --- a/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp +++ b/bindings/c/test/apitester/TesterAtomicOpsCorrectnessWorkload.cpp @@ -37,8 +37,8 @@ class AtomicOpsCorrectnessWorkload : public ApiWorkload { AtomicOpsCorrectnessWorkload(const WorkloadConfig& config) : ApiWorkload(config) {} private: - typedef std::function IntAtomicOpFunction; - typedef std::function AtomicOpFunction; + using IntAtomicOpFunction = std::function; + using AtomicOpFunction = std::function; enum OpType { OP_ATOMIC_ADD, @@ -57,7 +57,7 @@ class AtomicOpsCorrectnessWorkload : public ApiWorkload { }; void randomOperation(TTaskFct cont) override { - OpType txType = (OpType)Random::get().randomInt(0, OP_LAST); + auto txType = OpType(Random::get().randomInt(0, OP_LAST)); switch (txType) { case OP_ATOMIC_ADD: diff --git a/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp b/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp index deb77e16e47..c58b75a874d 100644 --- a/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp +++ b/bindings/c/test/apitester/TesterCorrectnessWorkload.cpp @@ -248,7 +248,7 @@ class ApiCorrectnessWorkload : public ApiWorkload { results->size())); } else { auto expected_kv = expected.begin(); - for (auto actual_kv : *results) { + for (const auto& actual_kv : *results) { if (actual_kv.key != expected_kv->key || actual_kv.value != expected_kv->value) { error(fmt::format( "randomGetRangeOp mismatch. expected key: {} actual key: {} expected value: " @@ -266,7 +266,7 @@ class ApiCorrectnessWorkload : public ApiWorkload { getTenant(tenantId)); } - void randomOperation(TTaskFct cont) { + void randomOperation(TTaskFct cont) override { std::optional tenantId = randomTenant(); OpType txType = (stores[tenantId].size() == 0) ? OP_INSERT : (OpType)Random::get().randomInt(0, OP_LAST); diff --git a/bindings/c/test/apitester/TesterKeyValueStore.cpp b/bindings/c/test/apitester/TesterKeyValueStore.cpp index af572d628bf..2a4e910b707 100644 --- a/bindings/c/test/apitester/TesterKeyValueStore.cpp +++ b/bindings/c/test/apitester/TesterKeyValueStore.cpp @@ -42,7 +42,7 @@ bool KeyValueStore::exists(fdb::KeyRef key) { fdb::Key KeyValueStore::getKey(fdb::KeyRef keyName, bool orEqual, int offset) const { std::unique_lock lock(mutex); // Begin by getting the start key referenced by the key selector - std::map::const_iterator mapItr = store.lower_bound(keyName); + auto mapItr = store.lower_bound(keyName); // Update the iterator position if necessary based on the value of orEqual int count = 0; @@ -92,16 +92,16 @@ std::vector KeyValueStore::getRange(fdb::KeyRef begin, fdb::KeyRe std::unique_lock lock(mutex); std::vector results; if (!reverse) { - std::map::const_iterator mapItr = store.lower_bound(begin); - - for (; mapItr != store.end() && mapItr->first < end && results.size() < limit; mapItr++) + for (auto mapItr = store.lower_bound(begin); + mapItr != store.end() && mapItr->first < end && results.size() < limit; + mapItr++) results.push_back(fdb::KeyValue{ mapItr->first, mapItr->second }); } // Support for reverse getRange queries is supported, but not tested at this time. This is because reverse range // queries have been disallowed by the database at the API level else { - std::map::const_iterator mapItr = store.lower_bound(end); + auto mapItr = store.lower_bound(end); if (mapItr == store.begin()) return results; diff --git a/bindings/c/test/apitester/TesterTestSpec.cpp b/bindings/c/test/apitester/TesterTestSpec.cpp index 2a8ab161691..f65784a7d29 100644 --- a/bindings/c/test/apitester/TesterTestSpec.cpp +++ b/bindings/c/test/apitester/TesterTestSpec.cpp @@ -137,7 +137,7 @@ TestSpec readTomlTestSpec(std::string fileName) { // Then parse each test const toml::array& tests = toml::find(conf, "test").as_array(); - if (tests.size() == 0) { + if (tests.empty()) { throw TesterError("Invalid test file. No [test] section found"); } else if (tests.size() > 1) { throw TesterError("Invalid test file. More than one [test] section found"); diff --git a/bindings/c/test/apitester/TesterTransactionExecutor.cpp b/bindings/c/test/apitester/TesterTransactionExecutor.cpp index 6bd4cf30d62..19cec9fe8fc 100644 --- a/bindings/c/test/apitester/TesterTransactionExecutor.cpp +++ b/bindings/c/test/apitester/TesterTransactionExecutor.cpp @@ -97,7 +97,7 @@ class TransactionContextBase : public ITransactionContext { } } - virtual ~TransactionContextBase() { ASSERT(txState == TxState::DONE); } + ~TransactionContextBase() override { ASSERT(txState == TxState::DONE); } // A state machine: // IN_PROGRESS -> (ON_ERROR -> IN_PROGRESS)* [-> ON_ERROR] -> DONE @@ -165,7 +165,7 @@ class TransactionContextBase : public ITransactionContext { } } - virtual void onError(fdb::Error err) override { + void onError(fdb::Error err) override { std::unique_lock lock(mutex); if (txState != TxState::IN_PROGRESS) { // Ignore further errors, if the transaction is in the error handing mode or completed @@ -438,7 +438,7 @@ class BlockingTransactionContext : public TransactionContextBase { onError(err); } - virtual void handleOnErrorFuture() override { + void handleOnErrorFuture() override { ASSERT(txState == TxState::ON_ERROR); auto start = timeNow(); @@ -503,7 +503,7 @@ class AsyncTransactionContext : public TransactionContextBase { static void futureReadyCallback(fdb::Future f, void* param) { try { - AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param; + auto* txCtx = (AsyncTransactionContext*)param; txCtx->onFutureReady(f); } catch (std::exception& err) { fmt::print("Unexpected exception in callback {}\n", err.what()); @@ -550,7 +550,7 @@ class AsyncTransactionContext : public TransactionContextBase { onError(err); } - virtual void handleOnErrorFuture() override { + void handleOnErrorFuture() override { ASSERT(txState == TxState::ON_ERROR); onErrorCallTimePoint = timeNow(); @@ -565,7 +565,7 @@ class AsyncTransactionContext : public TransactionContextBase { static void onErrorReadyCallback(fdb::Future f, void* param) { try { - AsyncTransactionContext* txCtx = (AsyncTransactionContext*)param; + auto* txCtx = (AsyncTransactionContext*)param; txCtx->onErrorReady(f); } catch (std::exception& err) { fmt::print("Unexpected exception in callback {}\n", err.what()); @@ -642,7 +642,7 @@ class TransactionExecutorBase : public ITransactionExecutor { public: TransactionExecutorBase(const TransactionExecutorOptions& options) : options(options), scheduler(nullptr) {} - ~TransactionExecutorBase() { + ~TransactionExecutorBase() override { if (tamperClusterFileThread.joinable()) { tamperClusterFileThread.join(); } diff --git a/bindings/c/test/apitester/TesterUtil.h b/bindings/c/test/apitester/TesterUtil.h index 4b5e6d41926..f8c96f63892 100644 --- a/bindings/c/test/apitester/TesterUtil.h +++ b/bindings/c/test/apitester/TesterUtil.h @@ -136,7 +136,8 @@ KeyRangeArray copyKeyRangeArray(fdb::future_var::KeyRangeRefArray::Type array); static_assert(__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__, "Do not support non-little-endian systems"); // Converts a little-endian encoded number into an integral type. -template ::value>> +template + requires(std::is_integral_v) static T toInteger(fdb::BytesRef value) { ASSERT(value.size() == sizeof(T)); T output; @@ -145,7 +146,8 @@ static T toInteger(fdb::BytesRef value) { } // Converts an integral type to a little-endian encoded byte string. -template ::value>> +template + requires(std::is_integral_v) static fdb::ByteString toByteString(T value) { fdb::ByteString output(sizeof(T), 0); memcpy(output.data(), (const uint8_t*)&value, sizeof(value)); diff --git a/bindings/c/test/apitester/TesterWorkload.cpp b/bindings/c/test/apitester/TesterWorkload.cpp index ec8ca8487d7..6faf51fc769 100644 --- a/bindings/c/test/apitester/TesterWorkload.cpp +++ b/bindings/c/test/apitester/TesterWorkload.cpp @@ -204,14 +204,14 @@ void WorkloadManager::run() { std::vector> initialWorkloads; { std::unique_lock lock(mutex); - for (auto iter : workloads) { + for (const auto& iter : workloads) { initialWorkloads.push_back(iter.second.ref); } } - for (auto iter : initialWorkloads) { + for (const auto& iter : initialWorkloads) { iter->init(this); } - for (auto iter : initialWorkloads) { + for (const auto& iter : initialWorkloads) { iter->start(); } scheduler->join(); @@ -265,7 +265,7 @@ void WorkloadManager::readControlInput(std::string pipeName) { // Open in binary mode and read char-by-char to avoid // any kind of buffering FILE* f = fopen(pipeName.c_str(), "rb"); - setbuf(f, NULL); + setbuf(f, nullptr); std::string line; while (true) { int ch = fgetc(f); @@ -291,7 +291,7 @@ void WorkloadManager::readControlInput(std::string pipeName) { void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) { statsTimer = scheduler->scheduleWithDelay(timeIntervalMs, [this, timeIntervalMs]() { - for (auto workload : getActiveWorkloads()) { + for (const auto& workload : getActiveWorkloads()) { workload->printStats(); } this->schedulePrintStatistics(timeIntervalMs); @@ -301,7 +301,7 @@ void WorkloadManager::schedulePrintStatistics(int timeIntervalMs) { std::vector> WorkloadManager::getActiveWorkloads() { std::unique_lock lock(mutex); std::vector> res; - for (auto iter : workloads) { + for (const auto& iter : workloads) { res.push_back(iter.second.ref); } return res; diff --git a/bindings/c/test/apitester/fdb_c_api_tester.cpp b/bindings/c/test/apitester/fdb_c_api_tester.cpp index 4ae10f8fc00..35f38b93d8a 100644 --- a/bindings/c/test/apitester/fdb_c_api_tester.cpp +++ b/bindings/c/test/apitester/fdb_c_api_tester.cpp @@ -325,7 +325,7 @@ void applyNetworkOptions(TesterOptions& options) { fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_RETAIN_CLIENT_LIBRARY_COPIES); } - for (auto knob : options.testSpec.knobs) { + for (const auto& knob : options.testSpec.knobs) { fmt::print(stderr, "Setting knob {}={}\n", knob.first.c_str(), knob.second.c_str()); fdb::network::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB, fmt::format("{}={}", knob.first.c_str(), knob.second.c_str())); diff --git a/bindings/c/test/mako/async.cpp b/bindings/c/test/mako/async.cpp index 5f9c1b058f5..25bd8640254 100644 --- a/bindings/c/test/mako/async.cpp +++ b/bindings/c/test/mako/async.cpp @@ -74,7 +74,7 @@ void ResumableStateForPopulate::runOneTick() { stats.incrOpCount(OP_COMMIT); stats.incrOpCount(OP_TRANSACTION); tx.reset(); - setTransactionTimeoutIfEnabled(args, tx); + setTransactionOptionsIfEnabled(args, tx); watch_tx.startFromStop(); key_checkpoint = i + 1; if (i != key_end) { @@ -165,7 +165,7 @@ void ResumableStateForRunWorkload::updateStepStats() { stats.addLatency(OP_COMMIT, step_latency); } tx.reset(); - setTransactionTimeoutIfEnabled(args, tx); + setTransactionOptionsIfEnabled(args, tx); stats.incrOpCount(OP_COMMIT); needs_commit = false; } @@ -211,7 +211,7 @@ void ResumableStateForRunWorkload::onTransactionSuccess() { stats.incrOpCount(OP_COMMIT); stats.incrOpCount(OP_TRANSACTION); tx.reset(); - setTransactionTimeoutIfEnabled(args, tx); + setTransactionOptionsIfEnabled(args, tx); watch_tx.startFromStop(); onIterationEnd(FutureRC::OK); } @@ -226,7 +226,7 @@ void ResumableStateForRunWorkload::onTransactionSuccess() { stats.incrOpCount(OP_TRANSACTION); watch_tx.startFromStop(); tx.reset(); - setTransactionTimeoutIfEnabled(args, tx); + setTransactionOptionsIfEnabled(args, tx); onIterationEnd(FutureRC::OK); } } @@ -238,6 +238,9 @@ void ResumableStateForRunWorkload::onIterationEnd(FutureRC rc) { if (ended()) { signalEnd(); } else { + if (rc == FutureRC::RETRY) { + setTransactionOptionsIfEnabled(args, tx); + } iter = getOpBegin(args); needs_commit = false; postNextTick(); diff --git a/bindings/c/test/mako/async.hpp b/bindings/c/test/mako/async.hpp index 26271b7aac7..66867f739d1 100644 --- a/bindings/c/test/mako/async.hpp +++ b/bindings/c/test/mako/async.hpp @@ -109,7 +109,7 @@ struct ResumableStateForRunWorkload : std::enable_shared_from_this= max_iters) || signal.load() == SIGNAL_RED; } diff --git a/bindings/c/test/mako/mako.cpp b/bindings/c/test/mako/mako.cpp index b628980bd48..01859e5300f 100644 --- a/bindings/c/test/mako/mako.cpp +++ b/bindings/c/test/mako/mako.cpp @@ -18,6 +18,7 @@ * limitations under the License. */ +#include #include #include #include @@ -38,6 +39,8 @@ #include #include +#include "flow/BooleanParam.h" + #include #include #include @@ -45,8 +48,6 @@ #include #include #include -#include -#include #include #include #include @@ -291,6 +292,9 @@ int runOneTransaction(Transaction& tx, if (future_rc == FutureRC::ABORT) { return -1; } + setTransactionOptionsIfEnabled(args, tx); + if (token) + tx.setOption(FDB_TR_OPTION_AUTHORIZATION_TOKEN, *token); // retry from first op op_iter = getOpBegin(args); needs_commit = false; @@ -304,6 +308,7 @@ int runOneTransaction(Transaction& tx, stats.addLatency(OP_COMMIT, step_latency); } tx.reset(); + setTransactionOptionsIfEnabled(args, tx); if (token) tx.setOption(FDB_TR_OPTION_AUTHORIZATION_TOKEN, *token); stats.incrOpCount(OP_COMMIT); @@ -331,8 +336,9 @@ int runOneTransaction(Transaction& tx, const auto rc = waitAndHandleError(tx, f, "COMMIT_AT_TX_END", args.isAnyTimeoutEnabled()); updateErrorStatsRunMode(stats, f.error(), OP_COMMIT); watch_commit.stop(); - auto tx_resetter = ExitGuard([&tx, &token]() { + auto tx_resetter = ExitGuard([&tx, &token, &args]() { tx.reset(); + setTransactionOptionsIfEnabled(args, tx); if (token) tx.setOption(FDB_TR_OPTION_AUTHORIZATION_TOKEN, *token); }); @@ -418,7 +424,7 @@ int runWorkload(Database db, if (current_tps > 0 || thread_tps == 0 /* throttling off */) { auto [tx, token] = createNewTransaction(db, args, -1); - setTransactionTimeoutIfEnabled(args, tx); + setTransactionOptionsIfEnabled(args, tx); /* enable transaction trace */ if (dotrace) { @@ -541,7 +547,7 @@ void runAsyncWorkload(Arguments const& args, while (shm.headerConst().signal.load() != SIGNAL_GREEN) usleep(1000); // launch [async_xacts] concurrent transactions - for (auto state : states) + for (const auto& state : states) state->postNextTick(); while (stopcount.load() != args.async_xacts) usleep(1000); @@ -573,7 +579,7 @@ void runAsyncWorkload(Arguments const& args, } while (shm.headerConst().signal.load() != SIGNAL_GREEN) usleep(1000); - for (auto state : states) + for (const auto& state : states) state->postNextTick(); logr.debug("Launched {} concurrent transactions", states.size()); while (stopcount.load() != args.async_xacts) @@ -787,6 +793,7 @@ Arguments::Arguments() { load_factor = 1.0; row_digits = digits(rows); seconds = 0; + warmup_seconds = 0; iteration = 0; tpsmax = 0; tpsmin = -1; @@ -823,6 +830,7 @@ Arguments::Arguments() { distributed_tracer_client = 0; transaction_timeout_db = 0; transaction_timeout_tx = 0; + max_grv_queue_delay_ms = -1; num_report_files = 0; } @@ -968,6 +976,9 @@ int parseTransaction(Arguments& args, char const* optarg) { } else if (strncmp(ptr, "g", 1) == 0) { op = OP_GET; ptr++; + } else if (strncmp(ptr, "sj", 2) == 0) { + op = OP_STATUSJSON; + ptr += 2; } else if (strncmp(ptr, "sgr", 3) == 0) { op = OP_SGETRANGE; rangeop = 1; @@ -1084,6 +1095,9 @@ void usage() { printf("%-24s %s\n", "-l, --load_factor=LOAD_FACTOR", "Specify load factor"); printf("%-24s %s\n", "-s, --seconds=SECONDS", "Specify the test duration in seconds\n"); printf("%-24s %s\n", "", "This option cannot be specified with --iteration."); + printf("%-24s %s\n", + " --warmup_seconds=SECONDS", + "Ignore the initial SECONDS of a run when computing aggregate throughput and count-based totals"); printf("%-24s %s\n", "-i, --iteration=ITERS", "Specify the number of iterations.\n"); printf("%-24s %s\n", "", "This option cannot be specified with --seconds."); printf("%-24s %s\n", " --keylen=LENGTH", "Specify the key lengths"); @@ -1135,6 +1149,9 @@ void usage() { printf("%-24s %s\n", " --transaction_timeout_tx=DURATION", "Duration in milliseconds after which a transaction times out in run mode. Set as transaction option"); + printf("%-24s %s\n", + " --max_grv_queue_delay=DURATION", + "Maximum estimated GRV proxy queue delay in milliseconds. Set as transaction option in run mode."); } /* parse benchmark parameters */ @@ -1147,59 +1164,64 @@ int parseArguments(int argc, char* argv[], Arguments& args) { static struct option long_options[] = { /* name, has_arg, flag, val */ /* options requiring an argument */ - { "api_version", required_argument, NULL, 'a' }, - { "cluster", required_argument, NULL, 'c' }, - { "num_databases", required_argument, NULL, 'd' }, - { "procs", required_argument, NULL, 'p' }, - { "threads", required_argument, NULL, 't' }, - { "async_xacts", required_argument, NULL, ARG_ASYNC }, - { "rows", required_argument, NULL, 'r' }, - { "load_factor", required_argument, NULL, 'l' }, - { "seconds", required_argument, NULL, 's' }, - { "iteration", required_argument, NULL, 'i' }, - { "keylen", required_argument, NULL, ARG_KEYLEN }, - { "vallen", required_argument, NULL, ARG_VALLEN }, - { "transaction", required_argument, NULL, 'x' }, - { "tps", required_argument, NULL, ARG_TPS }, - { "tpsmax", required_argument, NULL, ARG_TPSMAX }, - { "tpsmin", required_argument, NULL, ARG_TPSMIN }, - { "tpsinterval", required_argument, NULL, ARG_TPSINTERVAL }, - { "tpschange", required_argument, NULL, ARG_TPSCHANGE }, - { "sampling", required_argument, NULL, ARG_SAMPLING }, - { "verbose", required_argument, NULL, 'v' }, - { "mode", required_argument, NULL, 'm' }, - { "knobs", required_argument, NULL, ARG_KNOBS }, - { "loggroup", required_argument, NULL, ARG_LOGGROUP }, - { "tracepath", required_argument, NULL, ARG_TRACEPATH }, - { "trace_format", required_argument, NULL, ARG_TRACEFORMAT }, - { "streaming", required_argument, NULL, ARG_STREAMING_MODE }, - { "txntrace", required_argument, NULL, ARG_TXNTRACE }, - { "txntagging", required_argument, NULL, ARG_TXNTAGGING }, - { "txntagging_prefix", required_argument, NULL, ARG_TXNTAGGINGPREFIX }, - { "client_threads_per_version", required_argument, NULL, ARG_CLIENT_THREADS_PER_VERSION }, - { "bg_file_path", required_argument, NULL, ARG_BG_FILE_PATH }, - { "distributed_tracer_client", required_argument, NULL, ARG_DISTRIBUTED_TRACER_CLIENT }, - { "tls_certificate_file", required_argument, NULL, ARG_TLS_CERTIFICATE_FILE }, - { "tls_key_file", required_argument, NULL, ARG_TLS_KEY_FILE }, - { "tls_ca_file", required_argument, NULL, ARG_TLS_CA_FILE }, - { "authorization_keypair_id", required_argument, NULL, ARG_AUTHORIZATION_KEYPAIR_ID }, - { "authorization_private_key_pem_file", required_argument, NULL, ARG_AUTHORIZATION_PRIVATE_KEY_PEM_FILE }, - { "transaction_timeout_tx", required_argument, NULL, ARG_TRANSACTION_TIMEOUT_TX }, - { "transaction_timeout_db", required_argument, NULL, ARG_TRANSACTION_TIMEOUT_DB }, + { "api_version", required_argument, nullptr, 'a' }, + { "cluster", required_argument, nullptr, 'c' }, + { "num_databases", required_argument, nullptr, 'd' }, + { "procs", required_argument, nullptr, 'p' }, + { "threads", required_argument, nullptr, 't' }, + { "async_xacts", required_argument, nullptr, ARG_ASYNC }, + { "rows", required_argument, nullptr, 'r' }, + { "load_factor", required_argument, nullptr, 'l' }, + { "seconds", required_argument, nullptr, 's' }, + { "warmup_seconds", required_argument, nullptr, ARG_WARMUP_SECONDS }, + { "iteration", required_argument, nullptr, 'i' }, + { "keylen", required_argument, nullptr, ARG_KEYLEN }, + { "vallen", required_argument, nullptr, ARG_VALLEN }, + { "transaction", required_argument, nullptr, 'x' }, + { "tps", required_argument, nullptr, ARG_TPS }, + { "tpsmax", required_argument, nullptr, ARG_TPSMAX }, + { "tpsmin", required_argument, nullptr, ARG_TPSMIN }, + { "tpsinterval", required_argument, nullptr, ARG_TPSINTERVAL }, + { "tpschange", required_argument, nullptr, ARG_TPSCHANGE }, + { "sampling", required_argument, nullptr, ARG_SAMPLING }, + { "verbose", required_argument, nullptr, 'v' }, + { "mode", required_argument, nullptr, 'm' }, + { "knobs", required_argument, nullptr, ARG_KNOBS }, + { "loggroup", required_argument, nullptr, ARG_LOGGROUP }, + { "tracepath", required_argument, nullptr, ARG_TRACEPATH }, + { "trace_format", required_argument, nullptr, ARG_TRACEFORMAT }, + { "streaming", required_argument, nullptr, ARG_STREAMING_MODE }, + { "txntrace", required_argument, nullptr, ARG_TXNTRACE }, + { "txntagging", required_argument, nullptr, ARG_TXNTAGGING }, + { "txntagging_prefix", required_argument, nullptr, ARG_TXNTAGGINGPREFIX }, + { "client_threads_per_version", required_argument, nullptr, ARG_CLIENT_THREADS_PER_VERSION }, + { "bg_file_path", required_argument, nullptr, ARG_BG_FILE_PATH }, + { "distributed_tracer_client", required_argument, nullptr, ARG_DISTRIBUTED_TRACER_CLIENT }, + { "tls_certificate_file", required_argument, nullptr, ARG_TLS_CERTIFICATE_FILE }, + { "tls_key_file", required_argument, nullptr, ARG_TLS_KEY_FILE }, + { "tls_ca_file", required_argument, nullptr, ARG_TLS_CA_FILE }, + { "authorization_keypair_id", required_argument, nullptr, ARG_AUTHORIZATION_KEYPAIR_ID }, + { "authorization_private_key_pem_file", + required_argument, + nullptr, + ARG_AUTHORIZATION_PRIVATE_KEY_PEM_FILE }, + { "transaction_timeout_tx", required_argument, nullptr, ARG_TRANSACTION_TIMEOUT_TX }, + { "transaction_timeout_db", required_argument, nullptr, ARG_TRANSACTION_TIMEOUT_DB }, + { "max_grv_queue_delay", required_argument, nullptr, ARG_MAX_GRV_QUEUE_DELAY }, /* options which may or may not have an argument */ - { "json_report", optional_argument, NULL, ARG_JSON_REPORT }, - { "stats_export_path", optional_argument, NULL, ARG_EXPORT_PATH }, + { "json_report", optional_argument, nullptr, ARG_JSON_REPORT }, + { "stats_export_path", optional_argument, nullptr, ARG_EXPORT_PATH }, /* options without an argument */ - { "help", no_argument, NULL, 'h' }, - { "zipf", no_argument, NULL, 'z' }, - { "commitget", no_argument, NULL, ARG_COMMITGET }, - { "flatbuffers", no_argument, NULL, ARG_FLATBUFFERS }, - { "prefix_padding", no_argument, NULL, ARG_PREFIXPADDING }, - { "trace", no_argument, NULL, ARG_TRACE }, - { "version", no_argument, NULL, ARG_VERSION }, - { "disable_client_bypass", no_argument, NULL, ARG_DISABLE_CLIENT_BYPASS }, - { "disable_ryw", no_argument, NULL, ARG_DISABLE_RYW }, - { NULL, 0, NULL, 0 } + { "help", no_argument, nullptr, 'h' }, + { "zipf", no_argument, nullptr, 'z' }, + { "commitget", no_argument, nullptr, ARG_COMMITGET }, + { "flatbuffers", no_argument, nullptr, ARG_FLATBUFFERS }, + { "prefix_padding", no_argument, nullptr, ARG_PREFIXPADDING }, + { "trace", no_argument, nullptr, ARG_TRACE }, + { "version", no_argument, nullptr, ARG_VERSION }, + { "disable_client_bypass", no_argument, nullptr, ARG_DISABLE_CLIENT_BYPASS }, + { "disable_ryw", no_argument, nullptr, ARG_DISABLE_RYW }, + { nullptr, 0, nullptr, 0 } }; /* For optional arguments, optarg is only set when the argument is passed as "--option=[ARGUMENT]" but not as @@ -1228,9 +1250,9 @@ int parseArguments(int argc, char* argv[], Arguments& args) { case 'c': { const char delim[] = ","; char* cluster_file = strtok(optarg, delim); - while (cluster_file != NULL) { + while (cluster_file != nullptr) { strcpy(args.cluster_files[args.num_fdb_clusters++], cluster_file); - cluster_file = strtok(NULL, delim); + cluster_file = strtok(nullptr, delim); } break; } @@ -1276,8 +1298,7 @@ int parseArguments(int argc, char* argv[], Arguments& args) { args.mode = MODE_RUN; } else if (strcmp(optarg, "report") == 0) { args.mode = MODE_REPORT; - int i = optind; - for (; i < argc; i++) { + for (int i = optind; i < argc; i++) { if (argv[i][0] != '-') { const std::string report_file = argv[i]; strncpy(args.report_files[args.num_report_files], report_file.c_str(), report_file.size()); @@ -1461,6 +1482,12 @@ int parseArguments(int argc, char* argv[], Arguments& args) { case ARG_TRANSACTION_TIMEOUT_DB: args.transaction_timeout_db = atoi(optarg); break; + case ARG_MAX_GRV_QUEUE_DELAY: + args.max_grv_queue_delay_ms = atoi(optarg); + break; + case ARG_WARMUP_SECONDS: + args.warmup_seconds = atoi(optarg); + break; } } @@ -1555,12 +1582,36 @@ int Arguments::validate() { logr.error("--transaction_timeout_[tx|db] must be a non-negative integer"); return -1; } + if (max_grv_queue_delay_ms < -1 || max_grv_queue_delay_ms == 0) { + logr.error("--max_grv_queue_delay must be a positive integer"); + return -1; + } + if (warmup_seconds < 0) { + logr.error("--warmup_seconds must be a non-negative integer"); + return -1; + } + if (warmup_seconds > 0 && iteration > 0) { + logr.error("--warmup_seconds is only supported with --seconds"); + return -1; + } + if (seconds > 0 && warmup_seconds >= seconds) { + logr.error("--warmup_seconds must be smaller than --seconds"); + return -1; + } } if (mode != MODE_RUN && (transaction_timeout_db != 0 || transaction_timeout_tx != 0)) { logr.error("--transaction_timeout_[tx|db] only supported in run mode"); return -1; } + if (mode != MODE_RUN && max_grv_queue_delay_ms != -1) { + logr.error("--max_grv_queue_delay only supported in run mode"); + return -1; + } + if (mode != MODE_RUN && warmup_seconds != 0) { + logr.error("--warmup_seconds only supported in run mode"); + return -1; + } if (mode == MODE_RUN || mode == MODE_BUILD) { if (tpsmax > 0) { @@ -1654,7 +1705,37 @@ void printStats(Arguments const& args, WorkflowStatistics const* stats, double c prev = current; } -void printStatsHeader(Arguments const& args, bool show_commit, bool is_first_header_empty, bool show_op_stats) { +WorkflowStatistics aggregateWorkerStats(Arguments const& args, WorkflowStatistics const* worker_stats) { + auto aggregate = WorkflowStatistics{}; + const auto num_workers = args.async_xacts > 0 ? args.async_xacts : args.num_threads; + for (auto i = 0; i < args.num_processes * num_workers; i++) { + aggregate.combine(worker_stats[i]); + } + return aggregate; +} + +struct WarmupSnapshot { + WorkflowStatistics worker_stats; + double duration_sec; +}; + +std::optional maybeCaptureWarmupSnapshot(Arguments const& args, + WorkflowStatistics const* worker_stats, + double elapsed_sec) { + if (args.warmup_seconds == 0 || elapsed_sec < args.warmup_seconds) { + return std::nullopt; + } + return WarmupSnapshot{ aggregateWorkerStats(args, worker_stats), elapsed_sec }; +} + +FDB_BOOLEAN_PARAM(ShowCommit); +FDB_BOOLEAN_PARAM(IsFirstHeaderEmpty); +FDB_BOOLEAN_PARAM(ShowOpStats); + +void printStatsHeader(Arguments const& args, + ShowCommit show_commit, + IsFirstHeaderEmpty is_first_header_empty, + ShowOpStats show_op_stats) { /* header */ if (is_first_header_empty) putTitle(""); @@ -1709,7 +1790,7 @@ void printWorkerStats(WorkflowStatistics& final_stats, Arguments args, FILE* fp, } fmt::print("Latency (us)"); - printStatsHeader(args, true, false, true); + printStatsHeader(args, ShowCommit::True, IsFirstHeaderEmpty::False, ShowOpStats::True); /* Total Samples */ putTitle("Samples"); @@ -1944,16 +2025,18 @@ void printReport(Arguments const& args, WorkflowStatistics const* worker_stats, ThreadStatistics const* thread_stats, ProcessStatistics const* process_stats, - double const duration_sec, + double const run_duration_sec, + std::optional const& warmup_snapshot, pid_t pid_main, FILE* fp) { - auto final_worker_stats = WorkflowStatistics{}; - const auto num_workers = args.async_xacts > 0 ? args.async_xacts : args.num_threads; - - for (auto i = 0; i < args.num_processes * num_workers; i++) { - final_worker_stats.combine(worker_stats[i]); + auto final_worker_stats = aggregateWorkerStats(args, worker_stats); + auto measured_worker_stats = final_worker_stats; + if (warmup_snapshot.has_value()) { + measured_worker_stats.subtractCounters(warmup_snapshot->worker_stats); } + const auto warmup_duration_sec = warmup_snapshot.has_value() ? warmup_snapshot->duration_sec : 0.0; + const auto measurement_duration_sec = std::max(run_duration_sec - warmup_duration_sec, 1e-9); double cpu_time_worker_threads = std::accumulate(thread_stats, @@ -2002,7 +2085,9 @@ void printReport(Arguments const& args, (total_duration_local_fdb_networks); // assume that external networks have same total duration as local networks /* overall stats */ - fmt::printf("\n====== Total Duration %6.3f sec ======\n\n", duration_sec); + fmt::printf("\n====== Measured Duration %6.3f sec ======\n\n", measurement_duration_sec); + fmt::printf("Run Duration: %8.3f\n", run_duration_sec); + fmt::printf("Warmup Duration: %8.3f\n", warmup_duration_sec); fmt::printf("Total Processes: %8d\n", args.num_processes); fmt::printf("Total Threads: %8d\n", args.num_threads); fmt::printf("Total Async Xacts: %8d\n", args.async_xacts); @@ -2025,30 +2110,36 @@ void printReport(Arguments const& args, break; } } - const auto tps_f = final_worker_stats.getOpCount(OP_TRANSACTION) / duration_sec; + const auto tps_f = measured_worker_stats.getOpCount(OP_TRANSACTION) / measurement_duration_sec; const auto tps_i = static_cast(tps_f); - fmt::printf("Total Xacts: %8lu\n", final_worker_stats.getOpCount(OP_TRANSACTION)); - fmt::printf("Total Conflicts: %8lu\n", final_worker_stats.getConflictCount()); - fmt::printf("Total Errors: %8lu\n", final_worker_stats.getTotalErrorCount()); - fmt::printf("Total Timeouts: %8lu\n", final_worker_stats.getTotalTimeoutCount()); + fmt::printf("Total Xacts: %8lu\n", measured_worker_stats.getOpCount(OP_TRANSACTION)); + fmt::printf("Total Conflicts: %8lu\n", measured_worker_stats.getConflictCount()); + fmt::printf("Total Errors: %8lu\n", measured_worker_stats.getTotalErrorCount()); + fmt::printf("Total Timeouts: %8lu\n", measured_worker_stats.getTotalTimeoutCount()); fmt::printf("Overall TPS: %8lu\n\n", tps_i); fmt::printf("%%CPU Worker Processes: %6.2f \n", cpu_util_worker_processes); fmt::printf("%%CPU Worker Threads: %6.2f \n", cpu_util_worker_threads); fmt::printf("%%CPU Local Network Threads: %6.2f \n", cpu_util_local_fdb_networks); fmt::printf("%%CPU External Network Threads: %6.2f \n\n", cpu_util_external_fdb_networks); + if (warmup_duration_sec > 0) { + fmt::printf( + "Note: aggregate throughput and count-based totals exclude warmup; latency stats still include it.\n\n"); + } if (fp) { fmt::fprintf(fp, "\"results\": {"); - fmt::fprintf(fp, "\"totalDuration\": %6.3f,", duration_sec); + fmt::fprintf(fp, "\"runDuration\": %6.3f,", run_duration_sec); + fmt::fprintf(fp, "\"warmupDuration\": %6.3f,", warmup_duration_sec); + fmt::fprintf(fp, "\"totalDuration\": %6.3f,", measurement_duration_sec); fmt::fprintf(fp, "\"totalProcesses\": %d,", args.num_processes); fmt::fprintf(fp, "\"totalThreads\": %d,", args.num_threads); fmt::fprintf(fp, "\"totalAsyncXacts\": %d,", args.async_xacts); fmt::fprintf(fp, "\"targetTPS\": %d,", args.tpsmax); - fmt::fprintf(fp, "\"totalXacts\": %lu,", final_worker_stats.getOpCount(OP_TRANSACTION)); - fmt::fprintf(fp, "\"totalConflicts\": %lu,", final_worker_stats.getConflictCount()); - fmt::fprintf(fp, "\"totalErrors\": %lu,", final_worker_stats.getTotalErrorCount()); - fmt::fprintf(fp, "\"totalTimeouts\": %lu,", final_worker_stats.getTotalTimeoutCount()); + fmt::fprintf(fp, "\"totalXacts\": %lu,", measured_worker_stats.getOpCount(OP_TRANSACTION)); + fmt::fprintf(fp, "\"totalConflicts\": %lu,", measured_worker_stats.getConflictCount()); + fmt::fprintf(fp, "\"totalErrors\": %lu,", measured_worker_stats.getTotalErrorCount()); + fmt::fprintf(fp, "\"totalTimeouts\": %lu,", measured_worker_stats.getTotalTimeoutCount()); fmt::fprintf(fp, "\"overallTPS\": %lu,", tps_i); fmt::fprintf(fp, "\"workerProcesseCPU\": %.8f,", cpu_util_worker_processes); fmt::fprintf(fp, "\"workerThreadCPU\": %.8f,", cpu_util_worker_threads); @@ -2057,7 +2148,7 @@ void printReport(Arguments const& args, } /* per-op stats */ - printStatsHeader(args, true, true, false); + printStatsHeader(args, ShowCommit::True, IsFirstHeaderEmpty::True, ShowOpStats::False); /* OPS */ putTitle("Total OPS"); @@ -2067,24 +2158,24 @@ void printReport(Arguments const& args, auto first_op = true; for (auto op = 0; op < MAX_OP; op++) { if ((args.txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) { - putField(final_worker_stats.getOpCount(op)); + putField(measured_worker_stats.getOpCount(op)); if (fp) { if (first_op) { first_op = false; } else { fmt::fprintf(fp, ","); } - fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), final_worker_stats.getOpCount(op)); + fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), measured_worker_stats.getOpCount(op)); } } } /* TPS */ - const auto tps = final_worker_stats.getOpCount(OP_TRANSACTION) / duration_sec; + const auto tps = measured_worker_stats.getOpCount(OP_TRANSACTION) / measurement_duration_sec; putFieldFloat(tps, 2); /* Conflicts */ - const auto conflicts_rate = final_worker_stats.getConflictCount() / duration_sec; + const auto conflicts_rate = measured_worker_stats.getConflictCount() / measurement_duration_sec; putFieldFloat(conflicts_rate, 2); fmt::print("\n"); @@ -2097,14 +2188,14 @@ void printReport(Arguments const& args, first_op = true; for (auto op = 0; op < MAX_OP; op++) { if ((args.txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) { - putField(final_worker_stats.getErrorCount(op)); + putField(measured_worker_stats.getErrorCount(op)); if (fp) { if (first_op) { first_op = false; } else { fmt::fprintf(fp, ","); } - fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), final_worker_stats.getErrorCount(op)); + fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), measured_worker_stats.getErrorCount(op)); } } } @@ -2118,14 +2209,14 @@ void printReport(Arguments const& args, first_op = true; for (auto op = 0; op < MAX_OP; op++) { if ((args.txnspec.ops[op][OP_COUNT] > 0 && op != OP_TRANSACTION) || op == OP_COMMIT) { - putField(final_worker_stats.getTimeoutCount(op)); + putField(measured_worker_stats.getTimeoutCount(op)); if (fp) { if (first_op) { first_op = false; } else { fmt::fprintf(fp, ","); } - fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), final_worker_stats.getTimeoutCount(op)); + fmt::fprintf(fp, "\"%s\": %lu", getOpName(op), measured_worker_stats.getTimeoutCount(op)); } } } @@ -2176,6 +2267,7 @@ int statsProcessMain(Arguments const& args, std::atomic const& stopcount, pid_t pid_main) { bool first_stats = true; + auto warmup_snapshot = std::optional{}; /* wait until the signal turn on */ while (signal.load() == SIGNAL_OFF) { @@ -2183,9 +2275,9 @@ int statsProcessMain(Arguments const& args, } if (args.verbose >= VERBOSE_DEFAULT) - printStatsHeader(args, false, true, false); + printStatsHeader(args, ShowCommit::False, IsFirstHeaderEmpty::True, ShowOpStats::False); - FILE* fp = NULL; + FILE* fp = nullptr; if (args.json_output_path[0] != '\0') { fp = fopen(args.json_output_path, "w"); fmt::fprintf(fp, "{\"makoArgs\": {"); @@ -2198,6 +2290,7 @@ int statsProcessMain(Arguments const& args, fmt::fprintf(fp, "\"rows\": %d,", args.rows); fmt::fprintf(fp, "\"load_factor\": %lf,", args.load_factor); fmt::fprintf(fp, "\"seconds\": %d,", args.seconds); + fmt::fprintf(fp, "\"warmup_seconds\": %d,", args.warmup_seconds); fmt::fprintf(fp, "\"iteration\": %d,", args.iteration); fmt::fprintf(fp, "\"tpsmax\": %d,", args.tpsmax); fmt::fprintf(fp, "\"tpsmin\": %d,", args.tpsmin); @@ -2223,6 +2316,7 @@ int statsProcessMain(Arguments const& args, fmt::fprintf(fp, "\"disable_ryw\": %d,", args.disable_ryw); fmt::fprintf(fp, "\"transaction_timeout_db\": %d,", args.transaction_timeout_db); fmt::fprintf(fp, "\"transaction_timeout_tx\": %d,", args.transaction_timeout_tx); + fmt::fprintf(fp, "\"max_grv_queue_delay_ms\": %d,", args.max_grv_queue_delay_ms); fmt::fprintf(fp, "\"json_output_path\": \"%s\"", args.json_output_path); fmt::fprintf(fp, "},\"samples\": ["); } @@ -2235,6 +2329,10 @@ int statsProcessMain(Arguments const& args, /* print stats every (roughly) 1 sec */ if (toDoubleSeconds(time_now - time_prev) >= 1.0) { + if (!warmup_snapshot.has_value()) { + warmup_snapshot = + maybeCaptureWarmupSnapshot(args, worker_stats, toDoubleSeconds(time_now - time_start)); + } /* adjust throttle rate if needed */ if (args.tpsmax != args.tpsmin) { @@ -2290,11 +2388,14 @@ int statsProcessMain(Arguments const& args, /* print report */ if (args.verbose >= VERBOSE_DEFAULT) { auto time_now = steady_clock::now(); + const auto run_duration_sec = toDoubleSeconds(time_now - time_start); + if (!warmup_snapshot.has_value()) { + warmup_snapshot = maybeCaptureWarmupSnapshot(args, worker_stats, run_duration_sec); + } while (stopcount.load() < args.num_threads * args.num_processes) { usleep(10000); /* 10ms */ } - printReport( - args, worker_stats, thread_stats, process_stats, toDoubleSeconds(time_now - time_start), pid_main, fp); + printReport(args, worker_stats, thread_stats, process_stats, run_duration_sec, warmup_snapshot, pid_main, fp); } if (fp) { @@ -2360,7 +2461,7 @@ int main(int argc, char* argv[]) { if (args.mode == MODE_REPORT) { WorkflowStatistics stats = mergeSketchReport(args); - printWorkerStats(stats, args, NULL, true); + printWorkerStats(stats, args, nullptr, true); return 0; } @@ -2391,7 +2492,7 @@ int main(int argc, char* argv[]) { } /* map it */ - shm = mmap(NULL, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); + shm = mmap(nullptr, shmsize, PROT_READ | PROT_WRITE, MAP_SHARED, shmfd, 0); if (shm == MAP_FAILED) { logr.error("mmap (fd:{} size:{}) failed", shmfd, shmsize); return -1; diff --git a/bindings/c/test/mako/mako.hpp b/bindings/c/test/mako/mako.hpp index f02e7b6adf3..e4a7bf8708f 100644 --- a/bindings/c/test/mako/mako.hpp +++ b/bindings/c/test/mako/mako.hpp @@ -88,6 +88,8 @@ enum ArgKind { ARG_ENABLE_TOKEN_BASED_AUTHORIZATION, ARG_TRANSACTION_TIMEOUT_TX, ARG_TRANSACTION_TIMEOUT_DB, + ARG_WARMUP_SECONDS, + ARG_MAX_GRV_QUEUE_DELAY, }; constexpr const int OP_COUNT = 0; @@ -101,6 +103,7 @@ enum OpKind { OP_GETRANGE, OP_SGET, OP_SGETRANGE, + OP_STATUSJSON, OP_UPDATE, OP_INSERT, OP_INSERTRANGE, @@ -159,6 +162,7 @@ struct Arguments { double load_factor; int row_digits; int seconds; + int warmup_seconds; int iteration; int tpsmax; int tpsmin; @@ -203,13 +207,17 @@ struct Arguments { std::optional private_key_pem; int transaction_timeout_db; int transaction_timeout_tx; + int max_grv_queue_delay_ms; }; // helper functions -inline void setTransactionTimeoutIfEnabled(const Arguments& args, fdb::Transaction& tx) { +inline void setTransactionOptionsIfEnabled(const Arguments& args, fdb::Transaction& tx) { if (args.transaction_timeout_tx > 0) { tx.setOption(FDB_TR_OPTION_TIMEOUT, args.transaction_timeout_tx); } + if (args.max_grv_queue_delay_ms > 0) { + tx.setOption(FDB_TR_OPTION_MAX_GRV_QUEUE_DELAY, args.max_grv_queue_delay_ms); + } } } // namespace mako diff --git a/bindings/c/test/mako/mako.rst b/bindings/c/test/mako/mako.rst index c978ec1b85d..99781bfb274 100644 --- a/bindings/c/test/mako/mako.rst +++ b/bindings/c/test/mako/mako.rst @@ -68,6 +68,11 @@ Arguments | Test duration in seconds (Default: 30) | This option cannot be set with ``--iteration``. +- | ``--warmup_seconds `` + | Ignore the initial ```` of a run when computing aggregate throughput and count-based totals. + | Raw per-second samples are still emitted for the full run. + | This option is only supported with ``--seconds`` and must be smaller than ``--seconds``. + - | ``-i | --iteration `` | Specify the number of operations to be executed. | This option cannot be set with ``--seconds``. @@ -143,6 +148,9 @@ Arguments - | ``--transaction_timeout_db `` | Duration in milliseconds after which a transaction times out in run mode. Set as database option. +- | ``--max_grv_queue_delay `` + | Maximum estimated GRV proxy queue delay in milliseconds. Set as transaction option in run mode. + Transaction Specification ========================= | A transaction may contain multiple operations of various types. @@ -156,6 +164,7 @@ Operation Types - ``gr`` – GET RANGE - ``sg`` – Snapshot GET - ``sgr`` – Snapshot GET RANGE +- ``sj`` – GET ``\xff\xff/status/json`` from the special key space - ``u`` – Update (= GET followed by SET) - ``i`` – Insert (= SET with a new key) - ``ir`` – Insert Range (Sequential) diff --git a/bindings/c/test/mako/operations.cpp b/bindings/c/test/mako/operations.cpp index 7f7cf6a6b79..5c75fa8a599 100644 --- a/bindings/c/test/mako/operations.cpp +++ b/bindings/c/test/mako/operations.cpp @@ -111,6 +111,20 @@ const std::array opTable{ { } } }, 1, false }, + { "STATUSJSON", + { { StepKind::READ, + [](Transaction& tx, Arguments const&, ByteString&, ByteString&, ByteString&) { + static constexpr auto status_json_key = + std::string_view{ "\xff\xff/status/json", sizeof("\xff\xff/status/json") - 1 }; + return tx.get(toBytesRef(status_json_key), false /*snapshot*/).eraseType(); + }, + [](Future& f, Transaction&, Arguments const&, ByteString&, ByteString&, ByteString&) { + if (f && !f.error()) { + f.get(); + } + } } }, + 1, + false }, { "UPDATE", { { StepKind::READ, [](Transaction& tx, Arguments const& args, ByteString& key, ByteString&, ByteString&) { diff --git a/bindings/c/test/mako/stats.hpp b/bindings/c/test/mako/stats.hpp index f2498047737..6348da9515b 100644 --- a/bindings/c/test/mako/stats.hpp +++ b/bindings/c/test/mako/stats.hpp @@ -22,6 +22,7 @@ #define MAKO_STATS_HPP #include +#include #include #include #include @@ -174,6 +175,23 @@ class alignas(64) WorkflowStatistics { latency_us_total[op] += latency_us; } + void subtractCounters(const WorkflowStatistics& baseline) noexcept { + assert(conflicts >= baseline.conflicts); + assert(total_errors >= baseline.total_errors); + assert(total_timeouts >= baseline.total_timeouts); + conflicts -= baseline.conflicts; + total_errors -= baseline.total_errors; + total_timeouts -= baseline.total_timeouts; + for (auto op = 0; op < MAX_OP; op++) { + assert(ops[op] >= baseline.ops[op]); + assert(errors[op] >= baseline.errors[op]); + assert(timeouts[op] >= baseline.timeouts[op]); + ops[op] -= baseline.ops[op]; + errors[op] -= baseline.errors[op]; + timeouts[op] -= baseline.timeouts[op]; + } + } + void writeToFile(const std::string& filename, int op) const { rapidjson::StringBuffer ss; rapidjson::Writer writer(ss); diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index 736f74c75d6..ba4e78d0d0f 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -81,8 +81,7 @@ int runTest(struct RunResult (*testFxn)(struct ResultSet*, FDBTransaction*), const char* kpiName) { int numRuns = 25; int* results = malloc(sizeof(int) * numRuns); - int i = 0; - for (; i < numRuns; ++i) { + for (int i = 0; i < numRuns; ++i) { struct RunResult res = run(rs, db, testFxn); if (res.e) { logError(res.e, kpiName, rs); @@ -110,8 +109,7 @@ int runTestDb(struct RunResult (*testFxn)(struct ResultSet*, FDBDatabase*), const char* kpiName) { int numRuns = 25; int* results = malloc(sizeof(int) * numRuns); - int i = 0; - for (; i < numRuns; ++i) { + for (int i = 0; i < numRuns; ++i) { struct RunResult res = testFxn(rs, db); if (res.e) { logError(res.e, kpiName, rs); @@ -465,8 +463,9 @@ struct RunResult getRange(struct ResultSet* rs, FDBTransaction* tr) { } if (totalOut != GET_RANGE_COUNT) { - char* msg = (char*)malloc((sizeof(char)) * 200); - sprintf(msg, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT); + const size_t nbytes = 200; + char* msg = (char*)malloc(nbytes); + snprintf(msg, nbytes, "verifying out count (%d != %d)", totalOut, GET_RANGE_COUNT); logError(4100, msg, rs); free(msg); fdb_future_destroy(f); diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index f729f8949ff..2a20bcd46a8 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -50,8 +50,7 @@ int runTest(int (*testFxn)(FDBTransaction*, struct ResultSet*), const char* kpiName) { int numRuns = 25; int* results = malloc(sizeof(int) * numRuns); - int i = 0; - for (; i < numRuns; ++i) { + for (int i = 0; i < numRuns; ++i) { results[i] = testFxn(tr, rs); if (results[i] < 0) { free(results); @@ -231,24 +230,24 @@ int interleavedSetsGets(FDBTransaction* tr, struct ResultSet* rs) { int length; int i; - uint8_t* k = (uint8_t*)"foo"; - uint8_t v[10]; + char* k = "foo"; + char v[12]; int num = 1; double start = getTime(); - sprintf((char*)v, "%d", num); - fdb_transaction_set(tr, k, 3, v, strlen((char*)v)); + snprintf(v, sizeof(v), "%d", num); + fdb_transaction_set(tr, (uint8_t*)k, 3, (uint8_t*)v, strlen(v)); for (i = 0; i < 10000; ++i) { - FDBFuture* f = fdb_transaction_get(tr, k, 3, 0); + FDBFuture* f = fdb_transaction_get(tr, (uint8_t*)k, 3, 0); if (getError(fdb_future_block_until_ready(f), "InterleavedSetsGets (block for get)", rs)) return -1; if (getError(fdb_future_get_value(f, &present, &value, &length), "InterleavedSetsGets (get result)", rs)) return -1; fdb_future_destroy(f); - sprintf((char*)v, "%d", ++num); - fdb_transaction_set(tr, k, 3, v, strlen((char*)v)); + snprintf(v, sizeof(v), "%d", ++num); + fdb_transaction_set(tr, (uint8_t*)k, 3, (uint8_t*)v, strlen(v)); } double end = getTime(); diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index afb6e5010ac..3f314b885c5 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -42,8 +42,9 @@ double getTime() { } void writeKey(uint8_t** dest, int key, int keySize) { - *dest = (uint8_t*)malloc((sizeof(uint8_t)) * keySize); - sprintf((char*)*dest, "%0*d", keySize, key); + size_t bufsize = keySize + 1; + *dest = (uint8_t*)malloc(bufsize); + snprintf((char*)*dest, bufsize, "%0*d", keySize, key); } uint8_t** generateKeys(int numKeys, int keySize) { @@ -130,7 +131,7 @@ void addError(struct ResultSet* rs, const char* message) { void writeResultSet(struct ResultSet* rs) { uint64_t id = ((uint64_t)rand() << 32) + rand(); char name[100]; - sprintf(name, "fdb-c_result-%" SCNu64 ".json", id); + snprintf(name, sizeof(name), "fdb-c_result-%" SCNu64 ".json", id); FILE* fp = fopen(name, "w"); if (!fp) { fprintf(stderr, "Could not open results file %s\n", name); @@ -190,8 +191,9 @@ void freeResultSet(struct ResultSet* rs) { fdb_error_t getError(fdb_error_t err, const char* context, struct ResultSet* rs) { if (err) { - char* msg = (char*)malloc(strlen(context) + 100); - sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); + size_t nbytes = strlen(context) + 100; + char* msg = (char*)malloc(nbytes); + snprintf(msg, nbytes, "Error in %s: %s", context, fdb_get_error(err)); fprintf(stderr, "%s\n", msg); if (rs != NULL) { addError(rs, msg); @@ -214,8 +216,9 @@ void checkError(fdb_error_t err, const char* context, struct ResultSet* rs) { } fdb_error_t logError(fdb_error_t err, const char* context, struct ResultSet* rs) { - char* msg = (char*)malloc(strlen(context) + 100); - sprintf(msg, "Error in %s: %s", context, fdb_get_error(err)); + size_t nbytes = strlen(context) + 100; + char* msg = (char*)malloc(nbytes); + snprintf(msg, nbytes, "Error in %s: %s", context, fdb_get_error(err)); fprintf(stderr, "%s\n", msg); if (rs != NULL) { addError(rs, msg); diff --git a/bindings/c/test/workloads/workloads.cpp b/bindings/c/test/workloads/workloads.cpp index 5f53d87039d..5e49326fafc 100644 --- a/bindings/c/test/workloads/workloads.cpp +++ b/bindings/c/test/workloads/workloads.cpp @@ -20,7 +20,7 @@ #include "workloads.h" -FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() {} +FDBWorkloadFactoryImpl::~FDBWorkloadFactoryImpl() = default; std::map& FDBWorkloadFactoryImpl::factories() { static std::map _factories; diff --git a/bindings/flow/CMakeLists.txt b/bindings/flow/CMakeLists.txt index 5e5a31c6bd2..1a7bb07ce54 100644 --- a/bindings/flow/CMakeLists.txt +++ b/bindings/flow/CMakeLists.txt @@ -1,19 +1,19 @@ set(SRCS - DirectoryLayer.actor.cpp + DirectoryLayer.cpp DirectoryLayer.h DirectoryPartition.h DirectorySubspace.cpp DirectorySubspace.h FDBLoanerTypes.h - HighContentionAllocator.actor.cpp + HighContentionAllocator.cpp HighContentionAllocator.h IDirectory.h - Node.actor.cpp + Node.cpp Subspace.cpp Subspace.h Tuple.cpp Tuple.h - fdb_flow.actor.cpp + fdb_flow.cpp fdb_flow.h) add_flow_target(STATIC_LIBRARY NAME fdb_flow SRCS ${SRCS}) @@ -21,10 +21,8 @@ target_link_libraries(fdb_flow PUBLIC fdb_c) target_link_libraries(fdb_flow PUBLIC fdbclient) target_include_directories(fdb_flow PUBLIC "${CMAKE_SOURCE_DIR}" - "${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/tester" - "${CMAKE_CURRENT_BINARY_DIR}/tester" ) add_subdirectory(tester) diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.actor.cpp deleted file mode 100644 index c9b8ef3d488..00000000000 --- a/bindings/flow/DirectoryLayer.actor.cpp +++ /dev/null @@ -1,562 +0,0 @@ -/* - * DirectoryLayer.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "DirectoryLayer.h" -#include "DirectoryPartition.h" - -namespace FDB { -const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 }; -const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = "hca"_sr; -const StringRef DirectoryLayer::LAYER_KEY = "layer"_sr; -const StringRef DirectoryLayer::VERSION_KEY = "version"_sr; -const int64_t DirectoryLayer::SUB_DIR_KEY = 0; - -const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 }; - -const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = "\xfe"_sr; -const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX); -const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); -const StringRef DirectoryLayer::PARTITION_LAYER = "partition"_sr; - -DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) - : rootNode(nodeSubspace.get(nodeSubspace.key())), nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), - allocator(rootNode.get(HIGH_CONTENTION_KEY)), allowManualPrefixes(allowManualPrefixes) {} - -Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { - return nodeSubspace.get(prefix); -} - -template -Optional DirectoryLayer::nodeWithPrefix(Optional const& prefix) const { - if (!prefix.present()) { - return Optional(); - } - - return nodeWithPrefix(prefix.get()); -} - -ACTOR Future find(Reference dirLayer, - Reference tr, - IDirectory::Path path) { - state int pathIndex = 0; - state DirectoryLayer::Node node = DirectoryLayer::Node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path); - - for (; pathIndex != path.size(); ++pathIndex) { - ASSERT(node.subspace.present()); - Optional> val = - wait(tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key())); - - node.path.push_back(path[pathIndex]); - node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path); - - DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); - node = _node; - - if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) { - return node; - } - } - - if (!node.loadedMetadata) { - DirectoryLayer::Node _node = wait(node.loadMetadata(tr)); - node = _node; - } - - return node; -} - -IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const { - Path path; - - path.reserve(this->path.size() + subpath.size()); - path.insert(path.end(), this->path.begin(), this->path.end()); - path.insert(path.end(), subpath.begin(), subpath.end()); - - return path; -} - -Reference DirectoryLayer::contentsOfNode(Subspace const& node, - Path const& path, - Standalone const& layer) { - Standalone prefix = nodeSubspace.unpack(node.key()).getString(0); - - if (layer == PARTITION_LAYER) { - return Reference( - new DirectoryPartition(toAbsolutePath(path), prefix, Reference::addRef(this))); - } else { - return Reference( - new DirectorySubspace(toAbsolutePath(path), prefix, Reference::addRef(this), layer)); - } -} - -Reference DirectoryLayer::openInternal(Standalone const& layer, - Node const& existingNode, - bool allowOpen) { - if (!allowOpen) { - throw directory_already_exists(); - } - if (layer.size() > 0 && layer != existingNode.layer) { - throw mismatched_layer(); - } - - return existingNode.getContents(); -} - -Future> DirectoryLayer::open(Reference const& tr, - Path const& path, - Standalone const& layer) { - return createOrOpenInternal(tr, path, layer, Optional>(), false, true); -} - -void DirectoryLayer::initializeDirectory(Reference const& tr) const { - tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12)); -} - -ACTOR Future checkVersionInternal(const DirectoryLayer* dirLayer, Reference tr, bool writeAccess) { - Optional> versionBytes = - wait(tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY))); - - if (!versionBytes.present()) { - if (writeAccess) { - dirLayer->initializeDirectory(tr); - } - return Void(); - } else { - if (versionBytes.get().size() != 12) { - throw invalid_directory_layer_metadata(); - } - if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) { - throw incompatible_directory_version(); - } else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) { - throw incompatible_directory_version(); - } - } - - return Void(); -} - -Future DirectoryLayer::checkVersion(Reference const& tr, bool writeAccess) const { - return checkVersionInternal(this, tr, writeAccess); -} - -ACTOR Future> getPrefix(Reference dirLayer, - Reference tr, - Optional> prefix) { - if (!prefix.present()) { - Standalone allocated = wait(dirLayer->allocator.allocate(tr)); - state Standalone finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key()); - - FDBStandalone result = wait(tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1)); - - if (result.size() > 0) { - throw directory_prefix_not_empty(); - } - - return finalPrefix; - } - - return prefix.get(); -} - -ACTOR Future> nodeContainingKey(Reference dirLayer, - Reference tr, - Standalone key, - bool snapshot) { - if (key.startsWith(dirLayer->nodeSubspace.key())) { - return dirLayer->rootNode; - } - - KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key))); - FDBStandalone result = wait(tr->getRange(range, 1, snapshot, true)); - - if (result.size() > 0) { - Standalone prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0); - if (key.startsWith(prevPrefix)) { - return dirLayer->nodeWithPrefix(prevPrefix); - } - } - - return Optional(); -} - -ACTOR Future isPrefixFree(Reference dirLayer, - Reference tr, - Standalone prefix, - bool snapshot) { - if (!prefix.size()) { - return false; - } - - Optional node = wait(nodeContainingKey(dirLayer, tr, prefix, snapshot)); - if (node.present()) { - return false; - } - - FDBStandalone result = wait(tr->getRange( - KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot)); - return !result.size(); -} - -ACTOR Future getParentNode(Reference dirLayer, - Reference tr, - IDirectory::Path path) { - if (path.size() > 1) { - Reference parent = - wait(dirLayer->createOrOpenInternal(tr, - IDirectory::Path(path.begin(), path.end() - 1), - StringRef(), - Optional>(), - true, - true)); - return dirLayer->nodeWithPrefix(parent->key()); - } else { - return dirLayer->rootNode; - } -} - -ACTOR Future> createInternal(Reference dirLayer, - Reference tr, - IDirectory::Path path, - Standalone layer, - Optional> prefix, - bool allowCreate) { - if (!allowCreate) { - throw directory_does_not_exist(); - } - - wait(dirLayer->checkVersion(tr, true)); - - state Standalone newPrefix = wait(getPrefix(dirLayer, tr, prefix)); - bool isFree = wait(isPrefixFree(dirLayer, tr, newPrefix, !prefix.present())); - - if (!isFree) { - throw directory_prefix_in_use(); - } - - Subspace parentNode = wait(getParentNode(dirLayer, tr, path)); - Subspace node = dirLayer->nodeWithPrefix(newPrefix); - - tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix); - tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer); - return dirLayer->contentsOfNode(node, path, layer); -} - -ACTOR Future> _createOrOpenInternal(Reference dirLayer, - Reference tr, - IDirectory::Path path, - Standalone layer, - Optional> prefix, - bool allowCreate, - bool allowOpen) { - ASSERT(!prefix.present() || allowCreate); - wait(dirLayer->checkVersion(tr, false)); - - if (prefix.present() && !dirLayer->allowManualPrefixes) { - if (!dirLayer->getPath().size()) { - throw manual_prefixes_not_enabled(); - } else { - throw prefix_in_partition(); - } - } - - if (!path.size()) { - throw cannot_open_root_directory(); - } - - state DirectoryLayer::Node existingNode = wait(find(dirLayer, tr, path)); - if (existingNode.exists()) { - if (existingNode.isInPartition()) { - IDirectory::Path subpath = existingNode.getPartitionSubpath(); - Reference dirSpace = - wait(existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal( - tr, subpath, layer, prefix, allowCreate, allowOpen)); - return dirSpace; - } - return dirLayer->openInternal(layer, existingNode, allowOpen); - } else { - Reference dirSpace = wait(createInternal(dirLayer, tr, path, layer, prefix, allowCreate)); - return dirSpace; - } -} - -Future> DirectoryLayer::createOrOpenInternal(Reference const& tr, - Path const& path, - Standalone const& layer, - Optional> const& prefix, - bool allowCreate, - bool allowOpen) { - return _createOrOpenInternal( - Reference::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen); -} - -Future> DirectoryLayer::create(Reference const& tr, - Path const& path, - Standalone const& layer, - Optional> const& prefix) { - return createOrOpenInternal(tr, path, layer, prefix, true, false); -} - -Future> DirectoryLayer::createOrOpen(Reference const& tr, - Path const& path, - Standalone const& layer) { - return createOrOpenInternal(tr, path, layer, Optional>(), true, true); -} - -ACTOR Future>> listInternal(Reference dirLayer, - Reference tr, - IDirectory::Path path) { - wait(dirLayer->checkVersion(tr, false)); - - state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if (!node.exists()) { - throw directory_does_not_exist(); - } - if (node.isInPartition(true)) { - Standalone> partitionList = - wait(node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath())); - return partitionList; - } - - state Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY); - state Key begin = subdir.range().begin; - state Standalone> subdirectories; - - loop { - FDBStandalone subdirRange = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - - for (int i = 0; i < subdirRange.size(); ++i) { - subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0)); - } - - if (!subdirRange.more) { - return subdirectories; - } - - begin = keyAfter(subdirRange.back().key); - } -} - -Future>> DirectoryLayer::list(Reference const& tr, Path const& path) { - return listInternal(Reference::addRef(this), tr, path); -} - -bool pathsEqual(IDirectory::Path const& path1, - IDirectory::Path const& path2, - size_t maxElementsToCheck = std::numeric_limits::max()) { - if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { - return false; - } - for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) { - if (path1[i] != path2[i]) { - return false; - } - } - - return true; -} - -ACTOR Future removeFromParent(Reference dirLayer, - Reference tr, - IDirectory::Path path) { - ASSERT(path.size() >= 1); - DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1))); - if (parentNode.subspace.present()) { - tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key()); - } - - return Void(); -} - -ACTOR Future> moveInternal(Reference dirLayer, - Reference tr, - IDirectory::Path oldPath, - IDirectory::Path newPath) { - wait(dirLayer->checkVersion(tr, true)); - - if (oldPath.size() <= newPath.size()) { - if (pathsEqual(oldPath, newPath, oldPath.size())) { - throw invalid_destination_directory(); - } - } - - std::vector> futures; - futures.push_back(find(dirLayer, tr, oldPath)); - futures.push_back(find(dirLayer, tr, newPath)); - - std::vector nodes = wait(getAll(futures)); - - state DirectoryLayer::Node oldNode = nodes[0]; - state DirectoryLayer::Node newNode = nodes[1]; - - if (!oldNode.exists()) { - throw directory_does_not_exist(); - } - - if (oldNode.isInPartition() || newNode.isInPartition()) { - if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) { - throw cannot_move_directory_between_partitions(); - } - - Reference partitionMove = - wait(newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath())); - return partitionMove; - } - - if (newNode.exists() || newPath.empty()) { - throw directory_already_exists(); - } - - DirectoryLayer::Node parentNode = wait(find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1))); - if (!parentNode.exists()) { - throw parent_directory_does_not_exist(); - } - - tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), - dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0)); - wait(removeFromParent(dirLayer, tr, oldPath)); - - return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer); -} - -Future> DirectoryLayer::move(Reference const& tr, - Path const& oldPath, - Path const& newPath) { - return moveInternal(Reference::addRef(this), tr, oldPath, newPath); -} - -Future> DirectoryLayer::moveTo(Reference const& tr, - Path const& newAbsolutePath) { - throw cannot_modify_root_directory(); -} - -Future removeRecursive(Reference const&, Reference const&, Subspace const&); -ACTOR Future removeRecursive(Reference dirLayer, Reference tr, Subspace nodeSub) { - state Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY); - state Key begin = subdir.range().begin; - state std::vector> futures; - - loop { - FDBStandalone range = wait(tr->getRange(KeyRangeRef(begin, subdir.range().end))); - for (int i = 0; i < range.size(); ++i) { - Subspace subNode = dirLayer->nodeWithPrefix(range[i].value); - futures.push_back(removeRecursive(dirLayer, tr, subNode)); - } - - if (!range.more) { - break; - } - - begin = keyAfter(range.back().key); - } - - // waits are done concurrently - wait(waitForAll(futures)); - - Standalone nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0); - - tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix))); - tr->clear(nodeSub.range()); - - return Void(); -} - -Future removeInternal(Reference const&, - Reference const&, - IDirectory::Path const&, - bool const&); -ACTOR Future removeInternal(Reference dirLayer, - Reference tr, - IDirectory::Path path, - bool failOnNonexistent) { - wait(dirLayer->checkVersion(tr, true)); - - if (path.empty()) { - throw cannot_modify_root_directory(); - } - - state DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if (!node.exists()) { - if (failOnNonexistent) { - throw directory_does_not_exist(); - } else { - return false; - } - } - - if (node.isInPartition()) { - bool recurse = wait( - removeInternal(node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent)); - return recurse; - } - - state std::vector> futures; - futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get())); - futures.push_back(removeFromParent(dirLayer, tr, path)); - - wait(waitForAll(futures)); - - return true; -} - -Future DirectoryLayer::remove(Reference const& tr, Path const& path) { - return success(removeInternal(Reference::addRef(this), tr, path, true)); -} - -Future DirectoryLayer::removeIfExists(Reference const& tr, Path const& path) { - return removeInternal(Reference::addRef(this), tr, path, false); -} - -ACTOR Future existsInternal(Reference dirLayer, - Reference tr, - IDirectory::Path path) { - wait(dirLayer->checkVersion(tr, false)); - - DirectoryLayer::Node node = wait(find(dirLayer, tr, path)); - - if (!node.exists()) { - return false; - } - - if (node.isInPartition()) { - bool exists = wait(node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath())); - return exists; - } - - return true; -} - -Future DirectoryLayer::exists(Reference const& tr, Path const& path) { - return existsInternal(Reference::addRef(this), tr, path); -} - -Reference DirectoryLayer::getDirectoryLayer() { - return Reference::addRef(this); -} - -const Standalone DirectoryLayer::getLayer() const { - return StringRef(); -} - -const IDirectory::Path DirectoryLayer::getPath() const { - return path; -} -} // namespace FDB diff --git a/bindings/flow/DirectoryLayer.cpp b/bindings/flow/DirectoryLayer.cpp new file mode 100644 index 00000000000..86e5378b8f6 --- /dev/null +++ b/bindings/flow/DirectoryLayer.cpp @@ -0,0 +1,541 @@ +/* + * DirectoryLayer.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "DirectoryLayer.h" +#include "DirectoryPartition.h" + +namespace FDB { +const uint8_t DirectoryLayer::LITTLE_ENDIAN_LONG_ONE[8] = { 1, 0, 0, 0, 0, 0, 0, 0 }; +const StringRef DirectoryLayer::HIGH_CONTENTION_KEY = "hca"_sr; +const StringRef DirectoryLayer::LAYER_KEY = "layer"_sr; +const StringRef DirectoryLayer::VERSION_KEY = "version"_sr; +const int64_t DirectoryLayer::SUB_DIR_KEY = 0; + +const uint32_t DirectoryLayer::VERSION[3] = { 1, 0, 0 }; + +const StringRef DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX = "\xfe"_sr; +const Subspace DirectoryLayer::DEFAULT_NODE_SUBSPACE = Subspace(DEFAULT_NODE_SUBSPACE_PREFIX); +const Subspace DirectoryLayer::DEFAULT_CONTENT_SUBSPACE = Subspace(); +const StringRef DirectoryLayer::PARTITION_LAYER = "partition"_sr; + +DirectoryLayer::DirectoryLayer(Subspace nodeSubspace, Subspace contentSubspace, bool allowManualPrefixes) + : rootNode(nodeSubspace.get(nodeSubspace.key())), nodeSubspace(nodeSubspace), contentSubspace(contentSubspace), + allocator(rootNode.get(HIGH_CONTENTION_KEY)), allowManualPrefixes(allowManualPrefixes) {} + +Subspace DirectoryLayer::nodeWithPrefix(StringRef const& prefix) const { + return nodeSubspace.get(prefix); +} + +template +Optional DirectoryLayer::nodeWithPrefix(Optional const& prefix) const { + if (!prefix.present()) { + return Optional(); + } + + return nodeWithPrefix(prefix.get()); +} + +Future find(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + DirectoryLayer::Node node(dirLayer, dirLayer->rootNode, IDirectory::Path(), path); + + for (int pathIndex = 0; pathIndex != path.size(); ++pathIndex) { + ASSERT(node.subspace.present()); + Optional> val = + co_await tr->get(node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path[pathIndex], true).key()); + + node.path.push_back(path[pathIndex]); + node = DirectoryLayer::Node(dirLayer, dirLayer->nodeWithPrefix(val), node.path, path); + + node = co_await node.loadMetadata(tr); + + if (!node.exists() || node.layer == DirectoryLayer::PARTITION_LAYER) { + co_return node; + } + } + + if (!node.loadedMetadata) { + node = co_await node.loadMetadata(tr); + } + + co_return node; +} + +IDirectory::Path DirectoryLayer::toAbsolutePath(IDirectory::Path const& subpath) const { + Path path; + + path.reserve(this->path.size() + subpath.size()); + path.insert(path.end(), this->path.begin(), this->path.end()); + path.insert(path.end(), subpath.begin(), subpath.end()); + + return path; +} + +Reference DirectoryLayer::contentsOfNode(Subspace const& node, + Path const& path, + Standalone const& layer) { + Standalone prefix = nodeSubspace.unpack(node.key()).getString(0); + + if (layer == PARTITION_LAYER) { + return makeReference(toAbsolutePath(path), prefix, Reference::addRef(this)); + } else { + return makeReference( + toAbsolutePath(path), prefix, Reference::addRef(this), layer); + } +} + +Reference DirectoryLayer::openInternal(Standalone const& layer, + Node const& existingNode, + bool allowOpen) { + if (!allowOpen) { + throw directory_already_exists(); + } + if (!layer.empty() && layer != existingNode.layer) { + throw mismatched_layer(); + } + + return existingNode.getContents(); +} + +Future> DirectoryLayer::open(Reference const& tr, + Path const& path, + Standalone const& layer) { + return createOrOpenInternal(tr, path, layer, Optional>(), false, true); +} + +void DirectoryLayer::initializeDirectory(Reference const& tr) const { + tr->set(rootNode.pack(VERSION_KEY), StringRef((uint8_t*)VERSION, 12)); +} + +Future checkVersionInternal(const DirectoryLayer* dirLayer, Reference tr, bool writeAccess) { + Optional> versionBytes = + co_await tr->get(dirLayer->rootNode.pack(DirectoryLayer::VERSION_KEY)); + + if (!versionBytes.present()) { + if (writeAccess) { + dirLayer->initializeDirectory(tr); + } + co_return; + } else { + if (versionBytes.get().size() != 12) { + throw invalid_directory_layer_metadata(); + } + if (((uint32_t*)versionBytes.get().begin())[0] > DirectoryLayer::VERSION[0]) { + throw incompatible_directory_version(); + } else if (((uint32_t*)versionBytes.get().begin())[1] > DirectoryLayer::VERSION[1] && writeAccess) { + throw incompatible_directory_version(); + } + } +} + +Future DirectoryLayer::checkVersion(Reference const& tr, bool writeAccess) const { + return checkVersionInternal(this, tr, writeAccess); +} + +Future> getPrefix(Reference dirLayer, + Reference tr, + Optional> prefix) { + if (!prefix.present()) { + Standalone allocated = co_await dirLayer->allocator.allocate(tr); + Standalone finalPrefix = allocated.withPrefix(dirLayer->contentSubspace.key()); + + FDBStandalone result = co_await tr->getRange(KeyRangeRef(finalPrefix, strinc(finalPrefix)), 1); + + if (!result.empty()) { + throw directory_prefix_not_empty(); + } + + co_return finalPrefix; + } + + co_return prefix.get(); +} + +Future> nodeContainingKey(Reference dirLayer, + Reference tr, + Standalone key, + bool snapshot) { + if (key.startsWith(dirLayer->nodeSubspace.key())) { + co_return dirLayer->rootNode; + } + + KeyRange range = KeyRangeRef(dirLayer->nodeSubspace.range().begin, keyAfter(dirLayer->nodeSubspace.pack(key))); + FDBStandalone result = co_await tr->getRange(range, 1, snapshot, true); + + if (!result.empty()) { + Standalone prevPrefix = dirLayer->nodeSubspace.unpack(result[0].key).getString(0); + if (key.startsWith(prevPrefix)) { + co_return dirLayer->nodeWithPrefix(prevPrefix); + } + } + + co_return Optional(); +} + +Future isPrefixFree(Reference dirLayer, + Reference tr, + Standalone prefix, + bool snapshot) { + if (prefix.empty()) { + co_return false; + } + + Optional node = co_await nodeContainingKey(dirLayer, tr, prefix, snapshot); + if (node.present()) { + co_return false; + } + + FDBStandalone result = co_await tr->getRange( + KeyRangeRef(dirLayer->nodeSubspace.pack(prefix), dirLayer->nodeSubspace.pack(strinc(prefix))), 1, snapshot); + co_return result.empty(); +} + +Future getParentNode(Reference dirLayer, Reference tr, IDirectory::Path path) { + if (path.size() > 1) { + Reference parent = + co_await dirLayer->createOrOpenInternal(tr, + IDirectory::Path(path.begin(), path.end() - 1), + StringRef(), + Optional>(), + true, + true); + co_return dirLayer->nodeWithPrefix(parent->key()); + } else { + co_return dirLayer->rootNode; + } +} + +Future> createInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + Standalone layer, + Optional> prefix, + bool allowCreate) { + if (!allowCreate) { + throw directory_does_not_exist(); + } + + co_await dirLayer->checkVersion(tr, true); + + Standalone newPrefix = co_await getPrefix(dirLayer, tr, prefix); + bool isFree = co_await isPrefixFree(dirLayer, tr, newPrefix, !prefix.present()); + + if (!isFree) { + throw directory_prefix_in_use(); + } + + Subspace parentNode = co_await getParentNode(dirLayer, tr, path); + Subspace node = dirLayer->nodeWithPrefix(newPrefix); + + tr->set(parentNode.get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key(), newPrefix); + tr->set(node.get(DirectoryLayer::LAYER_KEY).key(), layer); + co_return dirLayer->contentsOfNode(node, path, layer); +} + +Future> _createOrOpenInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + Standalone layer, + Optional> prefix, + bool allowCreate, + bool allowOpen) { + ASSERT(!prefix.present() || allowCreate); + co_await dirLayer->checkVersion(tr, false); + + if (prefix.present() && !dirLayer->allowManualPrefixes) { + if (dirLayer->getPath().empty()) { + throw manual_prefixes_not_enabled(); + } else { + throw prefix_in_partition(); + } + } + + if (path.empty()) { + throw cannot_open_root_directory(); + } + + DirectoryLayer::Node existingNode = co_await find(dirLayer, tr, path); + if (existingNode.exists()) { + if (existingNode.isInPartition()) { + IDirectory::Path subpath = existingNode.getPartitionSubpath(); + Reference dirSpace = + co_await existingNode.getContents()->getDirectoryLayer()->createOrOpenInternal( + tr, subpath, layer, prefix, allowCreate, allowOpen); + co_return dirSpace; + } + co_return dirLayer->openInternal(layer, existingNode, allowOpen); + } else { + Reference dirSpace = co_await createInternal(dirLayer, tr, path, layer, prefix, allowCreate); + co_return dirSpace; + } +} + +Future> DirectoryLayer::createOrOpenInternal(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix, + bool allowCreate, + bool allowOpen) { + return _createOrOpenInternal( + Reference::addRef(this), tr, path, layer, prefix, allowCreate, allowOpen); +} + +Future> DirectoryLayer::create(Reference const& tr, + Path const& path, + Standalone const& layer, + Optional> const& prefix) { + return createOrOpenInternal(tr, path, layer, prefix, true, false); +} + +Future> DirectoryLayer::createOrOpen(Reference const& tr, + Path const& path, + Standalone const& layer) { + return createOrOpenInternal(tr, path, layer, Optional>(), true, true); +} + +Future>> listInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path) { + co_await dirLayer->checkVersion(tr, false); + + DirectoryLayer::Node node = co_await find(dirLayer, tr, path); + + if (!node.exists()) { + throw directory_does_not_exist(); + } + if (node.isInPartition(true)) { + Standalone> partitionList = + co_await node.getContents()->getDirectoryLayer()->list(tr, node.getPartitionSubpath()); + co_return partitionList; + } + + Subspace subdir = node.subspace.get().get(DirectoryLayer::SUB_DIR_KEY); + Key begin = subdir.range().begin; + Standalone> subdirectories; + + while (true) { + FDBStandalone subdirRange = co_await tr->getRange(KeyRangeRef(begin, subdir.range().end)); + + for (int i = 0; i < subdirRange.size(); ++i) { + subdirectories.push_back_deep(subdirectories.arena(), subdir.unpack(subdirRange[i].key).getString(0)); + } + + if (!subdirRange.more) { + co_return subdirectories; + } + + begin = keyAfter(subdirRange.back().key); + } +} + +Future>> DirectoryLayer::list(Reference const& tr, Path const& path) { + return listInternal(Reference::addRef(this), tr, path); +} + +bool pathsEqual(IDirectory::Path const& path1, + IDirectory::Path const& path2, + size_t maxElementsToCheck = std::numeric_limits::max()) { + if (std::min(path1.size(), maxElementsToCheck) != std::min(path2.size(), maxElementsToCheck)) { + return false; + } + for (int i = 0; i < path1.size() && i < maxElementsToCheck; ++i) { + if (path1[i] != path2[i]) { + return false; + } + } + + return true; +} + +Future removeFromParent(Reference dirLayer, Reference tr, IDirectory::Path path) { + ASSERT(!path.empty()); + DirectoryLayer::Node parentNode = co_await find(dirLayer, tr, IDirectory::Path(path.begin(), path.end() - 1)); + if (parentNode.subspace.present()) { + tr->clear(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(path.back(), true).key()); + } +} + +Future> moveInternal(Reference dirLayer, + Reference tr, + IDirectory::Path oldPath, + IDirectory::Path newPath) { + co_await dirLayer->checkVersion(tr, true); + + if (oldPath.size() <= newPath.size()) { + if (pathsEqual(oldPath, newPath, oldPath.size())) { + throw invalid_destination_directory(); + } + } + + std::vector> futures; + futures.push_back(find(dirLayer, tr, oldPath)); + futures.push_back(find(dirLayer, tr, newPath)); + + std::vector nodes = co_await getAll(futures); + + DirectoryLayer::Node oldNode = nodes[0]; + DirectoryLayer::Node newNode = nodes[1]; + + if (!oldNode.exists()) { + throw directory_does_not_exist(); + } + + if (oldNode.isInPartition() || newNode.isInPartition()) { + if (!oldNode.isInPartition() || !newNode.isInPartition() || !pathsEqual(oldNode.path, newNode.path)) { + throw cannot_move_directory_between_partitions(); + } + + Reference partitionMove = + co_await newNode.getContents()->move(tr, oldNode.getPartitionSubpath(), newNode.getPartitionSubpath()); + co_return partitionMove; + } + + if (newNode.exists() || newPath.empty()) { + throw directory_already_exists(); + } + + DirectoryLayer::Node parentNode = co_await find(dirLayer, tr, IDirectory::Path(newPath.begin(), newPath.end() - 1)); + if (!parentNode.exists()) { + throw parent_directory_does_not_exist(); + } + + tr->set(parentNode.subspace.get().get(DirectoryLayer::SUB_DIR_KEY).get(newPath.back(), true).key(), + dirLayer->nodeSubspace.unpack(oldNode.subspace.get().key()).getString(0)); + co_await removeFromParent(dirLayer, tr, oldPath); + + co_return dirLayer->contentsOfNode(oldNode.subspace.get(), newPath, oldNode.layer); +} + +Future> DirectoryLayer::move(Reference const& tr, + Path const& oldPath, + Path const& newPath) { + return moveInternal(Reference::addRef(this), tr, oldPath, newPath); +} + +Future> DirectoryLayer::moveTo(Reference const& tr, + Path const& newAbsolutePath) { + throw cannot_modify_root_directory(); +} + +Future removeRecursive(Reference dirLayer, Reference tr, Subspace nodeSub) { + Subspace subdir = nodeSub.get(DirectoryLayer::SUB_DIR_KEY); + Key begin = subdir.range().begin; + std::vector> futures; + + while (true) { + FDBStandalone range = co_await tr->getRange(KeyRangeRef(begin, subdir.range().end)); + for (int i = 0; i < range.size(); ++i) { + Subspace subNode = dirLayer->nodeWithPrefix(range[i].value); + futures.push_back(removeRecursive(dirLayer, tr, subNode)); + } + + if (!range.more) { + break; + } + + begin = keyAfter(range.back().key); + } + + // waits are done concurrently + co_await waitForAll(futures); + + Standalone nodePrefix = dirLayer->nodeSubspace.unpack(nodeSub.key()).getString(0); + + tr->clear(KeyRangeRef(nodePrefix, strinc(nodePrefix))); + tr->clear(nodeSub.range()); +} + +Future removeInternal(Reference dirLayer, + Reference tr, + IDirectory::Path path, + bool failOnNonexistent) { + co_await dirLayer->checkVersion(tr, true); + + if (path.empty()) { + throw cannot_modify_root_directory(); + } + + DirectoryLayer::Node node = co_await find(dirLayer, tr, path); + + if (!node.exists()) { + if (failOnNonexistent) { + throw directory_does_not_exist(); + } else { + co_return false; + } + } + + if (node.isInPartition()) { + bool recurse = co_await removeInternal( + node.getContents()->getDirectoryLayer(), tr, node.getPartitionSubpath(), failOnNonexistent); + co_return recurse; + } + + std::vector> futures; + futures.push_back(removeRecursive(dirLayer, tr, node.subspace.get())); + futures.push_back(removeFromParent(dirLayer, tr, path)); + + co_await waitForAll(futures); + + co_return true; +} + +Future DirectoryLayer::remove(Reference const& tr, Path const& path) { + return success(removeInternal(Reference::addRef(this), tr, path, true)); +} + +Future DirectoryLayer::removeIfExists(Reference const& tr, Path const& path) { + return removeInternal(Reference::addRef(this), tr, path, false); +} + +Future existsInternal(Reference dirLayer, Reference tr, IDirectory::Path path) { + co_await dirLayer->checkVersion(tr, false); + + DirectoryLayer::Node node = co_await find(dirLayer, tr, path); + + if (!node.exists()) { + co_return false; + } + + if (node.isInPartition()) { + bool exists = co_await node.getContents()->getDirectoryLayer()->exists(tr, node.getPartitionSubpath()); + co_return exists; + } + + co_return true; +} + +Future DirectoryLayer::exists(Reference const& tr, Path const& path) { + return existsInternal(Reference::addRef(this), tr, path); +} + +Reference DirectoryLayer::getDirectoryLayer() { + return Reference::addRef(this); +} + +const Standalone DirectoryLayer::getLayer() const { + return StringRef(); +} + +const IDirectory::Path DirectoryLayer::getPath() const { + return path; +} +} // namespace FDB diff --git a/bindings/flow/DirectoryPartition.h b/bindings/flow/DirectoryPartition.h index df4073290f4..f046ef754f3 100644 --- a/bindings/flow/DirectoryPartition.h +++ b/bindings/flow/DirectoryPartition.h @@ -32,12 +32,12 @@ class DirectoryPartition : public DirectorySubspace { public: DirectoryPartition(Path const& path, StringRef const& prefix, Reference parentDirectoryLayer) - : DirectorySubspace(path, - prefix, - Reference(new DirectoryLayer( - Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), - Subspace(prefix))), - DirectoryLayer::PARTITION_LAYER), + : DirectorySubspace( + path, + prefix, + makeReference(Subspace(DirectoryLayer::DEFAULT_NODE_SUBSPACE_PREFIX.withPrefix(prefix)), + Subspace(prefix)), + DirectoryLayer::PARTITION_LAYER), parentDirectoryLayer(parentDirectoryLayer) { this->directoryLayer->path = path; } diff --git a/bindings/flow/HighContentionAllocator.actor.cpp b/bindings/flow/HighContentionAllocator.actor.cpp deleted file mode 100644 index eebbada0ae2..00000000000 --- a/bindings/flow/HighContentionAllocator.actor.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * HighContentionAllocator.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "HighContentionAllocator.h" - -namespace FDB { -ACTOR Future> _allocate(Reference tr, Subspace counters, Subspace recent) { - state int64_t start = 0; - state int64_t window = 0; - - loop { - FDBStandalone range = wait(tr->getRange(counters.range(), 1, true, true)); - - if (range.size() > 0) { - start = counters.unpack(range[0].key).getInt(0); - } - - state bool windowAdvanced = false; - loop { - // if thread safety is needed, this should be locked { - if (windowAdvanced) { - tr->clear(KeyRangeRef(counters.key(), counters.get(start).key())); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); - tr->clear(KeyRangeRef(recent.key(), recent.get(start).key())); - } - - int64_t inc = 1; - tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD); - Future>> countFuture = tr->get(counters.get(start).key(), true); - // } - - Optional> countValue = wait(countFuture); - - int64_t count = 0; - if (countValue.present()) { - if (countValue.get().size() != 8) { - throw invalid_directory_layer_metadata(); - } - count = *(int64_t*)countValue.get().begin(); - } - - window = HighContentionAllocator::windowSize(start); - if (count * 2 < window) { - break; - } - - start += window; - windowAdvanced = true; - } - - loop { - state int64_t candidate = deterministicRandom()->randomInt(start, start + window); - - // if thread safety is needed, this should be locked { - state Future> latestCounter = tr->getRange(counters.range(), 1, true, true); - state Future>> candidateValue = tr->get(recent.get(candidate).key()); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); - tr->set(recent.get(candidate).key(), ValueRef()); - // } - - wait(success(latestCounter) && success(candidateValue)); - int64_t currentWindowStart = 0; - if (latestCounter.get().size() > 0) { - currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0); - } - - if (currentWindowStart > start) { - break; - } - - if (!candidateValue.get().present()) { - tr->addWriteConflictKey(recent.get(candidate).key()); - return Tuple().append(candidate).pack(); - } - } - } -} - -Future> HighContentionAllocator::allocate(Reference const& tr) const { - return _allocate(tr, counters, recent); -} - -int64_t HighContentionAllocator::windowSize(int64_t start) { - if (start < 255) { - return 64; - } - if (start < 65535) { - return 1024; - } - - return 8192; -} -} // namespace FDB diff --git a/bindings/flow/HighContentionAllocator.cpp b/bindings/flow/HighContentionAllocator.cpp new file mode 100644 index 00000000000..92f63a48c65 --- /dev/null +++ b/bindings/flow/HighContentionAllocator.cpp @@ -0,0 +1,110 @@ +/* + * HighContentionAllocator.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "HighContentionAllocator.h" + +namespace FDB { +Future> _allocate(Reference tr, Subspace counters, Subspace recent) { + int64_t start = 0; + int64_t window = 0; + + while (true) { + FDBStandalone range = co_await tr->getRange(counters.range(), 1, true, true); + + if (!range.empty()) { + start = counters.unpack(range[0].key).getInt(0); + } + + bool windowAdvanced = false; + while (true) { + // if thread safety is needed, this should be locked { + if (windowAdvanced) { + tr->clear(KeyRangeRef(counters.key(), counters.get(start).key())); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + tr->clear(KeyRangeRef(recent.key(), recent.get(start).key())); + } + + int64_t inc = 1; + tr->atomicOp(counters.get(start).key(), StringRef((uint8_t*)&inc, 8), FDB_MUTATION_TYPE_ADD); + Future>> countFuture = tr->get(counters.get(start).key(), true); + // } + + Optional> countValue = co_await countFuture; + + int64_t count = 0; + if (countValue.present()) { + if (countValue.get().size() != 8) { + throw invalid_directory_layer_metadata(); + } + count = *(int64_t*)countValue.get().begin(); + } + + window = HighContentionAllocator::windowSize(start); + if (count * 2 < window) { + break; + } + + start += window; + windowAdvanced = true; + } + + while (true) { + int64_t candidate = deterministicRandom()->randomInt(start, start + window); + + // if thread safety is needed, this should be locked { + Future> latestCounter = tr->getRange(counters.range(), 1, true, true); + Future>> candidateValue = tr->get(recent.get(candidate).key()); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + tr->set(recent.get(candidate).key(), ValueRef()); + // } + + co_await (success(latestCounter) && success(candidateValue)); + int64_t currentWindowStart = 0; + if (!latestCounter.get().empty()) { + currentWindowStart = counters.unpack(latestCounter.get()[0].key).getInt(0); + } + + if (currentWindowStart > start) { + break; + } + + if (!candidateValue.get().present()) { + tr->addWriteConflictKey(recent.get(candidate).key()); + co_return Tuple().append(candidate).pack(); + } + } + } +} + +Future> HighContentionAllocator::allocate(Reference const& tr) const { + return _allocate(tr, counters, recent); +} + +int64_t HighContentionAllocator::windowSize(int64_t start) { + if (start < 255) { + return 64; + } + if (start < 65535) { + return 1024; + } + + return 8192; +} +} // namespace FDB diff --git a/bindings/flow/Node.actor.cpp b/bindings/flow/Node.actor.cpp deleted file mode 100644 index 0be50d25532..00000000000 --- a/bindings/flow/Node.actor.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Node.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "DirectoryLayer.h" - -namespace FDB { -DirectoryLayer::Node::Node(Reference const& directoryLayer, - Optional const& subspace, - IDirectory::Path const& path, - IDirectory::Path const& targetPath) - : directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {} - -bool DirectoryLayer::Node::exists() const { - return subspace.present(); -} - -ACTOR Future loadMetadata(DirectoryLayer::Node* n, Reference tr) { - if (!n->exists()) { - n->loadedMetadata = true; - return *n; - } - - Optional> layer = wait(tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY))); - - n->layer = layer.present() ? layer.get() : Standalone(); - n->loadedMetadata = true; - - return *n; -} - -// Calls to loadMetadata must keep the Node alive while the future is outstanding -Future DirectoryLayer::Node::loadMetadata(Reference tr) { - return FDB::loadMetadata(this, tr); -} - -bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { - ASSERT(loadedMetadata); - return exists() && layer == DirectoryLayer::PARTITION_LAYER && - (includeEmptySubpath || targetPath.size() > path.size()); -} - -IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { - return Path(targetPath.begin() + path.size(), targetPath.end()); -} - -Reference DirectoryLayer::Node::getContents() const { - ASSERT(exists()); - ASSERT(loadedMetadata); - - return directoryLayer->contentsOfNode(subspace.get(), path, layer); -} -} // namespace FDB diff --git a/bindings/flow/Node.cpp b/bindings/flow/Node.cpp new file mode 100644 index 00000000000..55e13bd51bc --- /dev/null +++ b/bindings/flow/Node.cpp @@ -0,0 +1,69 @@ +/* + * Node.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "DirectoryLayer.h" + +namespace FDB { +DirectoryLayer::Node::Node(Reference const& directoryLayer, + Optional const& subspace, + IDirectory::Path const& path, + IDirectory::Path const& targetPath) + : directoryLayer(directoryLayer), subspace(subspace), path(path), targetPath(targetPath), loadedMetadata(false) {} + +bool DirectoryLayer::Node::exists() const { + return subspace.present(); +} + +Future loadMetadata(DirectoryLayer::Node* n, Reference tr) { + if (!n->exists()) { + n->loadedMetadata = true; + co_return *n; + } + + Optional> layer = co_await tr->get(n->subspace.get().pack(DirectoryLayer::LAYER_KEY)); + + n->layer = layer.present() ? layer.get() : Standalone(); + n->loadedMetadata = true; + + co_return *n; +} + +// Calls to loadMetadata must keep the Node alive while the future is outstanding +Future DirectoryLayer::Node::loadMetadata(Reference tr) { + return FDB::loadMetadata(this, tr); +} + +bool DirectoryLayer::Node::isInPartition(bool includeEmptySubpath) const { + ASSERT(loadedMetadata); + return exists() && layer == DirectoryLayer::PARTITION_LAYER && + (includeEmptySubpath || targetPath.size() > path.size()); +} + +IDirectory::Path DirectoryLayer::Node::getPartitionSubpath() const { + return Path(targetPath.begin() + path.size(), targetPath.end()); +} + +Reference DirectoryLayer::Node::getContents() const { + ASSERT(exists()); + ASSERT(loadedMetadata); + + return directoryLayer->contentsOfNode(subspace.get(), path, layer); +} +} // namespace FDB diff --git a/bindings/flow/Subspace.cpp b/bindings/flow/Subspace.cpp index 4f09408b43a..1605fd7a2fd 100644 --- a/bindings/flow/Subspace.cpp +++ b/bindings/flow/Subspace.cpp @@ -39,7 +39,7 @@ Subspace::Subspace(StringRef const& rawPrefix) { this->rawPrefix.append(this->rawPrefix.arena(), rawPrefix.begin(), rawPrefix.size()); } -Subspace::~Subspace() {} +Subspace::~Subspace() = default; Key Subspace::key() const { return StringRef(rawPrefix.begin(), rawPrefix.size()); @@ -89,4 +89,4 @@ Subspace Subspace::subspace(Tuple const& tuple) const { Subspace Subspace::get(Tuple const& tuple) const { return subspace(tuple); } -} // namespace FDB \ No newline at end of file +} // namespace FDB diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp deleted file mode 100644 index f9df8f5266e..00000000000 --- a/bindings/flow/fdb_flow.actor.cpp +++ /dev/null @@ -1,527 +0,0 @@ -/* - * fdb_flow.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdb_flow.h" - -#include -#include -#include - -#include "fmt/format.h" -#include "flow/DeterministicRandom.h" -#include "flow/SystemMonitor.h" -#include "flow/TLSConfig.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -using namespace FDB; - -THREAD_FUNC networkThread(void* fdb) { - ((FDB::API*)fdb)->runNetwork(); - THREAD_RETURN; -} - -ACTOR Future _test() { - API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION); - auto db = fdb->createDatabase(); - state Reference tr = db->createTransaction(); - - // tr->setVersion(1); - - Version ver = wait(tr->getReadVersion()); - fmt::print("{}\n", ver); - - state std::vector> versions; - - state double starttime = timer_monotonic(); - state int i; - // for (i = 0; i < 100000; i++) { - // Version v = wait( tr->getReadVersion() ); - // } - for (i = 0; i < 100000; i++) { - versions.push_back(tr->getReadVersion()); - } - for (i = 0; i < 100000; i++) { - Version v = wait(versions[i]); - } - // wait( waitForAllReady( versions ) ); - printf("Elapsed: %lf\n", timer_monotonic() - starttime); - - tr->set("foo"_sr, "bar"_sr); - - Optional> v = wait(tr->get("foo"_sr)); - if (v.present()) { - printf("%s\n", v.get().toString().c_str()); - } - - FDBStandalone r = wait(tr->getRange(KeyRangeRef("a"_sr, "z"_sr), 100)); - - for (auto kv : r) { - printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str()); - } - - g_network->stop(); - return Void(); -} - -void fdb_flow_test() { - API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION); - fdb->setupNetwork(); - startThread(networkThread, fdb); - - g_network = newNet2(TLSConfig()); - - openTraceFile({}, 1000000, 1000000, "."); - systemMonitor(); - uncancellable(recurring(&systemMonitor, 5.0, TaskPriority::FlushTrace)); - - Future t = _test(); - - g_network->run(); -} - -// FDB object used by bindings -namespace FDB { -class DatabaseImpl : public Database, NonCopyable { -public: - virtual ~DatabaseImpl() { fdb_database_destroy(db); } - - Reference createTransaction() override; - void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) override; - Future rebootWorker(const StringRef& address, bool check = false, int duration = 0) override; - Future forceRecoveryWithDataLoss(const StringRef& dcid) override; - Future createSnapshot(const StringRef& uid, const StringRef& snap_command) override; - -private: - FDBDatabase* db; - explicit DatabaseImpl(FDBDatabase* db) : db(db) {} - - friend class API; -}; - -class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated { - friend class DatabaseImpl; - -public: - virtual ~TransactionImpl() { - if (tr) { - fdb_transaction_destroy(tr); - } - } - - void setReadVersion(Version v) override; - Future getReadVersion() override; - - Future>> get(const Key& key, bool snapshot = false) override; - Future> getKey(const KeySelector& key, bool snapshot = false) override; - - Future watch(const Key& key) override; - - using Transaction::getRange; - Future> getRange(const KeySelector& begin, - const KeySelector& end, - GetRangeLimits limits = GetRangeLimits(), - bool snapshot = false, - bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; - - Future getEstimatedRangeSizeBytes(const KeyRange& keys) override; - Future>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override; - - void addReadConflictRange(KeyRangeRef const& keys) override; - void addReadConflictKey(KeyRef const& key) override; - void addWriteConflictRange(KeyRangeRef const& keys) override; - void addWriteConflictKey(KeyRef const& key) override; - - void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override; - void set(const KeyRef& key, const ValueRef& value) override; - void clear(const KeyRangeRef& range) override; - void clear(const KeyRef& key) override; - - Future commit() override; - Version getCommittedVersion() override; - Future> getVersionstamp() override; - - void setOption(FDBTransactionOption option, Optional value = Optional()) override; - - Future getApproximateSize() override; - Future onError(Error const& e) override; - - void cancel() override; - void reset() override; - - TransactionImpl() : tr(nullptr) {} - TransactionImpl(TransactionImpl&& r) noexcept { - tr = r.tr; - r.tr = nullptr; - } - TransactionImpl& operator=(TransactionImpl&& r) noexcept { - tr = r.tr; - r.tr = nullptr; - return *this; - } - -private: - FDBTransaction* tr; - - explicit TransactionImpl(FDBDatabase* db); -}; - -static inline void throw_on_error(fdb_error_t e) { - if (e) - throw Error(e); -} - -void CFuture::blockUntilReady() { - throw_on_error(fdb_future_block_until_ready(f)); -} - -void backToFutureCallback(FDBFuture* f, void* data) { - g_network->onMainThread(Promise((SAV*)data), - TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority -} - -// backToFuture( FDBFuture*, (FDBFuture* -> Type) ) -> Future -// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) -// and converts it into a Future (with callbacks working on this thread, cancellation etc). -// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type -ACTOR template -static Future backToFuture(FDBFuture* _f, Function convertValue) { - state Reference f(new CFuture(_f)); - - Promise ready; - Future onReady = ready.getFuture(); - - throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer())); - wait(onReady); - - return convertValue(f); -} - -void API::setNetworkOption(FDBNetworkOption option, Optional value) { - if (value.present()) - throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size())); - else - throw_on_error(fdb_network_set_option(option, nullptr, 0)); -} - -API* API::instance = nullptr; -API::API(int version) : version(version) {} - -API* API::selectAPIVersion(int apiVersion) { - if (API::instance) { - if (apiVersion != API::instance->version) { - throw api_version_already_set(); - } else { - return API::instance; - } - } - - if (apiVersion < 500 || apiVersion > FDB_API_VERSION) { - throw api_version_not_supported(); - } - - throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION)); - - API::instance = new API(apiVersion); - return API::instance; -} - -bool API::isAPIVersionSelected() { - return API::instance != nullptr; -} - -API* API::getInstance() { - if (API::instance == nullptr) { - throw api_version_unset(); - } else { - return API::instance; - } -} - -void API::setupNetwork() { - throw_on_error(fdb_setup_network()); -} - -void API::runNetwork() { - throw_on_error(fdb_run_network()); -} - -void API::stopNetwork() { - throw_on_error(fdb_stop_network()); -} - -bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) { - return fdb_error_predicate(pred, e.code()); -} - -Reference API::createDatabase(std::string const& connFilename) { - FDBDatabase* db; - throw_on_error(fdb_create_database(connFilename.c_str(), &db)); - return Reference(new DatabaseImpl(db)); -} - -int API::getAPIVersion() const { - return version; -} - -Reference DatabaseImpl::createTransaction() { - return Reference(new TransactionImpl(db)); -} - -void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional value) { - if (value.present()) - throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size())); - else - throw_on_error(fdb_database_set_option(db, option, nullptr, 0)); -} - -Future DatabaseImpl::rebootWorker(const StringRef& address, bool check, int duration) { - return backToFuture(fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration), - [](Reference f) { - int64_t res; - - throw_on_error(fdb_future_get_int64(f->f, &res)); - - return res; - }); -} - -Future DatabaseImpl::forceRecoveryWithDataLoss(const StringRef& dcid) { - return backToFuture(fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), - [](Reference f) { - throw_on_error(fdb_future_get_error(f->f)); - return Void(); - }); -} - -Future DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) { - return backToFuture( - fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()), - [](Reference f) { - throw_on_error(fdb_future_get_error(f->f)); - return Void(); - }); -} - -TransactionImpl::TransactionImpl(FDBDatabase* db) { - throw_on_error(fdb_database_create_transaction(db, &tr)); -} - -void TransactionImpl::setReadVersion(Version v) { - fdb_transaction_set_read_version(tr, v); -} - -Future TransactionImpl::getReadVersion() { - return backToFuture(fdb_transaction_get_read_version(tr), [](Reference f) { - Version value; - - throw_on_error(fdb_future_get_int64(f->f, &value)); - - return value; - }); -} - -Future>> TransactionImpl::get(const Key& key, bool snapshot) { - return backToFuture>>( - fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference f) { - fdb_bool_t present; - uint8_t const* value; - int value_length; - - throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length)); - - if (present) { - return Optional>(FDBStandalone(f, ValueRef(value, value_length))); - } else { - return Optional>(); - } - }); -} - -Future TransactionImpl::watch(const Key& key) { - return backToFuture(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference f) { - throw_on_error(fdb_future_get_error(f->f)); - return Void(); - }); -} - -Future> TransactionImpl::getKey(const KeySelector& key, bool snapshot) { - return backToFuture>( - fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot), - [](Reference f) { - uint8_t const* key; - int key_length; - - throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); - - return FDBStandalone(f, KeyRef(key, key_length)); - }); -} - -Future> TransactionImpl::getRange(const KeySelector& begin, - const KeySelector& end, - GetRangeLimits limits, - bool snapshot, - bool reverse, - FDBStreamingMode streamingMode) { - // FIXME: iteration - return backToFuture>( - fdb_transaction_get_range(tr, - begin.key.begin(), - begin.key.size(), - begin.orEqual, - begin.offset, - end.key.begin(), - end.key.size(), - end.orEqual, - end.offset, - limits.rows, - limits.bytes, - streamingMode, - 1, - snapshot, - reverse), - [](Reference f) { - FDBKeyValue const* kv; - int count; - fdb_bool_t more; - - throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more)); - - return FDBStandalone(f, - RangeResultRef(VectorRef((KeyValueRef*)kv, count), more)); - }); -} - -Future TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) { - return backToFuture(fdb_transaction_get_estimated_range_size_bytes( - tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), - [](Reference f) { - int64_t bytes; - throw_on_error(fdb_future_get_int64(f->f, &bytes)); - return bytes; - }); -} - -Future>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, - int64_t chunkSize) { - return backToFuture>>( - fdb_transaction_get_range_split_points( - tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), - [](Reference f) { - FDBKey const* ks; - int count; - throw_on_error(fdb_future_get_key_array(f->f, &ks, &count)); - - return FDBStandalone>(f, VectorRef((KeyRef*)ks, count)); - }); -} - -void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { - throw_on_error(fdb_transaction_add_conflict_range( - tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ)); -} - -void TransactionImpl::addReadConflictKey(KeyRef const& key) { - return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); -} - -void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) { - throw_on_error(fdb_transaction_add_conflict_range( - tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE)); -} - -void TransactionImpl::addWriteConflictKey(KeyRef const& key) { - return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); -} - -void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) { - fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType); -} - -void TransactionImpl::set(const KeyRef& key, const ValueRef& value) { - fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size()); -} - -void TransactionImpl::clear(const KeyRangeRef& range) { - fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size()); -} - -void TransactionImpl::clear(const KeyRef& key) { - fdb_transaction_clear(tr, key.begin(), key.size()); -} - -Future TransactionImpl::commit() { - return backToFuture(fdb_transaction_commit(tr), [](Reference f) { - throw_on_error(fdb_future_get_error(f->f)); - return Void(); - }); -} - -Version TransactionImpl::getCommittedVersion() { - Version v; - - throw_on_error(fdb_transaction_get_committed_version(tr, &v)); - return v; -} - -Future> TransactionImpl::getVersionstamp() { - return backToFuture>(fdb_transaction_get_versionstamp(tr), [](Reference f) { - uint8_t const* key; - int key_length; - - throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); - - return FDBStandalone(f, StringRef(key, key_length)); - }); -} - -void TransactionImpl::setOption(FDBTransactionOption option, Optional value) { - if (value.present()) { - throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size())); - } else { - throw_on_error(fdb_transaction_set_option(tr, option, nullptr, 0)); - } -} - -Future TransactionImpl::getApproximateSize() { - return backToFuture(fdb_transaction_get_approximate_size(tr), [](Reference f) { - int64_t size = 0; - throw_on_error(fdb_future_get_int64(f->f, &size)); - return size; - }); -} - -Future TransactionImpl::onError(Error const& e) { - return backToFuture(fdb_transaction_on_error(tr, e.code()), [](Reference f) { - throw_on_error(fdb_future_get_error(f->f)); - return Void(); - }); -} - -void TransactionImpl::cancel() { - fdb_transaction_cancel(tr); -} - -void TransactionImpl::reset() { - fdb_transaction_reset(tr); -} - -} // namespace FDB diff --git a/bindings/flow/fdb_flow.cpp b/bindings/flow/fdb_flow.cpp new file mode 100644 index 00000000000..9599cbece25 --- /dev/null +++ b/bindings/flow/fdb_flow.cpp @@ -0,0 +1,530 @@ +/* + * fdb_flow.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdb_flow.h" + +#include +#include +#include +#include + +#include "fmt/format.h" +#include "flow/DeterministicRandom.h" +#include "flow/SystemMonitor.h" +#include "flow/TLSConfig.h" + +using namespace FDB; + +THREAD_FUNC networkThread(void* fdb) { + ((FDB::API*)fdb)->runNetwork(); + THREAD_RETURN; +} + +Future _test() { + API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION); + auto db = fdb->createDatabase(); + Reference tr = db->createTransaction(); + + // tr->setVersion(1); + + Version ver = co_await tr->getReadVersion(); + fmt::print("{}\n", ver); + + std::vector> versions; + + double starttime = timer_monotonic(); + // for (int i = 0; i < 100000; i++) { + // Version v = wait( tr->getReadVersion() ); + // } + for (int i = 0; i < 100000; i++) { + versions.push_back(tr->getReadVersion()); + } + for (int i = 0; i < 100000; i++) { + co_await versions[i]; + } + // wait( waitForAllReady( versions ) ); + printf("Elapsed: %lf\n", timer_monotonic() - starttime); + + tr->set("foo"_sr, "bar"_sr); + + Optional> v = co_await tr->get("foo"_sr); + if (v.present()) { + printf("%s\n", v.get().toString().c_str()); + } + + FDBStandalone r = co_await tr->getRange(KeyRangeRef("a"_sr, "z"_sr), 100); + + for (auto kv : r) { + printf("%s is %s\n", kv.key.toString().c_str(), kv.value.toString().c_str()); + } + + g_network->stop(); +} + +void fdb_flow_test() { + API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION); + fdb->setupNetwork(); + startThread(networkThread, fdb); + + g_network = newNet2(TLSConfig()); + + openTraceFile({}, 1000000, 1000000, "."); + systemMonitor(); + uncancellable(recurring(&systemMonitor, 5.0, TaskPriority::FlushTrace)); + + Future t = _test(); + + g_network->run(); +} + +// FDB object used by bindings +namespace FDB { +class DatabaseImpl : public Database, NonCopyable { +public: + ~DatabaseImpl() override { fdb_database_destroy(db); } + + Reference createTransaction() override; + void setDatabaseOption(FDBDatabaseOption option, Optional value = Optional()) override; + Future rebootWorker(const StringRef& address, bool check = false, int duration = 0) override; + Future forceRecoveryWithDataLoss(const StringRef& dcid) override; + Future createSnapshot(const StringRef& uid, const StringRef& snap_command) override; + +private: + FDBDatabase* db; + explicit DatabaseImpl(FDBDatabase* db) : db(db) {} + + friend class API; +}; + +class TransactionImpl : public Transaction, private NonCopyable, public FastAllocated { + friend class DatabaseImpl; + +public: + ~TransactionImpl() override { + if (tr) { + fdb_transaction_destroy(tr); + } + } + + void setReadVersion(Version v) override; + Future getReadVersion() override; + + Future>> get(const Key& key, bool snapshot = false) override; + Future> getKey(const KeySelector& key, bool snapshot = false) override; + + Future watch(const Key& key) override; + + using Transaction::getRange; + Future> getRange(const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits = GetRangeLimits(), + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) override; + + Future getEstimatedRangeSizeBytes(const KeyRange& keys) override; + Future>> getRangeSplitPoints(const KeyRange& range, int64_t chunkSize) override; + + void addReadConflictRange(KeyRangeRef const& keys) override; + void addReadConflictKey(KeyRef const& key) override; + void addWriteConflictRange(KeyRangeRef const& keys) override; + void addWriteConflictKey(KeyRef const& key) override; + + void atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) override; + void set(const KeyRef& key, const ValueRef& value) override; + void clear(const KeyRangeRef& range) override; + void clear(const KeyRef& key) override; + + Future commit() override; + Version getCommittedVersion() override; + Future> getVersionstamp() override; + + void setOption(FDBTransactionOption option, Optional value = Optional()) override; + + Future getApproximateSize() override; + Future onError(Error const& e) override; + + void cancel() override; + void reset() override; + + TransactionImpl() : tr(nullptr) {} + TransactionImpl(TransactionImpl&& r) noexcept { + tr = r.tr; + r.tr = nullptr; + } + TransactionImpl& operator=(TransactionImpl&& r) noexcept { + tr = r.tr; + r.tr = nullptr; + return *this; + } + +private: + FDBTransaction* tr; + + explicit TransactionImpl(FDBDatabase* db); +}; + +static inline void throw_on_error(fdb_error_t e) { + if (e) + throw Error(e); +} + +void CFuture::blockUntilReady() { + throw_on_error(fdb_future_block_until_ready(f)); +} + +void backToFutureCallback(FDBFuture* f, void* data) { + g_network->onMainThread(Promise((SAV*)data), + TaskPriority::DefaultOnMainThread); // SOMEDAY: think about this priority +} + +// backToFuture( FDBFuture*, (FDBFuture* -> Type) ) -> Future +// Takes an FDBFuture (from the alien client world, with callbacks potentially firing on an alien thread) +// and converts it into a Future (with callbacks working on this thread, cancellation etc). +// You must pass as the second parameter a function which takes a ready FDBFuture* and returns a value of Type +template +static Future backToFuture(FDBFuture* _f, Function convertValue) { + Reference f(new CFuture(_f)); + + Promise ready; + Future onReady = ready.getFuture(); + + throw_on_error(fdb_future_set_callback(f->f, backToFutureCallback, ready.extractRawPointer())); + co_await onReady; + + if constexpr (std::is_same_v) { + convertValue(f); + co_return; + } else { + co_return convertValue(f); + } +} + +void API::setNetworkOption(FDBNetworkOption option, Optional value) { + if (value.present()) + throw_on_error(fdb_network_set_option(option, value.get().begin(), value.get().size())); + else + throw_on_error(fdb_network_set_option(option, nullptr, 0)); +} + +API* API::instance = nullptr; +API::API(int version) : version(version) {} + +API* API::selectAPIVersion(int apiVersion) { + if (API::instance) { + if (apiVersion != API::instance->version) { + throw api_version_already_set(); + } else { + return API::instance; + } + } + + if (apiVersion < 500 || apiVersion > FDB_API_VERSION) { + throw api_version_not_supported(); + } + + throw_on_error(fdb_select_api_version_impl(apiVersion, FDB_API_VERSION)); + + API::instance = new API(apiVersion); + return API::instance; +} + +bool API::isAPIVersionSelected() { + return API::instance != nullptr; +} + +API* API::getInstance() { + if (API::instance == nullptr) { + throw api_version_unset(); + } else { + return API::instance; + } +} + +void API::setupNetwork() { + throw_on_error(fdb_setup_network()); +} + +void API::runNetwork() { + throw_on_error(fdb_run_network()); +} + +void API::stopNetwork() { + throw_on_error(fdb_stop_network()); +} + +bool API::evaluatePredicate(FDBErrorPredicate pred, Error const& e) { + return fdb_error_predicate(pred, e.code()); +} + +Reference API::createDatabase(std::string const& connFilename) { + FDBDatabase* db; + throw_on_error(fdb_create_database(connFilename.c_str(), &db)); + return Reference(new DatabaseImpl(db)); +} + +int API::getAPIVersion() const { + return version; +} + +Reference DatabaseImpl::createTransaction() { + return Reference(new TransactionImpl(db)); +} + +void DatabaseImpl::setDatabaseOption(FDBDatabaseOption option, Optional value) { + if (value.present()) + throw_on_error(fdb_database_set_option(db, option, value.get().begin(), value.get().size())); + else + throw_on_error(fdb_database_set_option(db, option, nullptr, 0)); +} + +Future DatabaseImpl::rebootWorker(const StringRef& address, bool check, int duration) { + return backToFuture(fdb_database_reboot_worker(db, address.begin(), address.size(), check, duration), + [](Reference f) { + int64_t res; + + throw_on_error(fdb_future_get_int64(f->f, &res)); + + return res; + }); +} + +Future DatabaseImpl::forceRecoveryWithDataLoss(const StringRef& dcid) { + return backToFuture(fdb_database_force_recovery_with_data_loss(db, dcid.begin(), dcid.size()), + [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +Future DatabaseImpl::createSnapshot(const StringRef& uid, const StringRef& snap_command) { + return backToFuture( + fdb_database_create_snapshot(db, uid.begin(), uid.size(), snap_command.begin(), snap_command.size()), + [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +TransactionImpl::TransactionImpl(FDBDatabase* db) { + throw_on_error(fdb_database_create_transaction(db, &tr)); +} + +void TransactionImpl::setReadVersion(Version v) { + fdb_transaction_set_read_version(tr, v); +} + +Future TransactionImpl::getReadVersion() { + return backToFuture(fdb_transaction_get_read_version(tr), [](Reference f) { + Version value; + + throw_on_error(fdb_future_get_int64(f->f, &value)); + + return value; + }); +} + +Future>> TransactionImpl::get(const Key& key, bool snapshot) { + return backToFuture>>( + fdb_transaction_get(tr, key.begin(), key.size(), snapshot), [](Reference f) { + fdb_bool_t present; + uint8_t const* value; + int value_length; + + throw_on_error(fdb_future_get_value(f->f, &present, &value, &value_length)); + + if (present) { + return Optional>(FDBStandalone(f, ValueRef(value, value_length))); + } else { + return Optional>(); + } + }); +} + +Future TransactionImpl::watch(const Key& key) { + return backToFuture(fdb_transaction_watch(tr, key.begin(), key.size()), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +Future> TransactionImpl::getKey(const KeySelector& key, bool snapshot) { + return backToFuture>( + fdb_transaction_get_key(tr, key.key.begin(), key.key.size(), key.orEqual, key.offset, snapshot), + [](Reference f) { + uint8_t const* key; + int key_length; + + throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); + + return FDBStandalone(f, KeyRef(key, key_length)); + }); +} + +Future> TransactionImpl::getRange(const KeySelector& begin, + const KeySelector& end, + GetRangeLimits limits, + bool snapshot, + bool reverse, + FDBStreamingMode streamingMode) { + // FIXME: iteration + return backToFuture>( + fdb_transaction_get_range(tr, + begin.key.begin(), + begin.key.size(), + begin.orEqual, + begin.offset, + end.key.begin(), + end.key.size(), + end.orEqual, + end.offset, + limits.rows, + limits.bytes, + streamingMode, + 1, + snapshot, + reverse), + [](Reference f) { + FDBKeyValue const* kv; + int count; + fdb_bool_t more; + + throw_on_error(fdb_future_get_keyvalue_array(f->f, &kv, &count, &more)); + + return FDBStandalone(f, + RangeResultRef(VectorRef((KeyValueRef*)kv, count), more)); + }); +} + +Future TransactionImpl::getEstimatedRangeSizeBytes(const KeyRange& keys) { + return backToFuture(fdb_transaction_get_estimated_range_size_bytes( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size()), + [](Reference f) { + int64_t bytes; + throw_on_error(fdb_future_get_int64(f->f, &bytes)); + return bytes; + }); +} + +Future>> TransactionImpl::getRangeSplitPoints(const KeyRange& range, + int64_t chunkSize) { + return backToFuture>>( + fdb_transaction_get_range_split_points( + tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size(), chunkSize), + [](Reference f) { + FDBKey const* ks; + int count; + throw_on_error(fdb_future_get_key_array(f->f, &ks, &count)); + + return FDBStandalone>(f, VectorRef((KeyRef*)ks, count)); + }); +} + +void TransactionImpl::addReadConflictRange(KeyRangeRef const& keys) { + throw_on_error(fdb_transaction_add_conflict_range( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_READ)); +} + +void TransactionImpl::addReadConflictKey(KeyRef const& key) { + return addReadConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); +} + +void TransactionImpl::addWriteConflictRange(KeyRangeRef const& keys) { + throw_on_error(fdb_transaction_add_conflict_range( + tr, keys.begin.begin(), keys.begin.size(), keys.end.begin(), keys.end.size(), FDB_CONFLICT_RANGE_TYPE_WRITE)); +} + +void TransactionImpl::addWriteConflictKey(KeyRef const& key) { + return addWriteConflictRange(KeyRange(KeyRangeRef(key, keyAfter(key)))); +} + +void TransactionImpl::atomicOp(const KeyRef& key, const ValueRef& operand, FDBMutationType operationType) { + fdb_transaction_atomic_op(tr, key.begin(), key.size(), operand.begin(), operand.size(), operationType); +} + +void TransactionImpl::set(const KeyRef& key, const ValueRef& value) { + fdb_transaction_set(tr, key.begin(), key.size(), value.begin(), value.size()); +} + +void TransactionImpl::clear(const KeyRangeRef& range) { + fdb_transaction_clear_range(tr, range.begin.begin(), range.begin.size(), range.end.begin(), range.end.size()); +} + +void TransactionImpl::clear(const KeyRef& key) { + fdb_transaction_clear(tr, key.begin(), key.size()); +} + +Future TransactionImpl::commit() { + return backToFuture(fdb_transaction_commit(tr), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +Version TransactionImpl::getCommittedVersion() { + Version v; + + throw_on_error(fdb_transaction_get_committed_version(tr, &v)); + return v; +} + +Future> TransactionImpl::getVersionstamp() { + return backToFuture>(fdb_transaction_get_versionstamp(tr), [](Reference f) { + uint8_t const* key; + int key_length; + + throw_on_error(fdb_future_get_key(f->f, &key, &key_length)); + + return FDBStandalone(f, StringRef(key, key_length)); + }); +} + +void TransactionImpl::setOption(FDBTransactionOption option, Optional value) { + if (value.present()) { + throw_on_error(fdb_transaction_set_option(tr, option, value.get().begin(), value.get().size())); + } else { + throw_on_error(fdb_transaction_set_option(tr, option, nullptr, 0)); + } +} + +Future TransactionImpl::getApproximateSize() { + return backToFuture(fdb_transaction_get_approximate_size(tr), [](Reference f) { + int64_t size = 0; + throw_on_error(fdb_future_get_int64(f->f, &size)); + return size; + }); +} + +Future TransactionImpl::onError(Error const& e) { + return backToFuture(fdb_transaction_on_error(tr, e.code()), [](Reference f) { + throw_on_error(fdb_future_get_error(f->f)); + return Void(); + }); +} + +void TransactionImpl::cancel() { + fdb_transaction_cancel(tr); +} + +void TransactionImpl::reset() { + fdb_transaction_reset(tr); +} + +} // namespace FDB diff --git a/bindings/flow/tester/CMakeLists.txt b/bindings/flow/tester/CMakeLists.txt index dd7f8988de4..c1e67628fdf 100644 --- a/bindings/flow/tester/CMakeLists.txt +++ b/bindings/flow/tester/CMakeLists.txt @@ -1,6 +1,6 @@ set(TEST_SRCS - DirectoryTester.actor.cpp - Tester.actor.cpp - Tester.actor.h) + DirectoryTester.cpp + Tester.cpp + Tester.h) add_flow_target(EXECUTABLE NAME fdb_flow_tester SRCS ${TEST_SRCS}) target_link_libraries(fdb_flow_tester fdb_flow) diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.actor.cpp deleted file mode 100644 index a915b6b4266..00000000000 --- a/bindings/flow/tester/DirectoryTester.actor.cpp +++ /dev/null @@ -1,572 +0,0 @@ -/* - * DirectoryTester.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "Tester.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -using namespace FDB; - -ACTOR Future> popTuples(Reference data, int count = 1) { - state std::vector tuples; - - while (tuples.size() < count) { - Standalone sizeStr = wait(data->stack.pop()[0].value); - int size = Tuple::unpack(sizeStr).getInt(0); - - state std::vector tupleItems = data->stack.pop(size); - state Tuple tuple; - - state int index; - for (index = 0; index < tupleItems.size(); ++index) { - Standalone itemStr = wait(tupleItems[index].value); - tuple.append(Tuple::unpack(itemStr)); - } - - tuples.push_back(tuple); - } - - return tuples; -} - -ACTOR Future popTuple(Reference data) { - std::vector tuples = wait(popTuples(data)); - return tuples[0]; -} - -ACTOR Future> popPaths(Reference data, int count = 1) { - std::vector tuples = wait(popTuples(data, count)); - - std::vector paths; - for (auto& tuple : tuples) { - IDirectory::Path path; - for (int i = 0; i < tuple.size(); ++i) { - path.push_back(tuple.getString(i)); - } - - paths.push_back(path); - } - - return paths; -} - -ACTOR Future popPath(Reference data) { - std::vector paths = wait(popPaths(data)); - return paths[0]; -} - -std::string pathToString(IDirectory::Path const& path) { - std::string str; - str += "["; - for (int i = 0; i < path.size(); ++i) { - str += path[i].toString(); - if (i < path.size() - 1) { - str += ", "; - } - } - - return str + "]"; -} - -IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) { - IDirectory::Path outPath(path1.begin(), path1.end()); - for (auto p : path2) { - outPath.push_back(p); - } - - return outPath; -} - -void logOp(std::string message, bool force = false) { - if (LOG_OPS || force) { - printf("%s\n", message.c_str()); - fflush(stdout); - } -} - -// DIRECTORY_CREATE_SUBSPACE -struct DirectoryCreateSubspaceFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state Tuple path = wait(popTuple(data)); - Tuple rawPrefix = wait(data->stack.waitAndPop()); - - logOp(format( - "Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); - data->directoryData.push(new Subspace(path, rawPrefix.getString(0))); - return Void(); - } -}; -const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE"; -REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc); - -// DIRECTORY_CREATE_LAYER -struct DirectoryCreateLayerFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - std::vector args = wait(data->stack.waitAndPop(3)); - - int index1 = args[0].getInt(0); - int index2 = args[1].getInt(0); - bool allowManualPrefixes = args[2].getInt(0) != 0; - - if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { - logOp("Create directory layer: None"); - data->directoryData.push(); - } else { - Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get(); - Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get(); - logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, " - "allow_manual_prefixes = %d", - index1, - nodeSubspace->key().printable().c_str(), - index2, - nodeSubspace->key().printable().c_str(), - allowManualPrefixes)); - data->directoryData.push( - Reference(new DirectoryLayer(*nodeSubspace, *contentSubspace, allowManualPrefixes))); - } - - return Void(); - } -}; -const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER"; -REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc); - -// DIRECTORY_CHANGE -struct DirectoryChangeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple index = wait(data->stack.waitAndPop()); - data->directoryData.directoryListIndex = index.getInt(0); - ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size()); - - if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) { - data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex; - } - - if (LOG_DIRS) { - DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex]; - printf("Changed directory to %d (%s @\'%s\')\n", - data->directoryData.directoryListIndex, - d.typeString().c_str(), - d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() - : d.subspace.get()->key().printable().c_str()); - fflush(stdout); - } - - return Void(); - } -}; -const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE"; -REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc); - -// DIRECTORY_SET_ERROR_INDEX -struct DirectorySetErrorIndexFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple index = wait(data->stack.waitAndPop()); - data->directoryData.directoryErrorIndex = index.getInt(0); - - return Void(); - } -}; -const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX"; -REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc); - -// DIRECTORY_CREATE_OR_OPEN -struct DirectoryCreateOrOpenFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state IDirectory::Path path = wait(popPath(data)); - Tuple layerTuple = wait(data->stack.waitAndPop()); - Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); - - Reference directory = data->directoryData.directory(); - logOp(format("create_or_open %s: layer=%s", - pathToString(combinePaths(directory->getPath(), path)).c_str(), - layer.printable().c_str())); - - Reference dirSubspace = wait(executeMutation( - instruction, [this, directory, layer]() { return directory->createOrOpen(instruction->tr, path, layer); })); - - data->directoryData.push(dirSubspace); - - return Void(); - } -}; -const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN"; -REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc); - -// DIRECTORY_CREATE -struct DirectoryCreateFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state IDirectory::Path path = wait(popPath(data)); - std::vector args = wait(data->stack.waitAndPop(2)); - Standalone layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0); - Optional> prefix = - args[1].getType(0) == Tuple::NULL_TYPE ? Optional>() : args[1].getString(0); - - Reference directory = data->directoryData.directory(); - logOp(format("create %s: layer=%s, prefix=%s", - pathToString(combinePaths(directory->getPath(), path)).c_str(), - layer.printable().c_str(), - prefix.present() ? prefix.get().printable().c_str() : "")); - - Reference dirSubspace = - wait(executeMutation(instruction, [this, directory, layer, prefix]() { - return directory->create(instruction->tr, path, layer, prefix); - })); - - data->directoryData.push(dirSubspace); - - return Void(); - } -}; -const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE"; -REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc); - -// DIRECTORY_OPEN -struct DirectoryOpenFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state IDirectory::Path path = wait(popPath(data)); - Tuple layerTuple = wait(data->stack.waitAndPop()); - Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); - - Reference directory = data->directoryData.directory(); - logOp(format("open %s: layer=%s", - pathToString(combinePaths(directory->getPath(), path)).c_str(), - layer.printable().c_str())); - Reference dirSubspace = wait(directory->open(instruction->tr, path, layer)); - data->directoryData.push(dirSubspace); - - return Void(); - } -}; -const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN"; -REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc); - -// DIRECTORY_MOVE -struct DirectoryMoveFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - std::vector paths = wait(popPaths(data, 2)); - - Reference directory = data->directoryData.directory(); - logOp(format("move %s to %s", - pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), - pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); - - Reference dirSubspace = wait(executeMutation( - instruction, [this, directory, paths]() { return directory->move(instruction->tr, paths[0], paths[1]); })); - - data->directoryData.push(dirSubspace); - - return Void(); - } -}; -const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE"; -REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc); - -// DIRECTORY_MOVE_TO -struct DirectoryMoveToFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - IDirectory::Path path = wait(popPath(data)); - - Reference directory = data->directoryData.directory(); - logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str())); - - Reference dirSubspace = wait(executeMutation( - instruction, [this, directory, path]() { return directory->moveTo(instruction->tr, path); })); - - data->directoryData.push(dirSubspace); - - return Void(); - } -}; -const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO"; -REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc); - -// DIRECTORY_REMOVE -struct DirectoryRemoveFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple count = wait(data->stack.waitAndPop()); - state Reference directory = data->directoryData.directory(); - if (count.getInt(0) == 0) { - logOp(format("remove %s", pathToString(directory->getPath()).c_str())); - - wait(executeMutation(instruction, [this]() { return directory->remove(instruction->tr); })); - } else { - IDirectory::Path path = wait(popPath(data)); - logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); - - wait(executeMutation(instruction, [this, path]() { return directory->remove(instruction->tr, path); })); - } - - return Void(); - } -}; -const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE"; -REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc); - -// DIRECTORY_REMOVE_IF_EXISTS -struct DirectoryRemoveIfExistsFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple count = wait(data->stack.waitAndPop()); - state Reference directory = data->directoryData.directory(); - if (count.getInt(0) == 0) { - logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str())); - - wait( - success(executeMutation(instruction, [this]() { return directory->removeIfExists(instruction->tr); }))); - } else { - IDirectory::Path path = wait(popPath(data)); - logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); - - wait(success(executeMutation(instruction, - [this, path]() { return directory->removeIfExists(instruction->tr, path); }))); - } - - return Void(); - } -}; -const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS"; -REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc); - -// DIRECTORY_LIST -struct DirectoryListFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple count = wait(data->stack.waitAndPop()); - state Reference directory = data->directoryData.directory(); - state Standalone> subdirs; - if (count.getInt(0) == 0) { - logOp(format("list %s", pathToString(directory->getPath()).c_str())); - Standalone> _subdirs = wait(directory->list(instruction->tr)); - subdirs = _subdirs; - } else { - IDirectory::Path path = wait(popPath(data)); - logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); - Standalone> _subdirs = wait(directory->list(instruction->tr, path)); - subdirs = _subdirs; - } - - Tuple subdirTuple; - for (auto& sd : subdirs) { - subdirTuple.append(sd, true); - } - - data->stack.pushTuple(subdirTuple.pack()); - return Void(); - } -}; -const char* DirectoryListFunc::name = "DIRECTORY_LIST"; -REGISTER_INSTRUCTION_FUNC(DirectoryListFunc); - -// DIRECTORY_EXISTS -struct DirectoryExistsFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple count = wait(data->stack.waitAndPop()); - state Reference directory = data->directoryData.directory(); - state bool result; - if (count.getInt(0) == 0) { - bool _result = wait(directory->exists(instruction->tr)); - result = _result; - logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result)); - } else { - state IDirectory::Path path = wait(popPath(data)); - bool _result = wait(directory->exists(instruction->tr, path)); - result = _result; - logOp(format("exists %s: %d", pathToString(combinePaths(directory->getPath(), path)).c_str(), result)); - } - - data->stack.push(Tuple().append(result ? 1 : 0).pack()); - return Void(); - } -}; -const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS"; -REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc); - -// DIRECTORY_PACK_KEY -struct DirectoryPackKeyFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple tuple = wait(popTuple(data)); - data->stack.pushTuple(data->directoryData.subspace()->pack(tuple)); - - return Void(); - } -}; -const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY"; -REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc); - -// DIRECTORY_UNPACK_KEY -struct DirectoryUnpackKeyFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple key = wait(data->stack.waitAndPop()); - Subspace* subspace = data->directoryData.subspace(); - logOp(format("Unpack %s in subspace with prefix %s", - key.getString(0).printable().c_str(), - subspace->key().printable().c_str())); - Tuple tuple = subspace->unpack(key.getString(0)); - for (int i = 0; i < tuple.size(); ++i) { - data->stack.push(tuple.subTuple(i, i + 1).pack()); - } - - return Void(); - } -}; -const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY"; -REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc); - -// DIRECTORY_RANGE -struct DirectoryRangeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple tuple = wait(popTuple(data)); - KeyRange range = data->directoryData.subspace()->range(tuple); - data->stack.pushTuple(range.begin); - data->stack.pushTuple(range.end); - - return Void(); - } -}; -const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE"; -REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc); - -// DIRECTORY_CONTAINS -struct DirectoryContainsFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple key = wait(data->stack.waitAndPop()); - bool result = data->directoryData.subspace()->contains(key.getString(0)); - data->stack.push(Tuple().append(result ? 1 : 0).pack()); - - return Void(); - } -}; -const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS"; -REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc); - -// DIRECTORY_OPEN_SUBSPACE -struct DirectoryOpenSubspaceFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple tuple = wait(popTuple(data)); - Subspace* subspace = data->directoryData.subspace(); - logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str())); - Subspace* child = new Subspace(subspace->subspace(tuple)); - data->directoryData.push(child); - - return Void(); - } -}; -const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE"; -REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc); - -// DIRECTORY_LOG_SUBSPACE -struct DirectoryLogSubspaceFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple prefix = wait(data->stack.waitAndPop()); - Tuple tuple; - tuple.append(data->directoryData.directoryListIndex); - instruction->tr->set(Subspace(tuple, prefix.getString(0)).key(), data->directoryData.subspace()->key()); - - return Void(); - } -}; -const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE"; -REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc); - -// DIRECTORY_LOG_DIRECTORY -struct DirectoryLogDirectoryFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state Reference directory = data->directoryData.directory(); - state Tuple prefix = wait(data->stack.waitAndPop()); - state bool exists = wait(directory->exists(instruction->tr)); - - state Tuple childrenTuple; - if (exists) { - Standalone> children = wait(directory->list(instruction->tr)); - for (auto& c : children) { - childrenTuple.append(c, true); - } - } - - Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0)); - - Tuple pathTuple; - for (auto& p : directory->getPath()) { - pathTuple.append(p, true); - } - - instruction->tr->set(logSubspace.pack("path"_sr, true), pathTuple.pack()); - instruction->tr->set(logSubspace.pack("layer"_sr, true), Tuple().append(directory->getLayer()).pack()); - instruction->tr->set(logSubspace.pack("exists"_sr, true), Tuple().append(exists ? 1 : 0).pack()); - instruction->tr->set(logSubspace.pack("children"_sr, true), childrenTuple.pack()); - - return Void(); - } -}; -const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY"; -REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc); - -// DIRECTORY_STRIP_PREFIX -struct DirectoryStripPrefixFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Tuple str = wait(data->stack.waitAndPop()); - Subspace* subspace = data->directoryData.subspace(); - ASSERT(str.getString(0).startsWith(subspace->key())); - data->stack.pushTuple(str.getString(0).substr(subspace->key().size())); - return Void(); - } -}; -const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX"; -REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc); diff --git a/bindings/flow/tester/DirectoryTester.cpp b/bindings/flow/tester/DirectoryTester.cpp new file mode 100644 index 00000000000..e2c065c3592 --- /dev/null +++ b/bindings/flow/tester/DirectoryTester.cpp @@ -0,0 +1,539 @@ +/* + * DirectoryTester.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Tester.h" + +using namespace FDB; + +Future> popTuples(Reference data, int count = 1) { + std::vector tuples; + + while (tuples.size() < count) { + Standalone sizeStr = co_await data->stack.pop()[0].value; + int size = Tuple::unpack(sizeStr).getInt(0); + + std::vector tupleItems = data->stack.pop(size); + Tuple tuple; + + for (int index = 0; index < tupleItems.size(); ++index) { + Standalone itemStr = co_await tupleItems[index].value; + tuple.append(Tuple::unpack(itemStr)); + } + + tuples.push_back(tuple); + } + + co_return tuples; +} + +Future popTuple(Reference data) { + std::vector tuples = co_await popTuples(data); + co_return tuples[0]; +} + +Future> popPaths(Reference data, int count = 1) { + std::vector tuples = co_await popTuples(data, count); + + std::vector paths; + for (auto& tuple : tuples) { + IDirectory::Path path; + for (int i = 0; i < tuple.size(); ++i) { + path.push_back(tuple.getString(i)); + } + + paths.push_back(path); + } + + co_return paths; +} + +Future popPath(Reference data) { + std::vector paths = co_await popPaths(data); + co_return paths[0]; +} + +std::string pathToString(IDirectory::Path const& path) { + std::string str; + str += "["; + for (int i = 0; i < path.size(); ++i) { + str += path[i].toString(); + if (i < path.size() - 1) { + str += ", "; + } + } + + return str + "]"; +} + +IDirectory::Path combinePaths(IDirectory::Path const& path1, IDirectory::Path const& path2) { + IDirectory::Path outPath(path1.begin(), path1.end()); + for (const auto& p : path2) { + outPath.push_back(p); + } + + return outPath; +} + +void logOp(std::string message, bool force = false) { + if (LOG_OPS || force) { + printf("%s\n", message.c_str()); + fflush(stdout); + } +} + +// DIRECTORY_CREATE_SUBSPACE +struct DirectoryCreateSubspaceFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple path = co_await popTuple(data); + Tuple rawPrefix = co_await data->stack.waitAndPop(); + + logOp(format( + "Created subspace at %s: %s", tupleToString(path).c_str(), rawPrefix.getString(0).printable().c_str())); + data->directoryData.push(new Subspace(path, rawPrefix.getString(0))); + } +}; +const char* DirectoryCreateSubspaceFunc::name = "DIRECTORY_CREATE_SUBSPACE"; +REGISTER_INSTRUCTION_FUNC(DirectoryCreateSubspaceFunc); + +// DIRECTORY_CREATE_LAYER +struct DirectoryCreateLayerFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector args = co_await data->stack.waitAndPop(3); + + int index1 = args[0].getInt(0); + int index2 = args[1].getInt(0); + bool allowManualPrefixes = args[2].getInt(0) != 0; + + if (!data->directoryData.directoryList[index1].valid() || !data->directoryData.directoryList[index2].valid()) { + logOp("Create directory layer: None"); + data->directoryData.push(); + } else { + Subspace* nodeSubspace = data->directoryData.directoryList[index1].subspace.get(); + Subspace* contentSubspace = data->directoryData.directoryList[index2].subspace.get(); + logOp(format("Create directory layer: node_subspace (%d) = %s, content_subspace (%d) = %s, " + "allow_manual_prefixes = %d", + index1, + nodeSubspace->key().printable().c_str(), + index2, + nodeSubspace->key().printable().c_str(), + allowManualPrefixes)); + data->directoryData.push(Reference( + makeReference(*nodeSubspace, *contentSubspace, allowManualPrefixes))); + } + } +}; +const char* DirectoryCreateLayerFunc::name = "DIRECTORY_CREATE_LAYER"; +REGISTER_INSTRUCTION_FUNC(DirectoryCreateLayerFunc); + +// DIRECTORY_CHANGE +struct DirectoryChangeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple index = co_await data->stack.waitAndPop(); + data->directoryData.directoryListIndex = index.getInt(0); + ASSERT(data->directoryData.directoryListIndex < data->directoryData.directoryList.size()); + + if (!data->directoryData.directoryList[data->directoryData.directoryListIndex].valid()) { + data->directoryData.directoryListIndex = data->directoryData.directoryErrorIndex; + } + + if (LOG_DIRS) { + DirectoryOrSubspace d = data->directoryData.directoryList[data->directoryData.directoryListIndex]; + printf("Changed directory to %d (%s @\'%s\')\n", + data->directoryData.directoryListIndex, + d.typeString().c_str(), + d.directory.present() ? pathToString(d.directory.get()->getPath()).c_str() + : d.subspace.get()->key().printable().c_str()); + fflush(stdout); + } + } +}; +const char* DirectoryChangeFunc::name = "DIRECTORY_CHANGE"; +REGISTER_INSTRUCTION_FUNC(DirectoryChangeFunc); + +// DIRECTORY_SET_ERROR_INDEX +struct DirectorySetErrorIndexFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple index = co_await data->stack.waitAndPop(); + data->directoryData.directoryErrorIndex = index.getInt(0); + } +}; +const char* DirectorySetErrorIndexFunc::name = "DIRECTORY_SET_ERROR_INDEX"; +REGISTER_INSTRUCTION_FUNC(DirectorySetErrorIndexFunc); + +// DIRECTORY_CREATE_OR_OPEN +struct DirectoryCreateOrOpenFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + IDirectory::Path path = co_await popPath(data); + Tuple layerTuple = co_await data->stack.waitAndPop(); + Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); + + Reference directory = data->directoryData.directory(); + logOp(format("create_or_open %s: layer=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str())); + + Reference dirSubspace = + co_await executeMutation(instruction, [instruction, directory, path, layer]() { + return directory->createOrOpen(instruction->tr, path, layer); + }); + + data->directoryData.push(dirSubspace); + } +}; +const char* DirectoryCreateOrOpenFunc::name = "DIRECTORY_CREATE_OR_OPEN"; +REGISTER_INSTRUCTION_FUNC(DirectoryCreateOrOpenFunc); + +// DIRECTORY_CREATE +struct DirectoryCreateFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + IDirectory::Path path = co_await popPath(data); + std::vector args = co_await data->stack.waitAndPop(2); + Standalone layer = args[0].getType(0) == Tuple::NULL_TYPE ? StringRef() : args[0].getString(0); + Optional> prefix = + args[1].getType(0) == Tuple::NULL_TYPE ? Optional>() : args[1].getString(0); + + Reference directory = data->directoryData.directory(); + logOp(format("create %s: layer=%s, prefix=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str(), + prefix.present() ? prefix.get().printable().c_str() : "")); + + Reference dirSubspace = + co_await executeMutation(instruction, [instruction, directory, path, layer, prefix]() { + return directory->create(instruction->tr, path, layer, prefix); + }); + + data->directoryData.push(dirSubspace); + } +}; +const char* DirectoryCreateFunc::name = "DIRECTORY_CREATE"; +REGISTER_INSTRUCTION_FUNC(DirectoryCreateFunc); + +// DIRECTORY_OPEN +struct DirectoryOpenFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + IDirectory::Path path = co_await popPath(data); + Tuple layerTuple = co_await data->stack.waitAndPop(); + Standalone layer = layerTuple.getType(0) == Tuple::NULL_TYPE ? StringRef() : layerTuple.getString(0); + + Reference directory = data->directoryData.directory(); + logOp(format("open %s: layer=%s", + pathToString(combinePaths(directory->getPath(), path)).c_str(), + layer.printable().c_str())); + Reference dirSubspace = co_await directory->open(instruction->tr, path, layer); + data->directoryData.push(dirSubspace); + } +}; +const char* DirectoryOpenFunc::name = "DIRECTORY_OPEN"; +REGISTER_INSTRUCTION_FUNC(DirectoryOpenFunc); + +// DIRECTORY_MOVE +struct DirectoryMoveFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector paths = co_await popPaths(data, 2); + + Reference directory = data->directoryData.directory(); + logOp(format("move %s to %s", + pathToString(combinePaths(directory->getPath(), paths[0])).c_str(), + pathToString(combinePaths(directory->getPath(), paths[1])).c_str())); + + Reference dirSubspace = + co_await executeMutation(instruction, [instruction, directory, paths]() { + return directory->move(instruction->tr, paths[0], paths[1]); + }); + + data->directoryData.push(dirSubspace); + } +}; +const char* DirectoryMoveFunc::name = "DIRECTORY_MOVE"; +REGISTER_INSTRUCTION_FUNC(DirectoryMoveFunc); + +// DIRECTORY_MOVE_TO +struct DirectoryMoveToFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + IDirectory::Path path = co_await popPath(data); + + Reference directory = data->directoryData.directory(); + logOp(format("move %s to %s", pathToString(directory->getPath()).c_str(), pathToString(path).c_str())); + + Reference dirSubspace = co_await executeMutation( + instruction, [instruction, directory, path]() { return directory->moveTo(instruction->tr, path); }); + + data->directoryData.push(dirSubspace); + } +}; +const char* DirectoryMoveToFunc::name = "DIRECTORY_MOVE_TO"; +REGISTER_INSTRUCTION_FUNC(DirectoryMoveToFunc); + +// DIRECTORY_REMOVE +struct DirectoryRemoveFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple count = co_await data->stack.waitAndPop(); + Reference directory = data->directoryData.directory(); + if (count.getInt(0) == 0) { + logOp(format("remove %s", pathToString(directory->getPath()).c_str())); + + co_await executeMutation(instruction, + [instruction, directory]() { return directory->remove(instruction->tr); }); + } else { + IDirectory::Path path = co_await popPath(data); + logOp(format("remove %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); + + co_await executeMutation( + instruction, [instruction, directory, path]() { return directory->remove(instruction->tr, path); }); + } + } +}; +const char* DirectoryRemoveFunc::name = "DIRECTORY_REMOVE"; +REGISTER_INSTRUCTION_FUNC(DirectoryRemoveFunc); + +// DIRECTORY_REMOVE_IF_EXISTS +struct DirectoryRemoveIfExistsFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple count = co_await data->stack.waitAndPop(); + Reference directory = data->directoryData.directory(); + if (count.getInt(0) == 0) { + logOp(format("remove_if_exists %s", pathToString(directory->getPath()).c_str())); + + co_await executeMutation(instruction, + [instruction, directory]() { return directory->removeIfExists(instruction->tr); }); + } else { + IDirectory::Path path = co_await popPath(data); + logOp(format("remove_if_exists %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); + + co_await executeMutation(instruction, [instruction, directory, path]() { + return directory->removeIfExists(instruction->tr, path); + }); + } + } +}; +const char* DirectoryRemoveIfExistsFunc::name = "DIRECTORY_REMOVE_IF_EXISTS"; +REGISTER_INSTRUCTION_FUNC(DirectoryRemoveIfExistsFunc); + +// DIRECTORY_LIST +struct DirectoryListFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple count = co_await data->stack.waitAndPop(); + Reference directory = data->directoryData.directory(); + Standalone> subdirs; + if (count.getInt(0) == 0) { + logOp(format("list %s", pathToString(directory->getPath()).c_str())); + Standalone> _subdirs = co_await directory->list(instruction->tr); + subdirs = _subdirs; + } else { + IDirectory::Path path = co_await popPath(data); + logOp(format("list %s", pathToString(combinePaths(directory->getPath(), path)).c_str())); + Standalone> _subdirs = co_await directory->list(instruction->tr, path); + subdirs = _subdirs; + } + + Tuple subdirTuple; + for (auto& sd : subdirs) { + subdirTuple.append(sd, true); + } + + data->stack.pushTuple(subdirTuple.pack()); + } +}; +const char* DirectoryListFunc::name = "DIRECTORY_LIST"; +REGISTER_INSTRUCTION_FUNC(DirectoryListFunc); + +// DIRECTORY_EXISTS +struct DirectoryExistsFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple count = co_await data->stack.waitAndPop(); + Reference directory = data->directoryData.directory(); + bool result{ false }; + if (count.getInt(0) == 0) { + bool _result = co_await directory->exists(instruction->tr); + result = _result; + logOp(format("exists %s: %d", pathToString(directory->getPath()).c_str(), result)); + } else { + IDirectory::Path path = co_await popPath(data); + bool _result = co_await directory->exists(instruction->tr, path); + result = _result; + logOp(format("exists %s: %d", pathToString(combinePaths(directory->getPath(), path)).c_str(), result)); + } + + data->stack.push(Tuple().append(result ? 1 : 0).pack()); + } +}; +const char* DirectoryExistsFunc::name = "DIRECTORY_EXISTS"; +REGISTER_INSTRUCTION_FUNC(DirectoryExistsFunc); + +// DIRECTORY_PACK_KEY +struct DirectoryPackKeyFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple tuple = co_await popTuple(data); + data->stack.pushTuple(data->directoryData.subspace()->pack(tuple)); + } +}; +const char* DirectoryPackKeyFunc::name = "DIRECTORY_PACK_KEY"; +REGISTER_INSTRUCTION_FUNC(DirectoryPackKeyFunc); + +// DIRECTORY_UNPACK_KEY +struct DirectoryUnpackKeyFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple key = co_await data->stack.waitAndPop(); + Subspace* subspace = data->directoryData.subspace(); + logOp(format("Unpack %s in subspace with prefix %s", + key.getString(0).printable().c_str(), + subspace->key().printable().c_str())); + Tuple tuple = subspace->unpack(key.getString(0)); + for (int i = 0; i < tuple.size(); ++i) { + data->stack.push(tuple.subTuple(i, i + 1).pack()); + } + } +}; +const char* DirectoryUnpackKeyFunc::name = "DIRECTORY_UNPACK_KEY"; +REGISTER_INSTRUCTION_FUNC(DirectoryUnpackKeyFunc); + +// DIRECTORY_RANGE +struct DirectoryRangeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple tuple = co_await popTuple(data); + KeyRange range = data->directoryData.subspace()->range(tuple); + data->stack.pushTuple(range.begin); + data->stack.pushTuple(range.end); + } +}; +const char* DirectoryRangeFunc::name = "DIRECTORY_RANGE"; +REGISTER_INSTRUCTION_FUNC(DirectoryRangeFunc); + +// DIRECTORY_CONTAINS +struct DirectoryContainsFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple key = co_await data->stack.waitAndPop(); + bool result = data->directoryData.subspace()->contains(key.getString(0)); + data->stack.push(Tuple().append(result ? 1 : 0).pack()); + } +}; +const char* DirectoryContainsFunc::name = "DIRECTORY_CONTAINS"; +REGISTER_INSTRUCTION_FUNC(DirectoryContainsFunc); + +// DIRECTORY_OPEN_SUBSPACE +struct DirectoryOpenSubspaceFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple tuple = co_await popTuple(data); + Subspace* subspace = data->directoryData.subspace(); + logOp(format("open_subspace %s (at %s)", tupleToString(tuple).c_str(), subspace->key().printable().c_str())); + Subspace* child = new Subspace(subspace->subspace(tuple)); + data->directoryData.push(child); + } +}; +const char* DirectoryOpenSubspaceFunc::name = "DIRECTORY_OPEN_SUBSPACE"; +REGISTER_INSTRUCTION_FUNC(DirectoryOpenSubspaceFunc); + +// DIRECTORY_LOG_SUBSPACE +struct DirectoryLogSubspaceFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple prefix = co_await data->stack.waitAndPop(); + Tuple tuple; + tuple.append(data->directoryData.directoryListIndex); + instruction->tr->set(Subspace(tuple, prefix.getString(0)).key(), data->directoryData.subspace()->key()); + } +}; +const char* DirectoryLogSubspaceFunc::name = "DIRECTORY_LOG_SUBSPACE"; +REGISTER_INSTRUCTION_FUNC(DirectoryLogSubspaceFunc); + +// DIRECTORY_LOG_DIRECTORY +struct DirectoryLogDirectoryFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Reference directory = data->directoryData.directory(); + Tuple prefix = co_await data->stack.waitAndPop(); + bool exists = co_await directory->exists(instruction->tr); + + Tuple childrenTuple; + if (exists) { + Standalone> children = co_await directory->list(instruction->tr); + for (auto& c : children) { + childrenTuple.append(c, true); + } + } + + Subspace logSubspace(Tuple().append(data->directoryData.directoryListIndex), prefix.getString(0)); + + Tuple pathTuple; + for (auto& p : directory->getPath()) { + pathTuple.append(p, true); + } + + instruction->tr->set(logSubspace.pack("path"_sr, true), pathTuple.pack()); + instruction->tr->set(logSubspace.pack("layer"_sr, true), Tuple().append(directory->getLayer()).pack()); + instruction->tr->set(logSubspace.pack("exists"_sr, true), Tuple().append(exists ? 1 : 0).pack()); + instruction->tr->set(logSubspace.pack("children"_sr, true), childrenTuple.pack()); + } +}; +const char* DirectoryLogDirectoryFunc::name = "DIRECTORY_LOG_DIRECTORY"; +REGISTER_INSTRUCTION_FUNC(DirectoryLogDirectoryFunc); + +// DIRECTORY_STRIP_PREFIX +struct DirectoryStripPrefixFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Tuple str = co_await data->stack.waitAndPop(); + Subspace* subspace = data->directoryData.subspace(); + ASSERT(str.getString(0).startsWith(subspace->key())); + data->stack.pushTuple(str.getString(0).substr(subspace->key().size())); + } +}; +const char* DirectoryStripPrefixFunc::name = "DIRECTORY_STRIP_PREFIX"; +REGISTER_INSTRUCTION_FUNC(DirectoryStripPrefixFunc); diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp deleted file mode 100644 index f1aaf401cf6..00000000000 --- a/bindings/flow/tester/Tester.actor.cpp +++ /dev/null @@ -1,1950 +0,0 @@ -/* - * Tester.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "tester/Tester.actor.h" -#include -#ifdef __linux__ -#include -#endif - -#include "bindings/flow/Tuple.h" -#include "bindings/flow/FDBLoanerTypes.h" -#include "fdbrpc/fdbrpc.h" -#include "flow/DeterministicRandom.h" -#include "flow/TLSConfig.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -// Otherwise we have to type setupNetwork(), FDB::open(), etc. -using namespace FDB; - -std::map optionInfo; -std::set opsThatCreateDirectories; - -std::map, Reference> trMap; - -// NOTE: This was taken from within fdb_c.cpp (where it is defined as a static within the get_range function). -// If that changes, this will also have to be changed. -const int ITERATION_PROGRESSION[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; -const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION) / sizeof(int); - -static Future runTest(Reference const& data, - Reference const& db, - StringRef const& prefix); - -THREAD_FUNC networkThread(void* api) { - // This is the fdb_flow network we're running on a thread - ((API*)api)->runNetwork(); - THREAD_RETURN; -} - -bool hasEnding(std::string const& fullString, std::string const& ending) { - if (fullString.length() >= ending.length()) { - return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending)); - } else { - return false; - } -} - -ACTOR Future> waitAndPop(FlowTesterStack* self, int count) { - state std::vector tuples; - state std::vector items = self->pop(count); - - state int index; - for (index = 0; index < items.size(); ++index) { - Standalone itemStr = wait(items[index].value); - tuples.push_back(Tuple::unpack(itemStr)); - } - - return tuples; -} - -Future> FlowTesterStack::waitAndPop(int count) { - return ::waitAndPop(this, count); -} - -ACTOR Future waitAndPop(FlowTesterStack* self) { - std::vector tuples = wait(waitAndPop(self, 1)); - return tuples[0]; -} - -Future FlowTesterStack::waitAndPop() { - return ::waitAndPop(this); -} - -std::string tupleToString(Tuple const& tuple) { - std::string str = "("; - for (int i = 0; i < tuple.size(); ++i) { - Tuple::ElementType type = tuple.getType(i); - if (type == Tuple::NULL_TYPE) { - str += "NULL"; - } else if (type == Tuple::BYTES || type == Tuple::UTF8) { - if (type == Tuple::UTF8) { - str += "u"; - } - str += "\'" + tuple.getString(i).printable() + "\'"; - } else if (type == Tuple::INT) { - str += format("%ld", tuple.getInt(i)); - } else if (type == Tuple::FLOAT) { - str += format("%f", tuple.getFloat(i)); - } else if (type == Tuple::DOUBLE) { - str += format("%f", tuple.getDouble(i)); - } else if (type == Tuple::BOOL) { - str += tuple.getBool(i) ? "true" : "false"; - } else if (type == Tuple::UUID) { - Uuid u = tuple.getUuid(i); - str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8)); - } else if (type == Tuple::NESTED) { - str += tupleToString(tuple.getNested(i)); - } else if (type == Tuple::VERSIONSTAMP) { - Versionstamp versionstamp = tuple.getVersionstamp(i); - str += format("Transaction Version: '%ld', BatchNumber: '%hd', UserVersion : '%hd'", - versionstamp.getVersion(), - versionstamp.getBatchNumber(), - versionstamp.getUserVersion()); - } else { - ASSERT(false); - } - - if (i < tuple.size() - 1) { - str += ", "; - } - } - - str += ")"; - return str; -} - -ACTOR Future> getRange(Reference tr, - KeySelectorRef begin, - KeySelectorRef end, - int limits = 0, - bool snapshot = false, - bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - state KeySelector ks_begin(begin); - state KeySelector ks_end(end); - state Standalone results; - state int iteration = 1; - loop { - // printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), - // limits); - state FDBStandalone r; - if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { - int effective_iteration = std::min(iteration, MAX_ITERATION); - int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; - FDBStandalone rTemp = wait(tr->getRange(ks_begin, - ks_end, - GetRangeLimits(limits, bytes_limit), - snapshot, - reverse, - (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); - r = rTemp; - } else { - FDBStandalone rTemp = - wait(tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode)); - r = rTemp; - } - iteration += 1; - // printf("=====DB: count:%d\n", r.size()); - for (auto& s : r) { - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), - // printable(StringRef(s.value)).c_str()); - results.push_back_deep(results.arena(), s); - - if (reverse) - ks_end = KeySelector(firstGreaterOrEqual(s.key)); - else - ks_begin = KeySelector(firstGreaterThan(s.key)); - } - - ASSERT(limits == 0 || limits >= r.size()); - - if (!r.more || (limits > 0 && limits == r.size())) { - return results; - } - - if (limits > 0) { - limits -= r.size(); - } - } -} - -ACTOR Future> getRange(Reference tr, - KeyRange keys, - int limits = 0, - bool snapshot = false, - bool reverse = false, - FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { - state Key begin(keys.begin); - state Key end(keys.end); - state Standalone results; - state int iteration = 1; - loop { - // printf("=====DB: begin:%s, limits:%d\n", printable(begin).c_str(), limits); - KeyRange keyRange(KeyRangeRef(begin, end > begin ? end : begin)); - state FDBStandalone r; - if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { - int effective_iteration = std::min(iteration, MAX_ITERATION); - int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; - FDBStandalone rTemp = wait(tr->getRange(keyRange, - GetRangeLimits(limits, bytes_limit), - snapshot, - reverse, - (FDBStreamingMode)FDB_STREAMING_MODE_EXACT)); - r = rTemp; - } else { - FDBStandalone rTemp = - wait(tr->getRange(keyRange, limits, snapshot, reverse, streamingMode)); - r = rTemp; - } - iteration += 1; - // printf("=====DB: count:%d\n", r.size()); - for (auto& s : r) { - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), - // printable(StringRef(s.value)).c_str()); - results.push_back_deep(results.arena(), s); - - if (reverse) - end = s.key; - else - begin = keyAfter(s.key); - } - - ASSERT(limits == 0 || limits >= r.size()); - - if (!r.more || (limits > 0 && limits == r.size())) { - return results; - } - - if (limits > 0) { - limits -= r.size(); - } - } -} - -// ACTOR static Future debugPrintRange(Reference tr, std::string subspace, std::string msg) { -// if (!tr) -// return Void(); -// -// Standalone results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + -//'\xff')))); printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), -// StringRef(subspace).printable().c_str(), results.size()); -// for (auto & s : results) { -// printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str()); -// } -// -// return Void(); -//} - -ACTOR Future stackSub(FlowTesterStack* stack) { - if (stack->data.size() < 2) - return Void(); - - StackItem a = stack->data.back(); - stack->data.pop_back(); - state Standalone sa = wait(a.value); - - StackItem b = stack->data.back(); - stack->data.pop_back(); - Standalone sb = wait(b.value); - - int64_t c = Tuple::unpack(sa).getInt(0) - Tuple::unpack(sb).getInt(0); - Tuple f; - f.append(c); - stack->push(f.pack()); - - return Void(); -} - -ACTOR Future stackConcat(FlowTesterStack* stack) { - if (stack->data.size() < 2) - return Void(); - - StackItem a = stack->data.back(); - stack->data.pop_back(); - state Standalone sa = wait(a.value); - state Tuple ta = Tuple::unpack(sa); - - StackItem b = stack->data.back(); - stack->data.pop_back(); - Standalone sb = wait(b.value); - state Tuple tb = Tuple::unpack(sb); - - ASSERT(ta.getType(0) == tb.getType(0)); - stack->pushTuple(tb.getString(0).withPrefix(ta.getString(0)), ta.getType(0) == Tuple::ElementType::UTF8); - - return Void(); -} - -ACTOR Future stackSwap(FlowTesterStack* stack) { - if (stack->data.size() < 3) - return Void(); - - StackItem pop = stack->data.back(); - stack->data.pop_back(); - - Standalone sv = wait(pop.value); - int64_t idx = stack->data.size() - 1; - int64_t idx1 = idx - Tuple::unpack(sv).getInt(0); - if (idx1 < idx) { - // printf("=============SWAP:%d,%d\n", idx, stack->data.size()); - StackItem item = stack->data[idx]; - stack->data[idx] = stack->data[idx1]; - stack->data[idx1] = item; - } - return Void(); -} - -ACTOR Future printFlowTesterStack(FlowTesterStack* stack) { - // printf("====================stack item count:%ld\n", stack->data.size()); - state int idx; - for (idx = stack->data.size() - 1; idx >= 0; --idx) { - Standalone value = wait(stack->data[idx].value); - // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, - // value.printable().c_str()); - } - return Void(); -} - -// -// Data Operations -// - -struct PushFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - Tuple t = Tuple::unpack(instruction->instruction); - Standalone param = t.subTuple(1).pack(); - data->stack.push(param); - return Void(); - } -}; -const char* PushFunc::name = "PUSH"; -REGISTER_INSTRUCTION_FUNC(PushFunc); - -struct DupFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - data->stack.dup(); - return Void(); - } -}; -const char* DupFunc::name = "DUP"; -REGISTER_INSTRUCTION_FUNC(DupFunc); - -struct EmptyStackFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - // wait(printFlowTesterStack(&(data->stack))); - // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); - data->stack.clear(); - return Void(); - } -}; -const char* EmptyStackFunc::name = "EMPTY_STACK"; -REGISTER_INSTRUCTION_FUNC(EmptyStackFunc); - -struct SwapFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - wait(stackSwap(&(data->stack))); - return Void(); - } -}; -const char* SwapFunc::name = "SWAP"; -REGISTER_INSTRUCTION_FUNC(SwapFunc); - -struct PopFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - for (StackItem item : items) { - wait(success(item.value)); - } - return Void(); - } -}; -const char* PopFunc::name = "POP"; -REGISTER_INSTRUCTION_FUNC(PopFunc); - -struct SubFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - wait(stackSub(&(data->stack))); - return Void(); - } -}; -const char* SubFunc::name = "SUB"; -REGISTER_INSTRUCTION_FUNC(SubFunc); - -struct ConcatFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - wait(stackConcat(&(data->stack))); - return Void(); - } -}; -const char* ConcatFunc::name = "CONCAT"; -REGISTER_INSTRUCTION_FUNC(ConcatFunc); - -struct LogStackFunc : InstructionFunc { - static const char* name; - - ACTOR static Future logStack(Reference data, - std::map entries, - Standalone prefix) { - loop { - state Reference tr = data->db->createTransaction(); - try { - for (auto it : entries) { - Tuple tk; - tk.append(it.first); - tk.append((int64_t)it.second.index); - state Standalone pk = tk.pack().withPrefix(prefix); - Standalone pv = wait(it.second.value); - tr->set(pk, pv.substr(0, std::min(pv.size(), 40000))); - } - - wait(tr->commit()); - return Void(); - } catch (Error& e) { - wait(tr->onError(e)); - } - } - } - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.empty()) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone prefix = Tuple::unpack(s1).getString(0); - - state std::map entries; - while (data->stack.data.size() > 0) { - state std::vector it = data->stack.pop(); - ASSERT(it.size() == 1); - entries[data->stack.data.size()] = it.front(); - if (entries.size() == 100) { - wait(logStack(data, entries, prefix)); - entries.clear(); - } - } - wait(logStack(data, entries, prefix)); - - return Void(); - } -}; -const char* LogStackFunc::name = "LOG_STACK"; -REGISTER_INSTRUCTION_FUNC(LogStackFunc); - -// -// FoundationDB Operations -// -ACTOR Future> waitForVoid(Future f) { - try { - wait(f); - Tuple t; - t.append("RESULT_NOT_PRESENT"_sr); - return t.pack(); - } catch (Error& e) { - // printf("FDBError1:%d\n", e.code()); - Tuple t; - t.append("ERROR"_sr); - t.append(format("%d", e.code())); - // pack above as error string into another tuple - Tuple ret; - ret.append(t.pack()); - return ret.pack(); - } -} - -ACTOR Future> waitForValue(Future> f) { - try { - FDBStandalone value = wait(f); - Tuple t; - t.append(value); - return t.pack(); - } catch (Error& e) { - // printf("FDBError2:%d\n", e.code()); - Tuple t; - t.append("ERROR"_sr); - t.append(format("%d", e.code())); - // pack above as error string into another tuple - Tuple ret; - ret.append(t.pack()); - return ret.pack(); - } -} - -ACTOR Future> waitForValue(Future>> f) { - try { - Optional> value = wait(f); - Standalone str; - if (value.present()) - str = value.get(); - else - str = "RESULT_NOT_PRESENT"_sr; - - Tuple t; - t.append(str); - return t.pack(); - } catch (Error& e) { - // printf("FDBError3:%d\n", e.code()); - Tuple t; - t.append("ERROR"_sr); - t.append(format("%d", e.code())); - // pack above as error string into another tuple - Tuple ret; - ret.append(t.pack()); - return ret.pack(); - } -} - -ACTOR Future> getKey(Future> f, Standalone prefixFilter) { - try { - FDBStandalone key = wait(f); - Tuple t; - - if (key.startsWith(prefixFilter)) { - t.append(key); - } else if (key < prefixFilter) { - t.append(prefixFilter); - } else { - t.append(strinc(prefixFilter)); - } - - return t.pack(); - } catch (Error& e) { - // printf("FDBError4:%d\n", e.code()); - Tuple t; - t.append("ERROR"_sr); - t.append(format("%d", e.code())); - // pack above as error string into another tuple - Tuple ret; - ret.append(t.pack()); - return ret.pack(); - } -} - -struct NewTransactionFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - trMap[data->trName] = data->db->createTransaction(); - return Void(); - } -}; -const char* NewTransactionFunc::name = "NEW_TRANSACTION"; -REGISTER_INSTRUCTION_FUNC(NewTransactionFunc); - -struct UseTransactionFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - Standalone name = wait(items[0].value); - data->trName = name; - - if (trMap.count(data->trName) == 0) { - trMap[data->trName] = data->db->createTransaction(); - } - return Void(); - } -}; -const char* UseTransactionFunc::name = "USE_TRANSACTION"; -REGISTER_INSTRUCTION_FUNC(UseTransactionFunc); - -struct OnErrorFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.empty()) - return Void(); - - Standalone value = wait(items[0].value); - int err_code = Tuple::unpack(value).getInt(0); - // printf("OnError:%d:%d:%s\n", err_code, items[0].index, printable(value).c_str()); - - data->stack.push(waitForVoid(instruction->tr->onError(Error(err_code)))); - return Void(); - } -}; -const char* OnErrorFunc::name = "ON_ERROR"; -REGISTER_INSTRUCTION_FUNC(OnErrorFunc); - -struct SetFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(2); - if (items.size() != 2) - return Void(); - - Standalone sk = wait(items[0].value); - state Standalone key = Tuple::unpack(sk).getString(0); - // if (instruction->isDatabase) - // printf("SetDatabase:%s, isDatabase:%d\n", printable(key).c_str(), instruction->isDatabase); - Standalone sv = wait(items[1].value); - Standalone value = Tuple::unpack(sv).getString(0); - // printf("SetDatabase:%s:%s:%s\n", printable(key).c_str(), printable(sv).c_str(), printable(value).c_str()); - - Reference instructionCopy = instruction; - Standalone keyCopy = key; - - Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, value]() -> Future { - instructionCopy->tr->set(keyCopy, value); - return Void(); - }); - - if (instruction->isDatabase) { - data->stack.push(waitForVoid(mutation)); - } else { - wait(mutation); - } - - return Void(); - } -}; -const char* SetFunc::name = "SET"; -REGISTER_INSTRUCTION_FUNC(SetFunc); - -struct GetFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone sk = wait(items[0].value); - state Standalone key = Tuple::unpack(sk).getString(0); - - Future>> fk = instruction->tr->get(StringRef(key), instruction->isSnapshot); - data->stack.push(waitForValue(holdWhile(instruction->tr, fk))); - - return Void(); - } -}; -const char* GetFunc::name = "GET"; -REGISTER_INSTRUCTION_FUNC(GetFunc); - -struct GetEstimatedRangeSize : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(2); - if (items.size() != 2) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone beginKey = Tuple::unpack(s1).getString(0); - - Standalone s2 = wait(items[1].value); - state Standalone endKey = Tuple::unpack(s2).getString(0); - Future fsize = instruction->tr->getEstimatedRangeSizeBytes(KeyRangeRef(beginKey, endKey)); - int64_t size = wait(fsize); - data->stack.pushTuple("GOT_ESTIMATED_RANGE_SIZE"_sr); - - return Void(); - } -}; -const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE"; -REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize); - -struct GetRangeSplitPoints : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(3); - if (items.size() != 3) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone beginKey = Tuple::unpack(s1).getString(0); - - Standalone s2 = wait(items[1].value); - state Standalone endKey = Tuple::unpack(s2).getString(0); - - Standalone s3 = wait(items[2].value); - state int64_t chunkSize = Tuple::unpack(s3).getInt(0); - - Future>> fsplitPoints = - instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize); - FDBStandalone> splitPoints = wait(fsplitPoints); - data->stack.pushTuple("GOT_RANGE_SPLIT_POINTS"_sr); - - return Void(); - } -}; -const char* GetRangeSplitPoints::name = "GET_RANGE_SPLIT_POINTS"; -REGISTER_INSTRUCTION_FUNC(GetRangeSplitPoints); - -struct GetKeyFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(4); - if (items.size() != 4) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone key = Tuple::unpack(s1).getString(0); - - Standalone s2 = wait(items[1].value); - state int64_t or_equal = Tuple::unpack(s2).getInt(0); - - Standalone s3 = wait(items[2].value); - state int64_t offset = Tuple::unpack(s3).getInt(0); - - Standalone s4 = wait(items[3].value); - Standalone prefix = Tuple::unpack(s4).getString(0); - - // printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset); - Future> fk = - instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot); - data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix)); - - return Void(); - } -}; -const char* GetKeyFunc::name = "GET_KEY"; -REGISTER_INSTRUCTION_FUNC(GetKeyFunc); - -struct GetReadVersionFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - Version v = wait(instruction->tr->getReadVersion()); - data->lastVersion = v; - data->stack.pushTuple("GOT_READ_VERSION"_sr); - return Void(); - } -}; -const char* GetReadVersionFunc::name = "GET_READ_VERSION"; -REGISTER_INSTRUCTION_FUNC(GetReadVersionFunc); - -struct SetReadVersionFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - instruction->tr->setReadVersion(data->lastVersion); - return Void(); - } -}; -const char* SetReadVersionFunc::name = "SET_READ_VERSION"; -REGISTER_INSTRUCTION_FUNC(SetReadVersionFunc); - -// GET_COMMITTED_VERSION -struct GetCommittedVersionFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - data->lastVersion = instruction->tr->getCommittedVersion(); - data->stack.pushTuple("GOT_COMMITTED_VERSION"_sr); - return Void(); - } -}; -const char* GetCommittedVersionFunc::name = "GET_COMMITTED_VERSION"; -REGISTER_INSTRUCTION_FUNC(GetCommittedVersionFunc); - -// GET_APPROXIMATE_SIZE -struct GetApproximateSizeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - int64_t _ = wait(instruction->tr->getApproximateSize()); - (void)_; // disable unused variable warning - data->stack.pushTuple("GOT_APPROXIMATE_SIZE"_sr); - return Void(); - } -}; -const char* GetApproximateSizeFunc::name = "GET_APPROXIMATE_SIZE"; -REGISTER_INSTRUCTION_FUNC(GetApproximateSizeFunc); - -// GET_VERSIONSTAMP -struct GetVersionstampFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - data->stack.push(waitForValue(instruction->tr->getVersionstamp())); - return Void(); - } -}; -const char* GetVersionstampFunc::name = "GET_VERSIONSTAMP"; -REGISTER_INSTRUCTION_FUNC(GetVersionstampFunc); - -// COMMIT -struct CommitFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - data->stack.push(waitForVoid(holdWhile(instruction->tr, instruction->tr->commit()))); - return Void(); - } -}; -const char* CommitFunc::name = "COMMIT"; -REGISTER_INSTRUCTION_FUNC(CommitFunc); - -// WAIT_FUTURE -struct WaitFutureFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone sk = wait(items[0].value); - data->stack.push(StackItem(items[0].index, sk)); - return Void(); - } -}; -const char* WaitFutureFunc::name = "WAIT_FUTURE"; -REGISTER_INSTRUCTION_FUNC(WaitFutureFunc); - -// CLEAR -struct ClearFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone sk = wait(items[0].value); - Standalone key = Tuple::unpack(sk).getString(0); - - Reference instructionCopy = instruction; - - Future mutation = executeMutation(instruction, [instructionCopy, key]() -> Future { - instructionCopy->tr->clear(key); - return Void(); - }); - - if (instruction->isDatabase) { - data->stack.push(waitForVoid(mutation)); - } else { - wait(mutation); - } - - return Void(); - } -}; -const char* ClearFunc::name = "CLEAR"; -REGISTER_INSTRUCTION_FUNC(ClearFunc); - -// RESET -struct ResetFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - instruction->tr->reset(); - return Void(); - } -}; -const char* ResetFunc::name = "RESET"; -REGISTER_INSTRUCTION_FUNC(ResetFunc); - -// CANCEL -struct CancelFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - instruction->tr->cancel(); - return Void(); - } -}; -const char* CancelFunc::name = "CANCEL"; -REGISTER_INSTRUCTION_FUNC(CancelFunc); - -struct GetRangeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(5); - if (items.size() != 5) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone begin = Tuple::unpack(s1).getString(0); - - Standalone s2 = wait(items[1].value); - state Standalone end = Tuple::unpack(s2).getString(0); - - Standalone s3 = wait(items[2].value); - state int limit = Tuple::unpack(s3).getInt(0); - - Standalone s4 = wait(items[3].value); - state int reverse = Tuple::unpack(s4).getInt(0); - - Standalone s5 = wait(items[4].value); - FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0); - - // printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), - // printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot); - - Standalone results = wait(getRange(instruction->tr, - KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), - limit, - instruction->isSnapshot, - reverse, - mode)); - Tuple t; - for (auto& s : results) { - t.append(s.key); - t.append(s.value); - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), - // printable(StringRef(s.value)).c_str()); - } - // printf("=====Results Count:%d, size:%d\n", results.size(), str.size()); - - data->stack.push(Tuple().append(t.pack()).pack()); - return Void(); - } -}; -const char* GetRangeFunc::name = "GET_RANGE"; -REGISTER_INSTRUCTION_FUNC(GetRangeFunc); - -struct GetRangeStartsWithFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(4); - if (items.size() != 4) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone prefix = Tuple::unpack(s1).getString(0); - - Standalone s2 = wait(items[1].value); - state int limit = Tuple::unpack(s2).getInt(0); - - Standalone s3 = wait(items[2].value); - state int reverse = Tuple::unpack(s3).getInt(0); - - Standalone s4 = wait(items[3].value); - FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0); - - // printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, - // reverse, mode, isSnapshot); - Standalone results = wait(getRange(instruction->tr, - KeyRange(KeyRangeRef(prefix, strinc(prefix))), - limit, - instruction->isSnapshot, - reverse, - mode)); - Tuple t; - // printf("=====Results Count:%d\n", results.size()); - for (auto& s : results) { - t.append(s.key); - t.append(s.value); - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), - // printable(StringRef(s.value)).c_str()); - } - - data->stack.push(Tuple().append(t.pack()).pack()); - return Void(); - } -}; -const char* GetRangeStartsWithFunc::name = "GET_RANGE_STARTS_WITH"; -REGISTER_INSTRUCTION_FUNC(GetRangeStartsWithFunc); - -struct ClearRangeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(2); - if (items.size() != 2) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone begin = Tuple::unpack(s1).getString(0); - - Standalone s2 = wait(items[1].value); - Standalone end = Tuple::unpack(s2).getString(0); - - Reference instructionCopy = instruction; - Standalone beginCopy = begin; - - Future mutation = executeMutation(instruction, [instructionCopy, beginCopy, end]() -> Future { - instructionCopy->tr->clear(KeyRangeRef(beginCopy, end)); - return Void(); - }); - - if (instruction->isDatabase) { - data->stack.push(waitForVoid(mutation)); - } else { - wait(mutation); - } - - return Void(); - } -}; -const char* ClearRangeFunc::name = "CLEAR_RANGE"; -REGISTER_INSTRUCTION_FUNC(ClearRangeFunc); - -struct ClearRangeStartWithFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - Standalone begin = Tuple::unpack(s1).getString(0); - - Reference instructionCopy = instruction; - - Future mutation = executeMutation(instruction, [instructionCopy, begin]() -> Future { - instructionCopy->tr->clear(KeyRangeRef(begin, strinc(begin))); - return Void(); - }); - - if (instruction->isDatabase) { - data->stack.push(waitForVoid(mutation)); - } else { - wait(mutation); - } - - return Void(); - } -}; -const char* ClearRangeStartWithFunc::name = "CLEAR_RANGE_STARTS_WITH"; -REGISTER_INSTRUCTION_FUNC(ClearRangeStartWithFunc); - -struct GetRangeSelectorFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(10); - if (items.size() != 10) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone begin = Tuple::unpack(s1).getString(0); - - Standalone s2 = wait(items[1].value); - state bool begin_or_equal = Tuple::unpack(s2).getInt(0); - - Standalone s3 = wait(items[2].value); - state int64_t begin_offset = Tuple::unpack(s3).getInt(0); - - Standalone s4 = wait(items[3].value); - state Standalone end = Tuple::unpack(s4).getString(0); - - Standalone s5 = wait(items[4].value); - state bool end_or_equal = Tuple::unpack(s5).getInt(0); - - Standalone s6 = wait(items[5].value); - state int64_t end_offset = Tuple::unpack(s6).getInt(0); - - Standalone s7 = wait(items[6].value); - state int limit = Tuple::unpack(s7).getInt(0); - - Standalone s8 = wait(items[7].value); - state int reverse = Tuple::unpack(s8).getInt(0); - - Standalone s9 = wait(items[8].value); - state FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s9).getInt(0); - - Standalone s10 = wait(items[9].value); - state Optional> prefix; - Tuple t10 = Tuple::unpack(s10); - if (t10.getType(0) != Tuple::ElementType::NULL_TYPE) { - prefix = t10.getString(0); - } - - // printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", - // printable(begin).c_str(), begin_or_equal, begin_offset, printable(end).c_str(), end_or_equal, end_offset, - // limit, reverse, mode, instruction->isSnapshot, printable(prefix).c_str()); - Future> f = getRange(instruction->tr, - KeySelectorRef(begin, begin_or_equal, begin_offset), - KeySelectorRef(end, end_or_equal, end_offset), - limit, - instruction->isSnapshot, - reverse, - mode); - Standalone results = wait(holdWhile(instruction->tr, f)); - Tuple t; - // printf("=====Results Count:%d\n", results.size()); - for (auto& s : results) { - if (!prefix.present() || s.key.startsWith(prefix.get())) { - t.append(s.key); - t.append(s.value); - // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), - // printable(StringRef(s.value)).c_str()); - } - } - - data->stack.push(Tuple().append(t.pack()).pack()); - return Void(); - } -}; -const char* GetRangeSelectorFunc::name = "GET_RANGE_SELECTOR"; -REGISTER_INSTRUCTION_FUNC(GetRangeSelectorFunc); - -// Tuple Operations -// TUPLE_PACK -struct TuplePackFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - state int64_t count = Tuple::unpack(s1).getInt(0); - - state std::vector items1 = data->stack.pop(count); - if (items1.size() != count) - return Void(); - - state Tuple tuple; - state int i = 0; - for (; i < items1.size(); ++i) { - Standalone str = wait(items1[i].value); - Tuple itemTuple = Tuple::unpack(str); - if (deterministicRandom()->coinflip()) { - Tuple::ElementType type = itemTuple.getType(0); - if (type == Tuple::NULL_TYPE) { - tuple.appendNull(); - } else if (type == Tuple::INT) { - tuple << itemTuple.getInt(0); - } else if (type == Tuple::BYTES) { - tuple.append(itemTuple.getString(0), false); - } else if (type == Tuple::UTF8) { - tuple.append(itemTuple.getString(0), true); - } else if (type == Tuple::FLOAT) { - tuple << itemTuple.getFloat(0); - } else if (type == Tuple::DOUBLE) { - tuple << itemTuple.getDouble(0); - } else if (type == Tuple::BOOL) { - tuple << itemTuple.getBool(0); - } else if (type == Tuple::UUID) { - tuple << itemTuple.getUuid(0); - } else if (type == Tuple::NESTED) { - tuple.appendNested(itemTuple.getNested(0)); - } else if (type == Tuple::VERSIONSTAMP) { - tuple.appendVersionstamp(itemTuple.getVersionstamp(0)); - } else { - ASSERT(false); - } - } else { - tuple << itemTuple; - } - } - - data->stack.pushTuple(tuple.pack()); - return Void(); - } -}; -const char* TuplePackFunc::name = "TUPLE_PACK"; -REGISTER_INSTRUCTION_FUNC(TuplePackFunc); - -// TUPLE_UNPACK -struct TupleUnpackFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - Tuple t = Tuple::unpack(Tuple::unpack(s1).getString(0)); - - for (int i = 0; i < t.size(); ++i) { - Standalone str = t.subTuple(i, i + 1).pack(); - // printf("=====value:%s\n", printable(str).c_str()); - data->stack.pushTuple(str); - } - return Void(); - } -}; -const char* TupleUnpackFunc::name = "TUPLE_UNPACK"; -REGISTER_INSTRUCTION_FUNC(TupleUnpackFunc); - -// TUPLE_RANGE -struct TupleRangeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - state int64_t count = Tuple::unpack(s1).getInt(0); - - state std::vector items1 = data->stack.pop(count); - if (items1.size() != count) - return Void(); - - state Tuple tuple; - state size_t i = 0; - for (; i < items1.size(); ++i) { - Standalone str = wait(items1[i].value); - Tuple itemTuple = Tuple::unpack(str); - if (deterministicRandom()->coinflip()) { - Tuple::ElementType type = itemTuple.getType(0); - if (type == Tuple::NULL_TYPE) { - tuple.appendNull(); - } else if (type == Tuple::INT) { - tuple << itemTuple.getInt(0); - } else if (type == Tuple::BYTES) { - tuple.append(itemTuple.getString(0), false); - } else if (type == Tuple::UTF8) { - tuple.append(itemTuple.getString(0), true); - } else if (type == Tuple::FLOAT) { - tuple << itemTuple.getFloat(0); - } else if (type == Tuple::DOUBLE) { - tuple << itemTuple.getDouble(0); - } else if (type == Tuple::BOOL) { - tuple << itemTuple.getBool(0); - } else if (type == Tuple::UUID) { - tuple << itemTuple.getUuid(0); - } else if (type == Tuple::NESTED) { - tuple.appendNested(itemTuple.getNested(0)); - } else if (type == Tuple::VERSIONSTAMP) { - tuple.appendVersionstamp(itemTuple.getVersionstamp(0)); - } else { - ASSERT(false); - } - } else { - tuple << itemTuple; - } - } - - KeyRange range = tuple.range(); - - data->stack.pushTuple(range.begin); - data->stack.pushTuple(range.end); - return Void(); - } -}; -const char* TupleRangeFunc::name = "TUPLE_RANGE"; -REGISTER_INSTRUCTION_FUNC(TupleRangeFunc); - -// TUPLE_SORT -struct TupleSortFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - state int64_t count = Tuple::unpack(s1).getInt(0); - - state std::vector items1 = data->stack.pop(count); - if (items1.size() != count) - return Void(); - - state std::vector tuples; - state size_t i = 0; - for (; i < items1.size(); i++) { - Standalone value = wait(items1[i].value); - tuples.push_back(Tuple::unpack(value)); - } - - std::sort(tuples.begin(), tuples.end()); - for (Tuple const& t : tuples) { - data->stack.push(t.pack()); - } - - return Void(); - } -}; -const char* TupleSortFunc::name = "TUPLE_SORT"; -REGISTER_INSTRUCTION_FUNC(TupleSortFunc); - -// ENCODE_FLOAT -struct EncodeFloatFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - Standalone fBytes = Tuple::unpack(s1).getString(0); - ASSERT(fBytes.size() == 4); - - int32_t intVal = *(int32_t*)fBytes.begin(); - intVal = bigEndian32(intVal); - float fVal = *(float*)&intVal; - - Tuple t; - t.append(fVal); - data->stack.push(t.pack()); - - return Void(); - } -}; -const char* EncodeFloatFunc::name = "ENCODE_FLOAT"; -REGISTER_INSTRUCTION_FUNC(EncodeFloatFunc); - -// ENCODE_DOUBLE -struct EncodeDoubleFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - Standalone dBytes = Tuple::unpack(s1).getString(0); - ASSERT(dBytes.size() == 8); - - int64_t intVal = *(int64_t*)dBytes.begin(); - intVal = bigEndian64(intVal); - double dVal = *(double*)&intVal; - - Tuple t; - t.append(dVal); - data->stack.push(t.pack()); - - return Void(); - } -}; -const char* EncodeDoubleFunc::name = "ENCODE_DOUBLE"; -REGISTER_INSTRUCTION_FUNC(EncodeDoubleFunc); - -// DECODE_FLOAT -struct DecodeFloatFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - float fVal = Tuple::unpack(s1).getFloat(0); - int32_t intVal = *(int32_t*)&fVal; - intVal = bigEndian32(intVal); - - Tuple t; - t.append(StringRef((uint8_t*)&intVal, 4), false); - data->stack.push(t.pack()); - - return Void(); - } -}; -const char* DecodeFloatFunc::name = "DECODE_FLOAT"; -REGISTER_INSTRUCTION_FUNC(DecodeFloatFunc); - -// DECODE_DOUBLE -struct DecodeDoubleFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - double dVal = Tuple::unpack(s1).getDouble(0); - int64_t intVal = *(int64_t*)&dVal; - intVal = bigEndian64(intVal); - - Tuple t; - t.append(StringRef((uint8_t*)&intVal, 8), false); - data->stack.push(t.pack()); - return Void(); - } -}; -const char* DecodeDoubleFunc::name = "DECODE_DOUBLE"; -REGISTER_INSTRUCTION_FUNC(DecodeDoubleFunc); - -// Thread Operations -// START_THREAD -struct StartThreadFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone prefix = Tuple::unpack(s1).getString(0); - // printf("=========START_THREAD:%s\n", printable(prefix).c_str()); - - Reference newData = Reference(new FlowTesterData(data->api)); - data->subThreads.push_back(runTest(newData, data->db, prefix)); - - return Void(); - } -}; -const char* StartThreadFunc::name = "START_THREAD"; -REGISTER_INSTRUCTION_FUNC(StartThreadFunc); - -ACTOR template -Future()(Reference()).getValue())> read(Reference db, - Function func) { - state Reference tr = db->createTransaction(); - loop { - try { - state decltype(std::declval()(Reference()).getValue()) result = wait(func(tr)); - return result; - } catch (Error& e) { - wait(tr->onError(e)); - } - } -} - -// WAIT_EMPTY -struct WaitEmptyFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - Standalone prefix = Tuple::unpack(s1).getString(0); - // printf("=========WAIT_EMPTY:%s\n", printable(prefix).c_str()); - - wait(read(data->db, - [=](Reference tr) -> Future { return checkEmptyPrefix(tr, prefix); })); - - return Void(); - } - -private: - ACTOR static Future checkEmptyPrefix(Reference tr, Standalone prefix) { - FDBStandalone results = wait(tr->getRange(KeyRangeRef(prefix, strinc(prefix)), 1)); - if (results.size() > 0) { - throw not_committed(); - } - return Void(); - } -}; -const char* WaitEmptyFunc::name = "WAIT_EMPTY"; -REGISTER_INSTRUCTION_FUNC(WaitEmptyFunc); - -// DISABLE_WRITE_CONFLICT -struct DisableWriteConflictFunc : InstructionFunc { - static const char* name; - - static Future call(Reference const& data, Reference const& instruction) { - if (instruction->tr) { - instruction->tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); - } - return Void(); - } -}; -const char* DisableWriteConflictFunc::name = "DISABLE_WRITE_CONFLICT"; -REGISTER_INSTRUCTION_FUNC(DisableWriteConflictFunc); - -// READ_CONFLICT_KEY -struct ReadConflictKeyFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone key = Tuple::unpack(s1).getString(0); - // printf("=========READ_CONFLICT_KEY:%s\n", printable(key).c_str()); - instruction->tr->addReadConflictKey(key); - - data->stack.pushTuple("SET_CONFLICT_KEY"_sr); - return Void(); - } -}; -const char* ReadConflictKeyFunc::name = "READ_CONFLICT_KEY"; -REGISTER_INSTRUCTION_FUNC(ReadConflictKeyFunc); - -// WRITE_CONFLICT_KEY -struct WriteConflictKeyFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(); - if (items.size() != 1) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone key = Tuple::unpack(s1).getString(0); - // printf("=========WRITE_CONFLICT_KEY:%s\n", printable(key).c_str()); - instruction->tr->addWriteConflictKey(key); - - data->stack.pushTuple("SET_CONFLICT_KEY"_sr); - return Void(); - } -}; -const char* WriteConflictKeyFunc::name = "WRITE_CONFLICT_KEY"; -REGISTER_INSTRUCTION_FUNC(WriteConflictKeyFunc); - -// READ_CONFLICT_RANGE -struct ReadConflictRangeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(2); - if (items.size() != 2) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone begin = Tuple::unpack(s1).getString(0); - Standalone s2 = wait(items[1].value); - state Standalone end = Tuple::unpack(s2).getString(0); - - // printf("=========READ_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str()); - instruction->tr->addReadConflictRange(KeyRange(KeyRangeRef(begin, end))); - data->stack.pushTuple("SET_CONFLICT_RANGE"_sr); - return Void(); - } -}; -const char* ReadConflictRangeFunc::name = "READ_CONFLICT_RANGE"; -REGISTER_INSTRUCTION_FUNC(ReadConflictRangeFunc); - -// WRITE_CONFLICT_RANGE -struct WriteConflictRangeFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(2); - if (items.size() != 2) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone begin = Tuple::unpack(s1).getString(0); - Standalone s2 = wait(items[1].value); - state Standalone end = Tuple::unpack(s2).getString(0); - - // printf("=========WRITE_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str()); - instruction->tr->addWriteConflictRange(KeyRange(KeyRangeRef(begin, end))); - - data->stack.pushTuple("SET_CONFLICT_RANGE"_sr); - return Void(); - } -}; -const char* WriteConflictRangeFunc::name = "WRITE_CONFLICT_RANGE"; -REGISTER_INSTRUCTION_FUNC(WriteConflictRangeFunc); - -// ATOMIC_OP -struct AtomicOPFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - state std::vector items = data->stack.pop(3); - if (items.size() != 3) - return Void(); - - Standalone s1 = wait(items[0].value); - state Standalone op = Tuple::unpack(s1).getString(0); - Standalone s2 = wait(items[1].value); - state Standalone key = Tuple::unpack(s2).getString(0); - Standalone s3 = wait(items[2].value); - state Standalone value = Tuple::unpack(s3).getString(0); - - ASSERT(optionInfo.find(op.toString()) != optionInfo.end()); - - FDBMutationType atomicOp = optionInfo[op.toString()]; - - Reference instructionCopy = instruction; - Standalone keyCopy = key; - Standalone valueCopy = value; - - // printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), - // printable(value).c_str()); - Future mutation = - executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp]() -> Future { - instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp); - return Void(); - }); - - if (instruction->isDatabase) { - data->stack.push(waitForVoid(mutation)); - } else { - wait(mutation); - } - - return Void(); - } -}; -const char* AtomicOPFunc::name = "ATOMIC_OP"; -REGISTER_INSTRUCTION_FUNC(AtomicOPFunc); - -// UNIT_TESTS -struct UnitTestsFunc : InstructionFunc { - static const char* name; - - ACTOR static Future call(Reference data, Reference instruction) { - ASSERT(data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(1020))); - ASSERT(!data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(10))); - - ASSERT(API::isAPIVersionSelected()); - state API* fdb = API::getInstance(); - ASSERT(fdb->getAPIVersion() <= FDB_API_VERSION); - try { - API::selectAPIVersion(fdb->getAPIVersion() + 1); - ASSERT(false); - } catch (Error& e) { - ASSERT(e.code() == error_code_api_version_already_set); - } - try { - API::selectAPIVersion(fdb->getAPIVersion() - 1); - ASSERT(false); - } catch (Error& e) { - ASSERT(e.code() == error_code_api_version_already_set); - } - API::selectAPIVersion(fdb->getAPIVersion()); - - const uint64_t locationCacheSize = 100001; - const uint64_t maxWatches = 10001; - const uint64_t timeout = 60 * 1000; - const uint64_t noTimeout = 0; - const uint64_t retryLimit = 50; - const uint64_t noRetryLimit = -1; - const uint64_t maxRetryDelay = 100; - const uint64_t sizeLimit = 100000; - const uint64_t maxFieldLength = 1000; - - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, - Optional(StringRef((const uint8_t*)&locationCacheSize, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, - Optional(StringRef((const uint8_t*)&maxWatches, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional("dc_id"_sr)); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional("machine_id"_sr)); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, - Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, - Optional(StringRef((const uint8_t*)&timeout, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, - Optional(StringRef((const uint8_t*)&noTimeout, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, - Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, - Optional(StringRef((const uint8_t*)&sizeLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, - Optional(StringRef((const uint8_t*)&retryLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, - Optional(StringRef((const uint8_t*)&noRetryLimit, 8))); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY); - data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS); - - state Reference tr = data->db->createTransaction(); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_BATCH); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_CAUSAL_READ_RISKY); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_CAUSAL_WRITE_RISKY); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, - Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, - Optional(StringRef((const uint8_t*)&timeout, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, - Optional(StringRef((const uint8_t*)&retryLimit, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, - Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, - Optional("my_transaction"_sr)); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS); - tr->setOption(FDBTransactionOption::FDB_TR_OPTION_REPORT_CONFLICTING_KEYS); - - Optional> _ = wait(tr->get("\xff"_sr)); - tr->cancel(); - - return Void(); - } -}; -const char* UnitTestsFunc::name = "UNIT_TESTS"; -REGISTER_INSTRUCTION_FUNC(UnitTestsFunc); - -ACTOR static Future getInstructions(Reference data, StringRef prefix) { - state Reference tr = data->db->createTransaction(); - - // get test instructions - state Tuple testSpec; - testSpec.append(prefix); - loop { - try { - Standalone results = wait(getRange(tr, testSpec.range())); - data->instructions = results; - return Void(); - } catch (Error& e) { - wait(tr->onError(e)); - } - } -} - -ACTOR static Future doInstructions(Reference data) { - // printf("Total num instructions:%d\n", data->instructions.size()); - state size_t idx = 0; - for (; idx < data->instructions.size(); ++idx) { - Tuple opTuple = Tuple::unpack(data->instructions[idx].value); - state Standalone op = opTuple.getString(0); - - state bool isDatabase = op.endsWith("_DATABASE"_sr); - state bool isSnapshot = op.endsWith("_SNAPSHOT"_sr); - state bool isDirectory = op.startsWith("DIRECTORY_"_sr); - - try { - if (LOG_INSTRUCTIONS) { - if (op != "SWAP"_sr && op != "PUSH"_sr) { - printf("%zu. %s\n", idx, tupleToString(opTuple).c_str()); - fflush(stdout); - } - } - - if (isDatabase) - op = op.substr(0, op.size() - 9); - else if (isSnapshot) - op = op.substr(0, op.size() - 9); - - // printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n", - // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), - // StringRef(data->instructions[idx].value).printable().c_str(), isDatabase, isSnapshot, - // data->stack.data.size()); - - // wait(printFlowTesterStack(&(data->stack))); - // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); - - state Reference instruction = Reference( - new InstructionData(isDatabase, isSnapshot, data->instructions[idx].value, Reference())); - if (isDatabase) { - state Reference tr = data->db->createTransaction(); - instruction->tr = tr; - } else { - instruction->tr = trMap[data->trName]; - } - - // Flow directory operations don't support snapshot reads - ASSERT(!isDirectory || !isSnapshot); - - data->stack.index = idx; - wait(InstructionFunc::call(op.toString(), data, instruction)); - } catch (Error& e) { - if (LOG_ERRORS) { - printf("Error: %s (%d)\n", e.name(), e.code()); - fflush(stdout); - } - - if (isDirectory) { - if (opsThatCreateDirectories.count(op.toString())) { - data->directoryData.directoryList.push_back(DirectoryOrSubspace()); - } - data->stack.pushTuple("DIRECTORY_ERROR"_sr); - } else { - data->stack.pushError(e.code()); - } - } - } - // printf("Total num instructions:%d\n", data->instructions.size()); - return Void(); -} - -ACTOR static Future runTest(Reference data, Reference db, StringRef prefix) { - ASSERT(data); - try { - data->db = db; - wait(getInstructions(data, prefix)); - wait(doInstructions(data)); - wait(waitForAll(data->subThreads)); - } catch (Error& e) { - TraceEvent(SevError, "FlowTesterDataRunError").error(e); - } - - return Void(); -} - -void populateAtomicOpMap() { - optionInfo["ADD"] = FDBMutationType::FDB_MUTATION_TYPE_ADD; - optionInfo["AND"] = FDBMutationType::FDB_MUTATION_TYPE_AND; - optionInfo["BIT_AND"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_AND; - optionInfo["OR"] = FDBMutationType::FDB_MUTATION_TYPE_OR; - optionInfo["BIT_OR"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_OR; - optionInfo["XOR"] = FDBMutationType::FDB_MUTATION_TYPE_XOR; - optionInfo["BIT_XOR"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_XOR; - optionInfo["APPEND_IF_FITS"] = FDBMutationType::FDB_MUTATION_TYPE_APPEND_IF_FITS; - optionInfo["MAX"] = FDBMutationType::FDB_MUTATION_TYPE_MAX; - optionInfo["MIN"] = FDBMutationType::FDB_MUTATION_TYPE_MIN; - optionInfo["SET_VERSIONSTAMPED_KEY"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY; - optionInfo["SET_VERSIONSTAMPED_VALUE"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE; - optionInfo["BYTE_MIN"] = FDBMutationType::FDB_MUTATION_TYPE_BYTE_MIN; - optionInfo["BYTE_MAX"] = FDBMutationType::FDB_MUTATION_TYPE_BYTE_MAX; -} - -void populateOpsThatCreateDirectories() { - opsThatCreateDirectories.insert("DIRECTORY_CREATE_SUBSPACE"); - opsThatCreateDirectories.insert("DIRECTORY_CREATE_LAYER"); - opsThatCreateDirectories.insert("DIRECTORY_CREATE_OR_OPEN"); - opsThatCreateDirectories.insert("DIRECTORY_CREATE"); - opsThatCreateDirectories.insert("DIRECTORY_OPEN"); - opsThatCreateDirectories.insert("DIRECTORY_MOVE"); - opsThatCreateDirectories.insert("DIRECTORY_MOVE_TO"); - opsThatCreateDirectories.insert("DIRECTORY_OPEN_SUBSPACE"); -} - -ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersion) { - try { - populateAtomicOpMap(); // FIXME: NOOOOOO! - populateOpsThatCreateDirectories(); // FIXME - - // This is "our" network - g_network = newNet2(TLSConfig()); - - ASSERT(!API::isAPIVersionSelected()); - try { - API::getInstance(); - ASSERT(false); - } catch (Error& e) { - ASSERT(e.code() == error_code_api_version_unset); - } - - API* fdb = API::selectAPIVersion(apiVersion); - ASSERT(API::isAPIVersionSelected()); - ASSERT(fdb->getAPIVersion() == apiVersion); - // fdb->setNetworkOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE); - - // We have to start the fdb_flow network and thread separately! - fdb->setupNetwork(); - startThread(networkThread, fdb); - - // Connect to the default cluster/database, and create a transaction - auto db = fdb->createDatabase(clusterFilename); - - Reference data = Reference(new FlowTesterData(fdb)); - wait(runTest(data, db, prefix)); - - // Stopping the network returns from g_network->run() and allows - // the program to terminate - g_network->stop(); - } catch (Error& e) { - TraceEvent("ErrorRunningTest").error(e); - if (LOG_ERRORS) { - printf("Flow tester encountered error: %s\n", e.name()); - fflush(stdout); - } - flushAndExit(1); - } -} - -ACTOR void _test_versionstamp() { - try { - g_network = newNet2(TLSConfig()); - - API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION); - - fdb->setupNetwork(); - startThread(networkThread, fdb); - - auto db = fdb->createDatabase(); - state Reference tr = db->createTransaction(); - - state Future> ftrVersion = tr->getVersionstamp(); - - tr->atomicOp( - "foo"_sr, "blahblahbl\x00\x00\x00\x00"_sr, FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); - - wait(tr->commit()); // should use retry loop - - tr->reset(); - - Optional> optionalDbVersion = wait(tr->get("foo"_sr)); - state FDBStandalone dbVersion = optionalDbVersion.get(); - FDBStandalone trVersion = wait(ftrVersion); - - ASSERT(trVersion.compare(dbVersion) == 0); - - fprintf(stderr, "%s\n", trVersion.printable().c_str()); - - g_network->stop(); - } catch (Error& e) { - TraceEvent("ErrorRunningTest").error(e); - if (LOG_ERRORS) { - printf("Flow tester encountered error: %s\n", e.name()); - fflush(stdout); - } - flushAndExit(1); - } -} - -int main(int argc, char** argv) { - try { - platformInit(); - registerCrashHandler(); - setThreadLocalDeterministicRandomSeed(1); - - // Get arguments - if (argc < 3) { - fprintf(stderr, "Missing arguments! Usage: fdb_flow_tester prefix api_version [cluster_filename]\n"); - return 1; - - /*_test_versionstamp(); - g_network->run(); - - flushAndExit(FDB_EXIT_SUCCESS);*/ - } - StringRef prefix((const uint8_t*)argv[1], strlen(argv[1])); - int apiVersion; - sscanf(argv[2], "%d", &apiVersion); - std::string clusterFilename; - if (argc > 3) { - clusterFilename = std::string(argv[3]); - } - - // start test - startTest(clusterFilename, prefix, apiVersion); - - // Run the network until someone tells us to stop - g_network->run(); - - flushAndExit(FDB_EXIT_SUCCESS); - } catch (Error& e) { - fprintf(stderr, "Error: %s\n", e.name()); - TraceEvent(SevError, "MainError").error(e); - flushAndExit(FDB_EXIT_MAIN_ERROR); - } catch (std::exception& e) { - fprintf(stderr, "std::exception: %s\n", e.what()); - TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); - flushAndExit(FDB_EXIT_MAIN_EXCEPTION); - } -} diff --git a/bindings/flow/tester/Tester.actor.h b/bindings/flow/tester/Tester.actor.h deleted file mode 100644 index 96890bcfdcc..00000000000 --- a/bindings/flow/tester/Tester.actor.h +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Tester.actor.h - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// When actually compiled (NO_INTELLISENSE), include the generated version of this file. In intellisense use the source -// version. -#if defined(NO_INTELLISENSE) && !defined(FDB_FLOW_TESTER_TESTER_ACTOR_G_H) -#define FDB_FLOW_TESTER_TESTER_ACTOR_G_H -#include "tester/Tester.actor.g.h" -#elif !defined(FDB_FLOW_TESTER_TESTER_ACTOR_H) -#define FDB_FLOW_TESTER_TESTER_ACTOR_H - -#pragma once - -#include - -#include "flow/IDispatched.h" -#include "bindings/flow/fdb_flow.h" -#include "bindings/flow/IDirectory.h" -#include "bindings/flow/Subspace.h" -#include "bindings/flow/DirectoryLayer.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -constexpr bool LOG_ALL = false; -constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false; -constexpr bool LOG_OPS = LOG_ALL || false; -constexpr bool LOG_DIRS = LOG_ALL || false; -constexpr bool LOG_ERRORS = LOG_ALL || false; - -struct FlowTesterData; - -struct StackItem { - StackItem() : index(-1) {} - StackItem(uint32_t i, Future> v) : index(i), value(v) {} - StackItem(uint32_t i, Standalone v) : index(i), value(v) {} - uint32_t index; - Future> value; -}; - -struct FlowTesterStack { - uint32_t index; - std::vector data; - - void push(Future> value) { data.push_back(StackItem(index, value)); } - - void push(Standalone value) { push(Future>(value)); } - - void push(const StackItem& item) { data.push_back(item); } - - void pushTuple(StringRef value, bool utf8 = false) { - FDB::Tuple t; - t.append(value, utf8); - data.push_back(StackItem(index, t.pack())); - } - - void pushError(int errorCode) { - FDB::Tuple t; - t.append("ERROR"_sr); - t.append(format("%d", errorCode)); - // pack above as error string into another tuple - pushTuple(t.pack().toString()); - } - - std::vector pop(uint32_t count = 1) { - std::vector items; - while (!data.empty() && count > 0) { - items.push_back(data.back()); - data.pop_back(); - count--; - } - return items; - } - - Future> waitAndPop(int count); - Future waitAndPop(); - - void dup() { - if (data.empty()) - return; - data.push_back(data.back()); - } - - void clear() { data.clear(); } -}; - -struct InstructionData : public ReferenceCounted { - bool isDatabase; - bool isSnapshot; - StringRef instruction; - Reference tr; - - InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference _tr) - : isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {} -}; - -struct FlowTesterData; - -struct InstructionFunc - : IDispatched(Reference data, Reference instruction)>> { - static Future call(std::string op, Reference data, Reference instruction) { - ASSERT(data); - ASSERT(instruction); - - auto it = dispatches().find(op); - if (it == dispatches().end()) { - fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str()); - ASSERT(false); - } - - return dispatch(op)(data, instruction); - } -}; -#define REGISTER_INSTRUCTION_FUNC(Op) REGISTER_COMMAND(InstructionFunc, Op, name, call) - -struct DirectoryOrSubspace { - Optional> directory; - Optional subspace; - - DirectoryOrSubspace() {} - DirectoryOrSubspace(Reference directory) : directory(directory) {} - DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {} - DirectoryOrSubspace(Reference dirSubspace) - : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} - - bool valid() { return directory.present() || subspace.present(); } - - std::string typeString() { - if (directory.present() && subspace.present()) { - return "DirectorySubspace"; - } else if (directory.present()) { - return "IDirectory"; - } else if (subspace.present()) { - return "Subspace"; - } else { - return "InvalidDirectory"; - } - } -}; - -struct DirectoryTesterData { - std::vector directoryList; - int directoryListIndex; - int directoryErrorIndex; - - Reference directory() { - ASSERT(directoryListIndex < directoryList.size()); - ASSERT(directoryList[directoryListIndex].directory.present()); - return directoryList[directoryListIndex].directory.get(); - } - - FDB::Subspace* subspace() { - ASSERT(directoryListIndex < directoryList.size()); - ASSERT(directoryList[directoryListIndex].subspace.present()); - return directoryList[directoryListIndex].subspace.get(); - } - - DirectoryTesterData() : directoryListIndex(0), directoryErrorIndex(0) { - directoryList.push_back(Reference(new FDB::DirectoryLayer())); - } - - template - void push(T item) { - directoryList.push_back(DirectoryOrSubspace(item)); - if (LOG_DIRS) { - printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1); - fflush(stdout); - } - } - - void push() { push(DirectoryOrSubspace()); } -}; - -struct FlowTesterData : public ReferenceCounted { - FDB::API* api; - Reference db; - Standalone instructions; - Standalone trName; - FlowTesterStack stack; - FDB::Version lastVersion; - DirectoryTesterData directoryData; - - std::vector> subThreads; - - Future processInstruction(Reference instruction) { - return InstructionFunc::call( - instruction->instruction.toString(), Reference::addRef(this), instruction); - } - - FlowTesterData(FDB::API* api) { this->api = api; } -}; - -std::string tupleToString(FDB::Tuple const& tuple); - -ACTOR template -Future()().getValue())> executeMutation(Reference instruction, F func) { - loop { - try { - state decltype(std::declval()().getValue()) result = wait(func()); - if (instruction->isDatabase) { - wait(instruction->tr->commit()); - } - return result; - } catch (Error& e) { - if (instruction->isDatabase) { - wait(instruction->tr->onError(e)); - } else { - throw; - } - } - } -} - -#include "flow/unactorcompiler.h" -#endif diff --git a/bindings/flow/tester/Tester.cpp b/bindings/flow/tester/Tester.cpp new file mode 100644 index 00000000000..331f4f3fc09 --- /dev/null +++ b/bindings/flow/tester/Tester.cpp @@ -0,0 +1,1886 @@ +/* + * Tester.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tester/Tester.h" +#include +#ifdef __linux__ +#include +#endif + +#include "bindings/flow/Tuple.h" +#include "bindings/flow/FDBLoanerTypes.h" +#include "fdbrpc/fdbrpc.h" +#include "flow/DeterministicRandom.h" +#include "flow/TLSConfig.h" + +// Otherwise we have to type setupNetwork(), FDB::open(), etc. +using namespace FDB; + +std::map optionInfo; +std::set opsThatCreateDirectories; + +std::map, Reference> trMap; + +// NOTE: This was taken from within fdb_c.cpp (where it is defined as a static within the get_range function). +// If that changes, this will also have to be changed. +const int ITERATION_PROGRESSION[] = { 256, 1000, 4096, 6144, 9216, 13824, 20736, 31104, 46656, 69984, 80000 }; +const int MAX_ITERATION = sizeof(ITERATION_PROGRESSION) / sizeof(int); + +static Future runTest(Reference const& data, + Reference const& db, + StringRef const& prefix); + +THREAD_FUNC networkThread(void* api) { + // This is the fdb_flow network we're running on a thread + ((API*)api)->runNetwork(); + THREAD_RETURN; +} + +bool hasEnding(std::string const& fullString, std::string const& ending) { + if (fullString.length() >= ending.length()) { + return (0 == fullString.compare(fullString.length() - ending.length(), ending.length(), ending)); + } else { + return false; + } +} + +Future> waitAndPop(FlowTesterStack* self, int count) { + std::vector tuples; + std::vector items = self->pop(count); + + for (int index = 0; index < items.size(); ++index) { + Standalone itemStr = co_await items[index].value; + tuples.push_back(Tuple::unpack(itemStr)); + } + + co_return tuples; +} + +Future> FlowTesterStack::waitAndPop(int count) { + return ::waitAndPop(this, count); +} + +Future waitAndPop(FlowTesterStack* self) { + std::vector tuples = co_await waitAndPop(self, 1); + co_return tuples[0]; +} + +Future FlowTesterStack::waitAndPop() { + return ::waitAndPop(this); +} + +std::string tupleToString(Tuple const& tuple) { + std::string str = "("; + for (int i = 0; i < tuple.size(); ++i) { + Tuple::ElementType type = tuple.getType(i); + if (type == Tuple::NULL_TYPE) { + str += "NULL"; + } else if (type == Tuple::BYTES || type == Tuple::UTF8) { + if (type == Tuple::UTF8) { + str += "u"; + } + str += "\'" + tuple.getString(i).printable() + "\'"; + } else if (type == Tuple::INT) { + str += format("%ld", tuple.getInt(i)); + } else if (type == Tuple::FLOAT) { + str += format("%f", tuple.getFloat(i)); + } else if (type == Tuple::DOUBLE) { + str += format("%f", tuple.getDouble(i)); + } else if (type == Tuple::BOOL) { + str += tuple.getBool(i) ? "true" : "false"; + } else if (type == Tuple::UUID) { + Uuid u = tuple.getUuid(i); + str += format("%016llx%016llx", *(uint64_t*)u.getData().begin(), *(uint64_t*)(u.getData().begin() + 8)); + } else if (type == Tuple::NESTED) { + str += tupleToString(tuple.getNested(i)); + } else if (type == Tuple::VERSIONSTAMP) { + Versionstamp versionstamp = tuple.getVersionstamp(i); + str += format("Transaction Version: '%ld', BatchNumber: '%hd', UserVersion : '%hd'", + versionstamp.getVersion(), + versionstamp.getBatchNumber(), + versionstamp.getUserVersion()); + } else { + ASSERT(false); + } + + if (i < tuple.size() - 1) { + str += ", "; + } + } + + str += ")"; + return str; +} + +Future> getRange(Reference tr, + KeySelectorRef begin, + KeySelectorRef end, + int limits = 0, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + KeySelector ks_begin(begin); + KeySelector ks_end(end); + Standalone results; + int iteration = 1; + while (true) { + // printf("=====DB: begin:%s, end:%s, limits:%d\n", printable(begin.key).c_str(), printable(end.key).c_str(), + // limits); + FDBStandalone r; + if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { + int effective_iteration = std::min(iteration, MAX_ITERATION); + int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; + FDBStandalone rTemp = co_await tr->getRange(ks_begin, + ks_end, + GetRangeLimits(limits, bytes_limit), + snapshot, + reverse, + (FDBStreamingMode)FDB_STREAMING_MODE_EXACT); + r = rTemp; + } else { + FDBStandalone rTemp = + co_await tr->getRange(ks_begin, ks_end, limits, snapshot, reverse, streamingMode); + r = rTemp; + } + iteration += 1; + // printf("=====DB: count:%d\n", r.size()); + for (auto& s : r) { + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); + results.push_back_deep(results.arena(), s); + + if (reverse) + ks_end = KeySelector(firstGreaterOrEqual(s.key)); + else + ks_begin = KeySelector(firstGreaterThan(s.key)); + } + + ASSERT(limits == 0 || limits >= r.size()); + + if (!r.more || (limits > 0 && limits == r.size())) { + co_return results; + } + + if (limits > 0) { + limits -= r.size(); + } + } +} + +Future> getRange(Reference tr, + KeyRange keys, + int limits = 0, + bool snapshot = false, + bool reverse = false, + FDBStreamingMode streamingMode = FDB_STREAMING_MODE_SERIAL) { + Key begin(keys.begin); + Key end(keys.end); + Standalone results; + int iteration = 1; + while (true) { + // printf("=====DB: begin:%s, limits:%d\n", printable(begin).c_str(), limits); + KeyRange keyRange(KeyRangeRef(begin, end > begin ? end : begin)); + FDBStandalone r; + if (streamingMode == FDB_STREAMING_MODE_ITERATOR && iteration > 1) { + int effective_iteration = std::min(iteration, MAX_ITERATION); + int bytes_limit = ITERATION_PROGRESSION[effective_iteration - 1]; + FDBStandalone rTemp = co_await tr->getRange(keyRange, + GetRangeLimits(limits, bytes_limit), + snapshot, + reverse, + (FDBStreamingMode)FDB_STREAMING_MODE_EXACT); + r = rTemp; + } else { + FDBStandalone rTemp = + co_await tr->getRange(keyRange, limits, snapshot, reverse, streamingMode); + r = rTemp; + } + iteration += 1; + // printf("=====DB: count:%d\n", r.size()); + for (auto& s : r) { + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); + results.push_back_deep(results.arena(), s); + + if (reverse) + end = s.key; + else + begin = keyAfter(s.key); + } + + ASSERT(limits == 0 || limits >= r.size()); + + if (!r.more || (limits > 0 && limits == r.size())) { + co_return results; + } + + if (limits > 0) { + limits -= r.size(); + } + } +} + +// ACTOR static Future debugPrintRange(Reference tr, std::string subspace, std::string msg) { +// if (!tr) +// return Void(); +// +// Standalone results = wait(getRange(tr, KeyRange(KeyRangeRef(subspace + '\x00', subspace + +//'\xff')))); printf("==================================================DB:%s:%s, count:%d\n", msg.c_str(), +// StringRef(subspace).printable().c_str(), results.size()); +// for (auto & s : results) { +// printf("=====key:%s, value:%s\n", StringRef(s.key).printable().c_str(), StringRef(s.value).printable().c_str()); +// } +// +// return Void(); +//} + +Future stackSub(FlowTesterStack* stack) { + if (stack->data.size() < 2) + co_return; + + StackItem a = stack->data.back(); + stack->data.pop_back(); + Standalone sa = co_await a.value; + + StackItem b = stack->data.back(); + stack->data.pop_back(); + Standalone sb = co_await b.value; + + int64_t c = Tuple::unpack(sa).getInt(0) - Tuple::unpack(sb).getInt(0); + Tuple f; + f.append(c); + stack->push(f.pack()); +} + +Future stackConcat(FlowTesterStack* stack) { + if (stack->data.size() < 2) + co_return; + + StackItem a = stack->data.back(); + stack->data.pop_back(); + Standalone sa = co_await a.value; + Tuple ta = Tuple::unpack(sa); + + StackItem b = stack->data.back(); + stack->data.pop_back(); + Standalone sb = co_await b.value; + Tuple tb = Tuple::unpack(sb); + + ASSERT(ta.getType(0) == tb.getType(0)); + stack->pushTuple(tb.getString(0).withPrefix(ta.getString(0)), ta.getType(0) == Tuple::ElementType::UTF8); +} + +Future stackSwap(FlowTesterStack* stack) { + if (stack->data.size() < 3) + co_return; + + StackItem pop = stack->data.back(); + stack->data.pop_back(); + + Standalone sv = co_await pop.value; + int64_t idx = stack->data.size() - 1; + int64_t idx1 = idx - Tuple::unpack(sv).getInt(0); + if (idx1 < idx) { + // printf("=============SWAP:%d,%d\n", idx, stack->data.size()); + StackItem item = stack->data[idx]; + stack->data[idx] = stack->data[idx1]; + stack->data[idx1] = item; + } +} + +Future printFlowTesterStack(FlowTesterStack* stack) { + // printf("====================stack item count:%ld\n", stack->data.size()); + for (int idx = stack->data.size() - 1; idx >= 0; --idx) { + Standalone value = co_await stack->data[idx].value; + // printf("==========stack item:%d, index:%d, value:%s\n", idx, stack->data[idx].index, + // value.printable().c_str()); + } +} + +// +// Data Operations +// + +struct PushFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + Tuple t = Tuple::unpack(instruction->instruction); + Standalone param = t.subTuple(1).pack(); + data->stack.push(param); + return Void(); + } +}; +const char* PushFunc::name = "PUSH"; +REGISTER_INSTRUCTION_FUNC(PushFunc); + +struct DupFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + data->stack.dup(); + return Void(); + } +}; +const char* DupFunc::name = "DUP"; +REGISTER_INSTRUCTION_FUNC(DupFunc); + +struct EmptyStackFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + // wait(printFlowTesterStack(&(data->stack))); + // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); + data->stack.clear(); + return Void(); + } +}; +const char* EmptyStackFunc::name = "EMPTY_STACK"; +REGISTER_INSTRUCTION_FUNC(EmptyStackFunc); + +struct SwapFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + co_await stackSwap(&(data->stack)); + } +}; +const char* SwapFunc::name = "SWAP"; +REGISTER_INSTRUCTION_FUNC(SwapFunc); + +struct PopFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + for (const StackItem& item : items) { + co_await item.value; + } + } +}; +const char* PopFunc::name = "POP"; +REGISTER_INSTRUCTION_FUNC(PopFunc); + +struct SubFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + co_await stackSub(&(data->stack)); + } +}; +const char* SubFunc::name = "SUB"; +REGISTER_INSTRUCTION_FUNC(SubFunc); + +struct ConcatFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + co_await stackConcat(&(data->stack)); + } +}; +const char* ConcatFunc::name = "CONCAT"; +REGISTER_INSTRUCTION_FUNC(ConcatFunc); + +struct LogStackFunc : InstructionFunc { + static const char* name; + + static Future logStack(Reference data, + std::map entries, + Standalone prefix) { + while (true) { + Reference tr = data->db->createTransaction(); + Error err; + try { + for (const auto& it : entries) { + Tuple tk; + tk.append(it.first); + tk.append((int64_t)it.second.index); + Standalone pk = tk.pack().withPrefix(prefix); + Standalone pv = co_await it.second.value; + tr->set(pk, pv.substr(0, std::min(pv.size(), 40000))); + } + + co_await tr->commit(); + co_return; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } + } + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.empty()) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone prefix = Tuple::unpack(s1).getString(0); + + std::map entries; + while (!data->stack.data.empty()) { + std::vector it = data->stack.pop(); + ASSERT(it.size() == 1); + entries[data->stack.data.size()] = it.front(); + if (entries.size() == 100) { + co_await logStack(data, entries, prefix); + entries.clear(); + } + } + co_await logStack(data, entries, prefix); + } +}; +const char* LogStackFunc::name = "LOG_STACK"; +REGISTER_INSTRUCTION_FUNC(LogStackFunc); + +// +// FoundationDB Operations +// +Future> waitForVoid(Future f) { + try { + co_await f; + Tuple t; + t.append("RESULT_NOT_PRESENT"_sr); + co_return t.pack(); + } catch (Error& e) { + // printf("FDBError1:%d\n", e.code()); + Tuple t; + t.append("ERROR"_sr); + t.append(format("%d", e.code())); + // pack above as error string into another tuple + Tuple ret; + ret.append(t.pack()); + co_return ret.pack(); + } +} + +Future> waitForValue(Future> f) { + try { + FDBStandalone value = co_await f; + Tuple t; + t.append(value); + co_return t.pack(); + } catch (Error& e) { + // printf("FDBError2:%d\n", e.code()); + Tuple t; + t.append("ERROR"_sr); + t.append(format("%d", e.code())); + // pack above as error string into another tuple + Tuple ret; + ret.append(t.pack()); + co_return ret.pack(); + } +} + +Future> waitForValue(Future>> f) { + try { + Optional> value = co_await f; + Standalone str; + if (value.present()) + str = value.get(); + else + str = "RESULT_NOT_PRESENT"_sr; + + Tuple t; + t.append(str); + co_return t.pack(); + } catch (Error& e) { + // printf("FDBError3:%d\n", e.code()); + Tuple t; + t.append("ERROR"_sr); + t.append(format("%d", e.code())); + // pack above as error string into another tuple + Tuple ret; + ret.append(t.pack()); + co_return ret.pack(); + } +} + +Future> getKey(Future> f, Standalone prefixFilter) { + try { + FDBStandalone key = co_await f; + Tuple t; + + if (key.startsWith(prefixFilter)) { + t.append(key); + } else if (key < prefixFilter) { + t.append(prefixFilter); + } else { + t.append(strinc(prefixFilter)); + } + + co_return t.pack(); + } catch (Error& e) { + // printf("FDBError4:%d\n", e.code()); + Tuple t; + t.append("ERROR"_sr); + t.append(format("%d", e.code())); + // pack above as error string into another tuple + Tuple ret; + ret.append(t.pack()); + co_return ret.pack(); + } +} + +struct NewTransactionFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + trMap[data->trName] = data->db->createTransaction(); + return Void(); + } +}; +const char* NewTransactionFunc::name = "NEW_TRANSACTION"; +REGISTER_INSTRUCTION_FUNC(NewTransactionFunc); + +struct UseTransactionFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + Standalone name = co_await items[0].value; + data->trName = name; + + if (!trMap.contains(data->trName)) { + trMap[data->trName] = data->db->createTransaction(); + } + } +}; +const char* UseTransactionFunc::name = "USE_TRANSACTION"; +REGISTER_INSTRUCTION_FUNC(UseTransactionFunc); + +struct OnErrorFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.empty()) + co_return; + + Standalone value = co_await items[0].value; + int err_code = Tuple::unpack(value).getInt(0); + // printf("OnError:%d:%d:%s\n", err_code, items[0].index, printable(value).c_str()); + + data->stack.push(waitForVoid(instruction->tr->onError(Error(err_code)))); + } +}; +const char* OnErrorFunc::name = "ON_ERROR"; +REGISTER_INSTRUCTION_FUNC(OnErrorFunc); + +struct SetFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(2); + if (items.size() != 2) + co_return; + + Standalone sk = co_await items[0].value; + Standalone key = Tuple::unpack(sk).getString(0); + // if (instruction->isDatabase) + // printf("SetDatabase:%s, isDatabase:%d\n", printable(key).c_str(), instruction->isDatabase); + Standalone sv = co_await items[1].value; + Standalone value = Tuple::unpack(sv).getString(0); + // printf("SetDatabase:%s:%s:%s\n", printable(key).c_str(), printable(sv).c_str(), printable(value).c_str()); + + Reference instructionCopy = instruction; + Standalone keyCopy = key; + + Future mutation = executeMutation(instruction, [instructionCopy, keyCopy, value]() -> Future { + instructionCopy->tr->set(keyCopy, value); + return Void(); + }); + + if (instruction->isDatabase) { + data->stack.push(waitForVoid(mutation)); + } else { + co_await mutation; + } + } +}; +const char* SetFunc::name = "SET"; +REGISTER_INSTRUCTION_FUNC(SetFunc); + +struct GetFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone sk = co_await items[0].value; + Standalone key = Tuple::unpack(sk).getString(0); + + Future>> fk = instruction->tr->get(StringRef(key), instruction->isSnapshot); + data->stack.push(waitForValue(holdWhile(instruction->tr, fk))); + } +}; +const char* GetFunc::name = "GET"; +REGISTER_INSTRUCTION_FUNC(GetFunc); + +struct GetEstimatedRangeSize : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(2); + if (items.size() != 2) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone beginKey = Tuple::unpack(s1).getString(0); + + Standalone s2 = co_await items[1].value; + Standalone endKey = Tuple::unpack(s2).getString(0); + Future fsize = instruction->tr->getEstimatedRangeSizeBytes(KeyRangeRef(beginKey, endKey)); + co_await fsize; + data->stack.pushTuple("GOT_ESTIMATED_RANGE_SIZE"_sr); + } +}; +const char* GetEstimatedRangeSize::name = "GET_ESTIMATED_RANGE_SIZE"; +REGISTER_INSTRUCTION_FUNC(GetEstimatedRangeSize); + +struct GetRangeSplitPoints : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(3); + if (items.size() != 3) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone beginKey = Tuple::unpack(s1).getString(0); + + Standalone s2 = co_await items[1].value; + Standalone endKey = Tuple::unpack(s2).getString(0); + + Standalone s3 = co_await items[2].value; + int64_t chunkSize = Tuple::unpack(s3).getInt(0); + + Future>> fsplitPoints = + instruction->tr->getRangeSplitPoints(KeyRangeRef(beginKey, endKey), chunkSize); + FDBStandalone> splitPoints = co_await fsplitPoints; + data->stack.pushTuple("GOT_RANGE_SPLIT_POINTS"_sr); + } +}; +const char* GetRangeSplitPoints::name = "GET_RANGE_SPLIT_POINTS"; +REGISTER_INSTRUCTION_FUNC(GetRangeSplitPoints); + +struct GetKeyFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(4); + if (items.size() != 4) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone key = Tuple::unpack(s1).getString(0); + + Standalone s2 = co_await items[1].value; + int64_t or_equal = Tuple::unpack(s2).getInt(0); + + Standalone s3 = co_await items[2].value; + int64_t offset = Tuple::unpack(s3).getInt(0); + + Standalone s4 = co_await items[3].value; + Standalone prefix = Tuple::unpack(s4).getString(0); + + // printf("===================GET_KEY:%s, %ld, %ld\n", printable(key).c_str(), or_equal, offset); + Future> fk = + instruction->tr->getKey(KeySelector(KeySelectorRef(key, or_equal, offset)), instruction->isSnapshot); + data->stack.push(getKey(holdWhile(instruction->tr, fk), prefix)); + } +}; +const char* GetKeyFunc::name = "GET_KEY"; +REGISTER_INSTRUCTION_FUNC(GetKeyFunc); + +struct GetReadVersionFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + Version v = co_await instruction->tr->getReadVersion(); + data->lastVersion = v; + data->stack.pushTuple("GOT_READ_VERSION"_sr); + } +}; +const char* GetReadVersionFunc::name = "GET_READ_VERSION"; +REGISTER_INSTRUCTION_FUNC(GetReadVersionFunc); + +struct SetReadVersionFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + instruction->tr->setReadVersion(data->lastVersion); + return Void(); + } +}; +const char* SetReadVersionFunc::name = "SET_READ_VERSION"; +REGISTER_INSTRUCTION_FUNC(SetReadVersionFunc); + +// GET_COMMITTED_VERSION +struct GetCommittedVersionFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + data->lastVersion = instruction->tr->getCommittedVersion(); + data->stack.pushTuple("GOT_COMMITTED_VERSION"_sr); + return Void(); + } +}; +const char* GetCommittedVersionFunc::name = "GET_COMMITTED_VERSION"; +REGISTER_INSTRUCTION_FUNC(GetCommittedVersionFunc); + +// GET_APPROXIMATE_SIZE +struct GetApproximateSizeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + int64_t _ = co_await instruction->tr->getApproximateSize(); + (void)_; // disable unused variable warning + data->stack.pushTuple("GOT_APPROXIMATE_SIZE"_sr); + } +}; +const char* GetApproximateSizeFunc::name = "GET_APPROXIMATE_SIZE"; +REGISTER_INSTRUCTION_FUNC(GetApproximateSizeFunc); + +// GET_VERSIONSTAMP +struct GetVersionstampFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + data->stack.push(waitForValue(instruction->tr->getVersionstamp())); + return Void(); + } +}; +const char* GetVersionstampFunc::name = "GET_VERSIONSTAMP"; +REGISTER_INSTRUCTION_FUNC(GetVersionstampFunc); + +// COMMIT +struct CommitFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + data->stack.push(waitForVoid(holdWhile(instruction->tr, instruction->tr->commit()))); + return Void(); + } +}; +const char* CommitFunc::name = "COMMIT"; +REGISTER_INSTRUCTION_FUNC(CommitFunc); + +// WAIT_FUTURE +struct WaitFutureFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone sk = co_await items[0].value; + data->stack.push(StackItem(items[0].index, sk)); + } +}; +const char* WaitFutureFunc::name = "WAIT_FUTURE"; +REGISTER_INSTRUCTION_FUNC(WaitFutureFunc); + +// CLEAR +struct ClearFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone sk = co_await items[0].value; + Standalone key = Tuple::unpack(sk).getString(0); + + Reference instructionCopy = instruction; + + Future mutation = executeMutation(instruction, [instructionCopy, key]() -> Future { + instructionCopy->tr->clear(key); + return Void(); + }); + + if (instruction->isDatabase) { + data->stack.push(waitForVoid(mutation)); + } else { + co_await mutation; + } + } +}; +const char* ClearFunc::name = "CLEAR"; +REGISTER_INSTRUCTION_FUNC(ClearFunc); + +// RESET +struct ResetFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + instruction->tr->reset(); + return Void(); + } +}; +const char* ResetFunc::name = "RESET"; +REGISTER_INSTRUCTION_FUNC(ResetFunc); + +// CANCEL +struct CancelFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + instruction->tr->cancel(); + return Void(); + } +}; +const char* CancelFunc::name = "CANCEL"; +REGISTER_INSTRUCTION_FUNC(CancelFunc); + +struct GetRangeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(5); + if (items.size() != 5) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone begin = Tuple::unpack(s1).getString(0); + + Standalone s2 = co_await items[1].value; + Standalone end = Tuple::unpack(s2).getString(0); + + Standalone s3 = co_await items[2].value; + int limit = Tuple::unpack(s3).getInt(0); + + Standalone s4 = co_await items[3].value; + int reverse = Tuple::unpack(s4).getInt(0); + + Standalone s5 = co_await items[4].value; + FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s5).getInt(0); + + // printf("================GetRange: %s, %s, %d, %d, %d, %d\n", printable(begin).c_str(), + // printable(end).c_str(), limit, reverse, mode, instruction->isSnapshot); + + Standalone results = co_await getRange(instruction->tr, + KeyRange(KeyRangeRef(begin, end > begin ? end : begin)), + limit, + instruction->isSnapshot, + reverse, + mode); + Tuple t; + for (auto& s : results) { + t.append(s.key); + t.append(s.value); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); + } + // printf("=====Results Count:%d, size:%d\n", results.size(), str.size()); + + data->stack.push(Tuple().append(t.pack()).pack()); + } +}; +const char* GetRangeFunc::name = "GET_RANGE"; +REGISTER_INSTRUCTION_FUNC(GetRangeFunc); + +struct GetRangeStartsWithFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(4); + if (items.size() != 4) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone prefix = Tuple::unpack(s1).getString(0); + + Standalone s2 = co_await items[1].value; + int limit = Tuple::unpack(s2).getInt(0); + + Standalone s3 = co_await items[2].value; + int reverse = Tuple::unpack(s3).getInt(0); + + Standalone s4 = co_await items[3].value; + FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s4).getInt(0); + + // printf("================GetRangeStartsWithFunc: %s, %d, %d, %d, %d\n", printable(prefix).c_str(), limit, + // reverse, mode, isSnapshot); + Standalone results = co_await getRange(instruction->tr, + KeyRange(KeyRangeRef(prefix, strinc(prefix))), + limit, + instruction->isSnapshot, + reverse, + mode); + Tuple t; + // printf("=====Results Count:%d\n", results.size()); + for (auto& s : results) { + t.append(s.key); + t.append(s.value); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); + } + + data->stack.push(Tuple().append(t.pack()).pack()); + } +}; +const char* GetRangeStartsWithFunc::name = "GET_RANGE_STARTS_WITH"; +REGISTER_INSTRUCTION_FUNC(GetRangeStartsWithFunc); + +struct ClearRangeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(2); + if (items.size() != 2) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone begin = Tuple::unpack(s1).getString(0); + + Standalone s2 = co_await items[1].value; + Standalone end = Tuple::unpack(s2).getString(0); + + Reference instructionCopy = instruction; + Standalone beginCopy = begin; + + Future mutation = executeMutation(instruction, [instructionCopy, beginCopy, end]() -> Future { + instructionCopy->tr->clear(KeyRangeRef(beginCopy, end)); + return Void(); + }); + + if (instruction->isDatabase) { + data->stack.push(waitForVoid(mutation)); + } else { + co_await mutation; + } + } +}; +const char* ClearRangeFunc::name = "CLEAR_RANGE"; +REGISTER_INSTRUCTION_FUNC(ClearRangeFunc); + +struct ClearRangeStartWithFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone begin = Tuple::unpack(s1).getString(0); + + Reference instructionCopy = instruction; + + Future mutation = executeMutation(instruction, [instructionCopy, begin]() -> Future { + instructionCopy->tr->clear(KeyRangeRef(begin, strinc(begin))); + return Void(); + }); + + if (instruction->isDatabase) { + data->stack.push(waitForVoid(mutation)); + } else { + co_await mutation; + } + } +}; +const char* ClearRangeStartWithFunc::name = "CLEAR_RANGE_STARTS_WITH"; +REGISTER_INSTRUCTION_FUNC(ClearRangeStartWithFunc); + +struct GetRangeSelectorFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(10); + if (items.size() != 10) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone begin = Tuple::unpack(s1).getString(0); + + Standalone s2 = co_await items[1].value; + bool begin_or_equal = Tuple::unpack(s2).getInt(0); + + Standalone s3 = co_await items[2].value; + int64_t begin_offset = Tuple::unpack(s3).getInt(0); + + Standalone s4 = co_await items[3].value; + Standalone end = Tuple::unpack(s4).getString(0); + + Standalone s5 = co_await items[4].value; + bool end_or_equal = Tuple::unpack(s5).getInt(0); + + Standalone s6 = co_await items[5].value; + int64_t end_offset = Tuple::unpack(s6).getInt(0); + + Standalone s7 = co_await items[6].value; + int limit = Tuple::unpack(s7).getInt(0); + + Standalone s8 = co_await items[7].value; + int reverse = Tuple::unpack(s8).getInt(0); + + Standalone s9 = co_await items[8].value; + FDBStreamingMode mode = (FDBStreamingMode)Tuple::unpack(s9).getInt(0); + + Standalone s10 = co_await items[9].value; + Optional> prefix; + Tuple t10 = Tuple::unpack(s10); + if (t10.getType(0) != Tuple::ElementType::NULL_TYPE) { + prefix = t10.getString(0); + } + + // printf("================GetRangeSelectorFunc: %s, %d, %ld, %s, %d, %ld, %d, %d, %d, %d, %s\n", + // printable(begin).c_str(), begin_or_equal, begin_offset, printable(end).c_str(), end_or_equal, end_offset, + // limit, reverse, mode, instruction->isSnapshot, printable(prefix).c_str()); + Future> f = getRange(instruction->tr, + KeySelectorRef(begin, begin_or_equal, begin_offset), + KeySelectorRef(end, end_or_equal, end_offset), + limit, + instruction->isSnapshot, + reverse, + mode); + Standalone results = co_await holdWhile(instruction->tr, f); + Tuple t; + // printf("=====Results Count:%d\n", results.size()); + for (auto& s : results) { + if (!prefix.present() || s.key.startsWith(prefix.get())) { + t.append(s.key); + t.append(s.value); + // printf("=====key:%s, value:%s\n", printable(StringRef(s.key)).c_str(), + // printable(StringRef(s.value)).c_str()); + } + } + + data->stack.push(Tuple().append(t.pack()).pack()); + } +}; +const char* GetRangeSelectorFunc::name = "GET_RANGE_SELECTOR"; +REGISTER_INSTRUCTION_FUNC(GetRangeSelectorFunc); + +// Tuple Operations +// TUPLE_PACK +struct TuplePackFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + int64_t count = Tuple::unpack(s1).getInt(0); + + std::vector items1 = data->stack.pop(count); + if (items1.size() != count) + co_return; + + Tuple tuple; + for (int i = 0; i < items1.size(); ++i) { + Standalone str = co_await items1[i].value; + Tuple itemTuple = Tuple::unpack(str); + if (deterministicRandom()->coinflip()) { + Tuple::ElementType type = itemTuple.getType(0); + if (type == Tuple::NULL_TYPE) { + tuple.appendNull(); + } else if (type == Tuple::INT) { + tuple << itemTuple.getInt(0); + } else if (type == Tuple::BYTES) { + tuple.append(itemTuple.getString(0), false); + } else if (type == Tuple::UTF8) { + tuple.append(itemTuple.getString(0), true); + } else if (type == Tuple::FLOAT) { + tuple << itemTuple.getFloat(0); + } else if (type == Tuple::DOUBLE) { + tuple << itemTuple.getDouble(0); + } else if (type == Tuple::BOOL) { + tuple << itemTuple.getBool(0); + } else if (type == Tuple::UUID) { + tuple << itemTuple.getUuid(0); + } else if (type == Tuple::NESTED) { + tuple.appendNested(itemTuple.getNested(0)); + } else if (type == Tuple::VERSIONSTAMP) { + tuple.appendVersionstamp(itemTuple.getVersionstamp(0)); + } else { + ASSERT(false); + } + } else { + tuple << itemTuple; + } + } + + data->stack.pushTuple(tuple.pack()); + } +}; +const char* TuplePackFunc::name = "TUPLE_PACK"; +REGISTER_INSTRUCTION_FUNC(TuplePackFunc); + +// TUPLE_UNPACK +struct TupleUnpackFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Tuple t = Tuple::unpack(Tuple::unpack(s1).getString(0)); + + for (int i = 0; i < t.size(); ++i) { + Standalone str = t.subTuple(i, i + 1).pack(); + // printf("=====value:%s\n", printable(str).c_str()); + data->stack.pushTuple(str); + } + } +}; +const char* TupleUnpackFunc::name = "TUPLE_UNPACK"; +REGISTER_INSTRUCTION_FUNC(TupleUnpackFunc); + +// TUPLE_RANGE +struct TupleRangeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + int64_t count = Tuple::unpack(s1).getInt(0); + + std::vector items1 = data->stack.pop(count); + if (items1.size() != count) + co_return; + + Tuple tuple; + for (size_t i = 0; i < items1.size(); ++i) { + Standalone str = co_await items1[i].value; + Tuple itemTuple = Tuple::unpack(str); + if (deterministicRandom()->coinflip()) { + Tuple::ElementType type = itemTuple.getType(0); + if (type == Tuple::NULL_TYPE) { + tuple.appendNull(); + } else if (type == Tuple::INT) { + tuple << itemTuple.getInt(0); + } else if (type == Tuple::BYTES) { + tuple.append(itemTuple.getString(0), false); + } else if (type == Tuple::UTF8) { + tuple.append(itemTuple.getString(0), true); + } else if (type == Tuple::FLOAT) { + tuple << itemTuple.getFloat(0); + } else if (type == Tuple::DOUBLE) { + tuple << itemTuple.getDouble(0); + } else if (type == Tuple::BOOL) { + tuple << itemTuple.getBool(0); + } else if (type == Tuple::UUID) { + tuple << itemTuple.getUuid(0); + } else if (type == Tuple::NESTED) { + tuple.appendNested(itemTuple.getNested(0)); + } else if (type == Tuple::VERSIONSTAMP) { + tuple.appendVersionstamp(itemTuple.getVersionstamp(0)); + } else { + ASSERT(false); + } + } else { + tuple << itemTuple; + } + } + + KeyRange range = tuple.range(); + + data->stack.pushTuple(range.begin); + data->stack.pushTuple(range.end); + } +}; +const char* TupleRangeFunc::name = "TUPLE_RANGE"; +REGISTER_INSTRUCTION_FUNC(TupleRangeFunc); + +// TUPLE_SORT +struct TupleSortFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + int64_t count = Tuple::unpack(s1).getInt(0); + + std::vector items1 = data->stack.pop(count); + if (items1.size() != count) + co_return; + + std::vector tuples; + for (size_t i = 0; i < items1.size(); i++) { + Standalone value = co_await items1[i].value; + tuples.push_back(Tuple::unpack(value)); + } + + std::sort(tuples.begin(), tuples.end()); + for (Tuple const& t : tuples) { + data->stack.push(t.pack()); + } + } +}; +const char* TupleSortFunc::name = "TUPLE_SORT"; +REGISTER_INSTRUCTION_FUNC(TupleSortFunc); + +// ENCODE_FLOAT +struct EncodeFloatFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone fBytes = Tuple::unpack(s1).getString(0); + ASSERT(fBytes.size() == 4); + + int32_t intVal = *(int32_t*)fBytes.begin(); + intVal = bigEndian32(intVal); + float fVal = *(float*)&intVal; + + Tuple t; + t.append(fVal); + data->stack.push(t.pack()); + } +}; +const char* EncodeFloatFunc::name = "ENCODE_FLOAT"; +REGISTER_INSTRUCTION_FUNC(EncodeFloatFunc); + +// ENCODE_DOUBLE +struct EncodeDoubleFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone dBytes = Tuple::unpack(s1).getString(0); + ASSERT(dBytes.size() == 8); + + int64_t intVal = *(int64_t*)dBytes.begin(); + intVal = bigEndian64(intVal); + double dVal = *(double*)&intVal; + + Tuple t; + t.append(dVal); + data->stack.push(t.pack()); + } +}; +const char* EncodeDoubleFunc::name = "ENCODE_DOUBLE"; +REGISTER_INSTRUCTION_FUNC(EncodeDoubleFunc); + +// DECODE_FLOAT +struct DecodeFloatFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + float fVal = Tuple::unpack(s1).getFloat(0); + int32_t intVal = *(int32_t*)&fVal; + intVal = bigEndian32(intVal); + + Tuple t; + t.append(StringRef((uint8_t*)&intVal, 4), false); + data->stack.push(t.pack()); + } +}; +const char* DecodeFloatFunc::name = "DECODE_FLOAT"; +REGISTER_INSTRUCTION_FUNC(DecodeFloatFunc); + +// DECODE_DOUBLE +struct DecodeDoubleFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + double dVal = Tuple::unpack(s1).getDouble(0); + int64_t intVal = *(int64_t*)&dVal; + intVal = bigEndian64(intVal); + + Tuple t; + t.append(StringRef((uint8_t*)&intVal, 8), false); + data->stack.push(t.pack()); + } +}; +const char* DecodeDoubleFunc::name = "DECODE_DOUBLE"; +REGISTER_INSTRUCTION_FUNC(DecodeDoubleFunc); + +// Thread Operations +// START_THREAD +struct StartThreadFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone prefix = Tuple::unpack(s1).getString(0); + // printf("=========START_THREAD:%s\n", printable(prefix).c_str()); + + auto newData = makeReference(data->api); + data->subThreads.push_back(runTest(newData, data->db, prefix)); + } +}; +const char* StartThreadFunc::name = "START_THREAD"; +REGISTER_INSTRUCTION_FUNC(StartThreadFunc); + +template +Future()(Reference()).getValue())> read(Reference db, + Function func) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + co_return co_await func(tr); + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } +} + +// WAIT_EMPTY +struct WaitEmptyFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone prefix = Tuple::unpack(s1).getString(0); + // printf("=========WAIT_EMPTY:%s\n", printable(prefix).c_str()); + + co_await read(data->db, + [=](Reference tr) -> Future { return checkEmptyPrefix(tr, prefix); }); + } + +private: + static Future checkEmptyPrefix(Reference tr, Standalone prefix) { + FDBStandalone results = co_await tr->getRange(KeyRangeRef(prefix, strinc(prefix)), 1); + if (!results.empty()) { + throw not_committed(); + } + } +}; +const char* WaitEmptyFunc::name = "WAIT_EMPTY"; +REGISTER_INSTRUCTION_FUNC(WaitEmptyFunc); + +// DISABLE_WRITE_CONFLICT +struct DisableWriteConflictFunc : InstructionFunc { + static const char* name; + + static Future call(Reference const& data, Reference const& instruction) { + if (instruction->tr) { + instruction->tr->setOption(FDBTransactionOption::FDB_TR_OPTION_NEXT_WRITE_NO_WRITE_CONFLICT_RANGE); + } + return Void(); + } +}; +const char* DisableWriteConflictFunc::name = "DISABLE_WRITE_CONFLICT"; +REGISTER_INSTRUCTION_FUNC(DisableWriteConflictFunc); + +// READ_CONFLICT_KEY +struct ReadConflictKeyFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone key = Tuple::unpack(s1).getString(0); + // printf("=========READ_CONFLICT_KEY:%s\n", printable(key).c_str()); + instruction->tr->addReadConflictKey(key); + + data->stack.pushTuple("SET_CONFLICT_KEY"_sr); + } +}; +const char* ReadConflictKeyFunc::name = "READ_CONFLICT_KEY"; +REGISTER_INSTRUCTION_FUNC(ReadConflictKeyFunc); + +// WRITE_CONFLICT_KEY +struct WriteConflictKeyFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(); + if (items.size() != 1) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone key = Tuple::unpack(s1).getString(0); + // printf("=========WRITE_CONFLICT_KEY:%s\n", printable(key).c_str()); + instruction->tr->addWriteConflictKey(key); + + data->stack.pushTuple("SET_CONFLICT_KEY"_sr); + } +}; +const char* WriteConflictKeyFunc::name = "WRITE_CONFLICT_KEY"; +REGISTER_INSTRUCTION_FUNC(WriteConflictKeyFunc); + +// READ_CONFLICT_RANGE +struct ReadConflictRangeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(2); + if (items.size() != 2) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone begin = Tuple::unpack(s1).getString(0); + Standalone s2 = co_await items[1].value; + Standalone end = Tuple::unpack(s2).getString(0); + + // printf("=========READ_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str()); + instruction->tr->addReadConflictRange(KeyRange(KeyRangeRef(begin, end))); + data->stack.pushTuple("SET_CONFLICT_RANGE"_sr); + } +}; +const char* ReadConflictRangeFunc::name = "READ_CONFLICT_RANGE"; +REGISTER_INSTRUCTION_FUNC(ReadConflictRangeFunc); + +// WRITE_CONFLICT_RANGE +struct WriteConflictRangeFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(2); + if (items.size() != 2) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone begin = Tuple::unpack(s1).getString(0); + Standalone s2 = co_await items[1].value; + Standalone end = Tuple::unpack(s2).getString(0); + + // printf("=========WRITE_CONFLICT_RANGE:%s:%s\n", printable(begin).c_str(), printable(end).c_str()); + instruction->tr->addWriteConflictRange(KeyRange(KeyRangeRef(begin, end))); + + data->stack.pushTuple("SET_CONFLICT_RANGE"_sr); + } +}; +const char* WriteConflictRangeFunc::name = "WRITE_CONFLICT_RANGE"; +REGISTER_INSTRUCTION_FUNC(WriteConflictRangeFunc); + +// ATOMIC_OP +struct AtomicOPFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + std::vector items = data->stack.pop(3); + if (items.size() != 3) + co_return; + + Standalone s1 = co_await items[0].value; + Standalone op = Tuple::unpack(s1).getString(0); + Standalone s2 = co_await items[1].value; + Standalone key = Tuple::unpack(s2).getString(0); + Standalone s3 = co_await items[2].value; + Standalone value = Tuple::unpack(s3).getString(0); + + ASSERT(optionInfo.find(op.toString()) != optionInfo.end()); + + FDBMutationType atomicOp = optionInfo[op.toString()]; + + Reference instructionCopy = instruction; + Standalone keyCopy = key; + Standalone valueCopy = value; + + // printf("=========ATOMIC_OP:%s:%s:%s\n", printable(op).c_str(), printable(key).c_str(), + // printable(value).c_str()); + Future mutation = + executeMutation(instruction, [instructionCopy, keyCopy, valueCopy, atomicOp]() -> Future { + instructionCopy->tr->atomicOp(keyCopy, valueCopy, atomicOp); + return Void(); + }); + + if (instruction->isDatabase) { + data->stack.push(waitForVoid(mutation)); + } else { + co_await mutation; + } + } +}; +const char* AtomicOPFunc::name = "ATOMIC_OP"; +REGISTER_INSTRUCTION_FUNC(AtomicOPFunc); + +// UNIT_TESTS +struct UnitTestsFunc : InstructionFunc { + static const char* name; + + static Future call(Reference data, Reference instruction) { + ASSERT(data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(1020))); + ASSERT(!data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(10))); + + ASSERT(API::isAPIVersionSelected()); + API* fdb = API::getInstance(); + ASSERT(fdb->getAPIVersion() <= FDB_API_VERSION); + try { + API::selectAPIVersion(fdb->getAPIVersion() + 1); + ASSERT(false); + } catch (Error& e) { + ASSERT(e.code() == error_code_api_version_already_set); + } + try { + API::selectAPIVersion(fdb->getAPIVersion() - 1); + ASSERT(false); + } catch (Error& e) { + ASSERT(e.code() == error_code_api_version_already_set); + } + API::selectAPIVersion(fdb->getAPIVersion()); + + const uint64_t locationCacheSize = 100001; + const uint64_t maxWatches = 10001; + const uint64_t timeout = 60 * 1000; + const uint64_t noTimeout = 0; + const uint64_t retryLimit = 50; + const uint64_t noRetryLimit = -1; + const uint64_t maxRetryDelay = 100; + const uint64_t sizeLimit = 100000; + const uint64_t maxFieldLength = 1000; + + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_LOCATION_CACHE_SIZE, + Optional(StringRef((const uint8_t*)&locationCacheSize, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MAX_WATCHES, + Optional(StringRef((const uint8_t*)&maxWatches, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_DATACENTER_ID, Optional("dc_id"_sr)); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_MACHINE_ID, Optional("machine_id"_sr)); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_ENABLE); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_SNAPSHOT_RYW_DISABLE); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, + Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&noTimeout, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_MAX_RETRY_DELAY, + Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_SIZE_LIMIT, + Optional(StringRef((const uint8_t*)&sizeLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&noRetryLimit, 8))); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_CAUSAL_READ_RISKY); + data->db->setDatabaseOption(FDBDatabaseOption::FDB_DB_OPTION_TRANSACTION_INCLUDE_PORT_IN_ADDRESS); + + Reference tr = data->db->createTransaction(); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_BATCH); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_CAUSAL_READ_RISKY); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_CAUSAL_WRITE_RISKY); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_YOUR_WRITES_DISABLE); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_MAX_FIELD_LENGTH, + Optional(StringRef((const uint8_t*)&maxFieldLength, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_MAX_RETRY_DELAY, + Optional(StringRef((const uint8_t*)&maxRetryDelay, 8))); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_USED_DURING_COMMIT_PROTECTION_DISABLE); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_TRANSACTION_LOGGING_ENABLE, + Optional("my_transaction"_sr)); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_READ_LOCK_AWARE); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_LOCK_AWARE); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_INCLUDE_PORT_IN_ADDRESS); + tr->setOption(FDBTransactionOption::FDB_TR_OPTION_REPORT_CONFLICTING_KEYS); + + Optional> _ = co_await tr->get("\xff"_sr); + tr->cancel(); + } +}; +const char* UnitTestsFunc::name = "UNIT_TESTS"; +REGISTER_INSTRUCTION_FUNC(UnitTestsFunc); + +static Future getInstructions(Reference data, StringRef prefix) { + Reference tr = data->db->createTransaction(); + + // get test instructions + Tuple testSpec; + testSpec.append(prefix); + while (true) { + Error err; + try { + Standalone results = co_await getRange(tr, testSpec.range()); + data->instructions = results; + co_return; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } +} + +static Future doInstructions(Reference data) { + // printf("Total num instructions:%d\n", data->instructions.size()); + for (size_t idx = 0; idx < data->instructions.size(); ++idx) { + Tuple opTuple = Tuple::unpack(data->instructions[idx].value); + Standalone op = opTuple.getString(0); + + bool isDatabase = op.endsWith("_DATABASE"_sr); + bool isSnapshot = op.endsWith("_SNAPSHOT"_sr); + bool isDirectory = op.startsWith("DIRECTORY_"_sr); + + try { + if (LOG_INSTRUCTIONS) { + if (op != "SWAP"_sr && op != "PUSH"_sr) { + printf("%zu. %s\n", idx, tupleToString(opTuple).c_str()); + fflush(stdout); + } + } + + if (isDatabase) + op = op.substr(0, op.size() - 9); + else if (isSnapshot) + op = op.substr(0, op.size() - 9); + + // printf("[==========]%ld/%ld:%s:%s: isDatabase:%d, isSnapshot:%d, stack count:%ld\n", + // idx, data->instructions.size(), StringRef(data->instructions[idx].key).printable().c_str(), + // StringRef(data->instructions[idx].value).printable().c_str(), isDatabase, isSnapshot, + // data->stack.data.size()); + + // wait(printFlowTesterStack(&(data->stack))); + // wait(debugPrintRange(instruction->tr, "\x01test_results", "")); + + auto instruction = makeReference( + isDatabase, isSnapshot, data->instructions[idx].value, Reference()); + if (isDatabase) { + Reference tr = data->db->createTransaction(); + instruction->tr = tr; + } else { + instruction->tr = trMap[data->trName]; + } + + // Flow directory operations don't support snapshot reads + ASSERT(!isDirectory || !isSnapshot); + + data->stack.index = idx; + co_await InstructionFunc::call(op.toString(), data, instruction); + } catch (Error& e) { + if (LOG_ERRORS) { + printf("Error: %s (%d)\n", e.name(), e.code()); + fflush(stdout); + } + + if (isDirectory) { + if (opsThatCreateDirectories.contains(op.toString())) { + data->directoryData.directoryList.push_back(DirectoryOrSubspace()); + } + data->stack.pushTuple("DIRECTORY_ERROR"_sr); + } else { + data->stack.pushError(e.code()); + } + } + } + // printf("Total num instructions:%d\n", data->instructions.size()); +} + +static Future runTest(Reference const& data, + Reference const& db, + StringRef const& prefix) { + ASSERT(data); + try { + data->db = db; + co_await getInstructions(data, prefix); + co_await doInstructions(data); + co_await waitForAll(data->subThreads); + } catch (Error& e) { + TraceEvent(SevError, "FlowTesterDataRunError").error(e); + } +} + +void populateAtomicOpMap() { + optionInfo["ADD"] = FDBMutationType::FDB_MUTATION_TYPE_ADD; + optionInfo["AND"] = FDBMutationType::FDB_MUTATION_TYPE_AND; + optionInfo["BIT_AND"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_AND; + optionInfo["OR"] = FDBMutationType::FDB_MUTATION_TYPE_OR; + optionInfo["BIT_OR"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_OR; + optionInfo["XOR"] = FDBMutationType::FDB_MUTATION_TYPE_XOR; + optionInfo["BIT_XOR"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_XOR; + optionInfo["APPEND_IF_FITS"] = FDBMutationType::FDB_MUTATION_TYPE_APPEND_IF_FITS; + optionInfo["MAX"] = FDBMutationType::FDB_MUTATION_TYPE_MAX; + optionInfo["MIN"] = FDBMutationType::FDB_MUTATION_TYPE_MIN; + optionInfo["SET_VERSIONSTAMPED_KEY"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY; + optionInfo["SET_VERSIONSTAMPED_VALUE"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE; + optionInfo["BYTE_MIN"] = FDBMutationType::FDB_MUTATION_TYPE_BYTE_MIN; + optionInfo["BYTE_MAX"] = FDBMutationType::FDB_MUTATION_TYPE_BYTE_MAX; +} + +void populateOpsThatCreateDirectories() { + opsThatCreateDirectories.insert("DIRECTORY_CREATE_SUBSPACE"); + opsThatCreateDirectories.insert("DIRECTORY_CREATE_LAYER"); + opsThatCreateDirectories.insert("DIRECTORY_CREATE_OR_OPEN"); + opsThatCreateDirectories.insert("DIRECTORY_CREATE"); + opsThatCreateDirectories.insert("DIRECTORY_OPEN"); + opsThatCreateDirectories.insert("DIRECTORY_MOVE"); + opsThatCreateDirectories.insert("DIRECTORY_MOVE_TO"); + opsThatCreateDirectories.insert("DIRECTORY_OPEN_SUBSPACE"); +} + +Future startTest(Uncancellable, std::string clusterFilename, StringRef prefix, int apiVersion) { + try { + populateAtomicOpMap(); // FIXME: NOOOOOO! + populateOpsThatCreateDirectories(); // FIXME + + // This is "our" network + g_network = newNet2(TLSConfig()); + + ASSERT(!API::isAPIVersionSelected()); + try { + API::getInstance(); + ASSERT(false); + } catch (Error& e) { + ASSERT(e.code() == error_code_api_version_unset); + } + + API* fdb = API::selectAPIVersion(apiVersion); + ASSERT(API::isAPIVersionSelected()); + ASSERT(fdb->getAPIVersion() == apiVersion); + // fdb->setNetworkOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE); + + // We have to start the fdb_flow network and thread separately! + fdb->setupNetwork(); + startThread(networkThread, fdb); + + // Connect to the default cluster/database, and create a transaction + auto db = fdb->createDatabase(clusterFilename); + + auto data = makeReference(fdb); + co_await runTest(data, db, prefix); + + // Stopping the network returns from g_network->run() and allows + // the program to terminate + g_network->stop(); + } catch (Error& e) { + TraceEvent("ErrorRunningTest").error(e); + if (LOG_ERRORS) { + printf("Flow tester encountered error: %s\n", e.name()); + fflush(stdout); + } + flushAndExit(1); + } +} + +Future _test_versionstamp(Uncancellable) { + try { + g_network = newNet2(TLSConfig()); + + API* fdb = FDB::API::selectAPIVersion(FDB_API_VERSION); + + fdb->setupNetwork(); + startThread(networkThread, fdb); + + auto db = fdb->createDatabase(); + Reference tr = db->createTransaction(); + + Future> ftrVersion = tr->getVersionstamp(); + + tr->atomicOp( + "foo"_sr, "blahblahbl\x00\x00\x00\x00"_sr, FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_VALUE); + + co_await tr->commit(); // should use retry loop + + tr->reset(); + + Optional> optionalDbVersion = co_await tr->get("foo"_sr); + FDBStandalone dbVersion = optionalDbVersion.get(); + FDBStandalone trVersion = co_await ftrVersion; + + ASSERT(trVersion.compare(dbVersion) == 0); + + fprintf(stderr, "%s\n", trVersion.printable().c_str()); + + g_network->stop(); + } catch (Error& e) { + TraceEvent("ErrorRunningTest").error(e); + if (LOG_ERRORS) { + printf("Flow tester encountered error: %s\n", e.name()); + fflush(stdout); + } + flushAndExit(1); + } +} + +int main(int argc, char** argv) { + try { + platformInit(); + registerCrashHandler(); + setThreadLocalDeterministicRandomSeed(1); + + // Get arguments + if (argc < 3) { + fprintf(stderr, "Missing arguments! Usage: fdb_flow_tester prefix api_version [cluster_filename]\n"); + return 1; + + /*_test_versionstamp(); + g_network->run(); + + flushAndExit(FDB_EXIT_SUCCESS);*/ + } + StringRef prefix((const uint8_t*)argv[1], strlen(argv[1])); + int apiVersion; + sscanf(argv[2], "%d", &apiVersion); + std::string clusterFilename; + if (argc > 3) { + clusterFilename = std::string(argv[3]); + } + + // start test + startTest(Uncancellable(), clusterFilename, prefix, apiVersion); + + // Run the network until someone tells us to stop + g_network->run(); + + flushAndExit(FDB_EXIT_SUCCESS); + } catch (Error& e) { + fprintf(stderr, "Error: %s\n", e.name()); + TraceEvent(SevError, "MainError").error(e); + flushAndExit(FDB_EXIT_MAIN_ERROR); + } catch (std::exception& e) { + fprintf(stderr, "std::exception: %s\n", e.what()); + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + flushAndExit(FDB_EXIT_MAIN_EXCEPTION); + } +} diff --git a/bindings/flow/tester/Tester.h b/bindings/flow/tester/Tester.h new file mode 100644 index 00000000000..89bef41bafb --- /dev/null +++ b/bindings/flow/tester/Tester.h @@ -0,0 +1,238 @@ +/* + * Tester.h + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#include "flow/IDispatched.h" +#include "bindings/flow/fdb_flow.h" +#include "bindings/flow/IDirectory.h" +#include "bindings/flow/Subspace.h" +#include "bindings/flow/DirectoryLayer.h" + +constexpr bool LOG_ALL = false; +constexpr bool LOG_INSTRUCTIONS = LOG_ALL || false; +constexpr bool LOG_OPS = LOG_ALL || false; +constexpr bool LOG_DIRS = LOG_ALL || false; +constexpr bool LOG_ERRORS = LOG_ALL || false; + +struct FlowTesterData; + +struct StackItem { + StackItem() : index(-1) {} + StackItem(uint32_t i, Future> v) : index(i), value(v) {} + StackItem(uint32_t i, Standalone v) : index(i), value(v) {} + uint32_t index; + Future> value; +}; + +struct FlowTesterStack { + uint32_t index; + std::vector data; + + void push(Future> value) { data.push_back(StackItem(index, value)); } + + void push(Standalone value) { push(Future>(value)); } + + void push(const StackItem& item) { data.push_back(item); } + + void pushTuple(StringRef value, bool utf8 = false) { + FDB::Tuple t; + t.append(value, utf8); + data.push_back(StackItem(index, t.pack())); + } + + void pushError(int errorCode) { + FDB::Tuple t; + t.append("ERROR"_sr); + t.append(format("%d", errorCode)); + // pack above as error string into another tuple + pushTuple(t.pack().toString()); + } + + std::vector pop(uint32_t count = 1) { + std::vector items; + while (!data.empty() && count > 0) { + items.push_back(data.back()); + data.pop_back(); + count--; + } + return items; + } + + Future> waitAndPop(int count); + Future waitAndPop(); + + void dup() { + if (data.empty()) + return; + data.push_back(data.back()); + } + + void clear() { data.clear(); } +}; + +struct InstructionData : public ReferenceCounted { + bool isDatabase; + bool isSnapshot; + StringRef instruction; + Reference tr; + + InstructionData(bool _isDatabase, bool _isSnapshot, StringRef _instruction, Reference _tr) + : isDatabase(_isDatabase), isSnapshot(_isSnapshot), instruction(_instruction), tr(_tr) {} +}; + +struct FlowTesterData; + +struct InstructionFunc + : IDispatched(Reference data, Reference instruction)>> { + static Future call(std::string op, Reference data, Reference instruction) { + ASSERT(data); + ASSERT(instruction); + + auto it = dispatches().find(op); + if (it == dispatches().end()) { + fprintf(stderr, "Unrecognized instruction: %s\n", op.c_str()); + ASSERT(false); + } + + return dispatch(op)(data, instruction); + } +}; +#define REGISTER_INSTRUCTION_FUNC(Op) REGISTER_COMMAND(InstructionFunc, Op, name, call) + +struct DirectoryOrSubspace { + Optional> directory; + Optional subspace; + + DirectoryOrSubspace() {} + explicit DirectoryOrSubspace(Reference directory) : directory(directory) {} + explicit DirectoryOrSubspace(FDB::Subspace* subspace) : subspace(subspace) {} + explicit DirectoryOrSubspace(Reference dirSubspace) + : directory(dirSubspace), subspace(dirSubspace.getPtr()) {} + + bool valid() { return directory.present() || subspace.present(); } + + std::string typeString() { + if (directory.present() && subspace.present()) { + return "DirectorySubspace"; + } else if (directory.present()) { + return "IDirectory"; + } else if (subspace.present()) { + return "Subspace"; + } else { + return "InvalidDirectory"; + } + } +}; + +struct DirectoryTesterData { + std::vector directoryList; + int directoryListIndex; + int directoryErrorIndex; + + Reference directory() { + ASSERT(directoryListIndex < directoryList.size()); + ASSERT(directoryList[directoryListIndex].directory.present()); + return directoryList[directoryListIndex].directory.get(); + } + + FDB::Subspace* subspace() { + ASSERT(directoryListIndex < directoryList.size()); + ASSERT(directoryList[directoryListIndex].subspace.present()); + return directoryList[directoryListIndex].subspace.get(); + } + + DirectoryTesterData() : directoryListIndex(0), directoryErrorIndex(0) { + directoryList.push_back(DirectoryOrSubspace(Reference(makeReference()))); + } + + template + void push(T item) { + directoryList.push_back(DirectoryOrSubspace(item)); + if (LOG_DIRS) { + printf("Pushed %s at %lu\n", directoryList.back().typeString().c_str(), directoryList.size() - 1); + fflush(stdout); + } + } + + void push() { push(DirectoryOrSubspace()); } +}; + +struct FlowTesterData : public ReferenceCounted { + FDB::API* api; + Reference db; + Standalone instructions; + Standalone trName; + FlowTesterStack stack; + FDB::Version lastVersion; + DirectoryTesterData directoryData; + + std::vector> subThreads; + + Future processInstruction(Reference instruction) { + return InstructionFunc::call( + instruction->instruction.toString(), Reference::addRef(this), instruction); + } + + explicit FlowTesterData(FDB::API* api) { this->api = api; } +}; + +std::string tupleToString(FDB::Tuple const& tuple); + +template +using ExecuteMutationResult = decltype(std::declval()().getValue()); + +template +Future> executeMutation(Reference instruction, F func) { + while (true) { + Error err; + bool hasErr = false; + try { + if constexpr (std::is_same_v, Void>) { + co_await func(); + if (instruction->isDatabase) { + co_await instruction->tr->commit(); + } + co_return; + } else { + ExecuteMutationResult result = co_await func(); + if (instruction->isDatabase) { + co_await instruction->tr->commit(); + } + co_return result; + } + } catch (Error& e) { + err = e; + hasErr = true; + } + if (hasErr) { + if (instruction->isDatabase) { + co_await instruction->tr->onError(err); + } else { + throw err; + } + } + } +} diff --git a/bindings/go/CMakeLists.txt b/bindings/go/CMakeLists.txt index cae3ca3dc03..d4670e94967 100644 --- a/bindings/go/CMakeLists.txt +++ b/bindings/go/CMakeLists.txt @@ -22,7 +22,6 @@ set(SRCS src/fdb/directory/directory_subspace.go src/fdb/fdb_test.go src/fdb/snapshot.go - go.mod) set(GOPATH ${CMAKE_CURRENT_BINARY_DIR}) @@ -43,6 +42,7 @@ set(GO_PACKAGE_OUTDIR ${GOPATH}/pkg/${GOPLATFORM}/${GO_IMPORT_PATH}) file(MAKE_DIRECTORY ${GOPATH} ${GO_DEST}) set(go_options_file ${GO_DEST}/src/fdb/generated.go) +set(go_error_file ${GO_DEST}/src/fdb/error_codes_generated.go) set(go_env GOPATH=${GOPATH} CGO_CFLAGS="-I${CMAKE_BINARY_DIR}/bindings/c/foundationdb;-I${CMAKE_SOURCE_DIR}/bindings/c" @@ -62,14 +62,23 @@ foreach(src_file IN LISTS SRCS) endforeach() add_custom_target(copy_go_sources DEPENDS ${SRCS_OUT}) add_custom_command(OUTPUT ${go_options_file} - COMMAND ${GO_EXECUTABLE} run ${CMAKE_CURRENT_SOURCE_DIR}/src/_util/translate_fdb_options.go + COMMAND ${GO_EXECUTABLE} run ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/translate_fdb_options.go -in ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options -out ${go_options_file} - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/_util/translate_fdb_options.go + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/translate_fdb_options.go ${CMAKE_SOURCE_DIR}/fdbclient/vexillographer/fdb.options COMMENT "Generate FDBOptions for GO") add_custom_target(go_options_file DEPENDS ${go_options_file}) add_dependencies(go_options_file copy_go_sources) +add_custom_command(OUTPUT ${go_error_file} + COMMAND ${GO_EXECUTABLE} run ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/gen_errors/main.go + -in ${CMAKE_SOURCE_DIR}/flow/include/flow/error_definitions.h + -out ${go_error_file} + DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/src/internal/gen_errors/main.go + ${CMAKE_SOURCE_DIR}/flow/include/flow/error_definitions.h + COMMENT "Generate FDB errors for GO") +add_custom_target(go_error_file DEPENDS ${go_error_file}) +add_dependencies(go_error_file copy_go_sources) function(build_go_package) set(options LIBRARY EXECUTABLE INCLUDE_TEST) @@ -128,7 +137,7 @@ function(build_go_package) endfunction() build_go_package(LIBRARY NAME fdb_go PATH fdb INCLUDE_TEST) -add_dependencies(fdb_go fdb_c go_options_file) +add_dependencies(fdb_go fdb_c go_options_file go_error_file) build_go_package(LIBRARY NAME tuple_go PATH fdb/tuple INCLUDE_TEST) add_dependencies(tuple_go fdb_go) @@ -149,3 +158,23 @@ add_test( NAME update_bindings_go_src_fdb_generated_go COMMAND ${CMAKE_COMMAND} -E compare_files ${go_options_file} ${CMAKE_CURRENT_SOURCE_DIR}/src/fdb/generated.go ) + +# If this fails, then you need to update bindings/go/src/fdb/error_codes_generated.go +# Ideally this wouldn't be necessary, but it looks like we distribute the go +# bindings directly from the source on github. +add_test( + NAME update_generated_errors + COMMAND ${CMAKE_COMMAND} -E compare_files ${go_error_file} ${CMAKE_CURRENT_SOURCE_DIR}/src/fdb/error_codes_generated.go +) + +# gofmt should be installed with go, but it's better to check if the binary is available. +find_program(GOFMT_EXECUTABLE gofmt HINTS /usr/local/go/bin/) +if(GOFMT_EXECUTABLE) + add_test( + NAME fdb-go-fmt + # gofmt -d doesn't return a non-zero exit code, so we have to check if the output is empty. + COMMAND bash -c "out=$(\"${GOFMT_EXECUTABLE}\" -d \"${CMAKE_CURRENT_SOURCE_DIR}\"); test -z \"$out\" || { echo \"$out\"; exit 1; }" + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) +else() + message(WARNING "gofmt not found. The 'fdb-go-fmt' test will not be available.") +endif() diff --git a/bindings/go/README.md b/bindings/go/README.md index 884b5f1ffae..13e35307c04 100644 --- a/bindings/go/README.md +++ b/bindings/go/README.md @@ -45,3 +45,13 @@ make fdb_go ``` This will create binary packages for the appropriate platform within the "build" subdirectory of this folder. + +Generating options file +---------- + +The [generated.go](./src/fdb/generated.go) is generated based on the [fdb.options](../../fdbclient/vexillographer/fdb.options) file. +If you change the `fdb.options` file you can update the `generated.go` by running the following command from the root of this reporsitory: + +```bash +go run bindings/go/src/_util/translate_fdb_options.go < fdbclient/vexillographer/fdb.options > bindings/go/src/fdb/generated.go +``` diff --git a/bindings/go/src/fdb/database.go b/bindings/go/src/fdb/database.go index 610d51e9db3..47e8cdc21f4 100644 --- a/bindings/go/src/fdb/database.go +++ b/bindings/go/src/fdb/database.go @@ -115,6 +115,9 @@ func (d Database) CreateTransaction() (Transaction, error) { // process address is the form of IP:Port pair without the :tls suffix if the cluster is running // with TLS enabled. The address can also be multiple processes addresses concated by a comma, e.g. // "IP1:Port,IP2:port", in this case the RebootWorker will reboot all provided addresses concurrently. +// It's not recommened to close the Database directly after calling the RebootWorker method as the +// reboot commands will be queue and executed asynchronous. If the Database context is closed immediately +// some of the commands might not be delievered, even if the RebootWorker method returned successfully. func (d Database) RebootWorker(address string, checkFile bool, suspendDuration int) error { t := &futureInt64{ future: newFutureWithDb( @@ -144,7 +147,7 @@ func (d Database) RebootWorker(address string, checkFile bool, suspendDuration i // NOTE: ErrMultiVersionClientUnavailable will be returned if the Multi-Version client API was not enabled. func (d Database) GetClientStatus() ([]byte, error) { if apiVersion == 0 { - return nil, errAPIVersionUnset + return nil, ErrAPIVersionUnset } st := &futureByteSlice{ diff --git a/bindings/go/src/fdb/doc.go b/bindings/go/src/fdb/doc.go index 2ca154cea75..6601573df31 100644 --- a/bindings/go/src/fdb/doc.go +++ b/bindings/go/src/fdb/doc.go @@ -39,31 +39,33 @@ A basic interaction with the FoundationDB API is demonstrated below: package main import ( - "github.com/apple/foundationdb/bindings/go/src/fdb" - "log" - "fmt" + + "github.com/apple/foundationdb/bindings/go/src/fdb" + "log" + "fmt" + ) -func main() { - // Different API versions may expose different runtime behaviors. - fdb.MustAPIVersion(800) - - // Open the default database from the system cluster - db := fdb.MustOpenDefault() - - // Database reads and writes happen inside transactions - ret, err := db.Transact(func(tr fdb.Transaction) (interface{}, error) { - tr.Set(fdb.Key("hello"), []byte("world")) - return tr.Get(fdb.Key("foo")).MustGet(), nil - // db.Transact automatically commits (and if necessary, - // retries) the transaction - }) - if err != nil { - log.Fatalf("Unable to perform FDB transaction (%v)", err) - } - - fmt.Printf("hello is now world, foo was: %s\n", string(ret.([]byte))) -} + func main() { + // Different API versions may expose different runtime behaviors. + fdb.MustAPIVersion(800) + + // Open the default database from the system cluster + db := fdb.MustOpenDefault() + + // Database reads and writes happen inside transactions + ret, err := db.Transact(func(tr fdb.Transaction) (interface{}, error) { + tr.Set(fdb.Key("hello"), []byte("world")) + return tr.Get(fdb.Key("foo")).MustGet(), nil + // db.Transact automatically commits (and if necessary, + // retries) the transaction + }) + if err != nil { + log.Fatalf("Unable to perform FDB transaction (%v)", err) + } + + fmt.Printf("hello is now world, foo was: %s\n", string(ret.([]byte))) + } # Futures diff --git a/bindings/go/src/fdb/error_codes_generated.go b/bindings/go/src/fdb/error_codes_generated.go new file mode 100644 index 00000000000..69546fe6178 --- /dev/null +++ b/bindings/go/src/fdb/error_codes_generated.go @@ -0,0 +1,677 @@ +// Code generated by gen_errors. DO NOT EDIT. + +/* + * error_codes_generated.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fdb + +// To regenerate this file run this command: +// go run ./bindings/go/src/internal/gen_errors/main.go -in ./flow/include/flow/error_definitions.h -out ./bindings/go/src/fdb/error_codes_generated.go +// +// Error sentinel variables derived from flow/include/flow/error_definitions.h. +// Use with errors.Is to match a specific FoundationDB error code: +// +// errors.Is(err, fdb.ErrTransactionTooOld) +var ( + // Success + ErrSuccess = Error{Code: 0} + // End of stream + ErrEndOfStream = Error{Code: 1} + // Operation failed + ErrOperationFailed = Error{Code: 1000} + // Shard is not available from this server + ErrWrongShardServer = Error{Code: 1001} + // Operation result no longer necessary + ErrOperationObsolete = Error{Code: 1002} + // Cache server is not warm for this range + ErrColdCacheServer = Error{Code: 1003} + // Operation timed out + ErrTimedOut = Error{Code: 1004} + // Conflict occurred while changing coordination information + ErrCoordinatedStateConflict = Error{Code: 1005} + // All alternatives failed + ErrAllAlternativesFailed = Error{Code: 1006} + // Transaction is too old to perform reads or be committed + ErrTransactionTooOld = Error{Code: 1007} + // Not enough physical servers available + ErrNoMoreServers = Error{Code: 1008} + // Request for future version + ErrFutureVersion = Error{Code: 1009} + // Conflicting attempts to change data distribution + ErrMovekeysConflict = Error{Code: 1010} + // TLog stopped + ErrTLogStopped = Error{Code: 1011} + // Server request queue is full + ErrServerRequestQueueFull = Error{Code: 1012} + // Transaction not committed due to conflict with another transaction + ErrNotCommitted = Error{Code: 1020} + // Transaction may or may not have committed + ErrCommitUnknownResult = Error{Code: 1021} + // Idempotency id for transaction may have expired, so the commit status of the transaction cannot be determined + ErrCommitUnknownResultFatal = Error{Code: 1022} + // Operation aborted because the transaction was cancelled + ErrTransactionCancelled = Error{Code: 1025} + // Network connection failed + ErrConnectionFailed = Error{Code: 1026} + // Coordination servers have changed + ErrCoordinatorsChanged = Error{Code: 1027} + // New coordination servers did not respond in a timely way + ErrNewCoordinatorsTimedOut = Error{Code: 1028} + // Watch cancelled because storage server watch limit exceeded + ErrWatchCancelled = Error{Code: 1029} + // Request may or may not have been delivered + ErrRequestMaybeDelivered = Error{Code: 1030} + // Operation aborted because the transaction timed out + ErrTransactionTimedOut = Error{Code: 1031} + // Too many watches currently set + ErrTooManyWatches = Error{Code: 1032} + // Locality information not available + ErrLocalityInformationUnavailable = Error{Code: 1033} + // Watches cannot be set if read your writes is disabled + ErrWatchesDisabled = Error{Code: 1034} + // Default error for an ErrorOr object + ErrDefaultErrorOr = Error{Code: 1035} + // Read or wrote an unreadable key + ErrAccessedUnreadable = Error{Code: 1036} + // Storage process does not have recent mutations + ErrProcessBehind = Error{Code: 1037} + // Database is locked + ErrDatabaseLocked = Error{Code: 1038} + // The protocol version of the cluster has changed + ErrClusterVersionChanged = Error{Code: 1039} + // External client has already been loaded + ErrExternalClientAlreadyLoaded = Error{Code: 1040} + // DNS lookup failed + ErrLookupFailed = Error{Code: 1041} + // CommitProxy commit memory limit exceeded + ErrCommitProxyMemoryLimitExceeded = Error{Code: 1042} + // Operation no longer supported due to shutdown + ErrShutdownInProgress = Error{Code: 1043} + // Failed to deserialize an object + ErrSerializationFailed = Error{Code: 1044} + // No peer references for connection + ErrConnectionUnreferenced = Error{Code: 1048} + // Connection closed after idle timeout + ErrConnectionIdle = Error{Code: 1049} + // The disk queue adapter reset + ErrDiskAdapterReset = Error{Code: 1050} + // Batch GRV request rate limit exceeded + ErrBatchTransactionThrottled = Error{Code: 1051} + // Data distribution components cancelled + ErrDdCancelled = Error{Code: 1052} + // Data distributor not found + ErrDdNotFound = Error{Code: 1053} + // Connection file mismatch + ErrWrongConnectionFile = Error{Code: 1054} + // The requested changes have been compacted away + ErrVersionAlreadyCompacted = Error{Code: 1055} + // Local configuration file has changed. Restart and apply these changes + ErrLocalConfigChanged = Error{Code: 1056} + // Format version not supported + ErrUnsupportedFormatVersion = Error{Code: 1058} + // Change feed not found + ErrUnknownChangeFeed = Error{Code: 1059} + // Change feed not registered + ErrChangeFeedNotRegistered = Error{Code: 1060} + // Change feed was cancelled + ErrChangeFeedCancelled = Error{Code: 1062} + // Tried to read a version older than what has been popped from the change feed + ErrChangeFeedPopped = Error{Code: 1066} + // Page header does not match location on disk + ErrPageHeaderWrongPageID = Error{Code: 1068} + // Page header checksum failed + ErrPageHeaderChecksumFailed = Error{Code: 1069} + // Page header version is not supported + ErrPageHeaderVersionNotSupported = Error{Code: 1070} + // Page encoding type is not supported or not valid + ErrPageEncodingNotSupported = Error{Code: 1071} + // Page content decoding failed + ErrPageDecodingFailed = Error{Code: 1072} + // Page content decoding failed + ErrUnexpectedEncodingType = Error{Code: 1073} + // Encryption key not found + ErrEncryptionKeyNotFound = Error{Code: 1074} + // Data move was cancelled + ErrDataMoveCancelled = Error{Code: 1075} + // Dest team was not found for data move + ErrDataMoveDestTeamNotFound = Error{Code: 1076} + // GetReadVersion proxy memory limit exceeded + ErrGRVProxyMemoryLimitExceeded = Error{Code: 1078} + // Too many feed streams to a single storage server + ErrStorageTooManyFeedStreams = Error{Code: 1080} + // Storage engine was never successfully initialized. + ErrStorageEngineNotInitialized = Error{Code: 1081} + // Storage engine type is not recognized. + ErrUnknownStorageEngine = Error{Code: 1082} + // A duplicate snapshot request has been sent, the old request is discarded. + ErrDuplicateSnapshotRequest = Error{Code: 1083} + // DataDistribution configuration changed. + ErrDdConfigChanged = Error{Code: 1084} + // Consistency check urgent task is failed + ErrConsistencyCheckUrgentTaskFailed = Error{Code: 1085} + // Data move conflict in SS + ErrDataMoveConflict = Error{Code: 1086} + // Consistency check urgent got a duplicate request + ErrConsistencyCheckUrgentDuplicateRequest = Error{Code: 1087} + // Broken promise + ErrBrokenPromise = Error{Code: 1100} + // Asynchronous operation cancelled + ErrOperationCancelled = Error{Code: 1101} + // Future has been released + ErrFutureReleased = Error{Code: 1102} + // Connection object leaked + ErrConnectionLeaked = Error{Code: 1103} + // Never reply to the request + ErrNeverReply = Error{Code: 1104} + // Retry operation + ErrRetry = Error{Code: 1105} + // Recruitment of a server failed + ErrRecruitmentFailed = Error{Code: 1200} + // Attempt to move keys to a storage server that was removed + ErrMoveToRemovedServer = Error{Code: 1201} + // Normal worker shut down + ErrWorkerRemoved = Error{Code: 1202} + // Cluster recovery failed + ErrClusterRecoveryFailed = Error{Code: 1203} + // Master hit maximum number of versions in flight + ErrMasterMaxVersionsInFlight = Error{Code: 1204} + // Cluster recovery terminating because a TLog failed + ErrTLogFailed = Error{Code: 1205} + // Recovery of a worker process failed + ErrWorkerRecoveryFailed = Error{Code: 1206} + // Reboot of server process requested + ErrPleaseReboot = Error{Code: 1207} + // Reboot of server process requested, with deletion of state + ErrPleaseRebootDelete = Error{Code: 1208} + // Master terminating because a CommitProxy failed + ErrCommitProxyFailed = Error{Code: 1209} + // Cluster recovery terminating because a Resolver failed + ErrResolverFailed = Error{Code: 1210} + // Server is under too much load and cannot respond + ErrServerOverloaded = Error{Code: 1211} + // Cluster recovery terminating because a backup worker failed + ErrBackupWorkerFailed = Error{Code: 1212} + // Transaction tag is being throttled + ErrTagThrottled = Error{Code: 1213} + // Cluster recovery terminating because a GRVProxy failed + ErrGRVProxyFailed = Error{Code: 1214} + // The data distribution tracker has been cancelled + ErrDdTrackerCancelled = Error{Code: 1215} + // Process has failed to make sufficient progress + ErrFailedToProgress = Error{Code: 1216} + // Attempted to join cluster with a different cluster ID + ErrInvalidClusterID = Error{Code: 1217} + // Restart cluster controller process + ErrRestartClusterController = Error{Code: 1218} + // Need to reboot the storage engine + ErrPleaseRebootKVStore = Error{Code: 1219} + // Current software does not support database format + ErrIncompatibleSoftwareVersion = Error{Code: 1220} + // Validate storage consistency operation failed + ErrAuditStorageFailed = Error{Code: 1221} + // Exceeded the max number of allowed concurrent audit storage requests + ErrAuditStorageExceededRequestLimit = Error{Code: 1222} + // Exceeded maximum time allowed to read or write. + ErrKeyValueStoreDeadlineExceeded = Error{Code: 1224} + // Found data corruption + ErrAuditStorageError = Error{Code: 1226} + // Cluster recovery terminating because master has failed + ErrMasterFailed = Error{Code: 1227} + // Test failed + ErrTestFailed = Error{Code: 1228} + // Need background datamove cleanup + ErrRetryCleanUpDatamoveTombstoneAdded = Error{Code: 1229} + // Persist new audit metadata error + ErrPersistNewAuditMetadataError = Error{Code: 1230} + // Failed to cancel an audit + ErrCancelAuditStorageFailed = Error{Code: 1231} + // Audit has been cancelled + ErrAuditStorageCancelled = Error{Code: 1232} + // Found location metadata corruption + ErrLocationMetadataCorruption = Error{Code: 1233} + // Audit task is scheduled by an outdated DD + ErrAuditStorageTaskOutdated = Error{Code: 1234} + // Transaction throttled due to hot shard + ErrTransactionThrottledHotShard = Error{Code: 1235} + // Storage replicas not consistent + ErrStorageReplicaComparisonError = Error{Code: 1236} + // Storage replica cannot be reached + ErrUnreachableStorageReplica = Error{Code: 1237} + // Bulk loading task failed + ErrBulkloadTaskFailed = Error{Code: 1238} + // Bulk loading task outdated + ErrBulkloadTaskOutdated = Error{Code: 1239} + // Lock range failed + ErrRangeLockFailed = Error{Code: 1241} + // Transaction rejected due to range lock + ErrTransactionRejectedRangeLocked = Error{Code: 1242} + // Bulk dumping task failed + ErrBulkdumpTaskFailed = Error{Code: 1243} + // Bulk dumping task outdated + ErrBulkdumpTaskOutdated = Error{Code: 1244} + // Bulkload fileset provides invalid filepath + ErrBulkloadFilesetInvalidFilepath = Error{Code: 1245} + // Bulkload manifest string is failed to decode + ErrBulkloadManifestDecodeError = Error{Code: 1246} + // Range lock is rejected + ErrRangeLockReject = Error{Code: 1247} + // Range unlock is rejected + ErrRangeUnlockReject = Error{Code: 1248} + // Bulkload dataset does not cover the required range + ErrBulkloadDatasetNotCoverRequiredRange = Error{Code: 1249} + // BulkLoad requires cluster configuration with both shard_encode_location_metadata=1 and enable_read_lock_on_range=1 + ErrBulkloadInvalidConfiguration = Error{Code: 1250} + // GRV request rejected because estimated queue wait exceeds transaction limit + ErrTransactionGRVQueueRejected = Error{Code: 1251} + // finishMoveKeys exceeded retry limit + ErrFinishMoveKeysTooManyRetries = Error{Code: 1252} + // Platform error + ErrPlatformError = Error{Code: 1500} + // Large block allocation failed + ErrLargeAllocFailed = Error{Code: 1501} + // QueryPerformanceCounter error + ErrPerformanceCounterError = Error{Code: 1502} + // Null allocator was used to allocate memory + ErrBadAllocator = Error{Code: 1503} + // Disk i/o operation failed + ErrIOError = Error{Code: 1510} + // File not found + ErrFileNotFound = Error{Code: 1511} + // Unable to bind to network + ErrBindFailed = Error{Code: 1512} + // File could not be read + ErrFileNotReadable = Error{Code: 1513} + // File could not be written + ErrFileNotWritable = Error{Code: 1514} + // No cluster file found in current directory or default location + ErrNoClusterFileFound = Error{Code: 1515} + // File too large to be read + ErrFileTooLarge = Error{Code: 1516} + // Non sequential file operation not allowed + ErrNonSequentialOp = Error{Code: 1517} + // HTTP response was badly formed + ErrHTTPBadResponse = Error{Code: 1518} + // HTTP request not accepted + ErrHTTPNotAccepted = Error{Code: 1519} + // A data checksum failed + ErrChecksumFailed = Error{Code: 1520} + // A disk IO operation failed to complete in a timely manner + ErrIOTimeout = Error{Code: 1521} + // A structurally corrupt data file was detected + ErrFileCorrupt = Error{Code: 1522} + // HTTP response code not received or indicated failure + ErrHTTPRequestFailed = Error{Code: 1523} + // HTTP request failed due to bad credentials + ErrHTTPAuthFailed = Error{Code: 1524} + // HTTP response contained an unexpected X-Request-ID header + ErrHTTPBadRequestID = Error{Code: 1525} + // Invalid REST URI + ErrRestInvalidURI = Error{Code: 1526} + // Invalid RESTClient knob + ErrRestInvalidRestClientKnob = Error{Code: 1527} + // ConnectKey not found in connection pool + ErrRestConnectpoolKeyNotFound = Error{Code: 1528} + // Unable to lock the file + ErrLockFileFailure = Error{Code: 1529} + // Unsupported REST protocol + ErrRestUnsupportedProtocol = Error{Code: 1530} + // Malformed REST response + ErrRestMalformedResponse = Error{Code: 1531} + // Max BaseCipher length violation + ErrRestMaxBaseCipherLen = Error{Code: 1532} + // Requested resource was not found + ErrResourceNotFound = Error{Code: 1533} + // Invalid API call + ErrClientInvalidOperation = Error{Code: 2000} + // Commit with incomplete read + ErrCommitReadIncomplete = Error{Code: 2002} + // Invalid test specification + ErrTestSpecificationInvalid = Error{Code: 2003} + // Key outside legal range + ErrKeyOutsideLegalRange = Error{Code: 2004} + // Range begin key larger than end key + ErrInvertedRange = Error{Code: 2005} + // Option set with an invalid value + ErrInvalidOptionValue = Error{Code: 2006} + // Option not valid in this context + ErrInvalidOption = Error{Code: 2007} + // Action not possible before the network is configured + ErrNetworkNotSetup = Error{Code: 2008} + // Network can be configured only once + ErrNetworkAlreadySetup = Error{Code: 2009} + // Transaction already has a read version set + ErrReadVersionAlreadySet = Error{Code: 2010} + // Version not valid + ErrVersionInvalid = Error{Code: 2011} + // Range limits not valid + ErrRangeLimitsInvalid = Error{Code: 2012} + // Database name must be 'DB' + ErrInvalidDatabaseName = Error{Code: 2013} + // Attribute not found + ErrAttributeNotFound = Error{Code: 2014} + // Future not ready + ErrFutureNotSet = Error{Code: 2015} + // Future not an error + ErrFutureNotError = Error{Code: 2016} + // Operation issued while a commit was outstanding + ErrUsedDuringCommit = Error{Code: 2017} + // Unrecognized atomic mutation type + ErrInvalidMutationType = Error{Code: 2018} + // Attribute too large for type int + ErrAttributeTooLarge = Error{Code: 2019} + // Transaction does not have a valid commit version + ErrTransactionInvalidVersion = Error{Code: 2020} + // Transaction is read-only and therefore does not have a commit version + ErrNoCommitVersion = Error{Code: 2021} + // Environment variable network option could not be set + ErrEnvironmentVariableNetworkOptionFailed = Error{Code: 2022} + // Attempted to commit a transaction specified as read-only + ErrTransactionReadOnly = Error{Code: 2023} + // Invalid cache eviction policy, only random and lru are supported + ErrInvalidCacheEvictionPolicy = Error{Code: 2024} + // Network can only be started once + ErrNetworkCannotBeRestarted = Error{Code: 2025} + // Detected a deadlock in a callback called from the network thread + ErrBlockedFromNetworkThread = Error{Code: 2026} + // The index in K[] or V[] is not a valid number or out of range + ErrMapperBadIndex = Error{Code: 2030} + // A mapped key is not set in database + ErrMapperNoSuchKey = Error{Code: 2031} + // One of the mapped range queries is too large + ErrQuickGetKeyValuesHasMore = Error{Code: 2033} + // Found a mapped key that is not served in the same SS + ErrQuickGetValueMiss = Error{Code: 2034} + // Found a mapped range that is not served in the same SS + ErrQuickGetKeyValuesMiss = Error{Code: 2035} + // getMappedRange does not support continuation for now + ErrGetMappedKeyValuesHasMore = Error{Code: 2038} + // getMappedRange tries to read data that were previously written in the transaction + ErrGetMappedRangeReadsYourWrites = Error{Code: 2039} + // Checkpoint not found + ErrCheckpointNotFound = Error{Code: 2040} + // The key cannot be parsed as a tuple + ErrKeyNotTuple = Error{Code: 2041} + // The value cannot be parsed as a tuple + ErrValueNotTuple = Error{Code: 2042} + // The mapper cannot be parsed as a tuple + ErrMapperNotTuple = Error{Code: 2043} + // Invalid checkpoint format + ErrInvalidCheckpointFormat = Error{Code: 2044} + // Failed to create a checkpoint + ErrFailedToCreateCheckpoint = Error{Code: 2046} + // Failed to restore a checkpoint + ErrFailedToRestoreCheckpoint = Error{Code: 2047} + // Failed to dump shard metadata for a checkpoint to a sst file + ErrFailedToCreateCheckpointShardMetadata = Error{Code: 2048} + // Failed to parse address + ErrAddressParseError = Error{Code: 2049} + // Incompatible protocol version + ErrIncompatibleProtocolVersion = Error{Code: 2100} + // Transaction exceeds byte limit + ErrTransactionTooLarge = Error{Code: 2101} + // Key length exceeds limit + ErrKeyTooLarge = Error{Code: 2102} + // Value length exceeds limit + ErrValueTooLarge = Error{Code: 2103} + // Connection string invalid + ErrConnectionStringInvalid = Error{Code: 2104} + // Local address in use + ErrAddressInUse = Error{Code: 2105} + // Invalid local address + ErrInvalidLocalAddress = Error{Code: 2106} + // TLS error + ErrTLSError = Error{Code: 2107} + // Operation is not supported + ErrUnsupportedOperation = Error{Code: 2108} + // Too many tags set on transaction + ErrTooManyTags = Error{Code: 2109} + // Tag set on transaction is too long + ErrTagTooLong = Error{Code: 2110} + // Too many tag throttles have been created + ErrTooManyTagThrottles = Error{Code: 2111} + // Special key space range read crosses modules. Refer to the `special_key_space_relaxed' transaction option for more details. + ErrSpecialKeysCrossModuleRead = Error{Code: 2112} + // Special key space range read does not intersect a module. Refer to the `special_key_space_relaxed' transaction option for more details. + ErrSpecialKeysNoModuleFound = Error{Code: 2113} + // Special Key space is not allowed to write by default. Refer to the `special_key_space_enable_writes` transaction option for more details. + ErrSpecialKeysWriteDisabled = Error{Code: 2114} + // Special key space key or keyrange in set or clear does not intersect a module + ErrSpecialKeysNoWriteModuleFound = Error{Code: 2115} + // Special key space clear crosses modules + ErrSpecialKeysCrossModuleClear = Error{Code: 2116} + // Api call through special keys failed. For more information, call get on special key 0xff0xff/error_message to get a json string of the error message. + ErrSpecialKeysAPIFailure = Error{Code: 2117} + // Invalid client library metadata. + ErrClientLibInvalidMetadata = Error{Code: 2118} + // Client library with same identifier already exists on the cluster. + ErrClientLibAlreadyExists = Error{Code: 2119} + // Client library for the given identifier not found. + ErrClientLibNotFound = Error{Code: 2120} + // Client library exists, but is not available for download. + ErrClientLibNotAvailable = Error{Code: 2121} + // Invalid client library binary. + ErrClientLibInvalidBinary = Error{Code: 2122} + // No external client library provided. + ErrNoExternalClientProvided = Error{Code: 2123} + // All external clients have failed. + ErrAllExternalClientsFailed = Error{Code: 2124} + // None of the available clients match the protocol version of the cluster. + ErrIncompatibleClient = Error{Code: 2125} + // API version is not set + ErrAPIVersionUnset = Error{Code: 2200} + // API version may be set only once + ErrAPIVersionAlreadySet = Error{Code: 2201} + // API version not valid + ErrAPIVersionInvalid = Error{Code: 2202} + // API version not supported + ErrAPIVersionNotSupported = Error{Code: 2203} + // Failed to load a required FDB API function. + ErrAPIFunctionMissing = Error{Code: 2204} + // EXACT streaming mode requires limits, but none were given + ErrExactModeWithoutLimits = Error{Code: 2210} + // Unrecognized data type in packed tuple + ErrInvalidTupleDataType = Error{Code: 2250} + // Tuple does not have element at specified index + ErrInvalidTupleIndex = Error{Code: 2251} + // Cannot unpack key that is not in subspace + ErrKeyNotInSubspace = Error{Code: 2252} + // Cannot specify a prefix unless manual prefixes are enabled + ErrManualPrefixesNotEnabled = Error{Code: 2253} + // Cannot specify a prefix in a partition + ErrPrefixInPartition = Error{Code: 2254} + // Root directory cannot be opened + ErrCannotOpenRootDirectory = Error{Code: 2255} + // Directory already exists + ErrDirectoryAlreadyExists = Error{Code: 2256} + // Directory does not exist + ErrDirectoryDoesNotExist = Error{Code: 2257} + // Directory's parent does not exist + ErrParentDirectoryDoesNotExist = Error{Code: 2258} + // Directory has already been created with a different layer string + ErrMismatchedLayer = Error{Code: 2259} + // Invalid directory layer metadata + ErrInvalidDirectoryLayerMetadata = Error{Code: 2260} + // Directory cannot be moved between partitions + ErrCannotMoveDirectoryBetweenPartitions = Error{Code: 2261} + // Directory partition cannot be used as subspace + ErrCannotUsePartitionAsSubspace = Error{Code: 2262} + // Directory layer was created with an incompatible version + ErrIncompatibleDirectoryVersion = Error{Code: 2263} + // Database has keys stored at the prefix chosen by the automatic prefix allocator + ErrDirectoryPrefixNotEmpty = Error{Code: 2264} + // Directory layer already has a conflicting prefix + ErrDirectoryPrefixInUse = Error{Code: 2265} + // Target directory is invalid + ErrInvalidDestinationDirectory = Error{Code: 2266} + // Root directory cannot be modified + ErrCannotModifyRootDirectory = Error{Code: 2267} + // UUID is not sixteen bytes + ErrInvalidUUIDSize = Error{Code: 2268} + // Versionstamp is not exactly twelve bytes + ErrInvalidVersionstampSize = Error{Code: 2269} + // Backup error + ErrBackupError = Error{Code: 2300} + // Restore error + ErrRestoreError = Error{Code: 2301} + // Backup duplicate request + ErrBackupDuplicate = Error{Code: 2311} + // Backup unneeded request + ErrBackupUnneeded = Error{Code: 2312} + // Backup file block size too small + ErrBackupBadBlockSize = Error{Code: 2313} + // Backup Container URL invalid + ErrBackupInvalidURL = Error{Code: 2314} + // Backup Container info invalid + ErrBackupInvalidInfo = Error{Code: 2315} + // Cannot expire requested data from backup without violating minimum restorability + ErrBackupCannotExpire = Error{Code: 2316} + // Cannot find authentication details (such as a password or secret key) for the specified Backup Container URL + ErrBackupAuthMissing = Error{Code: 2317} + // Cannot read or parse one or more sources of authentication information for Backup Container URLs + ErrBackupAuthUnreadable = Error{Code: 2318} + // Backup does not exist + ErrBackupDoesNotExist = Error{Code: 2319} + // Backup before 6.3 cannot be filtered with key ranges + ErrBackupNotFilterableWithKeyRanges = Error{Code: 2320} + // Backup key ranges doesn't overlap with key ranges filter + ErrBackupNotOverlappedWithKeysFilter = Error{Code: 2321} + // bucket is not in the URL for backup + ErrBucketNotInURL = Error{Code: 2322} + // Invalid restore version + ErrRestoreInvalidVersion = Error{Code: 2361} + // Corrupted backup data + ErrRestoreCorruptedData = Error{Code: 2362} + // Missing backup data + ErrRestoreMissingData = Error{Code: 2363} + // Restore duplicate request + ErrRestoreDuplicateTag = Error{Code: 2364} + // Restore tag does not exist + ErrRestoreUnknownTag = Error{Code: 2365} + // Unknown backup/restore file type + ErrRestoreUnknownFileType = Error{Code: 2366} + // Unsupported backup file version + ErrRestoreUnsupportedFileVersion = Error{Code: 2367} + // Unexpected number of bytes read + ErrRestoreBadRead = Error{Code: 2368} + // Backup file has unexpected padding bytes + ErrRestoreCorruptedDataPadding = Error{Code: 2369} + // Attempted to restore into a non-empty destination database + ErrRestoreDestinationNotEmpty = Error{Code: 2370} + // Attempted to restore using a UID that had been used for an aborted restore + ErrRestoreDuplicateUID = Error{Code: 2371} + // Invalid task version + ErrTaskInvalidVersion = Error{Code: 2381} + // Task execution stopped due to timeout, abort, or completion by another worker + ErrTaskInterrupted = Error{Code: 2382} + // The provided encryption key file has invalid contents + ErrInvalidEncryptionKeyFile = Error{Code: 2383} + // Missing mutation logs + ErrBlobRestoreMissingLogs = Error{Code: 2384} + // Corrupted mutation logs + ErrBlobRestoreCorruptedLogs = Error{Code: 2385} + // Invalid manifest URL + ErrBlobRestoreInvalidManifestURL = Error{Code: 2386} + // Corrupted manifest + ErrBlobRestoreCorruptedManifest = Error{Code: 2387} + // Missing manifest + ErrBlobRestoreMissingManifest = Error{Code: 2388} + // Blob migrator is replaced + ErrBlobMigratorReplaced = Error{Code: 2389} + // BulkDump dataset incomplete. Use --rangefile flag + ErrRestoreBulkloadDatasetIncomplete = Error{Code: 2390} + // BulkLoad operation failed + ErrRestoreBulkloadFailed = Error{Code: 2391} + // BulkDump operation timed out + ErrBackupBulkdumpTimeout = Error{Code: 2392} + // BulkDump operation failed + ErrBackupBulkdumpFailed = Error{Code: 2393} + // Expected key is missing + ErrKeyNotFound = Error{Code: 2400} + // JSON string was malformed + ErrJSONMalformed = Error{Code: 2401} + // JSON string did not terminate where expected + ErrJSONEofExpected = Error{Code: 2402} + // Failed to disable tlog pops + ErrSnapDisableTLogPopFailed = Error{Code: 2500} + // Failed to snapshot storage nodes + ErrSnapStorageFailed = Error{Code: 2501} + // Failed to snapshot TLog nodes + ErrSnapTLogFailed = Error{Code: 2502} + // Failed to snapshot coordinator nodes + ErrSnapCoordFailed = Error{Code: 2503} + // Failed to enable tlog pops + ErrSnapEnableTLogPopFailed = Error{Code: 2504} + // Snapshot create binary path not whitelisted + ErrSnapPathNotWhitelisted = Error{Code: 2505} + // Unsupported when the cluster is not fully recovered + ErrSnapNotFullyRecoveredUnsupported = Error{Code: 2506} + // Unsupported when log anti quorum is configured + ErrSnapLogAntiQuorumUnsupported = Error{Code: 2507} + // Cluster recovery during snapshot operation not supported + ErrSnapWithRecoveryUnsupported = Error{Code: 2508} + // The given uid string is not a 32-length hex string + ErrSnapInvalidUIDString = Error{Code: 2509} + // Encryption operation error + ErrEncryptOpsError = Error{Code: 2700} + // Encryption header metadata mismatch + ErrEncryptHeaderMetadataMismatch = Error{Code: 2701} + // Expected encryption key is missing + ErrEncryptKeyNotFound = Error{Code: 2702} + // Expected encryption key TTL has expired + ErrEncryptKeyTTLExpired = Error{Code: 2703} + // Encryption header authentication token mismatch + ErrEncryptHeaderAuthtokenMismatch = Error{Code: 2704} + // Attempt to update encryption cipher key + ErrEncryptUpdateCipher = Error{Code: 2705} + // Invalid encryption cipher details + ErrEncryptInvalidID = Error{Code: 2706} + // Encryption keys fetch from external KMS failed + ErrEncryptKeysFetchFailed = Error{Code: 2707} + // Invalid encryption/kms configuration: discovery-url, validation-token, endpoint etc. + ErrEncryptInvalidKMSConfig = Error{Code: 2708} + // Encryption not supported + ErrEncryptUnsupported = Error{Code: 2709} + // Encryption mode mismatch with configuration + ErrEncryptModeMismatch = Error{Code: 2710} + // Encryption key-check-value mismatch + ErrEncryptKeyCheckValueMismatch = Error{Code: 2711} + // Max BaseCipher buffer length violation + ErrEncryptMaxBaseCipherLen = Error{Code: 2712} + // An unknown error occurred + ErrUnknownError = Error{Code: 4000} + // An internal error occurred + ErrInternalError = Error{Code: 4100} + // Not implemented yet + ErrNotImplemented = Error{Code: 4200} + // Client tried to access unauthorized data + ErrPermissionDenied = Error{Code: 6000} + // A untrusted client tried to send a message to a private endpoint + ErrUnauthorizedAttempt = Error{Code: 6001} + // Digital signature operation error + ErrDigitalSignatureOpsError = Error{Code: 6002} + // Failed to verify authorization token + ErrAuthorizationTokenVerifyFailed = Error{Code: 6003} + // Failed to decode public/private key + ErrPkeyDecodeError = Error{Code: 6004} + // Failed to encode public/private key + ErrPkeyEncodeError = Error{Code: 6005} + // gRPC Error + ErrGRPCError = Error{Code: 7000} +) diff --git a/bindings/go/src/fdb/errors.go b/bindings/go/src/fdb/errors.go index c40063f2f42..b72bc901e59 100644 --- a/bindings/go/src/fdb/errors.go +++ b/bindings/go/src/fdb/errors.go @@ -20,6 +20,8 @@ // FoundationDB Go API +//go:generate go run ./internal/gen_errors/main.go -in ../../../flow/include/flow/error_definitions.h -out fdb/error_codes_generated.go + package fdb // #define FDB_API_VERSION 800 @@ -34,10 +36,12 @@ import ( // Error may be returned by any FoundationDB API function that returns error, or // as a panic from any FoundationDB API function whose name ends with OrPanic. // -// You may compare the Code field of an Error against the list of FoundationDB -// error codes at https://apple.github.io/foundationdb/api-error-codes.html, -// but generally an Error should be passed to (Transaction).OnError. When using -// (Database).Transact, non-fatal errors will be retried automatically. +// Use errors.Is with the sentinel variables defined in error_codes_generated.go +// to check for specific error codes: +// +// errors.Is(err, fdb.ErrTransactionTooOld) +// +// When using (Database).Transact, non-fatal errors will be retried automatically. type Error struct { Code int } @@ -46,14 +50,15 @@ func (e Error) Error() string { return fmt.Sprintf("FoundationDB error code %d (%s)", e.Code, C.GoString(C.fdb_get_error(C.fdb_error_t(e.Code)))) } -// SOMEDAY: these (along with others) should be coming from fdb.options? - -var ( - errNetworkNotSetup = Error{2008} - errNetworkAlreadySetup = Error{2009} // currently unused - errNetworkCannotBeRestarted = Error{2025} // currently unused - - errAPIVersionUnset = Error{2200} - errAPIVersionAlreadySet = Error{2201} - errAPIVersionNotSupported = Error{2203} -) +// Is implements the errors.Is contract. It matches target if target is an Error +// or *Error with the same Code, enabling errors.Is(err, fdb.ErrTransactionTooOld). +func (e Error) Is(target error) bool { + switch t := target.(type) { + case Error: + return e.Code == t.Code + case *Error: + return t != nil && e.Code == t.Code + default: + return false + } +} diff --git a/bindings/go/src/fdb/errors_test.go b/bindings/go/src/fdb/errors_test.go index 15babf7f0d6..a6e054a7271 100644 --- a/bindings/go/src/fdb/errors_test.go +++ b/bindings/go/src/fdb/errors_test.go @@ -60,3 +60,30 @@ func TestErrorWrapping(t *testing.T) { } } } + +func TestErrorIs(t *testing.T) { + // value sentinel — direct match + if !errors.Is(Error{Code: 1007}, ErrTransactionTooOld) { + t.Error("expected match: Error{1007} vs ErrTransactionTooOld") + } + // pointer target (user takes address of sentinel var) + if !errors.Is(Error{Code: 1007}, &ErrTransactionTooOld) { + t.Error("expected match: Error{1007} vs &ErrTransactionTooOld") + } + // wrapped value + if !errors.Is(fmt.Errorf("wrap: %w", Error{Code: 1007}), ErrTransactionTooOld) { + t.Error("expected match through wrapping") + } + // wrapped pointer + if !errors.Is(fmt.Errorf("wrap: %w", &Error{Code: 1007}), ErrTransactionTooOld) { + t.Error("expected match through wrapped *Error") + } + // no false positive: wrong code + if errors.Is(Error{Code: 1008}, ErrTransactionTooOld) { + t.Error("expected no match: Error{1008} vs ErrTransactionTooOld") + } + // no false positive: unrelated error + if errors.Is(errors.New("other"), ErrTransactionTooOld) { + t.Error("expected no match: non-fdb error vs ErrTransactionTooOld") + } +} diff --git a/bindings/go/src/fdb/fdb.go b/bindings/go/src/fdb/fdb.go index 340a1874105..da86bc3c0aa 100644 --- a/bindings/go/src/fdb/fdb.go +++ b/bindings/go/src/fdb/fdb.go @@ -43,7 +43,7 @@ var ( // with the network thread while the network thread is no more running. ErrNetworkIsStopped = errors.New("network is stopped") - // ErrNetworkAlreadyStopped for a too early call to StopNetwork(). + // ErrNetworkNotStarted for a too early call to StopNetwork(). ErrNetworkNotStarted = errors.New("network has not been started") ) @@ -107,7 +107,7 @@ func (opt NetworkOptions) setOpt(code int, param []byte) error { defer networkMutex.Unlock() if apiVersion == 0 { - return errAPIVersionUnset + return ErrAPIVersionUnset } return setOpt(func(p *C.uint8_t, pl C.int) C.fdb_error_t { @@ -138,11 +138,11 @@ func APIVersion(version int) error { if apiVersion == version { return nil } - return errAPIVersionAlreadySet + return ErrAPIVersionAlreadySet } if version < 200 || version > headerVersion { - return errAPIVersionNotSupported + return ErrAPIVersionNotSupported } if e := C.fdb_select_api_version_impl(C.int(version), C.int(headerVersion)); e != 0 { @@ -181,7 +181,7 @@ func GetAPIVersion() (int, error) { if IsAPIVersionSelected() { return apiVersion, nil } - return 0, errAPIVersionUnset + return 0, ErrAPIVersionUnset } // MustAPIVersion is like APIVersion but panics if the API version is not @@ -266,7 +266,7 @@ func executeWithRunningNetworkThread(f func()) error { // StartNetwork does nothing, but it will ensure that the API version is set and return an error otherwise. func StartNetwork() error { if apiVersion == 0 { - return errAPIVersionUnset + return ErrAPIVersionUnset } return nil @@ -378,7 +378,7 @@ func MustOpen(clusterFile string, dbName []byte) Database { // Caller must call Close() to release resources. func createDatabase(clusterFile string) (Database, error) { if apiVersion == 0 { - return Database{}, errAPIVersionUnset + return Database{}, ErrAPIVersionUnset } var cf *C.char @@ -412,7 +412,7 @@ func createDatabase(clusterFile string) (Database, error) { // Caller must call Close() to release resources. func OpenWithConnectionString(connectionString string) (Database, error) { if apiVersion == 0 { - return Database{}, errAPIVersionUnset + return Database{}, ErrAPIVersionUnset } var cf *C.char @@ -451,11 +451,11 @@ func CreateCluster(clusterFile string) (Cluster, error) { defer networkMutex.Unlock() if apiVersion == 0 { - return Cluster{}, errAPIVersionUnset + return Cluster{}, ErrAPIVersionUnset } if !networkStarted { - return Cluster{}, errNetworkNotSetup + return Cluster{}, ErrNetworkNotSetup } if networkStopped { diff --git a/bindings/go/src/fdb/generated.go b/bindings/go/src/fdb/generated.go index c33ffc7feaf..76e39cca157 100644 --- a/bindings/go/src/fdb/generated.go +++ b/bindings/go/src/fdb/generated.go @@ -60,14 +60,14 @@ func (o NetworkOptions) SetTraceEnable(param string) error { return o.setOpt(30, []byte(param)) } -// Sets the maximum size in bytes of a single trace output file. This value should be in the range ``[0, INT64_MAX]``. If the value is set to 0, there is no limit on individual file size. The default is a maximum size of 10,485,760 bytes. +// Sets the maximum size in bytes of a single trace output file. This value should be in the range "[0, INT64_MAX]". If the value is set to 0, there is no limit on individual file size. The default is a maximum size of 10,485,760 bytes. // // Parameter: max size of a single trace output file func (o NetworkOptions) SetTraceRollSize(param int64) error { return o.setOpt(31, int64ToBytes(param)) } -// Sets the maximum size of all the trace output files put together. This value should be in the range ``[0, INT64_MAX]``. If the value is set to 0, there is no limit on the total size of the files. The default is a maximum size of 104,857,600 bytes. If the default roll size is used, this means that a maximum of 10 trace files will be written at a time. +// Sets the maximum size of all the trace output files put together. This value should be in the range "[0, INT64_MAX]". If the value is set to 0, there is no limit on the total size of the files. The default is a maximum size of 104,857,600 bytes. If the default roll size is used, this means that a maximum of 10 trace files will be written at a time. // // Parameter: max total size of trace files func (o NetworkOptions) SetTraceMaxLogsSize(param int64) error { @@ -336,7 +336,7 @@ func (o NetworkOptions) SetDistributedClientTracer(param string) error { // Sets the directory for storing temporary files created by FDB client, such as temporary copies of client libraries. Defaults to /tmp // -// Parameter: Client directory for temporary files. +// Parameter: Client directory for temporary files. func (o NetworkOptions) SetClientTmpDir(param string) error { return o.setOpt(91, []byte(param)) } @@ -379,35 +379,35 @@ func (o DatabaseOptions) SetSnapshotRywDisable() error { return o.setOpt(27, nil) } -// Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option. This sets the ``transaction_logging_max_field_length`` option of each transaction created by this database. See the transaction option description for more information. +// Sets the maximum escaped length of key and value fields to be logged to the trace file via the LOG_TRANSACTION option. This sets the "transaction_logging_max_field_length" option of each transaction created by this database. See the transaction option description for more information. // // Parameter: Maximum length of escaped key and value fields. func (o DatabaseOptions) SetTransactionLoggingMaxFieldLength(param int64) error { return o.setOpt(405, int64ToBytes(param)) } -// Set a timeout in milliseconds which, when elapsed, will cause each transaction automatically to be cancelled. This sets the ``timeout`` option of each transaction created by this database. See the transaction option description for more information. Using this option requires that the API version is 610 or higher. +// Set a timeout in milliseconds which, when elapsed, will cause each transaction automatically to be cancelled. This sets the "timeout" option of each transaction created by this database. See the transaction option description for more information. Using this option requires that the API version is 610 or higher. // // Parameter: value in milliseconds of timeout func (o DatabaseOptions) SetTransactionTimeout(param int64) error { return o.setOpt(500, int64ToBytes(param)) } -// Set a maximum number of retries after which additional calls to ``onError`` will throw the most recently seen error code. This sets the ``retry_limit`` option of each transaction created by this database. See the transaction option description for more information. +// Set a maximum number of retries after which additional calls to "onError" will throw the most recently seen error code. This sets the "retry_limit" option of each transaction created by this database. See the transaction option description for more information. // // Parameter: number of times to retry func (o DatabaseOptions) SetTransactionRetryLimit(param int64) error { return o.setOpt(501, int64ToBytes(param)) } -// Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. This sets the ``max_retry_delay`` option of each transaction created by this database. See the transaction option description for more information. +// Set the maximum amount of backoff delay incurred in the call to "onError" if the error is retryable. This sets the "max_retry_delay" option of each transaction created by this database. See the transaction option description for more information. // // Parameter: value in milliseconds of maximum delay func (o DatabaseOptions) SetTransactionMaxRetryDelay(param int64) error { return o.setOpt(502, int64ToBytes(param)) } -// Set the maximum transaction size in bytes. This sets the ``size_limit`` option on each transaction created by this database. See the transaction option description for more information. +// Set the maximum transaction size in bytes. This sets the "size_limit" option on each transaction created by this database. See the transaction option description for more information. // // Parameter: value in bytes func (o DatabaseOptions) SetTransactionSizeLimit(param int64) error { @@ -429,7 +429,7 @@ func (o DatabaseOptions) SetTransactionAutomaticIdempotency() error { return o.setOpt(506, nil) } -// Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. This sets the ``bypass_unreadable`` option of each transaction created by this database. See the transaction option description for more information. +// Allows "get" operations to read from sections of keyspace that have become unreadable because of versionstamp operations. This sets the "bypass_unreadable" option of each transaction created by this database. See the transaction option description for more information. func (o DatabaseOptions) SetTransactionBypassUnreadable() error { return o.setOpt(700, nil) } @@ -597,21 +597,21 @@ func (o TransactionOptions) SetServerRequestTracing() error { return o.setOpt(406, nil) } -// Set a timeout in milliseconds which, when elapsed, will cause the transaction automatically to be cancelled. Valid parameter values are ``[0, INT_MAX]``. If set to 0, will disable all timeouts. All pending and any future uses of the transaction will throw an exception. The transaction can be used again after it is reset. Prior to API version 610, like all other transaction options, the timeout must be reset after a call to ``onError``. If the API version is 610 or greater, the timeout is not reset after an ``onError`` call. This allows the user to specify a longer timeout on specific transactions than the default timeout specified through the ``transaction_timeout`` database option without the shorter database timeout cancelling transactions that encounter a retryable error. Note that at all API versions, it is safe and legal to set the timeout each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option. +// Set a timeout in milliseconds which, when elapsed, will cause the transaction automatically to be cancelled. Valid parameter values are "[0, INT_MAX]". If set to 0, will disable all timeouts. All pending and any future uses of the transaction will throw an exception. The transaction can be used again after it is reset. Prior to API version 610, like all other transaction options, the timeout must be reset after a call to "onError". If the API version is 610 or greater, the timeout is not reset after an "onError" call. This allows the user to specify a longer timeout on specific transactions than the default timeout specified through the "transaction_timeout" database option without the shorter database timeout cancelling transactions that encounter a retryable error. Note that at all API versions, it is safe and legal to set the timeout each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option. // // Parameter: value in milliseconds of timeout func (o TransactionOptions) SetTimeout(param int64) error { return o.setOpt(500, int64ToBytes(param)) } -// Set a maximum number of retries after which additional calls to ``onError`` will throw the most recently seen error code. Valid parameter values are ``[-1, INT_MAX]``. If set to -1, will disable the retry limit. Prior to API version 610, like all other transaction options, the retry limit must be reset after a call to ``onError``. If the API version is 610 or greater, the retry limit is not reset after an ``onError`` call. Note that at all API versions, it is safe and legal to set the retry limit each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option. +// Set a maximum number of retries after which additional calls to "onError" will throw the most recently seen error code. Valid parameter values are "[-1, INT_MAX]". If set to -1, will disable the retry limit. Prior to API version 610, like all other transaction options, the retry limit must be reset after a call to "onError". If the API version is 610 or greater, the retry limit is not reset after an "onError" call. Note that at all API versions, it is safe and legal to set the retry limit each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option. // // Parameter: number of times to retry func (o TransactionOptions) SetRetryLimit(param int64) error { return o.setOpt(501, int64ToBytes(param)) } -// Set the maximum amount of backoff delay incurred in the call to ``onError`` if the error is retryable. Defaults to 1000 ms. Valid parameter values are ``[0, INT_MAX]``. If the maximum retry delay is less than the current retry delay of the transaction, then the current retry delay will be clamped to the maximum retry delay. Prior to API version 610, like all other transaction options, the maximum retry delay must be reset after a call to ``onError``. If the API version is 610 or greater, the retry limit is not reset after an ``onError`` call. Note that at all API versions, it is safe and legal to set the maximum retry delay each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option. +// Set the maximum amount of backoff delay incurred in the call to "onError" if the error is retryable. Defaults to 1000 ms. Valid parameter values are "[0, INT_MAX]". If the maximum retry delay is less than the current retry delay of the transaction, then the current retry delay will be clamped to the maximum retry delay. Prior to API version 610, like all other transaction options, the maximum retry delay must be reset after a call to "onError". If the API version is 610 or greater, the retry limit is not reset after an "onError" call. Note that at all API versions, it is safe and legal to set the maximum retry delay each time the transaction begins, so most code written assuming the older behavior can be upgraded to the newer behavior without requiring any modification, and the caller is not required to implement special logic in retry loops to only conditionally set this option. // // Parameter: value in milliseconds of maximum delay func (o TransactionOptions) SetMaxRetryDelay(param int64) error { @@ -625,7 +625,7 @@ func (o TransactionOptions) SetSizeLimit(param int64) error { return o.setOpt(503, int64ToBytes(param)) } -// Automatically assign a random 16 byte idempotency id for this transaction. Prevents commits from failing with ``commit_unknown_result``. WARNING: If you are also using the multiversion client or transaction timeouts, if either cluster_version_changed or transaction_timed_out was thrown during a commit, then that commit may have already succeeded or may succeed in the future. This feature is in development and not ready for general use. +// Automatically assign a random 16 byte idempotency id for this transaction. Prevents commits from failing with "commit_unknown_result". WARNING: If you are also using the multiversion client or transaction timeouts, if either cluster_version_changed or transaction_timed_out was thrown during a commit, then that commit may have already succeeded or may succeed in the future. This feature is in development and not ready for general use. func (o TransactionOptions) SetAutomaticIdempotency() error { return o.setOpt(505, nil) } @@ -650,7 +650,7 @@ func (o TransactionOptions) SetUsedDuringCommitProtectionDisable() error { return o.setOpt(701, nil) } -// The transaction can read from locked databases. +// The transaction can read from locked databases. This option will make the transaction read-only if the lock_aware option is not set. func (o TransactionOptions) SetReadLockAware() error { return o.setOpt(702, nil) } @@ -689,7 +689,7 @@ func (o TransactionOptions) SetAutoThrottleTag(param string) error { return o.setOpt(801, []byte(param)) } -// Adds a parent to the Span of this transaction. Used for transaction tracing. A span can be identified with a 33 bytes serialized binary format which consists of: 8 bytes protocol version, e.g. ``0x0FDB00B073000000LL`` in little-endian format, 16 bytes trace id, 8 bytes span id, 1 byte set to 1 if sampling is enabled +// Adds a parent to the Span of this transaction. Used for transaction tracing. A span can be identified with a 33 bytes serialized binary format which consists of: 8 bytes protocol version, e.g. "0x0FDB00B073000000LL" in little-endian format, 16 bytes trace id, 8 bytes span id, 1 byte set to 1 if sampling is enabled // // Parameter: A serialized binary byte string of length 33 used to associate the span of this transaction with a parent func (o TransactionOptions) SetSpanParent(param []byte) error { @@ -701,7 +701,7 @@ func (o TransactionOptions) SetExpensiveClearCostEstimationEnable() error { return o.setOpt(1000, nil) } -// Allows ``get`` operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp. +// Allows "get" operations to read from sections of keyspace that have become unreadable because of versionstamp operations. These reads will view versionstamp operations as if they were set operations that did not fill in the versionstamp. func (o TransactionOptions) SetBypassUnreadable() error { return o.setOpt(1100, nil) } @@ -711,6 +711,13 @@ func (o TransactionOptions) SetUseGrvCache() error { return o.setOpt(1101, nil) } +// Rejects a GRV request with "transaction_grv_queue_rejected" if the GRV proxy estimates that ratekeeper throttling will keep the request queued longer than this limit. Valid parameter values are "[0, INT_MAX]". The estimate is advisory and only accounts for time spent queued at the GRV proxy due to ratekeeper throttling. +// +// Parameter: value in milliseconds of maximum estimated GRV queue delay +func (o TransactionOptions) SetMaxGrvQueueDelay(param int64) error { + return o.setOpt(1103, int64ToBytes(param)) +} + // Attach given authorization token to the transaction such that subsequent tenant-aware requests are authorized // // Parameter: A JSON Web Token authorized to access data belonging to one or more tenants, indicated by 'tenants' claim of the token's payload. @@ -744,13 +751,13 @@ const ( // order to minimize costs if the client doesn't read the entire range), and // as the caller iterates over more items in the range larger batches will // be transferred in order to minimize latency. After enough iterations, - // the iterator mode will eventually reach the same byte limit as “WANT_ALL“ + // the iterator mode will eventually reach the same byte limit as "WANT_ALL" StreamingModeIterator StreamingMode = 0 // Infrequently used. The client has passed a specific row limit and wants // that many rows delivered in a single batch. Because of iterator operation // in client drivers make request batches transparent to the user, consider - // “WANT_ALL“ StreamingMode instead. A row limit must be specified if this + // "WANT_ALL" StreamingMode instead. A row limit must be specified if this // mode is used. StreamingModeExact StreamingMode = 1 @@ -777,7 +784,7 @@ const ( StreamingModeSerial StreamingMode = 5 ) -// Performs an addition of little-endian integers. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The integers to be added must be stored in a little-endian representation. They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to ``param`` and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow. +// Performs an addition of little-endian integers. If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes. If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". The integers to be added must be stored in a little-endian representation. They can be signed in two's complement representation or unsigned. You can add to an integer at a known offset in the value by prepending the appropriate number of zero bytes to "param" and padding with zero bytes to match the length of the value. However, this offset technique requires that you know the addition will not cause the integer field within the value to overflow. func (t Transaction) Add(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 2) } @@ -787,7 +794,7 @@ func (t Transaction) And(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 6) } -// Performs a bitwise ``and`` operation. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. +// Performs a bitwise "and" operation. If the existing value in the database is not present, then "param" is stored in the database. If the existing value in the database is shorter than "param", it is first extended to the length of "param" with zero bytes. If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". func (t Transaction) BitAnd(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 6) } @@ -797,7 +804,7 @@ func (t Transaction) Or(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 7) } -// Performs a bitwise ``or`` operation. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. +// Performs a bitwise "or" operation. If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes. If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". func (t Transaction) BitOr(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 7) } @@ -807,47 +814,47 @@ func (t Transaction) Xor(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 8) } -// Performs a bitwise ``xor`` operation. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. +// Performs a bitwise "xor" operation. If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes. If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". func (t Transaction) BitXor(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 8) } -// Appends ``param`` to the end of the existing value already in the database at the given key (or creates the key and sets the value to ``param`` if the key is empty). This will only append the value if the final concatenated value size is less than or equal to the maximum value size (i.e., if it fits). WARNING: No error is surfaced back to the user if the final value is too large because the mutation will not be applied until after the transaction has been committed. Therefore, it is only safe to use this mutation type if one can guarantee that one will keep the total value size under the maximum size. +// Appends "param" to the end of the existing value already in the database at the given key (or creates the key and sets the value to "param" if the key is empty). This will only append the value if the final concatenated value size is less than or equal to the maximum value size (i.e., if it fits). WARNING: No error is surfaced back to the user if the final value is too large because the mutation will not be applied until after the transaction has been committed. Therefore, it is only safe to use this mutation type if one can guarantee that one will keep the total value size under the maximum size. func (t Transaction) AppendIfFits(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 9) } -// Performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The larger of the two values is then stored in the database. +// Performs a little-endian comparison of byte strings. If the existing value in the database is not present or shorter than "param", it is first extended to the length of "param" with zero bytes. If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". The larger of the two values is then stored in the database. func (t Transaction) Max(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 12) } -// Performs a little-endian comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored in the database. If the existing value in the database is shorter than ``param``, it is first extended to the length of ``param`` with zero bytes. If ``param`` is shorter than the existing value in the database, the existing value is truncated to match the length of ``param``. The smaller of the two values is then stored in the database. +// Performs a little-endian comparison of byte strings. If the existing value in the database is not present, then "param" is stored in the database. If the existing value in the database is shorter than "param", it is first extended to the length of "param" with zero bytes. If "param" is shorter than the existing value in the database, the existing value is truncated to match the length of "param". The smaller of the two values is then stored in the database. func (t Transaction) Min(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 13) } -// Transforms ``key`` using a versionstamp for the transaction. Sets the transformed key in the database to ``param``. The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the key from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes. +// Transforms "key" using a versionstamp for the transaction. Sets the transformed key in the database to "param". The key is transformed by removing the final four bytes from the key and reading those as a little-Endian 32-bit integer to get a position "pos". The 10 bytes of the key from "pos" to "pos + 10" are replaced with the versionstamp of the transaction used. The first byte of the key is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the offset was computed from only the final two bytes rather than the final four bytes. func (t Transaction) SetVersionstampedKey(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 14) } -// Transforms ``param`` using a versionstamp for the transaction. Sets the ``key`` given to the transformed ``param``. The parameter is transformed by removing the final four bytes from ``param`` and reading those as a little-Endian 32-bit integer to get a position ``pos``. The 10 bytes of the parameter from ``pos`` to ``pos + 10`` are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset. +// Transforms "param" using a versionstamp for the transaction. Sets the "key" given to the transformed "param". The parameter is transformed by removing the final four bytes from "param" and reading those as a little-Endian 32-bit integer to get a position "pos". The 10 bytes of the parameter from "pos" to "pos + 10" are replaced with the versionstamp of the transaction used. The first byte of the parameter is position 0. A versionstamp is a 10 byte, unique, monotonically (but not sequentially) increasing value for each committed transaction. The first 8 bytes are the committed version of the database (serialized in big-Endian order). The last 2 bytes are monotonic in the serialization order for transactions. WARNING: At this time, versionstamps are compatible with the Tuple layer only in the Java, Python, and Go bindings. Also, note that prior to API version 520, the versionstamp was always placed at the beginning of the parameter rather than computing an offset. func (t Transaction) SetVersionstampedValue(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 15) } -// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the smaller of the two values is then stored in the database. +// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then "param" is stored. Otherwise the smaller of the two values is then stored in the database. func (t Transaction) ByteMin(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 16) } -// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then ``param`` is stored. Otherwise the larger of the two values is then stored in the database. +// Performs lexicographic comparison of byte strings. If the existing value in the database is not present, then "param" is stored. Otherwise the larger of the two values is then stored in the database. func (t Transaction) ByteMax(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 17) } -// Performs an atomic ``compare and clear`` operation. If the existing value in the database is equal to the given value, then given key is cleared. +// Performs an atomic "compare and clear" operation. If the existing value in the database is equal to the given value, then given key is cleared. func (t Transaction) CompareAndClear(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, 20) } @@ -867,15 +874,15 @@ type ErrorPredicate int const ( - // Returns “true“ if the error indicates the operations in the transactions + // Returns "true" if the error indicates the operations in the transactions // should be retried because of transient error. ErrorPredicateRetryable ErrorPredicate = 50000 - // Returns “true“ if the error indicates the transaction may have succeeded, + // Returns "true" if the error indicates the transaction may have succeeded, // though not in a way the system can verify. ErrorPredicateMaybeCommitted ErrorPredicate = 50001 - // Returns “true“ if the error indicates the transaction has not committed, + // Returns "true" if the error indicates the transaction has not committed, // though in a way that can be retried. ErrorPredicateRetryableNotCommitted ErrorPredicate = 50002 ) diff --git a/bindings/go/src/internal/gen_errors/main.go b/bindings/go/src/internal/gen_errors/main.go new file mode 100644 index 00000000000..429403a52b6 --- /dev/null +++ b/bindings/go/src/internal/gen_errors/main.go @@ -0,0 +1,176 @@ +//go:build ignore +// +build ignore + +/* + * main.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// gen_errors reads flow/include/flow/error_definitions.h and generates a Go +// source file containing integer constants for every FDB error code. +// +// Usage: +// +// go run internal/gen_errors/main.go -in -out +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strings" + "text/template" + "unicode" +) + +var errorRe = regexp.MustCompile(`^\s*ERROR\(\s*(\w+)\s*,\s*(\d+)\s*,\s*"([^"]+)"\s*\)`) + +// initialism maps lower-case words to their expected go words. Most of them will be upper-case with TLog being an exception. +var initialism = map[string]string{ + "api": "API", + "db": "DB", + "dns": "DNS", + "grpc": "GRPC", + "grv": "GRV", + "http": "HTTP", + "id": "ID", + "ids": "IDs", + "io": "IO", + "ip": "IP", + "json": "JSON", + "kms": "KMS", + "kv": "KV", + "ssl": "SSL", + "tlog": "TLog", + "tls": "TLS", + "ttl": "TTL", + "uid": "UID", + "uri": "URI", + "url": "URL", + "uuid": "UUID", +} + +type errorDef struct { + Name string // exported Go constant name, e.g. ErrCodeTransactionTooOld + Code string // numeric string, e.g. "1007" + Description string // human-readable description from the macro +} + +func toPascalCase(s string) string { + parts := strings.Split(s, "_") + var b strings.Builder + for _, p := range parts { + if len(p) == 0 { + continue + } + if upper, ok := initialism[strings.ToLower(p)]; ok { + b.WriteString(upper) + continue + } + runes := []rune(p) + b.WriteRune(unicode.ToUpper(runes[0])) + b.WriteString(string(runes[1:])) + } + return b.String() +} + +func main() { + var inFile string + var outFile string + flag.StringVar(&inFile, "in", "stdin", "Input file") + flag.StringVar(&outFile, "out", "stdout", "Output file") + flag.Parse() + + f, err := os.Open(inFile) + if err != nil { + fmt.Fprintf(os.Stderr, "open %s: %v\n", inFile, err) + os.Exit(1) + } + defer f.Close() + + var defs []errorDef + scanner := bufio.NewScanner(f) + for scanner.Scan() { + m := errorRe.FindStringSubmatch(scanner.Text()) + if m == nil { + continue + } + defs = append(defs, errorDef{ + Name: "Err" + toPascalCase(m[1]), + Code: m[2], + Description: m[3], + }) + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "scan: %v\n", err) + os.Exit(1) + } + + out, err := os.Create(outFile) + if err != nil { + fmt.Fprintf(os.Stderr, "create %s: %v\n", outFile, err) + os.Exit(1) + } + defer out.Close() + + if err := tmpl.Execute(out, defs); err != nil { + fmt.Fprintf(os.Stderr, "template: %v\n", err) + os.Exit(1) + } +} + +var tmpl = template.Must(template.New("").Parse(`// Code generated by gen_errors. DO NOT EDIT. + +/* + * error_codes_generated.go + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package fdb + +// To regenerate this file run this command: +// go run ./bindings/go/src/internal/gen_errors/main.go -in ./flow/include/flow/error_definitions.h -out ./bindings/go/src/fdb/error_codes_generated.go +// +// Error sentinel variables derived from flow/include/flow/error_definitions.h. +// Use with errors.Is to match a specific FoundationDB error code: +// +// errors.Is(err, fdb.ErrTransactionTooOld) +var ( +{{- range .}} + // {{.Description}} + {{.Name}} = Error{Code: {{.Code}}} +{{- end}} +) +`)) diff --git a/bindings/go/src/_util/translate_fdb_options.go b/bindings/go/src/internal/translate_fdb_options.go similarity index 92% rename from bindings/go/src/_util/translate_fdb_options.go rename to bindings/go/src/internal/translate_fdb_options.go index f62d9493aec..58b6687837c 100644 --- a/bindings/go/src/_util/translate_fdb_options.go +++ b/bindings/go/src/internal/translate_fdb_options.go @@ -42,14 +42,23 @@ type Option struct { Description string `xml:"description,attr"` Hidden bool `xml:"hidden,attr"` } + type Scope struct { Name string `xml:"name,attr"` Option []Option } + type Options struct { Scope []Scope } +// sanitize normalizes description text to be gofmt-compliant: replaces RST +// double-backtick spans with Go-style double-quoted strings and removes +// trailing whitespace. +func sanitize(s string) string { + return strings.TrimRight(strings.ReplaceAll(s, "``", "\""), " \t") +} + func writeOptString(w io.Writer, receiver string, function string, opt Option) { fmt.Fprintf(w, `func (o %s) %s(param string) error { return o.setOpt(%d, []byte(param)) @@ -84,9 +93,9 @@ func writeOpt(w io.Writer, receiver string, opt Option) { fmt.Fprintln(w) if opt.Description != "" { - fmt.Fprintf(w, "// %s\n", opt.Description) + fmt.Fprintf(w, "// %s\n", sanitize(opt.Description)) if opt.ParamDesc != "" { - fmt.Fprintf(w, "//\n// Parameter: %s\n", opt.ParamDesc) + fmt.Fprintf(w, "//\n// Parameter: %s\n", sanitize(opt.ParamDesc)) } } else { fmt.Fprintf(w, "// Not yet implemented.\n") @@ -117,14 +126,13 @@ func writeMutation(w io.Writer, opt Option) { func (t Transaction) %s(key KeyConvertible, param []byte) { t.atomicOp(key.FDBKey(), param, %d) } -`, opt.Description, tname, opt.Code) +`, sanitize(opt.Description), tname, opt.Code) } func writeEnum(w io.Writer, scope Scope, opt Option, delta int) { fmt.Fprintln(w) if opt.Description != "" { - doc.ToText(w, opt.Description, "\t// ", "", 73) - // fmt.Printf(" // %s\n", opt.Description) + doc.ToText(w, sanitize(opt.Description), "\t// ", "", 73) } fmt.Fprintf(w, " %s %s = %d\n", scope.Name+translateName(opt.Name), scope.Name, opt.Code+delta) } diff --git a/bindings/java/JavaWorkload.cpp b/bindings/java/JavaWorkload.cpp index c14d6e2ee66..a10a8d0c0bc 100644 --- a/bindings/java/JavaWorkload.cpp +++ b/bindings/java/JavaWorkload.cpp @@ -90,17 +90,17 @@ void printTrace(JNIEnv* env, jclass, jlong logger, jint severity, jstring messag } jlong getProcessID(JNIEnv* env, jclass, jlong self) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); return jlong(context->getProcessID()); } void setProcessID(JNIEnv* env, jclass, jlong self, jlong processID) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); context->setProcessID(processID); } jboolean getOptionBool(JNIEnv* env, jclass, jlong self, jstring name, jboolean defaultValue) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); jboolean isCopy = true; const char* utf = env->GetStringUTFChars(name, &isCopy); auto res = jboolean(context->getOption(utf, bool(defaultValue))); @@ -111,7 +111,7 @@ jboolean getOptionBool(JNIEnv* env, jclass, jlong self, jstring name, jboolean d } jlong getOptionLong(JNIEnv* env, jclass, jlong self, jstring name, jlong defaultValue) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); jboolean isCopy = true; const char* utf = env->GetStringUTFChars(name, &isCopy); auto res = jlong(context->getOption(utf, long(defaultValue))); @@ -122,7 +122,7 @@ jlong getOptionLong(JNIEnv* env, jclass, jlong self, jstring name, jlong default } jdouble getOptionDouble(JNIEnv* env, jclass, jlong self, jstring name, jdouble defaultValue) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); jboolean isCopy = true; const char* utf = env->GetStringUTFChars(name, &isCopy); auto res = jdouble(context->getOption(utf, double(defaultValue))); @@ -133,7 +133,7 @@ jdouble getOptionDouble(JNIEnv* env, jclass, jlong self, jstring name, jdouble d } jstring getOptionString(JNIEnv* env, jclass, jlong self, jstring name, jstring defaultValue) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); jboolean isCopy; jboolean defIsCopy; const char* nameStr = env->GetStringUTFChars(name, &isCopy); @@ -149,17 +149,17 @@ jstring getOptionString(JNIEnv* env, jclass, jlong self, jstring name, jstring d } jint getClientID(JNIEnv* env, jclass, jlong self) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); return jint(context->clientId()); } jint getClientCount(JNIEnv* env, jclass, jlong self) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); return jint(context->clientCount()); } jlong getSharedRandomNumber(JNIEnv* env, jclass, jlong self) { - FDBWorkloadContext* context = reinterpret_cast(self); + auto* context = reinterpret_cast(self); return jlong(context->sharedRandomNumber()); } @@ -324,7 +324,7 @@ struct JVM { if (!env) { throw JNIError{}; } - if (classPath.count(path) > 0) { + if (classPath.contains(path)) { // already added return; } diff --git a/bindings/java/fdbJNI.cpp b/bindings/java/fdbJNI.cpp index 81e4f391161..451cf8d1555 100644 --- a/bindings/java/fdbJNI.cpp +++ b/bindings/java/fdbJNI.cpp @@ -448,7 +448,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureKeyRangeArray_Future return JNI_NULL; } - jobjectArray kr_values = jenv->NewObjectArray(count, keyrange_class, NULL); + jobjectArray kr_values = jenv->NewObjectArray(count, keyrange_class, nullptr); if (!kr_values) { if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); @@ -600,7 +600,7 @@ JNIEXPORT jobject JNICALL Java_com_apple_foundationdb_FutureMappedResults_Future return JNI_NULL; } - jobjectArray mrr_values = jenv->NewObjectArray(count, mapped_key_value_class, NULL); + jobjectArray mrr_values = jenv->NewObjectArray(count, mapped_key_value_class, nullptr); if (!mrr_values) { if (!jenv->ExceptionOccurred()) throwOutOfMem(jenv); @@ -753,7 +753,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1create throwParamNotNull(jenv); return 0; } - FDBDatabase* database = (FDBDatabase*)dbPtr; + auto* database = (FDBDatabase*)dbPtr; FDBTransaction* tr; fdb_error_t err = fdb_database_create_transaction(database, &tr); if (err) { @@ -780,12 +780,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1setOpti throwParamNotNull(jenv); return; } - FDBDatabase* c = (FDBDatabase*)dPtr; + auto* c = (FDBDatabase*)dPtr; uint8_t* barr = nullptr; int size = 0; if (value != JNI_NULL) { - barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); + barr = (decltype(barr))jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { throwRuntimeEx(jenv, "Error getting handle to native resources"); return; @@ -810,7 +810,7 @@ JNIEXPORT jdouble JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getM throwParamNotNull(jenv); return 0; } - FDBDatabase* database = (FDBDatabase*)dbPtr; + auto* database = (FDBDatabase*)dbPtr; return (jdouble)fdb_database_get_main_thread_busyness(database); } @@ -822,7 +822,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBDatabase_Database_1getCli return 0; } - FDBDatabase* database = (FDBDatabase*)dbPtr; + auto* database = (FDBDatabase*)dbPtr; FDBFuture* f = fdb_database_get_client_status(database); return (jlong)f; @@ -869,7 +869,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s throwParamNotNull(jenv); return; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; fdb_transaction_set_read_version(tr, version); } @@ -880,7 +880,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; FDBFuture* f = fdb_transaction_get_read_version(tr); return (jlong)f; } @@ -894,9 +894,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + auto* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); if (!barr) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); @@ -919,9 +919,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + auto* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); if (!barr) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); @@ -953,16 +953,16 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); + auto* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); if (!barrBegin) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - uint8_t* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); + auto* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); if (!barrEnd) { jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); if (!jenv->ExceptionOccurred()) @@ -1010,16 +1010,16 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); + auto* barrBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); if (!barrBegin) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - uint8_t* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); + auto* barrEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); if (!barrEnd) { jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); if (!jenv->ExceptionOccurred()) @@ -1027,7 +1027,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 return 0; } - uint8_t* barrMapper = (uint8_t*)jenv->GetByteArrayElements(mapperBytes, JNI_NULL); + auto* barrMapper = (uint8_t*)jenv->GetByteArrayElements(mapperBytes, JNI_NULL); if (!barrMapper) { jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrBegin, JNI_ABORT); jenv->ReleaseByteArrayElements(keyEndBytes, (jbyte*)barrEnd, JNI_ABORT); @@ -1076,7 +1076,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FutureResults_FutureResults_1 return; } - FDBFuture* f = (FDBFuture*)future; + auto* f = (FDBFuture*)future; const FDBKeyValue* kvs; int count; fdb_bool_t more; @@ -1153,7 +1153,7 @@ Java_com_apple_foundationdb_FutureMappedResults_FutureMappedResults_1getDirect(J return; } - FDBFuture* f = (FDBFuture*)future; + auto* f = (FDBFuture*)future; const FDBMappedKeyValue* kvms; int count; fdb_bool_t more; @@ -1219,16 +1219,16 @@ Java_com_apple_foundationdb_FDBTransaction_Transaction_1getEstimatedRangeSizeByt throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); + auto* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); if (!startKey) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + auto* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); if (!endKey) { jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); if (!jenv->ExceptionOccurred()) @@ -1254,16 +1254,16 @@ Java_com_apple_foundationdb_FDBTransaction_Transaction_1getRangeSplitPoints(JNIE throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); + auto* startKey = (uint8_t*)jenv->GetByteArrayElements(beginKeyBytes, JNI_NULL); if (!startKey) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); return 0; } - uint8_t* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); + auto* endKey = (uint8_t*)jenv->GetByteArrayElements(endKeyBytes, JNI_NULL); if (!endKey) { jenv->ReleaseByteArrayElements(beginKeyBytes, (jbyte*)startKey, JNI_ABORT); if (!jenv->ExceptionOccurred()) @@ -1287,16 +1287,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s throwParamNotNull(jenv); return; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + auto* barrKey = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); if (!barrKey) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(valueBytes, JNI_NULL); + auto* barrValue = (uint8_t*)jenv->GetByteArrayElements(valueBytes, JNI_NULL); if (!barrValue) { jenv->ReleaseByteArrayElements(keyBytes, (jbyte*)barrKey, JNI_ABORT); if (!jenv->ExceptionOccurred()) @@ -1317,9 +1317,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c throwParamNotNull(jenv); return; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); + auto* barr = (uint8_t*)jenv->GetByteArrayElements(keyBytes, JNI_NULL); if (!barr) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); @@ -1339,16 +1339,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1c throwParamNotNull(jenv); return; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barrKeyBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); + auto* barrKeyBegin = (uint8_t*)jenv->GetByteArrayElements(keyBeginBytes, JNI_NULL); if (!barrKeyBegin) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - uint8_t* barrKeyEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); + auto* barrKeyEnd = (uint8_t*)jenv->GetByteArrayElements(keyEndBytes, JNI_NULL); if (!barrKeyEnd) { jenv->ReleaseByteArrayElements(keyBeginBytes, (jbyte*)barrKeyBegin, JNI_ABORT); if (!jenv->ExceptionOccurred()) @@ -1372,16 +1372,16 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1m throwParamNotNull(jenv); return; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barrKey = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); + auto* barrKey = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barrKey) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); return; } - uint8_t* barrValue = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); + auto* barrValue = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); if (!barrValue) { jenv->ReleaseByteArrayElements(key, (jbyte*)barrKey, JNI_ABORT); if (!jenv->ExceptionOccurred()) @@ -1403,7 +1403,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; FDBFuture* f = fdb_transaction_commit(tr); return (jlong)f; } @@ -1417,12 +1417,12 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1s throwParamNotNull(jenv); return; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; uint8_t* barr = nullptr; int size = 0; if (value != JNI_NULL) { - barr = (uint8_t*)jenv->GetByteArrayElements(value, JNI_NULL); + barr = (decltype(barr))jenv->GetByteArrayElements(value, JNI_NULL); if (!barr) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); @@ -1445,7 +1445,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; int64_t version; fdb_error_t err = fdb_transaction_get_committed_version(tr, &version); if (err) { @@ -1462,7 +1462,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBFuture* f = fdb_transaction_get_approximate_size((FDBTransaction*)tPtr); + auto* f = fdb_transaction_get_approximate_size((FDBTransaction*)tPtr); return (jlong)f; } @@ -1473,7 +1473,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; FDBFuture* f = fdb_transaction_get_versionstamp(tr); return (jlong)f; } @@ -1486,9 +1486,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); + auto* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barr) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); @@ -1510,7 +1510,7 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; FDBFuture* f = fdb_transaction_on_error(tr, (fdb_error_t)errorCode); return (jlong)f; } @@ -1543,9 +1543,9 @@ JNIEXPORT jlong JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1 throwParamNotNull(jenv); return 0; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); + auto* barr = (uint8_t*)jenv->GetByteArrayElements(key, JNI_NULL); if (!barr) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); @@ -1578,9 +1578,9 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a throwParamNotNull(jenv); return; } - FDBTransaction* tr = (FDBTransaction*)tPtr; + auto* tr = (FDBTransaction*)tPtr; - uint8_t* begin_barr = (uint8_t*)jenv->GetByteArrayElements(keyBegin, JNI_NULL); + auto* begin_barr = (uint8_t*)jenv->GetByteArrayElements(keyBegin, JNI_NULL); if (!begin_barr) { if (!jenv->ExceptionOccurred()) throwRuntimeEx(jenv, "Error getting handle to native resources"); @@ -1588,7 +1588,7 @@ JNIEXPORT void JNICALL Java_com_apple_foundationdb_FDBTransaction_Transaction_1a } int begin_size = jenv->GetArrayLength(keyBegin); - uint8_t* end_barr = (uint8_t*)jenv->GetByteArrayElements(keyEnd, JNI_NULL); + auto* end_barr = (uint8_t*)jenv->GetByteArrayElements(keyEnd, JNI_NULL); if (!end_barr) { jenv->ReleaseByteArrayElements(keyBegin, (jbyte*)begin_barr, JNI_ABORT); if (!jenv->ExceptionOccurred()) diff --git a/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java b/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java index 437a194ce13..e9881719e3f 100644 --- a/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java +++ b/bindings/java/src/integration/com/apple/foundationdb/MappedRangeQueryIntegrationTest.java @@ -152,74 +152,67 @@ public interface RangeQueryWithIndex { } RangeQueryWithIndex rangeQueryAndThenRangeQueries = (int begin, int end, Database db) -> db.run(tr -> { - try { - List kvs = tr.getRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)), - KeySelector.firstGreaterOrEqual(indexEntryKey(end)), - ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL) - .asList() - .get(); - Assertions.assertEquals(end - begin, kvs.size()); - - // Get the records of each index entry IN PARALLEL. - List>> resultFutures = new ArrayList<>(); - // In reality, we need to get the record key by parsing the index entry key. But considering this is a - // performance test, we just ignore the returned key and simply generate it from recordKey. + // Let Database.run retry transient FDB errors instead of turning them into assertion failures. + List kvs = tr.getRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)), + KeySelector.firstGreaterOrEqual(indexEntryKey(end)), + ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL) + .asList() + .join(); + Assertions.assertEquals(end - begin, kvs.size()); + + // Get the records of each index entry IN PARALLEL. + List>> resultFutures = new ArrayList<>(); + // In reality, we need to get the record key by parsing the index entry key. But considering this is a + // performance test, we just ignore the returned key and simply generate it from recordKey. + for (int id = begin; id < end; id++) { + resultFutures.add(tr.getRange(Range.startsWith(recordKeyPrefix(id)), + ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL).asList()); + } + AsyncUtil.whenAll(resultFutures).join(); + + if (validate) { + final Iterator indexes = kvs.iterator(); + final Iterator>> records = resultFutures.iterator(); for (int id = begin; id < end; id++) { - resultFutures.add(tr.getRange(Range.startsWith(recordKeyPrefix(id)), - ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL).asList()); - } - AsyncUtil.whenAll(resultFutures).get(); - - if (validate) { - final Iterator indexes = kvs.iterator(); - final Iterator>> records = resultFutures.iterator(); - for (int id = begin; id < end; id++) { - Assertions.assertTrue(indexes.hasNext()); - assertByteArrayEquals(indexEntryKey(id), indexes.next().getKey()); - - Assertions.assertTrue(records.hasNext()); - List rangeResult = records.next().get(); - validateRangeResult(id, rangeResult); - } - Assertions.assertFalse(indexes.hasNext()); - Assertions.assertFalse(records.hasNext()); + Assertions.assertTrue(indexes.hasNext()); + assertByteArrayEquals(indexEntryKey(id), indexes.next().getKey()); + + Assertions.assertTrue(records.hasNext()); + List rangeResult = records.next().join(); + validateRangeResult(id, rangeResult); } - } catch (Exception e) { - Assertions.fail("Unexpected exception", e); + Assertions.assertFalse(indexes.hasNext()); + Assertions.assertFalse(records.hasNext()); } return null; }); RangeQueryWithIndex mappedRangeQuery = (int begin, int end, Database db) -> db.run(tr -> { - try { - List kvs = - tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)), - KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER, - ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL) - .asList() - .get(); - Assertions.assertEquals(end - begin, kvs.size()); - - if (validate) { - final Iterator results = kvs.iterator(); - for (int id = begin; id < end; id++) { - Assertions.assertTrue(results.hasNext()); - MappedKeyValue mappedKeyValue = results.next(); - assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey()); - assertByteArrayEquals(EMPTY, mappedKeyValue.getValue()); - assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey()); - byte[] prefix = recordKeyPrefix(id); - assertByteArrayEquals(prefix, mappedKeyValue.getRangeBegin()); - prefix[prefix.length - 1] = (byte)0x01; - assertByteArrayEquals(prefix, mappedKeyValue.getRangeEnd()); - - List rangeResult = mappedKeyValue.getRangeResult(); - validateRangeResult(id, rangeResult); - } - Assertions.assertFalse(results.hasNext()); + List kvs = + tr.getMappedRange(KeySelector.firstGreaterOrEqual(indexEntryKey(begin)), + KeySelector.firstGreaterOrEqual(indexEntryKey(end)), MAPPER, + ReadTransaction.ROW_LIMIT_UNLIMITED, false, StreamingMode.WANT_ALL) + .asList() + .join(); + Assertions.assertEquals(end - begin, kvs.size()); + + if (validate) { + final Iterator results = kvs.iterator(); + for (int id = begin; id < end; id++) { + Assertions.assertTrue(results.hasNext()); + MappedKeyValue mappedKeyValue = results.next(); + assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey()); + assertByteArrayEquals(EMPTY, mappedKeyValue.getValue()); + assertByteArrayEquals(indexEntryKey(id), mappedKeyValue.getKey()); + byte[] prefix = recordKeyPrefix(id); + assertByteArrayEquals(prefix, mappedKeyValue.getRangeBegin()); + prefix[prefix.length - 1] = (byte)0x01; + assertByteArrayEquals(prefix, mappedKeyValue.getRangeEnd()); + + List rangeResult = mappedKeyValue.getRangeResult(); + validateRangeResult(id, rangeResult); } - } catch (Exception e) { - Assertions.fail("Unexpected exception", e); + Assertions.assertFalse(results.hasNext()); } return null; }); diff --git a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java index 8cdecd172c2..28068d9c4de 100644 --- a/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/FDBTransaction.java @@ -640,6 +640,11 @@ public CompletableFuture commit() { @Override public Long getCommittedVersion() { + return getCommittedVersionAsPrimitive(); + } + + @Override + public long getCommittedVersionAsPrimitive() { if (eventKeeper != null) { eventKeeper.increment(Events.JNI_CALL); } diff --git a/bindings/java/src/main/com/apple/foundationdb/Transaction.java b/bindings/java/src/main/com/apple/foundationdb/Transaction.java index 3e07779cf99..a194e521fea 100644 --- a/bindings/java/src/main/com/apple/foundationdb/Transaction.java +++ b/bindings/java/src/main/com/apple/foundationdb/Transaction.java @@ -253,6 +253,24 @@ public interface Transaction extends AutoCloseable, ReadTransaction, Transaction */ Long getCommittedVersion(); + /** + * Gets the version number at which a successful commit modified the database, + * as a primitive {@code long} to avoid the overhead of autoboxing. + * This must be called only after the successful (non-error) completion of a call + * to {@link #commit()} on this {@code Transaction}, or the behavior is undefined. + * Read-only transactions do not modify the database when committed and will have + * a committed version of -1. Keep in mind that a transaction which reads keys and + * then sets them to their current values may be optimized to a read-only transaction. + * + *

This method is equivalent to {@link #getCommittedVersion()} but avoids the + * allocation of a {@link Long} object on every call.

+ * + * @return the database version at which the commit succeeded, as a primitive {@code long} + */ + default long getCommittedVersionAsPrimitive() { + return getCommittedVersion(); + } + /** * Returns a future which will contain the versionstamp which was used by any versionstamp * operations in this transaction. The future will be ready only after the successful diff --git a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java index 2eb9d674f98..438a26cb474 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/ConcurrentGetSetGet.java @@ -65,7 +65,6 @@ public void apply(Database db) { System.out.println(" sem: " + semaphore); System.out.println(); } catch (InterruptedException e) { - // TODO Auto-generated catch block e.printStackTrace(); } diff --git a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java index 6a53f871b9b..20f7890add7 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/SerialIteration.java @@ -62,7 +62,6 @@ private static double runThreadedTest(Database database, int threadCount) { t.join(); rowsPerSecond += t.getAverageRowsPerSecond(); } catch (InterruptedException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } @@ -90,7 +89,6 @@ public void run() { try { Thread.sleep(r.nextInt(1000)); } catch (InterruptedException e) { - // TODO Auto-generated catch block e.printStackTrace(); } long totalTime = 0; diff --git a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java index a28b78a86fa..23dcca2c0af 100644 --- a/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java +++ b/bindings/java/src/test/com/apple/foundationdb/test/WatchTest.java @@ -104,7 +104,6 @@ public static void raceTest(Database db) { Thread.sleep(1); } catch(InterruptedException e1) { - // TODO Auto-generated catch block e1.printStackTrace(); } } diff --git a/cmake/AddFdbServerLinkTest.cmake b/cmake/AddFdbServerLinkTest.cmake new file mode 100644 index 00000000000..71ab3b81558 --- /dev/null +++ b/cmake/AddFdbServerLinkTest.cmake @@ -0,0 +1,8 @@ +function(add_fdbserver_link_test target_name) + set(link_libs ${ARGN}) + list(GET link_libs 0 primary_lib) + list(REMOVE_AT link_libs 0) + file(RELATIVE_PATH link_test_src "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_SOURCE_DIR}/fdbclient/LinkTest.cpp") + add_flow_target(LINK_TEST NAME ${target_name} SRCS ${link_test_src}) + target_link_libraries(${target_name} PRIVATE "$" ${link_libs} rapidxml) +endfunction() diff --git a/cmake/CompileBoost.cmake b/cmake/CompileBoost.cmake index 925bae405f8..98b3b3084e2 100644 --- a/cmake/CompileBoost.cmake +++ b/cmake/CompileBoost.cmake @@ -38,10 +38,6 @@ function(compile_boost) set(BOOST_LINK_FLAGS "") if(APPLE OR ICX OR USE_LIBCXX) list(APPEND BOOST_COMPILER_FLAGS -stdlib=libc++ -nostdlib++) - if (APPLE) - # Remove this after boost 1.81 or above is used - list(APPEND BOOST_COMPILER_FLAGS -D_LIBCPP_ENABLE_CXX17_REMOVED_UNARY_BINARY_FUNCTION) - endif() list(APPEND BOOST_LINK_FLAGS -lc++ -lc++abi) if (NOT APPLE) list(APPEND BOOST_LINK_FLAGS -static-libgcc) @@ -164,11 +160,10 @@ set(Boost_USE_STATIC_LIBS ON) if (UNIX AND CMAKE_CXX_COMPILER_ID MATCHES "Clang$" AND USE_LIBCXX) list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_86_0_clang) set(BOOST_HINT_PATHS /opt/boost_1_86_0_clang) - message(STATUS "Using Clang version of boost") + message(STATUS "Preferring _clang build of boost ...") else () list(APPEND CMAKE_PREFIX_PATH /opt/boost_1_86_0) set(BOOST_HINT_PATHS /opt/boost_1_86_0) - message(STATUS "Using g++ version of boost") endif () if(BOOST_ROOT) @@ -178,10 +173,18 @@ endif() if(WIN32) # Use CONFIG mode to prefer Boost's BoostConfig.cmake over deprecated FindBoost module # This is required for CMake 3.30+ compatibility (policy CMP0167) - find_package(Boost 1.86.0 QUIET COMPONENTS filesystem iostreams serialization system program_options url CONFIG PATHS ${BOOST_HINT_PATHS}) + find_package(Boost 1.86.0 QUIET + COMPONENTS filesystem iostreams serialization system program_options url + CONFIG PATHS ${BOOST_HINT_PATHS}) add_library(boost_target INTERFACE) - target_link_libraries(boost_target INTERFACE Boost::boost Boost::filesystem Boost::iostreams Boost::serialization Boost::system Boost::url) - + target_link_libraries(boost_target + INTERFACE Boost::boost + Boost::filesystem + Boost::iostreams + Boost::serialization + Boost::system + Boost::url + ) # Add zstd library since boost::iostreams may depend on it find_package(PkgConfig QUIET) if(PkgConfig_FOUND) @@ -191,15 +194,17 @@ if(WIN32) target_link_directories(boost_target INTERFACE ${ZSTD_LIBRARY_DIRS}) endif() endif() - add_library(boost_target_program_options INTERFACE) target_link_libraries(boost_target_program_options INTERFACE Boost::boost Boost::program_options) return() endif() -find_package(Boost 1.86.0 EXACT QUIET COMPONENTS context filesystem iostreams program_options serialization system url CONFIG PATHS ${BOOST_HINT_PATHS}) -set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores any installed boost") +find_package(Boost 1.86.0 EXACT QUIET + COMPONENTS context filesystem iostreams program_options serialization system url + CONFIG PATHS ${BOOST_HINT_PATHS}) + +set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores any installed boost") # The precompiled boost silently broke in CI. While investigating, I considered extending # the old check with something like this, so that it would fail loudly if it found a bad # pre-existing boost. It turns out the error messages we get from CMake explain what is @@ -214,9 +219,17 @@ set(FORCE_BOOST_BUILD OFF CACHE BOOL "Forces cmake to build boost and ignores an # message(FATAL_ERROR "Unacceptable precompiled boost found") # if(Boost_FOUND AND NOT FORCE_BOOST_BUILD) + message(STATUS "Found Boost: ${Boost_DIR}") add_library(boost_target INTERFACE) - target_link_libraries(boost_target INTERFACE Boost::boost Boost::context Boost::filesystem Boost::iostreams Boost::serialization Boost::system Boost::url) - + target_link_libraries(boost_target + INTERFACE Boost::boost + Boost::context + Boost::filesystem + Boost::iostreams + Boost::serialization + Boost::system + Boost::url + ) # Add zstd library since boost::iostreams may depend on it when compiled with homebrew on macOS if(APPLE) find_package(PkgConfig QUIET) @@ -228,7 +241,6 @@ if(Boost_FOUND AND NOT FORCE_BOOST_BUILD) endif() endif() endif() - add_library(boost_target_program_options INTERFACE) target_link_libraries(boost_target_program_options INTERFACE Boost::boost Boost::program_options) elseif(WIN32) diff --git a/cmake/CompileRocksDB.cmake b/cmake/CompileRocksDB.cmake index 99ae2581a3c..c50b1dacef6 100644 --- a/cmake/CompileRocksDB.cmake +++ b/cmake/CompileRocksDB.cmake @@ -64,18 +64,25 @@ else() set(FDB_ROCKSDB_GIT_HASH "") endif() -# Generate FDBRocksDBVersion.h from template into build directory +# Generate FDBRocksDBVersion.h from template into the core build include tree # This file is NOT source-controlled - it's regenerated at cmake configure time -file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/fdbserver/include/fdbserver) +file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/fdbserver/core/include/fdbserver/core) configure_file( ${CMAKE_CURRENT_LIST_DIR}/FDBRocksDBVersion.h.in - ${CMAKE_BINARY_DIR}/fdbserver/include/fdbserver/FDBRocksDBVersion.h + ${CMAKE_BINARY_DIR}/fdbserver/core/include/fdbserver/core/FDBRocksDBVersion.h @ONLY) message(STATUS "Generated FDBRocksDBVersion.h with version ${FDB_ROCKSDB_MAJOR}.${FDB_ROCKSDB_MINOR}.${FDB_ROCKSDB_PATCH}") # Only try to find system RocksDB if not using a specific commit hash if(NOT ROCKSDB_GIT_HASH) + # Save the configured version - FindRocksDB.cmake may overwrite ROCKSDB_VERSION + # with the system version even when find_package fails + set(_FDB_ROCKSDB_VERSION_CONFIGURED ${ROCKSDB_VERSION}) find_package(RocksDB ${ROCKSDB_VERSION}) + if(NOT ROCKSDB_FOUND) + # Restore configured version since FindRocksDB overwrote it + set(ROCKSDB_VERSION ${_FDB_ROCKSDB_VERSION_CONFIGURED}) + endif() endif() include(ExternalProject) diff --git a/cmake/ConfigureCompiler.cmake b/cmake/ConfigureCompiler.cmake index 321ac4a22a9..11a1d0b3062 100644 --- a/cmake/ConfigureCompiler.cmake +++ b/cmake/ConfigureCompiler.cmake @@ -13,6 +13,9 @@ env_set(USE_UBSAN OFF BOOL "Compile with undefined behavior sanitizer") env_set(FDB_RELEASE_CANDIDATE OFF BOOL "This is a building of a release candidate") env_set(FDB_RELEASE OFF BOOL "This is a building of a final release") env_set(USE_CCACHE OFF BOOL "Use ccache for compilation if available") +env_set(USE_CLANG_TIDY OFF BOOL "Run clang-tidy during C/C++ compilation") +env_set(CLANG_TIDY "" STRING "Path to clang-tidy executable (empty to auto-detect)") +env_set(CLANG_TIDY_EXTRA_ARGS "" STRING "Additional clang-tidy arguments (space-separated)") env_set(RELATIVE_DEBUG_PATHS OFF BOOL "Use relative file paths in debug info") env_set(USE_WERROR OFF BOOL "Compile with -Werror. Recommended for local development and CI.") default_linker(_use_ld) @@ -102,6 +105,33 @@ set(CMAKE_CXX_EXTENSIONS OFF) set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) +if(USE_CLANG_TIDY) + if(CLANG_TIDY) + set(_clang_tidy_program "${CLANG_TIDY}") + else() + find_program(_clang_tidy_program NAMES clang-tidy) + endif() + + if(NOT _clang_tidy_program) + message( + FATAL_ERROR + "USE_CLANG_TIDY is enabled, but clang-tidy was not found. Install clang-tidy or set CLANG_TIDY=/path/to/clang-tidy.") + endif() + + set(_clang_tidy_command "${_clang_tidy_program}") + list(APPEND _clang_tidy_command "--config-file=${CMAKE_SOURCE_DIR}/.clang-tidy") + if(CLANG_TIDY_EXTRA_ARGS) + separate_arguments(_clang_tidy_extra_args NATIVE_COMMAND "${CLANG_TIDY_EXTRA_ARGS}") + list(APPEND _clang_tidy_command ${_clang_tidy_extra_args}) + endif() + + set(CMAKE_C_CLANG_TIDY ${_clang_tidy_command}) + set(CMAKE_CXX_CLANG_TIDY ${_clang_tidy_command}) + + string(REPLACE ";" " " _clang_tidy_command_display "${_clang_tidy_command}") + message(STATUS "clang-tidy enabled for C/C++ compilation: ${_clang_tidy_command_display}") +endif() + if(NOT OPEN_FOR_IDE) add_compile_definitions(NO_INTELLISENSE) endif() diff --git a/cmake/FDBBenchmark.cmake b/cmake/FDBBenchmark.cmake new file mode 100644 index 00000000000..5c0ddb4c04c --- /dev/null +++ b/cmake/FDBBenchmark.cmake @@ -0,0 +1,67 @@ +function(fdb_setup_googlebenchmark) + if(TARGET fdb_google_benchmark) + return() + endif() + + if(NOT benchmark_ROOT) + if(EXISTS /opt/googlebenchmark-f91b6b AND CLANG AND USE_LIBCXX) + set(benchmark_ROOT /opt/googlebenchmark-f91b6b) + elseif(EXISTS /opt/googlebenchmark-f91b6b-g++ AND NOT USE_LIBCXX) + set(benchmark_ROOT /opt/googlebenchmark-f91b6b-g++) + endif() + endif() + + find_package(benchmark) + + add_library(fdb_google_benchmark INTERFACE) + + if(benchmark_FOUND) + target_link_libraries(fdb_google_benchmark INTERFACE benchmark::benchmark) + target_include_directories( + fdb_google_benchmark + INTERFACE $) + return() + endif() + + if(NOT TARGET benchmark) + set(googlebenchmark_root ${CMAKE_BINARY_DIR}/googlebenchmark-download) + + configure_file( + ${CMAKE_SOURCE_DIR}/cmake/benchmark-download.cmake + ${googlebenchmark_root}/CMakeLists.txt + COPYONLY) + + execute_process( + COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . + RESULT_VARIABLE results + WORKING_DIRECTORY ${googlebenchmark_root}) + if(results) + message(FATAL_ERROR "Configuration step for Benchmark has Failed. ${results}") + endif() + + execute_process( + COMMAND ${CMAKE_COMMAND} --build . --config Release + RESULT_VARIABLE results + WORKING_DIRECTORY ${googlebenchmark_root}) + if(results) + message(FATAL_ERROR "Build step for Benchmark has Failed. ${results}") + endif() + + if(CMAKE_THREAD_LIBS_INIT) + list(APPEND BENCHMARK_CXX_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) + elseif(UNIX) + list(APPEND BENCHMARK_CXX_LIBRARIES -pthread) + endif() + + set(BENCHMARK_ENABLE_TESTING OFF) + add_subdirectory( + ${googlebenchmark_root}/googlebenchmark-src + ${googlebenchmark_root}/googlebenchmark-build + EXCLUDE_FROM_ALL) + endif() + + target_include_directories( + fdb_google_benchmark + INTERFACE ${CMAKE_BINARY_DIR}/googlebenchmark-download/googlebenchmark-src/include) + target_link_libraries(fdb_google_benchmark INTERFACE benchmark) +endfunction() diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake index 59023fc03f4..8a2c7ea1bc1 100644 --- a/cmake/FDBComponents.cmake +++ b/cmake/FDBComponents.cmake @@ -159,7 +159,7 @@ if(BUILD_SWIFT_BINDING AND NOT WITH_C_BINDING) message(WARNING "Swift binding depends on C binding, but C binding is not enabled") endif() -if(NOT BUILD_SWIFT_BINDING OR NOT BUILD_C_BINDING OR OPEN_FOR_IDE) +if(NOT BUILD_SWIFT_BINDING OR NOT BUILD_C_BINDING OR OPEN_FOR_IDE OR NOT WITH_SWIFT) set(WITH_SWIFT_BINDING OFF) else() find_program(SWIFT_EXECUTABLE swift) @@ -262,28 +262,15 @@ set(WITH_LIBURING OFF CACHE BOOL "Build with liburing enabled") # Set this to ON # TOML11 ################################################################################ -# TOML can download and install itself into the binary directory, so it should -# always be available. -find_package(toml11 3.4.0) -if(toml11_FOUND) - message(STATUS "Using TOML11 from system") - add_library(toml11_target INTERFACE) - target_link_libraries(toml11_target INTERFACE toml11) -else() - include(ExternalProject) - ExternalProject_add(toml11Project - URL "https://github.com/ToruNiina/toml11/archive/v3.4.0.tar.gz" - URL_HASH SHA256=bc6d733efd9216af8c119d8ac64a805578c79cc82b813e4d1d880ca128bd154d - CMAKE_CACHE_ARGS - -DCMAKE_BUILD_TYPE:STRING=${CMAKE_BUILD_TYPE} - -DCMAKE_C_COMPILER:FILEPATH=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER:FILEPATH=${CMAKE_CXX_COMPILER} - -DCMAKE_INSTALL_PREFIX:PATH=${CMAKE_CURRENT_BINARY_DIR}/toml11 - -Dtoml11_BUILD_TEST:BOOL=OFF - BUILD_ALWAYS ON) - add_library(toml11_target INTERFACE) - add_dependencies(toml11_target toml11Project) - target_include_directories(toml11_target SYSTEM INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/toml11/include) +find_package(toml11 3.8.1 EXACT CONFIG) +if(NOT toml11_FOUND) + include(FetchContent) + FetchContent_Declare( + toml11 + URL "https://github.com/ToruNiina/toml11/archive/v3.8.1.tar.gz" + URL_HASH SHA256=6a3d20080ecca5ea42102c078d3415bef80920f6c4ea2258e87572876af77849 + ) + FetchContent_MakeAvailable(toml11) endif() ################################################################################ diff --git a/cmake/FindCoroutines.cmake b/cmake/FindCoroutines.cmake index 9391ddcaed9..b7411ed2f79 100644 --- a/cmake/FindCoroutines.cmake +++ b/cmake/FindCoroutines.cmake @@ -249,7 +249,7 @@ if(CXX_COROUTINES_HAVE_COROUTINES) # Try to compile a simple coroutines program without any compiler flags set(CMAKE_REQUIRED_LIBRARIES ${CMAKE_THREAD_LIBS_INIT}) - + check_cxx_source_compiles("${code}" CXX_COROUTINES_NO_AWAIT_NEEDED) set(can_link ${CXX_COROUTINES_NO_AWAIT_NEEDED}) diff --git a/cmake/GetFmt.cmake b/cmake/GetFmt.cmake index ecb890896af..d197dd8765d 100644 --- a/cmake/GetFmt.cmake +++ b/cmake/GetFmt.cmake @@ -1,30 +1,11 @@ -# Try to find fmt using pkg-config first (for homebrew installations) -find_package(PkgConfig QUIET) -if(PkgConfig_FOUND) - pkg_check_modules(FMT QUIET fmt) - if(FMT_FOUND) - # Create an imported target for fmt - add_library(fmt::fmt INTERFACE IMPORTED) - target_link_libraries(fmt::fmt INTERFACE ${FMT_LIBRARIES}) - target_link_directories(fmt::fmt INTERFACE ${FMT_LIBRARY_DIRS}) - target_include_directories(fmt::fmt INTERFACE ${FMT_INCLUDE_DIRS}) - target_compile_options(fmt::fmt INTERFACE ${FMT_CFLAGS_OTHER}) - set(fmt_FOUND TRUE) - endif() -endif() - -# Fall back to standard find_package -if(NOT fmt_FOUND) - find_package(fmt 11.0.2 EXACT QUIET CONFIG) -endif() +find_package(fmt 11.1.4 EXACT CONFIG) -# Fall back to FetchContent if still not found if(NOT fmt_FOUND) include(FetchContent) FetchContent_Declare( fmt GIT_REPOSITORY https://github.com/fmtlib/fmt - GIT_TAG 11.0.2 + GIT_TAG 11.1.4 ) FetchContent_MakeAvailable(fmt) endif() diff --git a/cmake/RocksDBVersion.cmake b/cmake/RocksDBVersion.cmake index 3b0c349640f..51f5e5d3512 100644 --- a/cmake/RocksDBVersion.cmake +++ b/cmake/RocksDBVersion.cmake @@ -5,7 +5,7 @@ # Choose ONE of the two options below by uncommenting the appropriate section. # Do NOT set both - CMake will error if both are configured. # -# CMake will automatically generate fdbserver/include/fdbserver/FDBRocksDBVersion.h +# CMake will automatically generate fdbserver/core/include/fdbserver/core/FDBRocksDBVersion.h # based on the version specified here. Do NOT edit that file manually. # # To get SHA256: diff --git a/cmake/benchmark-download.cmake b/cmake/benchmark-download.cmake new file mode 100644 index 00000000000..410842e2cfd --- /dev/null +++ b/cmake/benchmark-download.cmake @@ -0,0 +1,35 @@ +cmake_minimum_required(VERSION 3.13) +project(googlebenchmark-download NONE) + +include(ExternalProject) +ExternalProject_Add(googlebenchmark + GIT_REPOSITORY https://github.com/google/benchmark.git + # If you change this, then be sure to also update the directory name (which contains this SHA) + # in FDB's build environment and the prebuilt googlebenchmark package. + GIT_TAG f91b6b42b1b9854772a90ae9501464a161707d1e # v1.6.0 + GIT_SHALLOW ON + GIT_CONFIG advice.detachedHead=false + SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-src" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-build" + CMAKE_ARGS "-DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_LTO=true -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) + +include(ExternalProject) +ExternalProject_Add(googletest DEPENDS googlebenchmark + GIT_REPOSITORY https://github.com/google/googletest.git + # If you change this, be sure to update the prebuilt googlebenchmark package as well. + GIT_TAG 2fe3bd994b3189899d93f1d5a881e725e046fdc2 # release-1.8.1 + GIT_SHALLOW ON + GIT_CONFIG advice.detachedHead=false + SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-src/googletest" + BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/googlebenchmark-build/googletest" + CMAKE_ARGS "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) diff --git a/cmake/utils.cmake b/cmake/utils.cmake index 6a907285028..e90811ec16b 100644 --- a/cmake/utils.cmake +++ b/cmake/utils.cmake @@ -110,4 +110,9 @@ function(generate_grpc_protobuf pkg_name) APPEND_PATH ${out_rel_path} PROTOS ${proto_files} ) + + get_target_property(_generated_proto_sources ${target_name} SOURCES) + if(_generated_proto_sources) + set_source_files_properties(${_generated_proto_sources} PROPERTIES SKIP_LINTING ON) + endif() endfunction() diff --git a/contrib/Joshua/scripts/correctnessTest.sh b/contrib/Joshua/scripts/correctnessTest.sh index 32f5cd8b3c1..54c9ad7be58 100755 --- a/contrib/Joshua/scripts/correctnessTest.sh +++ b/contrib/Joshua/scripts/correctnessTest.sh @@ -99,10 +99,21 @@ # Maps to: --long-running # Values: true/false (default: false) # +# TH_DISABLE_CODE_PROBES Disable code probe collection and ensemble coverage checks +# Maps to: --disable-code-probes +# Values: true/false (default: false) +# # TH_RANDOM_SEED Force specific random seed for debugging # Maps to: --random-seed # Default: auto-generated # +# TH_FDBSERVER_MEMORY Pass --memory SIZE to fdbserver +# Maps to: --fdbserver-memory +# Example: 12288MiB +# Default: unset +# Note: if unset, TestHarness2 automatically uses +# 12288MiB for ASAN/UBSAN-instrumented binaries +# # TH_OUTPUT_FORMAT Output format for test results # Maps to: --output-format # Values: xml/json (default: xml) diff --git a/contrib/TestHarness2/README.md b/contrib/TestHarness2/README.md index f9a180558e7..5e82f8bb624 100644 --- a/contrib/TestHarness2/README.md +++ b/contrib/TestHarness2/README.md @@ -76,11 +76,17 @@ TestHarness2 supports environment variables for configuration. Most variables fo - **`TH_BUGGIFY`**: Buggify mode (`on`, `off`, or `random`, default: `random`) - **`TH_USE_VALGRIND`**: Run tests under valgrind (`true`/`false`, default: `false`) - **`TH_LONG_RUNNING`**: Enable long-running test mode (`true`/`false`, default: `false`) +- **`TH_DISABLE_CODE_PROBES`**: Disable code probe collection and ensemble coverage checks (`true`/`false`, default: `false`) +- **`TH_FDBSERVER_MEMORY`**: Pass `--memory SIZE` to `fdbserver` (for example `12288MiB`) **Optional Configuration:** - **`TH_RANDOM_SEED`**: Force specific random seed for debugging - **`TH_OUTPUT_FORMAT`**: Output format (`xml` or `json`, default: `xml`) +When `TH_FDBSERVER_MEMORY` is unset, TestHarness2 automatically runs sanitizer-instrumented +`fdbserver` binaries with `--memory 12288MiB` so ASAN/UBSAN correctness jobs do not stay +pinned to the default 8 GiB simulator limit. + For a complete list of all variables, run: `python3 -m test_harness.app --help` ## Output Format diff --git a/contrib/TestHarness2/test_harness/config.py b/contrib/TestHarness2/test_harness/config.py index 54c80c0cbb7..7f92d0ce6fe 100644 --- a/contrib/TestHarness2/test_harness/config.py +++ b/contrib/TestHarness2/test_harness/config.py @@ -216,10 +216,21 @@ def __init__(self): } self.print_coverage = False self.print_coverage_args = {"action": "store_true"} + self.disable_code_probes: bool = False + self.disable_code_probes_args = { + "action": "store_true", + "help": "Disable code probe collection and ensemble coverage checks", + } self.binary = Path("bin") / ( "fdbserver.exe" if os.name == "nt" else "fdbserver" ) self.binary_args = {"help": "Path to executable"} + self.fdbserver_memory: str | None = None + self.fdbserver_memory_args = { + "type": str, + "required": False, + "help": "Pass --memory SIZE to fdbserver (for example 12288MiB)", + } self.hit_per_runs_ratio: int = 20000 self.hit_per_runs_ratio_args = { "help": "Maximum test runs before each code probe hit at least once" @@ -352,7 +363,23 @@ def _read_env(self): # Use the env var to supply the default value, so that if the # environment variable is set and the corresponding command line # flag is not, the environment variable has an effect. - self._config_map[attr].kwargs["default"] = attr_type(e) + self._config_map[attr].kwargs["default"] = self._parse_env_value( + attr_type, env_name, e + ) + + def _parse_env_value(self, attr_type: type, env_name: str, value: str): + if attr_type is bool: + normalized = value.lower() + if normalized in ("true", "1", "yes", "on"): + return True + if normalized in ("false", "0", "no", "off"): + return False + raise ValueError( + "Invalid boolean value {} for {} -- use true or false".format( + value, env_name + ) + ) + return attr_type(value) def build_arguments(self, parser: argparse.ArgumentParser): for val in self._config_map.values(): diff --git a/contrib/TestHarness2/test_harness/results.py b/contrib/TestHarness2/test_harness/results.py index c0124ce080c..4d31ddbafae 100644 --- a/contrib/TestHarness2/test_harness/results.py +++ b/contrib/TestHarness2/test_harness/results.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import collections import io import json import re @@ -28,9 +29,14 @@ def __init__(self, cluster_file: str | None, ensemble_id: str): self.fdb_path = ("joshua", "ensembles", "results", "application", ensemble_id) self.coverage_path = self.fdb_path + ("coverage",) self.statistics = test_harness.fdb.Statistics(cluster_file, self.fdb_path) - coverage_dict: OrderedDict[Coverage, int] = test_harness.fdb.read_coverage( - cluster_file, self.coverage_path - ) + self.code_probe_tracking_enabled = not config.disable_code_probes + coverage_dict: OrderedDict[Coverage, int] + if self.code_probe_tracking_enabled: + coverage_dict = test_harness.fdb.read_coverage( + cluster_file, self.coverage_path + ) + else: + coverage_dict = collections.OrderedDict() self.coverage: List[Tuple[Coverage, int]] = [] self.min_coverage_hit: int | None = None self.ratio = self.global_statistics.total_test_runs / config.hit_per_runs_ratio @@ -54,7 +60,9 @@ def __init__(self, cluster_file: str | None, ensemble_id: str): self.global_statistics.total_cpu_time += v.runtime self.stats.append((k, v.runtime, v.run_count)) self.stats.sort(key=lambda x: x[1], reverse=True) - if self.min_coverage_hit is not None: + if not self.code_probe_tracking_enabled: + self.coverage_ok = True + elif self.min_coverage_hit is not None: self.coverage_ok = self.min_coverage_hit > self.ratio else: self.coverage_ok = False @@ -64,6 +72,9 @@ def dump(self, prefix: str): out = SummaryTree("EnsembleResults") out.attributes["TotalRuntime"] = str(self.global_statistics.total_cpu_time) out.attributes["TotalTestRuns"] = str(self.global_statistics.total_test_runs) + out.attributes["CodeProbeTrackingEnabled"] = ( + "1" if self.code_probe_tracking_enabled else "0" + ) out.attributes["TotalProbesHit"] = str(self.global_statistics.total_probes_hit) out.attributes["MinProbeHit"] = str(self.min_coverage_hit) out.attributes["TotalProbes"] = str(len(self.coverage)) diff --git a/contrib/TestHarness2/test_harness/run.py b/contrib/TestHarness2/test_harness/run.py index d0133e588ab..90b5eed9dd1 100644 --- a/contrib/TestHarness2/test_harness/run.py +++ b/contrib/TestHarness2/test_harness/run.py @@ -22,6 +22,10 @@ from test_harness.summarize import Summary, SummaryTree +DEFAULT_SANITIZER_MEMORY = "12288MiB" +SANITIZER_MARKERS = (b"__asan_init", b"__ubsan_handle_") +_SANITIZER_BINARY_CACHE: Dict[Path, bool] = {} + def parse_test_args_file(args_file: Path) -> tuple[Path, int, bool, List[str]]: """ @@ -85,6 +89,44 @@ def parse_test_args_file(args_file: Path) -> tuple[Path, int, bool, List[str]]: return test_file, random_seed, buggify_enabled, extra_args +def extra_args_include_memory(extra_args: List[str]) -> bool: + return any(arg in ("-m", "--memory") for arg in extra_args) + + +def binary_uses_sanitizers(binary: Path) -> bool: + cached = _SANITIZER_BINARY_CACHE.get(binary) + if cached is not None: + return cached + + found = False + max_marker_len = max(len(marker) for marker in SANITIZER_MARKERS) + overlap = max_marker_len - 1 + with binary.open("rb") as infile: + tail = b"" + while True: + chunk = infile.read(1024 * 1024) + if not chunk: + break + haystack = tail + chunk + if any(marker in haystack for marker in SANITIZER_MARKERS): + found = True + break + tail = haystack[-overlap:] if overlap else b"" + + _SANITIZER_BINARY_CACHE[binary] = found + return found + + +def resolve_fdbserver_memory(binary: Path, extra_args: List[str]) -> str | None: + if extra_args_include_memory(extra_args): + return None + if config.fdbserver_memory is not None: + return config.fdbserver_memory + if binary_uses_sanitizers(binary): + return DEFAULT_SANITIZER_MEMORY + return None + + @total_ordering class TestDescription: def __init__(self, path: Path, name: str, priority: float): @@ -432,6 +474,7 @@ def __init__( extra_args: List[str] | None = None, ): self.binary = binary + self.is_old_binary = False self.test_file = test_file self.random_seed = random_seed self.uid = uid @@ -553,6 +596,9 @@ def run(self): "-s", str(self.random_seed), ] + memory_limit = resolve_fdbserver_memory(self.binary, self.extra_args) + if memory_limit is not None: + command += ["--memory", memory_limit] if self.trace_format is not None: command += ["--trace_format", self.trace_format] if self.use_tls_plugin: diff --git a/contrib/TestHarness2/test_harness/summarize.py b/contrib/TestHarness2/test_harness/summarize.py index 94526018637..d8e11329134 100644 --- a/contrib/TestHarness2/test_harness/summarize.py +++ b/contrib/TestHarness2/test_harness/summarize.py @@ -390,7 +390,11 @@ def summarize(self, trace_dir: Path, command: str): return self.summarize_files(trace_files[0]) # Skip write_coverage for old binaries in restarting tests - if config.joshua_dir is not None and not self.is_old_binary: + if ( + config.joshua_dir is not None + and not self.is_old_binary + and not config.disable_code_probes + ): import test_harness.fdb test_harness.fdb.write_coverage( @@ -424,7 +428,7 @@ def ok(self): return (not self.error) != self.is_negative_test def done(self): - if config.print_coverage: + if config.print_coverage and not config.disable_code_probes: for k, v in self.coverage.items(): child = SummaryTree("CodeCoverage") child.attributes["File"] = k.file @@ -500,6 +504,13 @@ def done(self): ): # When running ASAN we expect to see this message. Boost coroutine should be using the correct asan annotations so that it shouldn't produce any false positives. continue + if "WARNING: ASan is ignoring requested __asan_handle_no_return: stack type:" in line: + # ASAN emits this with makecontext/swapcontext coroutine stacks. + continue + if line == "False positive error reports may follow": + continue + if line == "For details see https://github.com/google/sanitizers/issues/189": + continue if line.endswith("Warning: unimplemented fcntl command: 1036"): # Valgrind produces this warning when F_SET_RW_HINT is used continue @@ -695,6 +706,8 @@ def parse_error(attrs: Dict[str, str]): self.handler.add_handler(("Severity", "40"), parse_error) def coverage(attrs: Dict[str, str]): + if config.disable_code_probes: + return covered = True if "Covered" in attrs: covered = int(attrs["Covered"]) != 0 diff --git a/contrib/benchmark_comparison.py b/contrib/benchmark_comparison.py index ada345f691c..b5776ad558e 100644 --- a/contrib/benchmark_comparison.py +++ b/contrib/benchmark_comparison.py @@ -3,11 +3,11 @@ Generate comprehensive actor vs coroutine benchmark comparison report. USAGE: - cd build_output # or your build directory with bin/flowbench + cd build_output # or your build directory with bin/flow_bench python3 ../contrib/benchmark_comparison.py REQUIREMENTS: - - Working bin/flowbench executable + - Working bin/flow_bench executable - Both actor and coroutine benchmarks available: * bench_net2, coroutine_net2 * bench_delay.*DELAY.*, coroutine_delay_bench @@ -33,7 +33,7 @@ def geomean(values): def get_benchmark_data(filter_name): """Get benchmark data and parse results""" - result = subprocess.run(['./bin/flowbench', f'--benchmark_filter={filter_name}', '--benchmark_format=json'], + result = subprocess.run(['./bin/flow_bench', f'--benchmark_filter={filter_name}', '--benchmark_format=json'], capture_output=True, text=True, cwd='/root/build_output') if result.returncode == 0 and result.stdout.strip(): return json.loads(result.stdout)['benchmarks'] @@ -87,7 +87,7 @@ def main(): print(f"OVERALL_GEOMEAN {geom:+7.4f} {geom:+7.4f} 0 0 0 0") print("") - print("Comparing bench_net2 to coroutine_net2 (from ./build_output/bin/flowbench)") + print("Comparing bench_net2 to coroutine_net2 (from ./build_output/bin/flow_bench)") print("Benchmark Time CPU Time Old Time New CPU Old CPU New") print("----------------------------------------------------------------------------------------------------------------------------") @@ -114,7 +114,7 @@ def main(): print(f"OVERALL_GEOMEAN {geom:+7.4f} {geom:+7.4f} 0 0 0 0") print("") - print("Comparing bench_callback to coroutine_callback (from ./build_output/bin/flowbench)") + print("Comparing bench_callback to coroutine_callback (from ./build_output/bin/flow_bench)") print("Benchmark Time CPU Time Old Time New CPU Old CPU New") print("---------------------------------------------------------------------------------------------------------------------------------------------") @@ -147,4 +147,4 @@ def main(): print(f"OVERALL_GEOMEAN {geom:+7.4f} {geom:+7.4f} 0 0 0 0") if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/contrib/joshua_logtool.py b/contrib/joshua_logtool.py index 87d579ab3f5..8f296de9998 100755 --- a/contrib/joshua_logtool.py +++ b/contrib/joshua_logtool.py @@ -20,7 +20,7 @@ from typing import List, Union -# Defined in SimulatedCluster.actor.cpp:SimulationConfig::setStorageEngine +# Defined in SimulatedCluster.cpp:SimulationConfig::setStorageEngine ROCKSDB_TRACEEVENT_STRING = ["RocksDBNonDeterminism", "ShardedRocksDBNonDeterminism"] # e.g. /var/joshua/ensembles/20230221-051349-xiaogesu-c9fc5b230dcd91cf diff --git a/contrib/linenoise/CMakeLists.txt b/contrib/linenoise/CMakeLists.txt index 52487a6f228..9191ea44432 100644 --- a/contrib/linenoise/CMakeLists.txt +++ b/contrib/linenoise/CMakeLists.txt @@ -1,2 +1,3 @@ add_library(linenoise STATIC linenoise.c) target_include_directories(linenoise PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") +set_target_properties(linenoise PROPERTIES C_CLANG_TIDY "") diff --git a/contrib/lsan.suppressions b/contrib/lsan.suppressions index 3d3d40e59eb..70fd84f298d 100644 --- a/contrib/lsan.suppressions +++ b/contrib/lsan.suppressions @@ -3,3 +3,14 @@ # Not all incoming connections are cleanly shut down in client API tests leak:ConnectionReaderActorState + +# Peer objects in TransportData::peers are intentionally not freed at shutdown. +# FlowTransport is allocated into a global slot and never deleted because its +# destructor triggers actor cancellation cascades that access freed state. +# Peer::Peer covers allocations during Peer construction (DDSketch, AsyncTrigger). +# connectionKeeper and connectionMonitor cover Promise allocations made during +# the Peer's lifetime that are leaked along with the Peer. +leak:TransportData::getOrOpenPeer +leak:Peer::Peer +leak:connectionKeeper +leak:connectionMonitor diff --git a/contrib/mockkms/CMakeLists.txt b/contrib/mockkms/CMakeLists.txt deleted file mode 100644 index 6a0b75639c2..00000000000 --- a/contrib/mockkms/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -if(WITH_GO_BINDING) - set(MOCK_KMS_SRC fault_injection.go get_encryption_keys.go mock_kms.go utils.go) - set(MOCK_KMS_TEST_SRC ${MOCK_KMS_SRC} mockkms_test.go) - add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms - COMMAND ${GO_EXECUTABLE} build -o ${CMAKE_BINARY_DIR}/bin/mockkms ${MOCK_KMS_SRC} - DEPENDS ${MOCK_KMS_SRC} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - add_custom_target(mockkms ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms) - fdb_install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/mockkms DESTINATION bin COMPONENT server) - - add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/bin/mockkms_test - COMMAND ${GO_EXECUTABLE} test -c -o ${CMAKE_BINARY_DIR}/bin/mockkms_test ${MOCK_KMS_TEST_SRC} - DEPENDS ${MOCK_KMS_TEST_SRC} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) - add_custom_target(mockkms_test ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/mockkms_test) - add_test(NAME mockkms COMMAND ${CMAKE_BINARY_DIR}/bin/mockkms_test) - -endif() diff --git a/contrib/mockkms/fault_injection.go b/contrib/mockkms/fault_injection.go deleted file mode 100644 index 8b5ff01b194..00000000000 --- a/contrib/mockkms/fault_injection.go +++ /dev/null @@ -1,179 +0,0 @@ -/* - * fault_injection.go - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Interface supports client to inject fault(s) -// Module enables a client to update { FaultLocation -> FaultStatus } mapping in a -// thread-safe manner, however, client is responsible to synchronize fault status -// updates across 'getEncryptionKeys' REST requests to obtain predictable results. - -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "sync" -) - -type Fault struct { - Location int `json:"fault_location"` - Enable bool `json:"enable_fault"` -} - -type FaultInjectionRequest struct { - Faults []Fault `json:"faults"` -} - -type FaultInjectionResponse struct { - Faults []Fault `json:"faults"` -} - -type faultLocMap struct { - locMap map[int]bool - rwLock sync.RWMutex -} - -var ( - faultLocMapInstance *faultLocMap // Singleton mapping of { FaultLocation -> FaultStatus } -) - -// Caller is responsible for thread synchronization. Recommended to be invoked during package::init() -func NewFaultLocMap() *faultLocMap { - if faultLocMapInstance == nil { - faultLocMapInstance = &faultLocMap{} - - faultLocMapInstance.rwLock = sync.RWMutex{} - faultLocMapInstance.locMap = map[int]bool { - READ_HTTP_REQUEST_BODY : false, - UNMARSHAL_REQUEST_BODY_JSON : false, - UNSUPPORTED_QUERY_MODE : false, - PARSE_HTTP_REQUEST : false, - MARSHAL_RESPONSE : false, - } - } - return faultLocMapInstance -} - -func getLocFaultStatus(loc int) (val bool, found bool) { - if faultLocMapInstance == nil { - panic("FaultLocMap not intialized") - os.Exit(1) - } - - faultLocMapInstance.rwLock.RLock() - defer faultLocMapInstance.rwLock.RUnlock() - val, found = faultLocMapInstance.locMap[loc] - if !found { - return - } - return -} - -func updateLocFaultStatuses(faults []Fault) (updated []Fault, err error) { - if faultLocMapInstance == nil { - panic("FaultLocMap not intialized") - os.Exit(1) - } - - updated = []Fault{} - err = nil - - faultLocMapInstance.rwLock.Lock() - defer faultLocMapInstance.rwLock.Unlock() - for i := 0; i < len(faults); i++ { - fault := faults[i] - - oldVal, found := faultLocMapInstance.locMap[fault.Location] - if !found { - err = fmt.Errorf("Unknown fault_location '%d'", fault.Location) - return - } - faultLocMapInstance.locMap[fault.Location] = fault.Enable - log.Printf("Update Location '%d' oldVal '%t' newVal '%t'", fault.Location, oldVal, fault.Enable) - } - - // return the updated faultLocMap - for loc, enable := range faultLocMapInstance.locMap { - var f Fault - f.Location = loc - f.Enable = enable - updated = append(updated, f) - } - return -} - -func jsonifyFaultArr(w http.ResponseWriter, faults []Fault) (jResp string) { - resp := FaultInjectionResponse{ - Faults: faults, - } - - mResp, err := json.Marshal(resp) - if err != nil { - log.Printf("Error marshaling response '%s'", err.Error()) - sendErrorResponse(w, err) - return - } - jResp = string(mResp) - return -} - -func updateFaultLocMap(w http.ResponseWriter, faults []Fault) { - updated , err := updateLocFaultStatuses(faults) - if err != nil { - sendErrorResponse(w, err) - return - } - - fmt.Fprintf(w, jsonifyFaultArr(w, updated)) -} - -func shouldInjectFault(loc int) bool { - status, found := getLocFaultStatus(loc) - if !found { - log.Printf("Unknown fault_location '%d'", loc) - return false - } - return status -} - -func handleUpdateFaultInjection(w http.ResponseWriter, r *http.Request) { - byteArr, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Printf("Http request body read error '%s'", err.Error()) - sendErrorResponse(w, err) - return - } - - req := FaultInjectionRequest{} - err = json.Unmarshal(byteArr, &req) - if err != nil { - log.Printf("Error parsing FaultInjectionRequest '%s'", string(byteArr)) - sendErrorResponse(w, err) - } - updateFaultLocMap(w, req.Faults) -} - -func initFaultLocMap() { - faultLocMapInstance = NewFaultLocMap() - log.Printf("FaultLocMap int done") -} \ No newline at end of file diff --git a/contrib/mockkms/get_encryption_keys.go b/contrib/mockkms/get_encryption_keys.go deleted file mode 100644 index 1f48e0f371d..00000000000 --- a/contrib/mockkms/get_encryption_keys.go +++ /dev/null @@ -1,321 +0,0 @@ -/* - * get_encryption_keys.go - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// GetEncryptionKeys handler -// Handler is resposible for the following: -// 1. Parse the incoming HttpRequest and validate JSON request structural sanity -// 2. Ability to handle getEncryptionKeys by 'KeyId' or 'DomainId' as requested -// 3. Ability to inject faults if requested - -package main - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "log" - "math/rand" - "net/http" -) - -type CipherDetailRes struct { - BaseCipherId uint64 `json:"base_cipher_id"` - EncryptDomainId int64 `json:"encrypt_domain_id"` - BaseCipher string `json:"base_cipher"` -} - -type ValidationToken struct { - TokenName string `json:"token_name"` - TokenValue string `json:"token_value"` -} - -type CipherDetailReq struct { - BaseCipherId uint64 `json:"base_cipher_id"` - EncryptDomainId int64 `json:"encrypt_domain_id"` -} - -type GetEncryptKeysResponse struct { - CipherDetails []CipherDetailRes `json:"cipher_key_details"` - KmsUrls []string `json:"kms_urls"` -} - -type GetEncryptKeysRequest struct { - QueryMode string `json:"query_mode"` - CipherDetails []CipherDetailReq `json:"cipher_key_details"` - ValidationTokens []ValidationToken `json:"validation_tokens"` - RefreshKmsUrls bool `json:"refresh_kms_urls"` -} - -type cipherMapInstanceSingleton map[uint64][]byte - -const ( - READ_HTTP_REQUEST_BODY = iota - UNMARSHAL_REQUEST_BODY_JSON - UNSUPPORTED_QUERY_MODE - PARSE_HTTP_REQUEST - MARSHAL_RESPONSE -) - -const ( - maxCipherKeys = uint64(1024 * 1024) // Max cipher keys - maxCipherSize = 16 // Max cipher buffer size -) - -var ( - cipherMapInstance cipherMapInstanceSingleton // Singleton mapping of { baseCipherId -> baseCipher } -) - -// const mapping of { Location -> errorString } -func errStrMap() func(int) string { - _errStrMap := map[int]string{ - READ_HTTP_REQUEST_BODY: "Http request body read error", - UNMARSHAL_REQUEST_BODY_JSON: "Http request body unmarshal error", - UNSUPPORTED_QUERY_MODE: "Unsupported query_mode", - PARSE_HTTP_REQUEST: "Error parsing GetEncryptionKeys request", - MARSHAL_RESPONSE: "Error marshaling response", - } - - return func(key int) string { - return _errStrMap[key] - } -} - -// Caller is responsible for thread synchronization. Recommended to be invoked during package::init() -func NewCipherMap(maxKeys uint64, cipherSize int) cipherMapInstanceSingleton { - if cipherMapInstance == nil { - cipherMapInstance = make(map[uint64][]byte) - - for i := uint64(1); i <= maxKeys; i++ { - cipher := make([]byte, cipherSize) - rand.Read(cipher) - cipherMapInstance[i] = cipher - } - log.Printf("KMS cipher map populate done, maxCiphers '%d'", maxCipherKeys) - } - return cipherMapInstance -} - -func getKmsUrls() (urls []string) { - urlCount := rand.Intn(5) + 1 - for i := 1; i <= urlCount; i++ { - url := fmt.Sprintf("https://KMS/%d:%d:%d:%d", i, i, i, i) - urls = append(urls, url) - } - return -} - -func isEncryptDomainIdValid(id int64) bool { - if id > 0 || id == -1 || id == -2 { - return true - } - return false -} - -func abs(x int64) int64 { - if x < 0 { - return -x - } - return x -} - -func getBaseCipherIdFromDomainId(domainId int64) (baseCipherId uint64) { - baseCipherId = uint64(1) + uint64(abs(domainId))%maxCipherKeys - return -} - -func getEncryptionKeysByKeyIds(w http.ResponseWriter, byteArr []byte) { - req := GetEncryptKeysRequest{} - err := json.Unmarshal(byteArr, &req) - if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) { - var err error - if shouldInjectFault(PARSE_HTTP_REQUEST) { - err = fmt.Errorf("[FAULT] %s %s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr)) - } else { - err = fmt.Errorf("%s %s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err) - } - log.Println(err.Error()) - sendErrorResponse(w, err) - return - } - - var details []CipherDetailRes - for i := 0; i < len(req.CipherDetails); i++ { - var baseCipherId = uint64(req.CipherDetails[i].BaseCipherId) - - var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId) - if !isEncryptDomainIdValid(encryptDomainId) { - err := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId) - sendErrorResponse(w, err) - return - } - - cipher, found := cipherMapInstance[baseCipherId] - if !found { - err := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId) - sendErrorResponse(w, err) - return - } - - var detail = CipherDetailRes{ - BaseCipherId: baseCipherId, - EncryptDomainId: encryptDomainId, - BaseCipher: string(cipher), - } - details = append(details, detail) - } - - var urls []string - if req.RefreshKmsUrls { - urls = getKmsUrls() - } - - resp := GetEncryptKeysResponse{ - CipherDetails: details, - KmsUrls: urls, - } - - mResp, err := json.Marshal(resp) - if err != nil || shouldInjectFault(MARSHAL_RESPONSE) { - var err error - if shouldInjectFault(MARSHAL_RESPONSE) { - err = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE)) - } else { - err = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err) - } - log.Println(err.Error()) - sendErrorResponse(w, err) - return - } - - fmt.Fprintf(w, string(mResp)) -} - -func getEncryptionKeysByDomainIds(w http.ResponseWriter, byteArr []byte) { - req := GetEncryptKeysRequest{} - err := json.Unmarshal(byteArr, &req) - if err != nil || shouldInjectFault(PARSE_HTTP_REQUEST) { - var err error - if shouldInjectFault(PARSE_HTTP_REQUEST) { - err = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr)) - } else { - err = fmt.Errorf("%s '%s' err '%v'", errStrMap()(PARSE_HTTP_REQUEST), string(byteArr), err) - } - log.Println(err.Error()) - sendErrorResponse(w, err) - return - } - - var details []CipherDetailRes - for i := 0; i < len(req.CipherDetails); i++ { - var encryptDomainId = int64(req.CipherDetails[i].EncryptDomainId) - if !isEncryptDomainIdValid(encryptDomainId) { - err := fmt.Errorf("EncryptDomainId not valid '%d'", encryptDomainId) - sendErrorResponse(w, err) - return - } - - var baseCipherId = getBaseCipherIdFromDomainId(encryptDomainId) - cipher, found := cipherMapInstance[baseCipherId] - if !found { - err := fmt.Errorf("BaseCipherId not found '%d'", baseCipherId) - sendErrorResponse(w, err) - return - } - - var detail = CipherDetailRes{ - BaseCipherId: baseCipherId, - EncryptDomainId: encryptDomainId, - BaseCipher: string(cipher), - } - details = append(details, detail) - } - - var urls []string - if req.RefreshKmsUrls { - urls = getKmsUrls() - } - - resp := GetEncryptKeysResponse{ - CipherDetails: details, - KmsUrls: urls, - } - - mResp, err := json.Marshal(resp) - if err != nil || shouldInjectFault(MARSHAL_RESPONSE) { - var err error - if shouldInjectFault(MARSHAL_RESPONSE) { - err = fmt.Errorf("[FAULT] %s", errStrMap()(MARSHAL_RESPONSE)) - } else { - err = fmt.Errorf("%s err '%v'", errStrMap()(MARSHAL_RESPONSE), err) - } - log.Println(err.Error()) - sendErrorResponse(w, err) - return - } - - fmt.Fprintf(w, string(mResp)) -} - -func handleGetEncryptionKeys(w http.ResponseWriter, r *http.Request) { - byteArr, err := ioutil.ReadAll(r.Body) - if err != nil || shouldInjectFault(READ_HTTP_REQUEST_BODY) { - var err error - if shouldInjectFault(READ_HTTP_REQUEST_BODY) { - err = fmt.Errorf("[FAULT] %s", errStrMap()(READ_HTTP_REQUEST_BODY)) - } else { - err = fmt.Errorf("%s err '%v'", errStrMap()(READ_HTTP_REQUEST_BODY), err) - } - log.Println(err.Error()) - sendErrorResponse(w, err) - return - } - - var arbitrary_json map[string]interface{} - err = json.Unmarshal(byteArr, &arbitrary_json) - if err != nil || shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) { - var err error - if shouldInjectFault(UNMARSHAL_REQUEST_BODY_JSON) { - err = fmt.Errorf("[FAULT] %s", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON)) - } else { - err = fmt.Errorf("%s err '%v'", errStrMap()(UNMARSHAL_REQUEST_BODY_JSON), err) - } - log.Println(err.Error()) - sendErrorResponse(w, err) - return - } - - if shouldInjectFault(UNSUPPORTED_QUERY_MODE) { - err = fmt.Errorf("[FAULT] %s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"]) - sendErrorResponse(w, err) - return - } else if arbitrary_json["query_mode"] == "lookupByKeyId" { - getEncryptionKeysByKeyIds(w, byteArr) - } else if arbitrary_json["query_mode"] == "lookupByDomainId" { - getEncryptionKeysByDomainIds(w, byteArr) - } else { - err = fmt.Errorf("%s '%s'", errStrMap()(UNSUPPORTED_QUERY_MODE), arbitrary_json["query_mode"]) - sendErrorResponse(w, err) - return - } -} - -func initEncryptCipherMap() { - cipherMapInstance = NewCipherMap(maxCipherKeys, maxCipherSize) -} diff --git a/contrib/mockkms/mock_kms.go b/contrib/mockkms/mock_kms.go deleted file mode 100644 index 52d72539f72..00000000000 --- a/contrib/mockkms/mock_kms.go +++ /dev/null @@ -1,75 +0,0 @@ -/* - * mock_kms.go - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// FoundationDB Mock KMS (Key Management Solution/Service) interface -// Interface runs an HTTP server handling REST calls simulating FDB communications -// with an external KMS. - -package main - -// Importing module enable .note.gnu.build-id insertion in the ELF executable. -// Few drawbacks of the scheme are: -// 1. Potentially slower builds -// 2. No cross-compilation support -// 3. Limited `go tools` availability. -// TODO: Replace with a better scheme if possible. - -import "C" - -import ( - "log" - "math/rand" - "net/http" - "sync" - "time" -) - -// KMS supported endpoints -const ( - getEncryptionKeysEndpoint = "/getEncryptionKeys" - updateFaultInjectionEndpoint = "/updateFaultInjection" -) - -// Routine is responsible to instantiate data-structures necessary for MockKMS functioning -func init () { - var wg sync.WaitGroup - - wg.Add(2) - go func(){ - initEncryptCipherMap() - wg.Done() - }() - go func(){ - initFaultLocMap() - wg.Done() - }() - - wg.Wait() - - rand.Seed(time.Now().UTC().UnixNano()) -} - -func main() { - http.NewServeMux() - http.HandleFunc(getEncryptionKeysEndpoint, handleGetEncryptionKeys) - http.HandleFunc(updateFaultInjectionEndpoint, handleUpdateFaultInjection) - - log.Fatal(http.ListenAndServe(":5001", nil)) -} \ No newline at end of file diff --git a/contrib/mockkms/mockkms_test.go b/contrib/mockkms/mockkms_test.go deleted file mode 100644 index 38b901bd8cc..00000000000 --- a/contrib/mockkms/mockkms_test.go +++ /dev/null @@ -1,302 +0,0 @@ -/* - * mockkms_test.go - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// MockKMS unit tests, the coverage includes: -// 1. Mock HttpRequest creation and HttpResponse writer. -// 2. Construct fake request to validate the following scenarios: -// 2.1. Request with "unsupported query mode" -// 2.2. Get encryption keys by KeyIds; with and without 'RefreshKmsUrls' flag. -// 2.2. Get encryption keys by DomainIds; with and without 'RefreshKmsUrls' flag. -// 2.3. Random fault injection and response validation - -package main - -import ( - "encoding/json" - "io/ioutil" - "math/rand" - "net/http" - "net/http/httptest" - "strings" - "testing" -) - -const ( - ByKeyIdReqWithRefreshUrls = `{ - "query_mode": "lookupByKeyId", - "cipher_key_details": [ - { - "base_cipher_id": 77, - "encrypt_domain_id": 76 - }, - { - "base_cipher_id": 2, - "encrypt_domain_id": -1 - } - ], - "validation_tokens": [ - { - "token_name": "1", - "token_value":"12344" - }, - { - "token_name": "2", - "token_value":"12334" - } - ], - "refresh_kms_urls": true - }` - ByKeyIdReqWithoutRefreshUrls = `{ - "query_mode": "lookupByKeyId", - "cipher_key_details": [ - { - "base_cipher_id": 77, - "encrypt_domain_id": 76 - }, - { - "base_cipher_id": 2, - "encrypt_domain_id": -1 - } - ], - "validation_tokens": [ - { - "token_name": "1", - "token_value":"12344" - }, - { - "token_name": "2", - "token_value":"12334" - } - ], - "refresh_kms_urls": false - }` - ByDomainIdReqWithRefreshUrls = `{ - "query_mode": "lookupByDomainId", - "cipher_key_details": [ - { - "encrypt_domain_id": 76 - }, - { - "encrypt_domain_id": -1 - } - ], - "validation_tokens": [ - { - "token_name": "1", - "token_value":"12344" - }, - { - "token_name": "2", - "token_value":"12334" - } - ], - "refresh_kms_urls": true - }` - ByDomainIdReqWithoutRefreshUrls = `{ - "query_mode": "lookupByDomainId", - "cipher_key_details": [ - { - "encrypt_domain_id": 76 - }, - { - "encrypt_domain_id": -1 - } - ], - "validation_tokens": [ - { - "token_name": "1", - "token_value":"12344" - }, - { - "token_name": "2", - "token_value":"12334" - } - ], - "refresh_kms_urls": false - }` - UnsupportedQueryMode= `{ - "query_mode": "foo_mode", - "cipher_key_details": [ - { - "encrypt_domain_id": 76 - }, - { - "encrypt_domain_id": -1 - } - ], - "validation_tokens": [ - { - "token_name": "1", - "token_value":"12344" - }, - { - "token_name": "2", - "token_value":"12334" - } - ], - "refresh_kms_urls": false - }` -) - -func unmarshalValidResponse(data []byte, t *testing.T) (resp GetEncryptKeysResponse) { - resp = GetEncryptKeysResponse{} - err := json.Unmarshal(data, &resp) - if err != nil { - t.Errorf("Error unmarshaling valid response '%s' error '%v'", string(data), err) - t.Fail() - } - return -} - -func unmarshalErrorResponse(data []byte, t *testing.T) (resp ErrorResponse) { - resp = ErrorResponse{} - err := json.Unmarshal(data, &resp) - if err != nil { - t.Errorf("Error unmarshaling error response resp '%s' error '%v'", string(data), err) - t.Fail() - } - return -} - -func checkGetEncyptKeysResponseValidity(resp GetEncryptKeysResponse, t *testing.T) { - if len(resp.CipherDetails) != 2 { - t.Errorf("Unexpected CipherDetails count, expected '%d' actual '%d'", 2, len(resp.CipherDetails)) - t.Fail() - } - - baseCipherIds := [...]uint64 {uint64(77), uint64(2)} - encryptDomainIds := [...]int64 {int64(76), int64(-1)} - - for i := 0; i < len(resp.CipherDetails); i++ { - if resp.CipherDetails[i].BaseCipherId != baseCipherIds[i] { - t.Errorf("Mismatch BaseCipherId, expected '%d' actual '%d'", baseCipherIds[i], resp.CipherDetails[i].BaseCipherId) - t.Fail() - } - if resp.CipherDetails[i].EncryptDomainId != encryptDomainIds[i] { - t.Errorf("Mismatch EncryptDomainId, expected '%d' actual '%d'", encryptDomainIds[i], resp.CipherDetails[i].EncryptDomainId) - t.Fail() - } - if len(resp.CipherDetails[i].BaseCipher) == 0 { - t.Error("Empty BaseCipher") - t.Fail() - } - } -} - -func runQueryExpectingErrorResponse(payload string, url string, errSubStr string, t *testing.T) { - body := strings.NewReader(payload) - req := httptest.NewRequest(http.MethodPost, url, body) - w := httptest.NewRecorder() - handleGetEncryptionKeys(w, req) - res := w.Result() - defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Errorf("Error %v", err) - } - - resp := unmarshalErrorResponse(data, t) - if !strings.Contains(resp.Err.Detail, errSubStr) { - t.Errorf("Unexpected error response '%s'", resp.Err.Detail) - t.Fail() - } -} - -func runQueryExpectingValidResponse(payload string, url string, t *testing.T) { - body := strings.NewReader(payload) - req := httptest.NewRequest(http.MethodPost, url, body) - w := httptest.NewRecorder() - handleGetEncryptionKeys(w, req) - res := w.Result() - defer res.Body.Close() - data, err := ioutil.ReadAll(res.Body) - if err != nil { - t.Errorf("Error %v", err) - } - - resp := unmarshalValidResponse(data, t) - checkGetEncyptKeysResponseValidity(resp, t) -} - -func TestUnsupportedQueryMode(t *testing.T) { - runQueryExpectingErrorResponse(UnsupportedQueryMode, getEncryptionKeysEndpoint, errStrMap()(UNSUPPORTED_QUERY_MODE), t) -} - -func TestGetEncryptionKeysByKeyIdsWithRefreshUrls(t *testing.T) { - runQueryExpectingValidResponse(ByKeyIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t) -} - -func TestGetEncryptionKeysByKeyIdsWithoutRefreshUrls(t *testing.T) { - runQueryExpectingValidResponse(ByKeyIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t) -} - -func TestGetEncryptionKeysByDomainIdsWithRefreshUrls(t *testing.T) { - runQueryExpectingValidResponse(ByDomainIdReqWithRefreshUrls, getEncryptionKeysEndpoint, t) -} - -func TestGetEncryptionKeysByDomainIdsWithoutRefreshUrls(t *testing.T) { - runQueryExpectingValidResponse(ByDomainIdReqWithoutRefreshUrls, getEncryptionKeysEndpoint, t) -} - -func TestFaultInjection(t *testing.T) { - numIterations := rand.Intn(701) + 86 - - for i := 0; i < numIterations; i++ { - loc := rand.Intn(MARSHAL_RESPONSE + 1) - f := Fault{} - f.Location = loc - f.Enable = true - - var faults []Fault - faults = append(faults, f) - fW := httptest.NewRecorder() - body := strings.NewReader(jsonifyFaultArr(fW, faults)) - fReq := httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body) - handleUpdateFaultInjection(fW, fReq) - if !shouldInjectFault(loc) { - t.Errorf("Expected fault enabled for loc '%d'", loc) - t.Fail() - } - - var payload string - lottery := rand.Intn(100) - if lottery < 25 { - payload = ByKeyIdReqWithRefreshUrls - } else if lottery >= 25 && lottery < 50 { - payload = ByKeyIdReqWithoutRefreshUrls - } else if lottery >= 50 && lottery < 75 { - payload = ByDomainIdReqWithRefreshUrls - } else { - payload = ByDomainIdReqWithoutRefreshUrls - } - runQueryExpectingErrorResponse(payload, getEncryptionKeysEndpoint, errStrMap()(loc), t) - - // reset Fault - faults[0].Enable = false - fW = httptest.NewRecorder() - body = strings.NewReader(jsonifyFaultArr(fW, faults)) - fReq = httptest.NewRequest(http.MethodPost, updateFaultInjectionEndpoint, body) - handleUpdateFaultInjection(fW, fReq) - if shouldInjectFault(loc) { - t.Errorf("Expected fault disabled for loc '%d'", loc) - t.Fail() - } - } -} \ No newline at end of file diff --git a/contrib/mockkms/utils.go b/contrib/mockkms/utils.go deleted file mode 100644 index e811c91658e..00000000000 --- a/contrib/mockkms/utils.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - * utils.go - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package main - -import ( - "encoding/json" - "fmt" - "log" - "net/http" -) - -type ErrorDetail struct { - Detail string `json:"details"` -} - -type ErrorResponse struct { - Err ErrorDetail `json:"error"` -} - -func sendErrorResponse(w http.ResponseWriter, err error) { - e := ErrorDetail{} - e.Detail = fmt.Sprintf("Error: %s", err.Error()) - resp := ErrorResponse{ - Err: e, - } - - mResp, err := json.Marshal(resp) - if err != nil { - log.Printf("Error marshalling error response %s", err.Error()) - panic(err) - } - fmt.Fprintf(w, string(mResp)) -} diff --git a/contrib/monitoring/actor_flamegraph.cpp b/contrib/monitoring/actor_flamegraph.cpp index 58bd0d2c862..b97aaa360b8 100644 --- a/contrib/monitoring/actor_flamegraph.cpp +++ b/contrib/monitoring/actor_flamegraph.cpp @@ -113,11 +113,11 @@ class Traces { actor->stack.push_back(currentStack.top()->name); } } else if (op == OP_DESTROY) { - if (actors.count(id)) { + if (actors.contains(id)) { actors.erase(id); } } else if (op == OP_ENTER) { - if (actors.count(id) == 0) { + if (!actors.contains(id)) { actors[id] = std::make_shared(results, id, name); } currentStack.push(actors[id]); diff --git a/contrib/pkg_tester/requirements.txt b/contrib/pkg_tester/requirements.txt index af94b20328e..d56e3045ff4 100644 --- a/contrib/pkg_tester/requirements.txt +++ b/contrib/pkg_tester/requirements.txt @@ -6,7 +6,7 @@ packaging==20.9 pluggy==0.13.1 py==1.10.0 pyparsing==2.4.7 -pytest==6.2.4 +pytest==9.0.3 syrupy==1.2.3 toml==0.10.2 typing-extensions==3.10.0.0 diff --git a/contrib/replay/CMakeLists.txt b/contrib/replay/CMakeLists.txt index 85e393c707d..7a113685ced 100644 --- a/contrib/replay/CMakeLists.txt +++ b/contrib/replay/CMakeLists.txt @@ -25,27 +25,23 @@ file(MAKE_DIRECTORY ${REPLAY_OUTPUT_DIR}) # Get all Go source files for dependency tracking file(GLOB REPLAY_GO_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/*.go") +set(REPLAY_GO_MOD "${CMAKE_CURRENT_SOURCE_DIR}/go.mod") +set(REPLAY_GO_SUM "${CMAKE_CURRENT_SOURCE_DIR}/go.sum") -# Custom target to build replay -# This handles all Go dependencies automatically - users just need Go installed -# ALL means it builds by default (but only if Go is available - see check above) -add_custom_target(replay ALL - COMMAND ${CMAKE_COMMAND} -E echo "Downloading Go dependencies..." - COMMAND ${GO_EXECUTABLE} mod download - COMMAND ${CMAKE_COMMAND} -E echo "Tidying Go modules..." - COMMAND ${GO_EXECUTABLE} mod tidy - COMMAND ${CMAKE_COMMAND} -E echo "Building replay..." +# Custom command to build replay - only rebuilds when source files change +# go build handles dependencies automatically (downloads if needed) +add_custom_command( + OUTPUT ${REPLAY_BINARY} COMMAND ${GO_EXECUTABLE} build -o ${REPLAY_BINARY} . WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMENT "Building FDB trace replay tool (all dependencies handled automatically)" - BYPRODUCTS ${REPLAY_BINARY} - SOURCES ${REPLAY_GO_SOURCES} + DEPENDS ${REPLAY_GO_SOURCES} ${REPLAY_GO_MOD} ${REPLAY_GO_SUM} + COMMENT "Building FDB trace replay tool" ) -# Make sure the binary is executable -add_custom_command(TARGET replay POST_BUILD - COMMAND chmod +x ${REPLAY_BINARY} - COMMENT "Making replay executable" +# Target that depends on the binary - only triggers rebuild when deps change +add_custom_target(replay ALL + DEPENDS ${REPLAY_BINARY} + SOURCES ${REPLAY_GO_SOURCES} ) message(STATUS "replay will be built by default (Go found)") diff --git a/contrib/sqlite/CMakeLists.txt b/contrib/sqlite/CMakeLists.txt index 71ca7e6666c..41ae5a42781 100644 --- a/contrib/sqlite/CMakeLists.txt +++ b/contrib/sqlite/CMakeLists.txt @@ -7,6 +7,8 @@ add_library(sqlite STATIC sqliteLimit.h sqlite3.amalgamation.c) +set_target_properties(sqlite PROPERTIES C_CLANG_TIDY "") + target_include_directories(sqlite PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) # Suppress warnings in sqlite since it's third party @@ -18,4 +20,4 @@ endif() # SQLite won't build with -ffast-math if(ICX) target_compile_options(sqlite PRIVATE -fno-fast-math) -endif() \ No newline at end of file +endif() diff --git a/contrib/stacktrace/CMakeLists.txt b/contrib/stacktrace/CMakeLists.txt index f82b4b6a6c3..6d958db451f 100644 --- a/contrib/stacktrace/CMakeLists.txt +++ b/contrib/stacktrace/CMakeLists.txt @@ -1,4 +1,5 @@ add_library(stacktrace STATIC stacktrace.amalgamation.cpp) +set_target_properties(stacktrace PROPERTIES CXX_CLANG_TIDY "") target_include_directories(stacktrace PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include") if (USE_ASAN) target_compile_definitions(stacktrace PRIVATE ADDRESS_SANITIZER) diff --git a/design/AI-generated/FDB_NETWORK_PROTOCOL.md b/design/AI-generated/FDB_NETWORK_PROTOCOL.md new file mode 100644 index 00000000000..6e4ca2c82bd --- /dev/null +++ b/design/AI-generated/FDB_NETWORK_PROTOCOL.md @@ -0,0 +1,2666 @@ +# FoundationDB Network Protocol Message Reference + +This document is a comprehensive enumeration of every serialized message type in the +FoundationDB network protocol. These types are not documented in any single place in +the source; they are defined implicitly by `serialize`/`serializer` methods on C++ +structs throughout the codebase. + +*NOTE: By design FDB protocols can change between minor versions. Information in +this file is very detailed and is subject to being out of date. However it is +probably straightforward to ask state of the art models to update the documentation +for you if needed.* + +**Conventions in this document:** +- Every struct referenced in any message is fully defined with all serialized fields. +- Types are resolved recursively down to C/C++ base types. +- "→ base" annotations after a type show what it resolves to at the wire level. +- Fields marked "(not serialized)" are present in the C++ struct but do not appear on the wire. +- `Optional` is serialized as: `bool present` + (if true) the value of type `T`. +- `std::vector` / `VectorRef` are serialized as: `int32_t count` + count × T. +- `Arena` fields provide memory backing for `Ref` types; they are serialized last. + +--- + +## Table of Contents + +- [1. Base Types and Aliases](#1-base-types-and-aliases) +- [2. Foundational Struct Definitions](#2-foundational-struct-definitions) +- [3. Coordination Protocol](#3-coordination-protocol) +- [4. Cluster Controller Protocol](#4-cluster-controller-protocol) +- [5. Master Protocol](#5-master-protocol) +- [6. GRV Proxy Protocol](#6-grv-proxy-protocol) +- [7. Commit Proxy Protocol](#7-commit-proxy-protocol) +- [8. Resolver Protocol](#8-resolver-protocol) +- [9. Transaction Log (TLog) Protocol](#9-transaction-log-tlog-protocol) +- [10. Storage Server Protocol](#10-storage-server-protocol) +- [11. Data Distributor Protocol](#11-data-distributor-protocol) +- [12. Ratekeeper Protocol](#12-ratekeeper-protocol) +- [13. Worker Protocol](#13-worker-protocol) +- [14. Backup / Consistency Scan / Test Protocols](#14-backup--consistency-scan--test-protocols) +- [15. Client Worker / Debug / Process Protocols](#15-client-worker--debug--process-protocols) +- [Appendix A: Log System Configuration Types](#appendix-a-log-system-configuration-types) +- [Appendix B: In-Band Log Messages](#appendix-b-in-band-log-messages) +- [Appendix C: Backpressure Protocol](#appendix-c-backpressure-protocol) +- [Appendix D: Client Library Protocol Flows](#appendix-d-client-library-protocol-flows) +- [Appendix E: fdbcli Protocol Usage](#appendix-e-fdbcli-protocol-usage) +- [Appendix F: Server-Side Protocol Flows](#appendix-f-server-side-protocol-flows) + +--- + +## Architecture Overview + +FDB uses an RPC framework built on its "Flow" actor model. Each server role exposes +an **interface** struct containing `RequestStream` fields — typed endpoints. +Request messages carry a `ReplyPromise` (or `ReplyPromiseStream` for streaming) +that the receiver fulfills with the reply. + +- **Endpoint multiplexing**: many interfaces serialize only one `RequestStream` and + reconstruct the others via `getAdjustedEndpoint(offset)` during deserialization. +- **Wire format**: `ProtocolVersion` (uint64) header, then fields serialized + sequentially via `serializer(ar, field1, field2, ...)` with no inter-field framing. + +--- + +## 1. Base Types and Aliases + +These are the leaf types that all struct fields ultimately resolve to. + +| Wire Type | C++ Type | Size | Notes | +|-----------|----------|------|-------| +| `bool` | `bool` | 1 byte | | +| `int8_t` / `uint8_t` | integer | 1 byte | | +| `int16_t` / `uint16_t` | integer | 2 bytes | little-endian | +| `int32_t` / `uint32_t` / `int` | integer | 4 bytes | little-endian | +| `int64_t` / `uint64_t` | integer | 8 bytes | little-endian | +| `double` | IEEE 754 | 8 bytes | little-endian | +| `StringRef` | length-prefixed bytes | `int32_t` len + bytes | | +| `std::string` | length-prefixed bytes | `int32_t` len + bytes | | +| `ProtocolVersion` | `uint64_t` | 8 bytes | version with feature flags | +| enums | underlying integer type | varies | typically `uint8_t` or `uint32_t` | + +**Type aliases** used throughout the protocol (all are simple typedefs): + +| Alias | Resolves To | Purpose | +|-------|-------------|---------| +| `Version` | `int64_t` | MVCC version number | +| `LogEpoch` | `uint64_t` | TLog generation epoch | +| `Generation` | `int64_t` | Cluster controller generation | +| `Sequence` | `uint64_t` | Sequence number | +| `DBRecoveryCount` | `uint64_t` | Sequential recovery counter | +| `Key` | `Standalone` → `StringRef` | Key bytes (arena-owned) | +| `KeyRef` | `StringRef` | Key bytes (ref) | +| `Value` | `Standalone` → `StringRef` | Value bytes (arena-owned) | +| `ValueRef` | `StringRef` | Value bytes (ref) | +| `TransactionTag` | `Standalone` → `StringRef` | Transaction tag string | +| `TransactionTagRef` | `StringRef` | Transaction tag ref | +| `SpanID` | `UID` | Tracing span identifier | + +**Generic containers** (serialization is count + elements): + +| Container | Wire Format | +|-----------|-------------| +| `std::vector` | `int32_t` count, then count × T | +| `VectorRef` | `int32_t` count, then count × T | +| `std::set` | `int32_t` count, then count × T | +| `std::map` | `int32_t` count, then count × (K, V) pairs | +| `std::unordered_map` | `int32_t` count, then count × (K, V) pairs | +| `std::deque` | `int32_t` count, then count × T | +| `Optional` | `bool` present, then (if true) T | +| `std::variant` | `uint8_t` index, then the active variant | +| `Standalone` | same as T (Arena is implicit) | + +--- + +## 2. Foundational Struct Definitions + +Every struct used as a field in protocol messages is defined here, in dependency +order. Each definition shows all serialized fields and their fully-resolved types. + +### UID +Universally unique identifier. Serialized as two `uint64_t` values. + +| Field | Type | Purpose | +|-------|------|---------| +| part[0] | `uint64_t` | First 8 bytes | +| part[1] | `uint64_t` | Second 8 bytes | + +### IPAddress +IP address supporting both IPv4 and IPv6. + +Serialized as: `bool isV6`, then either `uint32_t` (v4) or `std::array` (v6). + +### NetworkAddress +A network endpoint address. + +| Field | Type | Purpose | +|-------|------|---------| +| ip | `IPAddress` | IP address (see above) | +| port | `uint16_t` | Port number | +| flags | `uint16_t` | Flags (TLS, public, etc.) | +| fromHostname | `bool` | Whether resolved from hostname (conditional on protocol version) | + +### NetworkAddressList +A primary + optional secondary address for multi-homed processes. + +| Field | Type | Purpose | +|-------|------|---------| +| address | `NetworkAddress` | Primary address | +| secondaryAddress | `Optional` | Secondary address | + +### Hostname + +| Field | Type | Purpose | +|-------|------|---------| +| host | `std::string` | Hostname string | +| service | `std::string` | Service/port string | +| isTLS | `bool` | Whether TLS is enabled | + +### AddressExclusion +An IP address + optional port for excluding servers. + +| Field | Type | Purpose | +|-------|------|---------| +| ip | `IPAddress` | IP address | +| port | `int` (int32_t) | Port (0 = all ports on this IP) | + +### Endpoint +A network-addressable RPC endpoint. + +| Field | Type | Purpose | +|-------|------|---------| +| addresses | `NetworkAddressList` | Network addresses | +| token | `UID` | Endpoint token (two `uint64_t`) | + +### Tag +Identifies which storage server(s) should receive a mutation. + +| Field | Type | Purpose | +|-------|------|---------| +| locality | `int8_t` | Locality group (-1 = special, 0+ = DC index) | +| id | `uint16_t` | Tag ID within the locality | + +### KeyRangeRef +A contiguous range of keys [begin, end). + +| Field | Type | Purpose | +|-------|------|---------| +| begin | `KeyRef` → `StringRef` | Start key (inclusive) | +| end | `KeyRef` → `StringRef` | End key (exclusive) | + +**Note:** `KeyRange` is `Standalone` (same wire format, arena-owned). + +### KeyValueRef +A single key-value pair. + +| Field | Type | Purpose | +|-------|------|---------| +| key | `KeyRef` → `StringRef` | Key bytes | +| value | `ValueRef` → `StringRef` | Value bytes | + +**Note:** `KeyValue` is `Standalone`. + +### KeySelectorRef +A relative key reference: "the first/last key ≥/≤ `key`, plus `offset`." + +| Field | Type | Purpose | +|-------|------|---------| +| key | `KeyRef` → `StringRef` | Anchor key | +| orEqual | `bool` | Whether to include the anchor key | +| offset | `int` (int32_t) | Offset from anchor | + +### RangeResultRef +Result of a range read. Inherits from `VectorRef` (vector of KV pairs). + +| Field | Type | Purpose | +|-------|------|---------| +| (inherited) | `VectorRef` | Key-value pairs | +| more | `bool` | Whether more results exist | +| readThrough | `Optional` | Read-through key | +| readToBegin | `bool` | Read reached beginning | +| readThroughEnd | `bool` | Read reached end | + +### MutationRef +A single database mutation. + +| Field | Type | Purpose | +|-------|------|---------| +| type | `uint8_t` | Mutation type (SetValue=0, ClearRange=1, AddValue=2, ...) | +| param1 | `StringRef` | First parameter (key or range start) | +| param2 | `StringRef` | Second parameter (value or range end) | +| checksum | `Optional` | Optional checksum (conditional on protocol version) | +| accumulativeChecksumIndex | `Optional` | Checksum index (conditional on protocol version) | + +### CommitTransactionRef +A complete transaction to be committed. + +| Field | Type | Purpose | +|-------|------|---------| +| read_conflict_ranges | `VectorRef` | Read conflict ranges | +| write_conflict_ranges | `VectorRef` | Write conflict ranges | +| mutations | `VectorRef` | Mutations to apply | +| read_snapshot | `Version` → `int64_t` | Read snapshot version | +| report_conflicting_keys | `bool` | Whether to report conflicts | +| lock_aware | `bool` | Whether lock-aware | +| spanContext | `Optional` | Tracing context | +| tenantIds | `Optional>` | Tenant IDs (conditional on protocol version) | + +### MutationsAndVersionRef +A set of mutations at a particular version. + +| Field | Type | Purpose | +|-------|------|---------| +| mutations | `VectorRef` | Mutations | +| version | `Version` → `int64_t` | Version | +| knownCommittedVersion | `Version` → `int64_t` | Known committed version | + +### SpanContext +OpenTelemetry-compatible distributed tracing context. + +| Field | Type | Purpose | +|-------|------|---------| +| traceID | `UID` (two `uint64_t`) | Trace identifier | +| spanID | `uint64_t` | Span identifier | +| m_Flags | `uint8_t` | Trace flags | + +### FailureStatus + +| Field | Type | Purpose | +|-------|------|---------| +| failed | `bool` | Whether the process is failed | + +### SystemFailureStatus + +| Field | Type | Purpose | +|-------|------|---------| +| addresses | `NetworkAddressList` | Process addresses | +| status | `FailureStatus` | Failure status (→ `bool failed`) | + +### LocalityData +Locality key-value map for a process. Serialized as a `uint64_t` map size followed +by (key, value) pairs. + +| Field | Type | Purpose | +|-------|------|---------| +| _data | `std::map, Optional>>` | Locality key-value pairs | + +### ProcessClass +Classification of a process (role fitness). + +| Field | Type | Purpose | +|-------|------|---------| +| _class | `int16_t` | Class type (unset, storage, transaction, resolution, ...) | +| _source | `int16_t` | Source (command line, configure, ...) | + +### ClusterControllerPriorityInfo + +| Field | Type | Purpose | +|-------|------|---------| +| processClassFitness | `uint8_t` | Fitness for cluster controller role | +| isExcluded | `bool` | Whether the process is excluded | +| dcFitness | `uint8_t` | Data center fitness | + +### StorageMetrics +Load metrics for a storage range. + +| Field | Type | Purpose | +|-------|------|---------| +| bytes | `int64_t` | Bytes stored | +| bytesWrittenPerKSecond | `int64_t` | Write rate | +| iosPerKSecond | `int64_t` | I/O rate | +| bytesReadPerKSecond | `int64_t` | Read rate | +| opsReadPerKSecond | `int64_t` | Read operation rate | + +### StorageBytes +Disk space information. + +| Field | Type | Purpose | +|-------|------|---------| +| free | `int64_t` | Free bytes | +| total | `int64_t` | Total bytes | +| used | `int64_t` | Used bytes | +| available | `int64_t` | Available bytes | + +### HealthMetrics +Cluster-wide health metrics. + +| Field | Type | Purpose | +|-------|------|---------| +| worstStorageQueue | `int64_t` | Worst SS queue size | +| limitingStorageQueue | `int64_t` | Limiting SS queue size | +| worstStorageDurabilityLag | `int64_t` | Worst SS durability lag | +| limitingStorageDurabilityLag | `int64_t` | Limiting SS durability lag | +| worstTLogQueue | `int64_t` | Worst TLog queue size | +| tpsLimit | `double` | Current TPS limit | +| batchLimited | `bool` | Whether batch is limited | +| storageStats | `std::map` | Per-SS stats | +| tLogQueue | `std::map` | Per-TLog queue sizes | + +#### HealthMetrics::StorageStats + +| Field | Type | Purpose | +|-------|------|---------| +| storageQueue | `int64_t` | Queue size | +| storageDurabilityLag | `int64_t` | Durability lag | +| diskUsage | `double` | Disk usage fraction | +| cpuUsage | `double` | CPU usage fraction | + +### BusyTagInfo +Information about a busy transaction tag on a storage server. + +| Field | Type | Purpose | +|-------|------|---------| +| tag | `TransactionTag` → `StringRef` | Tag string | +| rate | `double` | Tag rate | +| fractionalBusyness | `double` | Fraction of busyness | + +### DDMetricsRef +Data distribution shard metrics. + +| Field | Type | Purpose | +|-------|------|---------| +| shardBytes | `int64_t` | Bytes in shard | +| shardBytesPerKSecond | `int64_t` | Write rate | +| beginKey | `KeyRef` → `StringRef` | Shard begin key | + +### LifetimeToken +Identifies a master's lifetime for fencing. + +| Field | Type | Purpose | +|-------|------|---------| +| ccID | `UID` (two `uint64_t`) | Cluster controller ID | +| count | `int64_t` | Monotonic counter | + +### UniqueGeneration +Paxos generation identifier. + +| Field | Type | Purpose | +|-------|------|---------| +| generation | `uint64_t` | Generation number | +| uid | `UID` (two `uint64_t`) | Unique ID for tie-breaking | + +### LeaderInfo +Information about the current cluster controller leader. + +| Field | Type | Purpose | +|-------|------|---------| +| changeID | `UID` (two `uint64_t`) | Unique ID for this leader epoch | +| serializedInfo | `Value` → `StringRef` | Serialized `ClusterControllerClientInterface` | +| forward | `bool` | If true, serializedInfo is a forwarding address | + +### ClusterConnectionString + +| Field | Type | Purpose | +|-------|------|---------| +| coords | `std::vector` | Coordinator addresses | +| hostnames | `std::vector` | Coordinator hostnames | +| key | `Key` → `StringRef` | Cluster description key | +| keyDesc | `Key` → `StringRef` | Cluster description | + +### ClientVersionRef + +| Field | Type | Purpose | +|-------|------|---------| +| clientVersion | `StringRef` | Client version string | +| sourceVersion | `StringRef` | Source version hash | +| protocolVersion | `StringRef` | Protocol version string | + +### VersionHistory +Global configuration change record. + +| Field | Type | Purpose | +|-------|------|---------| +| version | `Version` → `int64_t` | Version of this config change | +| mutations | `Standalone>` | Config mutations | + +### ReadOptions +Options for read operations. + +| Field | Type | Purpose | +|-------|------|---------| +| type | `ReadType` → `uint8_t` | Read type (NORMAL, EAGER, FETCH) | +| cacheResult | `bool` | Whether to cache the result | +| debugID | `Optional` | Debug trace ID | +| consistencyCheckStartVersion | `Optional` | Consistency check version | +| lockAware | `bool` | Whether lock-aware | + +### TagSet +A set of transaction tags. Custom serialization. + +Serialized as a vector of `TransactionTagRef` (`StringRef`) entries. + +### ClientTagThrottleLimits + +| Field | Type | Purpose | +|-------|------|---------| +| tpsRate | `double` | Transactions per second rate limit | +| duration | `double` | Duration of throttle | + +### TransactionCommitCostEstimation + +| Field | Type | Purpose | +|-------|------|---------| +| opsSum | `int` (int32_t) | Total operation count | +| costSum | `uint64_t` | Total cost | + +### IdempotencyIdRef +A 16-byte idempotency identifier. Serialized as raw bytes with custom encoding. + +| Field | Type | Purpose | +|-------|------|---------| +| first | `uint64_t` | First 8 bytes (or size indicator) | +| second | `uint64_t` | Second 8 bytes (or pointer to data) | + +### KeyValueStoreType +Storage engine type enum. + +| Field | Type | Purpose | +|-------|------|---------| +| type | `uint32_t` | Engine type (SSD=1, MEMORY=2, SSD_ROCKSDB=4, ...) | + +### TLogVersion + +| Field | Type | Purpose | +|-------|------|---------| +| version | `uint32_t` | TLog version (V2=2, ..., V8=8) | + +### TLogSpillType + +| Field | Type | Purpose | +|-------|------|---------| +| type | `uint32_t` | Spill type (DEFAULT=0, VALUE=1, REFERENCE=2) | + +### EncryptionAtRestModeDeprecated + +| Field | Type | Purpose | +|-------|------|---------| +| mode | `uint32_t` | Encryption mode (DISABLED=0, DOMAIN_AWARE=1, CLUSTER_AWARE=2) | + +### MoveKeysLock + +| Field | Type | Purpose | +|-------|------|---------| +| prevOwner | `UID` (two `uint64_t`) | Previous lock owner | +| myOwner | `UID` (two `uint64_t`) | Current lock owner | +| prevWrite | `UID` (two `uint64_t`) | Previous write | + +### LogMessageVersion + +| Field | Type | Purpose | +|-------|------|---------| +| version | `Version` → `int64_t` | Version | +| sub | `uint32_t` | Sub-version within the version | + +### Versionstamp +Serialized in big-endian for ordered comparison. + +| Field | Type | Purpose | +|-------|------|---------| +| version | `int64_t` | Version (big-endian on wire) | +| batchNumber | `int16_t` | Batch number (big-endian on wire) | + +### StorageMetadataType + +| Field | Type | Purpose | +|-------|------|---------| +| createdTime | `double` | Creation timestamp | +| storeType | `KeyValueStoreType` → `uint32_t` | Storage engine type | + +### RecoveryState (enum) +Serialized as its underlying integer type. + +| Value | Name | Meaning | +|-------|------|---------| +| 0 | UNINITIALIZED | Not started | +| 1 | READING_CSTATE | Reading coordinated state | +| 2 | LOCKING_CSTATE | Locking coordinated state | +| 3 | RECRUITING | Recruiting transaction system | +| 4 | RECOVERY_TRANSACTION | Writing recovery transaction | +| 5 | WRITING_CSTATE | Writing coordinated state | +| 6 | ACCEPTING_COMMITS | Accepting commits | +| 7 | ALL_LOGS_RECRUITED | All logs recruited | +| 8 | STORAGE_RECOVERED | Storage servers recovered | +| 9 | FULLY_RECOVERED | Fully recovered | + +### ClusterType (enum) + +| Value | Name | +|-------|------| +| 0 | STANDALONE | +| 1 | LEGACY_UNUSED_METACLUSTER_MANAGEMENT | +| 2 | LEGACY_UNUSED_METACLUSTER_DATA | + +### LogSystemType (enum) + +| Value | Name | +|-------|------| +| -1 | unset | +| 0 | empty | +| 2 | tagPartitioned | + +### ResolverMoveRef +Describes a shard reassignment between resolvers. + +| Field | Type | Purpose | +|-------|------|---------| +| range | `KeyRangeRef` | Key range being moved | +| dest | `int` (int32_t) | Destination resolver index | + +### StateTransactionRef +A state transaction with its commit status. + +| Field | Type | Purpose | +|-------|------|---------| +| committed | `bool` | Whether committed | +| mutations | `VectorRef` | State mutations | + +### UnknownCommittedVersions +Tracks version ranges whose commit status is unknown during recovery. + +| Field | Type | Purpose | +|-------|------|---------| +| version | `Version` → `int64_t` | Version | +| prev | `Version` → `int64_t` | Previous version | +| tLogLocIds | `std::vector` | TLog locality IDs | + +### StorageServerShard +Physical shard metadata on a storage server. + +| Field | Type | Purpose | +|-------|------|---------| +| range | `KeyRange` | Shard key range | +| version | `Version` → `int64_t` | Shard version | +| id | `uint64_t` | Shard ID | +| desiredId | `uint64_t` | Desired shard ID | +| shardState | `int8_t` | Shard state (ReadWrite, Adding, etc.) | +| moveInShardId | `Optional` | Move-in shard ID if being moved | + +### CheckpointMetaData +Metadata describing a storage checkpoint. + +| Field | Type | Purpose | +|-------|------|---------| +| version | `Version` → `int64_t` | Checkpoint version | +| ranges | `std::vector` | Key ranges in checkpoint | +| format | `int16_t` | Checkpoint format | +| state | `int16_t` | Checkpoint state | +| checkpointID | `UID` (two `uint64_t`) | Checkpoint identifier | +| src | `std::vector` | Source servers | +| serializedCheckpoint | `Standalone` | Serialized checkpoint data | +| actionId | `Optional` | Action ID | +| bytesSampleFile | `Optional` | Sample file path | +| dir | `std::string` | Directory path | + +### OverlappingChangeFeedEntry +An entry describing a change feed that overlaps a key range. + +| Field | Type | Purpose | +|-------|------|---------| +| feedId | `KeyRef` → `StringRef` | Change feed identifier | +| range | `KeyRangeRef` | Feed's key range | +| emptyVersion | `Version` → `int64_t` | Version at which feed was empty | +| stopVersion | `Version` → `int64_t` | Version at which feed was stopped | +| feedMetadataVersion | `Version` → `int64_t` | Feed metadata version | + +### ReadHotRangeWithMetrics + +| Field | Type | Purpose | +|-------|------|---------| +| keys | `KeyRangeRef` | Hot key range | +| density | `double` | Read density | +| readBandwidthSec | `double` | Read bandwidth (bytes/sec) | +| bytes | `int64_t` | Bytes in range | +| readOpsSec | `double` | Read operations per second | + +### CheckSumMetaData + +| Field | Type | Purpose | +|-------|------|---------| +| range | `KeyRange` | Key range | +| version | `Version` → `int64_t` | Version | +| checkSumValue | `StringRef` | Checksum bytes | + +### BulkDumpState +State of a bulk dump job. + +| Field | Type | Purpose | +|-------|------|---------| +| jobId | `UID` (two `uint64_t`) | Job identifier | +| jobRange | `KeyRange` | Key range being dumped | +| phase | `uint8_t` | Dump phase | +| taskId | `Optional` | Task identifier | +| manifest | `BulkLoadManifest` | Manifest data (complex, serialized as bytes) | + +### AuditStorageState + +| Field | Type | Purpose | +|-------|------|---------| +| id | `UID` (two `uint64_t`) | Audit ID | +| auditServerId | `UID` (two `uint64_t`) | Server performing audit | +| range | `KeyRange` | Audited key range | +| type | `uint8_t` | Audit type | +| phase | `uint8_t` | Audit phase | +| error | `std::string` | Error message | +| ddId | `UID` (two `uint64_t`) | Data distributor ID | +| engineType | `KeyValueStoreType` → `uint32_t` | Storage engine type | + +### MappedKeyValueRef +A key-value pair with a secondary index lookup result. +Inherits from `KeyValueRef`. + +| Field | Type | Purpose | +|-------|------|---------| +| key | `KeyRef` → `StringRef` | Primary key (inherited) | +| value | `ValueRef` → `StringRef` | Primary value (inherited) | +| reqAndResult | `std::variant` | Mapped result | + +#### GetValueReqAndResultRef + +| Field | Type | Purpose | +|-------|------|---------| +| key | `KeyRef` → `StringRef` | Looked-up key | +| result | `Optional` | Looked-up value | + +#### GetRangeReqAndResultRef + +| Field | Type | Purpose | +|-------|------|---------| +| begin | `KeySelectorRef` | Range begin selector | +| end | `KeySelectorRef` | Range end selector | +| result | `RangeResultRef` | Range result | + +### DatabaseConfiguration +Database configuration. Serialized as a key-value list. + +| Field | Type | Purpose | +|-------|------|---------| +| rawConfiguration | `Standalone>` | Configuration key-value pairs | + +The key-value pairs encode redundancy mode, storage engine, proxy/resolver/log counts, +region configuration, and other tuning parameters. + +### OpenDatabaseRequest::Samples +Sampling data for client issue/version tracking. + +| Field | Type | Purpose | +|-------|------|---------| +| count | `int` (int32_t) | Total count | +| samples | `std::set>` | Sampled (address, key) pairs | + +### DiskFailureCommand (nested in SetFailureInjection) + +| Field | Type | Purpose | +|-------|------|---------| +| stallInterval | `double` | Interval between stalls | +| stallPeriod | `double` | Duration of each stall | +| throttlePeriod | `double` | Throttle period | + +### FlipBitsCommand (nested in SetFailureInjection) + +| Field | Type | Purpose | +|-------|------|---------| +| percentBitFlips | `double` | Percentage of bits to flip | + +### SerializedSample +An actor lineage sample. + +| Field | Type | Purpose | +|-------|------|---------| +| time | `double` | Sample timestamp | +| data | `std::unordered_map` | Wait state → data map | + +`WaitState` is serialized as its underlying integer type. + +### PerfMetric + +| Field | Type | Purpose | +|-------|------|---------| +| m_name | `std::string` | Metric name | +| m_format_code | `std::string` | Format code | +| m_value | `double` | Metric value | +| m_averaged | `bool` | Whether averaged | + +--- + +## 3. Coordination Protocol + +**Source:** `fdbclient/include/fdbclient/CoordinationInterface.h`, +`fdbserver/core/include/fdbserver/core/CoordinationInterface.h` + +Coordinators run Paxos-based leader election and store the `DBCoreState`. + +### ProtocolInfoRequest +Handshake message to discover a peer's protocol version. + +| Field | Type | Purpose | +|-------|------|---------| +| reply | `ReplyPromise` → `UID` token | Where to send reply | + +**Reply: ProtocolInfoReply** + +| Field | Type | Purpose | +|-------|------|---------| +| version | `ProtocolVersion` → `uint64_t` | Protocol version of the responder | + +### GetLeaderRequest +Ask a coordinator who the current leader (cluster controller) is. + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Cluster description key | +| knownLeader | `UID` | Last known leader change ID (long-poll) | +| reply | `ReplyPromise>` | Where to send reply | + +**Reply:** `Optional` — see [LeaderInfo](#leaderinfo) definition. + +### OpenDatabaseCoordRequest +Client requests current `ClientDBInfo` from coordinators. + +| Field | Type | Purpose | +|-------|------|---------| +| traceLogGroup | `Key` → `StringRef` | Client's trace log group | +| issues | `Standalone>` | Client-reported issues | +| supportedVersions | `Standalone>` | Client library versions | +| knownClientInfoID | `UID` | Last known DB info ID (long-poll) | +| clusterKey | `Key` → `StringRef` | Cluster connection key | +| hostnames | `std::vector` | Coordinator hostnames | +| coordinators | `std::vector` | Coordinator addresses | +| internal | `bool` | Whether this is an internal request | +| reply | `ReplyPromise>` | Where to send reply | + +**Reply:** `ClientDBInfo` — see [Section 4](#4-cluster-controller-protocol). + +### CheckDescriptorMutableRequest + +| Field | Type | Purpose | +|-------|------|---------| +| reply | `ReplyPromise` | Where to send reply | + +**Reply: CheckDescriptorMutableReply** — `bool isMutable`. + +### CandidacyRequest +Propose a candidate for cluster controller during leader election. + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Cluster description key | +| myInfo | `LeaderInfo` | Candidate's leader info | +| knownLeader | `UID` | Currently known leader | +| prevChangeID | `UID` | Previous change ID | +| reply | `ReplyPromise>` | Election result | + +### ElectionResultRequest +Wait for the result of an election. + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Cluster description key | +| hostnames | `std::vector` | Coordinator hostnames | +| coordinators | `std::vector` | Coordinator addresses | +| knownLeader | `UID` | Currently known leader | +| reply | `ReplyPromise>` | Election result | + +### LeaderHeartbeatRequest +Leader sends heartbeats to maintain leadership. + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Cluster description key | +| myInfo | `LeaderInfo` | Leader's info | +| prevChangeID | `UID` | Previous change ID | +| reply | `ReplyPromise` | Heartbeat ack | + +**Reply: LeaderHeartbeatReply** — `bool value` (accepted or not). + +### ForwardRequest +Tell a coordinator to forward clients to a new connection string. + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Cluster description key | +| conn | `Value` → `StringRef` | New cluster connection string | +| reply | `ReplyPromise` | Acknowledgment | + +### GenerationRegReadRequest +Read from the generation register (Paxos storage layer). + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Register key | +| gen | `UniqueGeneration` | Generation to read at | +| reply | `ReplyPromise` | Register value | + +**Reply: GenerationRegReadReply** + +| Field | Type | Purpose | +|-------|------|---------| +| value | `Optional` | Stored value | +| gen | `UniqueGeneration` | Write generation | +| rgen | `UniqueGeneration` | Read generation | + +### GenerationRegWriteRequest +Write to the generation register. + +| Field | Type | Purpose | +|-------|------|---------| +| kv | `KeyValue` (→ `StringRef` key + `StringRef` value) | Key-value to write | +| gen | `UniqueGeneration` | Generation for this write | +| reply | `ReplyPromise` | Actual generation after write | + +--- + +## 4. Cluster Controller Protocol + +**Source:** `fdbclient/include/fdbclient/ClusterInterface.h` + +The cluster controller manages failure detection, role recruiting, and distributes +`ClientDBInfo` to clients. + +### Interface: ClusterInterface + +| Endpoint | Request Type | +|----------|-------------| +| openDatabase | `OpenDatabaseRequest` | +| failureMonitoring | `FailureMonitoringRequest` | +| databaseStatus | `StatusRequest` | +| ping | `ReplyPromise` | +| getClientWorkers | `GetClientWorkersRequest` | +| forceRecovery | `ForceRecoveryRequest` | +| moveShard | `MoveShardRequest` | +| repairSystemData | `RepairSystemDataRequest` | +| splitShard | `SplitShardRequest` | +| triggerAudit | `TriggerAuditRequest` | + +### OpenDatabaseRequest + +| Field | Type | Purpose | +|-------|------|---------| +| clientCount | `int` (int32_t) | Number of clients | +| issues | `std::map` | Client-reported issues | +| supportedVersions | `std::map, Samples>` | Client versions | +| maxProtocolSupported | `std::map` | Max protocol per client | +| knownClientInfoID | `UID` | Last known info ID (long-poll) | +| reply | `ReplyPromise` | Current database info | + +### ClientDBInfo +Returned to clients; contains everything needed to reach proxies. + +| Field | Type | Purpose | +|-------|------|---------| +| grvProxies | `std::vector` | GRV proxy interfaces | +| commitProxies | `std::vector` | Commit proxy interfaces | +| id | `UID` | Unique ID for this info snapshot | +| forward | `Optional` → `Optional` | Forwarding connection string | +| history | `std::vector` | Configuration version history | +| clusterId | `UID` | Cluster ID | +| clusterType | `ClusterType` → enum (int) | Cluster type | + +Note: the struct also contains `firstCommitProxy` (`Optional`) +but this field is **not serialized** — it is reconstructed locally from `commitProxies`. + +### FailureMonitoringRequest + +| Field | Type | Purpose | +|-------|------|---------| +| senderStatus | `Optional` | Sender's own status | +| failureInformationVersion | `Version` → `int64_t` | Last known failure info version | +| addresses | `NetworkAddressList` | Sender's addresses | +| reply | `ReplyPromise` | Failure updates | + +**Reply: FailureMonitoringReply** + +| Field | Type | Purpose | +|-------|------|---------| +| changes | `VectorRef` | Changed failure statuses | +| failureInformationVersion | `Version` → `int64_t` | Updated version | +| allOthersFailed | `bool` | Whether all other processes are failed | +| clientRequestIntervalMS | `int` (int32_t) | Suggested poll interval | +| considerServerFailedTimeoutMS | `int` (int32_t) | Failure timeout threshold | +| arena | `Arena` | Memory arena | + +### StatusRequest + +| Field | Type | Purpose | +|-------|------|---------| +| statusField | `std::string` | Optional field filter | +| reply | `ReplyPromise` | JSON status | + +**Reply: StatusReply** — serializes `statusStr` (`std::string`, JSON); `statusObj` is reconstructed on deserialization. + +### TriggerAuditRequest + +| Field | Type | Purpose | +|-------|------|---------| +| type | `uint8_t` | Audit type | +| range | `KeyRange` | Range to audit | +| id | `UID` | Audit ID | +| cancel | `bool` | Whether to cancel | +| engineType | `KeyValueStoreType` → `uint32_t` | Storage engine type | +| reply | `ReplyPromise` | Audit ID | + +### Other Cluster Controller Requests +- **GetClientWorkersRequest**: `reply` → `std::vector` +- **ForceRecoveryRequest**: `Key dcId`, `reply` → `Void` +- **MoveShardRequest**: `KeyRange shard`, `std::vector addresses`, `reply` → `Void` +- **RepairSystemDataRequest**: `reply` → `Void` +- **SplitShardRequest**: `KeyRange shard`, `int num`, `reply` → **SplitShardReply** {`std::vector shards`} + +--- + +## 5. Master Protocol + +**Source:** `fdbserver/core/include/fdbserver/core/MasterInterface.h` + +The master coordinates recovery and hands out commit versions to proxies. + +### Interface: MasterInterface +Serializes `locality` and `waitFailure`; other endpoints reconstructed via offsets. + +| Endpoint | Request Type | +|----------|-------------| +| waitFailure | `ReplyPromise` | +| getCommitVersion | `GetCommitVersionRequest` | +| getLiveCommittedVersion | `GetRawCommittedVersionRequest` | +| reportLiveCommittedVersion | `ReportRawCommittedVersionRequest` | +| updateRecoveryData | `UpdateRecoveryDataRequest` | + +### GetCommitVersionRequest + +| Field | Type | Purpose | +|-------|------|---------| +| spanContext | `SpanContext` | Tracing context | +| requestNum | `uint64_t` | Monotonic request number | +| mostRecentProcessedRequestNum | `uint64_t` | Most recent processed request | +| requestingProxy | `UID` | Proxy ID | +| reply | `ReplyPromise` | Assigned version | + +**Reply: GetCommitVersionReply** + +| Field | Type | Purpose | +|-------|------|---------| +| resolverChanges | `Standalone>` | Resolver shard reassignments | +| resolverChangesVersion | `Version` → `int64_t` | Version of changes | +| version | `Version` → `int64_t` | Assigned commit version | +| prevVersion | `Version` → `int64_t` | Previous commit version | +| requestNum | `uint64_t` | Echoed request number | + +### ReportRawCommittedVersionRequest + +| Field | Type | Purpose | +|-------|------|---------| +| version | `Version` → `int64_t` | Committed version | +| locked | `bool` | Database locked | +| metadataVersion | `Optional` | Metadata version | +| minKnownCommittedVersion | `Version` → `int64_t` | Min known committed | +| prevVersion | `Optional` | Previous version | +| writtenTags | `Optional>` | Tags written | +| reply | `ReplyPromise` | Acknowledgment | + +### UpdateRecoveryDataRequest + +| Field | Type | Purpose | +|-------|------|---------| +| recoveryTransactionVersion | `Version` → `int64_t` | Recovery txn version | +| lastEpochEnd | `Version` → `int64_t` | End of last epoch | +| commitProxies | `std::vector` | Proxy interfaces | +| resolvers | `std::vector` | Resolver interfaces | +| versionEpoch | `Optional` | Version epoch | +| primaryLocality | `int8_t` | Primary DC locality | +| reply | `ReplyPromise` | Acknowledgment | + +### ChangeCoordinatorsRequest + +| Field | Type | Purpose | +|-------|------|---------| +| newConnectionString | `Standalone` | New connection string | +| masterId | `UID` | Master ID | +| reply | `ReplyPromise` | Acknowledgment | + +--- + +## 6. GRV Proxy Protocol + +**Source:** `fdbclient/include/fdbclient/GrvProxyInterface.h` + +### Interface: GrvProxyInterface +Serializes `processId` (`Optional`), `provisional` (`bool`), and +`getConsistentReadVersion`; other endpoints reconstructed via offsets. + +### GetReadVersionRequest + +| Field | Type | Purpose | +|-------|------|---------| +| spanContext | `SpanContext` | Tracing context | +| transactionCount | `uint32_t` | Transactions in batch | +| flags | `uint32_t` | Priority flags | +| tags | `TransactionTagMap` → `std::unordered_map` | Tag counts | +| debugID | `Optional` | Debug trace ID | +| maxVersion | `Version` → `int64_t` | Max acceptable version | +| reply | `ReplyPromise` | Read version | + +**Reply: GetReadVersionReply** + +| Field | Type | Purpose | +|-------|------|---------| +| processBusyTime | `double` | Proxy busy time | +| version | `Version` → `int64_t` | Assigned read version | +| locked | `bool` | Database locked | +| metadataVersion | `Optional` | Metadata version | +| tagThrottleInfo | `TransactionTagMap` | Per-tag throttle limits | +| midShardSize | `int64_t` | Median shard size | +| rkDefaultThrottled | `bool` | Ratekeeper throttling default | +| rkBatchThrottled | `bool` | Ratekeeper throttling batch | +| ssVersionVectorDelta | `VersionVector` | SS version vector delta (custom serialization) | +| proxyId | `UID` | Proxy ID | + +### GlobalConfigRefreshRequest + +| Field | Type | Purpose | +|-------|------|---------| +| lastKnown | `Version` → `int64_t` | Last known config version | +| reply | `ReplyPromise` | Updated config | + +**Reply: GlobalConfigRefreshReply**: `Version version`, `RangeResultRef result`, `Arena arena`. + +--- + +## 7. Commit Proxy Protocol + +**Source:** `fdbclient/include/fdbclient/CommitProxyInterface.h` + +### Interface: CommitProxyInterface +Serializes `processId` (`Optional`), `provisional` (`bool`), and `commit`; +11 other endpoints reconstructed via offsets 1–10 and 13 (offsets 11–12 are +reserved/unused). + +### CommitTransactionRequest + +| Field | Type | Purpose | +|-------|------|---------| +| spanContext | `SpanContext` | Tracing context | +| transaction | `CommitTransactionRef` | Transaction to commit | +| flags | `uint32_t` | Commit flags | +| debugID | `Optional` | Debug trace ID | +| commitCostEstimation | `Optional` | Cost estimate (see below) | +| tagSet | `Optional` | Transaction tags | +| idempotencyId | `IdempotencyIdRef` | Idempotency identifier | +| arena | `Arena` | Memory arena | +| reply | `ReplyPromise` | Commit result | + +#### ClientTrCommitCostEstimation + +| Field | Type | Purpose | +|-------|------|---------| +| opsCount | `int` (int32_t) | Operation count | +| writeCosts | `uint64_t` | Write cost estimate | +| clearIdxCosts | `std::deque>` | Clear index costs | +| expensiveCostEstCount | `uint32_t` | Expensive estimate count | + +**Reply: CommitID** + +| Field | Type | Purpose | +|-------|------|---------| +| version | `Version` → `int64_t` | Committed version | +| txnBatchId | `uint16_t` | Batch identifier | +| metadataVersion | `Optional` | Metadata version | +| conflictingKRIndices | `Optional>>` | Conflicting key range indices | + +### GetKeyServerLocationsRequest + +| Field | Type | Purpose | +|-------|------|---------| +| spanContext | `SpanContext` | Tracing context | +| begin | `KeyRef` → `StringRef` | Range begin key | +| end | `Optional` | Range end key | +| limit | `int` (int32_t) | Max results | +| reverse | `bool` | Reverse iteration | +| legacyVersion | `Version` → `int64_t` | Legacy version | +| arena | `Arena` | Memory arena | +| reply | `ReplyPromise` | Locations | + +**Reply: GetKeyServerLocationsReply** + +| Field | Type | Purpose | +|-------|------|---------| +| results | `std::vector>>` | Range→SS mapping | +| resultsTssMapping | `std::vector>` | TSS pairs | +| resultsTagMapping | `std::vector>` | SS→Tag mapping | +| arena | `Arena` | Memory arena | + +### Other Commit Proxy Messages +- **GetRawCommittedVersionRequest**: `SpanContext`, `Optional debugID`, `Version maxVersion`, `reply` → **GetRawCommittedVersionReply** {`Optional debugID`, `Version version`, `bool locked`, `Optional metadataVersion`, `Version minKnownCommittedVersion`, `VersionVector ssVersionVectorDelta`} +- **GetStorageServerRejoinInfoRequest**: `UID id`, `Optional dcId`, `reply` → **GetStorageServerRejoinInfoReply** {`Version version`, `Tag tag`, `Optional newTag`, `bool newLocality`, `std::vector> history`, `EncryptionAtRestModeDeprecated encryptMode`} +- **TxnStateRequest**: `VectorRef data`, `Sequence sequence` (uint64_t), `bool last`, `std::vector broadcastInfo`, `Arena arena`, `reply` → `Void` +- **GetHealthMetricsRequest**: `bool detailed`, `reply` → **GetHealthMetricsReply** {`Standalone serialized`} +- **GetDDMetricsRequest**: `KeyRange keys`, `int shardLimit`, `reply` → **GetDDMetricsReply** {`Standalone> storageMetricsList`} +- **ProxySnapRequest**: `StringRef snapPayload`, `UID snapUID`, `Optional debugID`, `Arena arena`, `reply` → `Void` +- **ExclusionSafetyCheckRequest**: `std::vector exclusions`, `reply` → **ExclusionSafetyCheckReply** {`bool safe`} +- **SetThrottledShardRequest**: `std::vector throttledShards`, `double expirationTime`, `reply` → **SetThrottledShardReply** {} +- **ExpireIdempotencyIdRequest** (fire-and-forget): `Version commitVersion`, `uint8_t batchIndexHighByte` +- **GlobalConfigRefreshRequest**: `Version lastKnown`, `reply` → **GlobalConfigRefreshReply** {`Version version`, `RangeResultRef result`, `Arena arena`} + +--- + +## 8. Resolver Protocol + +**Source:** `fdbserver/core/include/fdbserver/core/ResolverInterface.h` + +### Interface: ResolverInterface +All endpoints serialized directly: `uniqueID` (`UID`), `locality` (`LocalityData`), +`resolve`, `metrics`, `split`, `waitFailure`, `txnState`. + +### ResolveTransactionBatchRequest + +| Field | Type | Purpose | +|-------|------|---------| +| spanContext | `SpanContext` | Tracing context | +| prevVersion | `Version` → `int64_t` | Previous batch version | +| version | `Version` → `int64_t` | This batch's version | +| lastReceivedVersion | `Version` → `int64_t` | Last received by this resolver | +| transactions | `VectorRef` | Transactions to check | +| txnStateTransactions | `VectorRef` | Indices of state transactions | +| debugID | `Optional` | Debug trace ID | +| writtenTags | `std::set` | Tags written | +| lastShardMove | `Version` → `int64_t` | Last shard move version | +| arena | `Arena` | Memory arena | +| reply | `ReplyPromise` | Results | + +**Reply: ResolveTransactionBatchReply** + +| Field | Type | Purpose | +|-------|------|---------| +| committed | `VectorRef` | Per-txn status (0=ok, 1=conflict, ...) | +| stateMutations | `VectorRef>` | State mutations | +| debugID | `Optional` | Debug trace ID | +| conflictingKeyRangeMap | `std::map>` | Conflict details | +| privateMutations | `VectorRef` | Private mutations | +| privateMutationCount | `uint32_t` | Count | +| tpcvMap | `std::unordered_map` | Per-proxy committed versions | +| writtenTags | `std::set` | Tags written | +| lastShardMove | `Version` → `int64_t` | Last shard move | +| arena | `Arena` | Memory arena | + +### Other Resolver Messages +- **ResolutionMetricsRequest**: `reply` → **ResolutionMetricsReply** {`int64_t value`} +- **ResolutionSplitRequest**: `KeyRange range`, `int64_t offset`, `bool front`, `reply` → **ResolutionSplitReply** {`Key key`, `int64_t used`} + +--- + +## 9. Transaction Log (TLog) Protocol + +**Source:** `fdbserver/core/include/fdbserver/core/TLogInterface.h` + +### Interface: TLogInterface +Serializes `uniqueID` (`UID`), `sharedTLogID` (`UID`), `filteredLocality` (`LocalityData`), +and `peekMessages`; 12 other endpoints reconstructed via offsets. + +### TLogCommitRequest + +| Field | Type | Purpose | +|-------|------|---------| +| spanContext | `SpanContext` | Tracing context | +| prevVersion | `Version` → `int64_t` | Previous version | +| version | `Version` → `int64_t` | Commit version | +| knownCommittedVersion | `Version` → `int64_t` | Known committed version | +| minKnownCommittedVersion | `Version` → `int64_t` | Min known committed | +| seqPrevVersion | `Version` → `int64_t` | Sequential previous version | +| messages | `StringRef` | Serialized mutation messages | +| tLogCount | `uint16_t` | Number of TLogs | +| tLogLocIds | `std::vector` | TLog locality IDs | +| debugID | `Optional` | Debug trace ID | +| arena | `Arena` | Memory arena | +| reply | `ReplyPromise` | Acknowledgment | + +**Reply: TLogCommitReply** — `Version version` (int64_t). + +### TLogPeekRequest + +| Field | Type | Purpose | +|-------|------|---------| +| begin | `Version` → `int64_t` | Start version | +| tag | `Tag` | Tag to peek | +| returnIfBlocked | `bool` | Return immediately if blocked | +| onlySpilled | `bool` | Only spilled data | +| sequence | `Optional>` | Ordering sequence | +| end | `Optional` | End version | +| returnEmptyIfStopped | `Optional` | Return empty if stopped | +| reply | `ReplyPromise` | Mutations | + +**Reply: TLogPeekReply** + +| Field | Type | Purpose | +|-------|------|---------| +| messages | `StringRef` | Serialized mutations | +| end | `Version` → `int64_t` | End version | +| popped | `Optional` | Popped version | +| maxKnownVersion | `Version` → `int64_t` | Max known version | +| minKnownCommittedVersion | `Version` → `int64_t` | Min known committed | +| begin | `Optional` | Actual begin | +| onlySpilled | `bool` | Only from spill | +| arena | `Arena` | Memory arena | + +### TLogPeekStreamRequest + +| Field | Type | Purpose | +|-------|------|---------| +| begin | `Version` → `int64_t` | Start version | +| tag | `Tag` | Tag to peek | +| returnIfBlocked | `bool` | Return immediately | +| limitBytes | `int` (int32_t) | Byte limit per reply | +| end | `Optional` | End version | +| returnEmptyIfStopped | `Optional` | Return empty if stopped | +| reply | `ReplyPromiseStream` | Streaming reply | + +**Reply: TLogPeekStreamReply** (extends ReplyPromiseStreamReply) + +| Field | Type | Purpose | +|-------|------|---------| +| acknowledgeToken | `Optional` | Backpressure token | +| sequence | `uint16_t` | Sequence number | +| rep | `TLogPeekReply` | Peek data (see above) | + +### Other TLog Messages +- **TLogPopRequest**: `Version to`, `Version durableKnownCommittedVersion`, `Tag tag`, `reply` → `Void` +- **TLogQueuingMetricsRequest**: `reply` → **TLogQueuingMetricsReply** {`double localTime`, `int64_t instanceID`, `int64_t bytesDurable`, `int64_t bytesInput`, `StorageBytes storageBytes`, `Version v`} +- **TLogConfirmRunningRequest**: `Optional debugID`, `reply` → `Void` +- **TLogRecoveryFinishedRequest**: `reply` → `Void` +- **TLogDisablePopRequest**: `UID snapUID`, `Optional debugID`, `reply` → `Void` +- **TLogEnablePopRequest**: `UID snapUID`, `Optional debugID`, `reply` → `Void` +- **TLogSnapRequest**: `StringRef snapPayload`, `UID snapUID`, `StringRef role`, `Arena arena`, `reply` → `Void` +- **TrackTLogRecoveryRequest**: `Version oldestGenRecoverAtVersion`, `reply` → **TrackTLogRecoveryReply** {`Version oldestUnrecoveredStartVersion`} +- **TLogLockResult** (reply to `lock` endpoint): `Version end`, `Version knownCommittedVersion`, `std::deque unknownCommittedVersions`, `UID id`, `UID logId` + +--- + +## 10. Storage Server Protocol + +**Source:** `fdbclient/include/fdbclient/StorageServerInterface.h` + +### Interface: StorageServerInterface +Serializes `uniqueID` (`UID`), `locality` (`LocalityData`), `getValue`, +`tssPairID` (`Optional`), and `acceptingRequests` (`bool`); 30+ other +endpoints reconstructed via offsets. + +### GetValueRequest + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Key to read | +| version | `Version` → `int64_t` | Read version | +| tags | `Optional` | Transaction tags | +| reply | `ReplyPromise` | Value | +| spanContext | `SpanContext` | Tracing context | +| options | `Optional` | Read options | +| ssLatestCommitVersions | `VersionVector` | SS version vector | + +**Reply: GetValueReply** + +| Field | Type | Purpose | +|-------|------|---------| +| penalty | `double` | Load balancing penalty | +| error | `Optional` | Error (serialized as error code) | +| value | `Optional` → `Optional` | The value | +| cached | `bool` | From cache | + +### GetKeyRequest + +| Field | Type | Purpose | +|-------|------|---------| +| sel | `KeySelectorRef` | Key selector | +| version | `Version` → `int64_t` | Read version | +| tags | `Optional` | Transaction tags | +| reply | `ReplyPromise` | Resolved key | +| spanContext | `SpanContext` | Tracing context | +| options | `Optional` | Read options | +| ssLatestCommitVersions | `VersionVector` | SS version vector | +| arena | `Arena` | Memory arena | + +**Reply: GetKeyReply** — `double penalty`, `Optional error`, `KeySelector sel`, `bool cached`. + +### GetKeyValuesRequest + +| Field | Type | Purpose | +|-------|------|---------| +| begin | `KeySelectorRef` | Range begin | +| end | `KeySelectorRef` | Range end | +| version | `Version` → `int64_t` | Read version | +| limit | `int` (int32_t) | Row limit | +| limitBytes | `int` (int32_t) | Byte limit | +| tags | `Optional` | Transaction tags | +| reply | `ReplyPromise` | Results | +| spanContext | `SpanContext` | Tracing context | +| options | `Optional` | Read options | +| ssLatestCommitVersions | `VersionVector` | SS version vector | +| taskID | `Optional` → `Optional` | Task priority | +| arena | `Arena` | Memory arena | + +**Reply: GetKeyValuesReply** — `double penalty`, `Optional error`, `VectorRef data`, `Version version`, `bool more`, `bool cached`, `Arena arena`. + +### GetMappedKeyValuesRequest +Same as `GetKeyValuesRequest` plus `KeyRef mapper`. Reply: `VectorRef data`, `Version version`, `bool more`, `bool cached`, `Arena arena`. + +### GetKeyValuesStreamRequest +Same fields as `GetKeyValuesRequest` (without `taskID`) but with `ReplyPromiseStream reply`. + +**Reply: GetKeyValuesStreamReply** — `Optional acknowledgeToken`, `uint16_t sequence`, `VectorRef data`, `Version version`, `bool more`, `bool cached`, `Arena arena`. + +### WatchValueRequest + +| Field | Type | Purpose | +|-------|------|---------| +| key | `Key` → `StringRef` | Key to watch | +| value | `Optional` | Expected current value | +| version | `Version` → `int64_t` | Version of expected value | +| tags | `Optional` | Transaction tags | +| debugID | `Optional` | Debug trace ID | +| reply | `ReplyPromise` | Watch trigger | +| spanContext | `SpanContext` | Tracing context | + +**Reply: WatchValueReply** — `Version version`, `bool cached`. + +### ChangeFeedStreamRequest + +| Field | Type | Purpose | +|-------|------|---------| +| rangeID | `Key` → `StringRef` | Change feed ID | +| begin | `Version` → `int64_t` | Start version | +| end | `Version` → `int64_t` | End version | +| range | `KeyRange` | Key range | +| reply | `ReplyPromiseStream` | Streaming reply | +| spanContext | `SpanContext` | Tracing context | +| replyBufferSize | `int` (int32_t) | Buffer size | +| canReadPopped | `bool` | Can read popped data | +| id | `UID` | Request ID | +| options | `Optional` | Read options | +| encrypted | `bool` | Whether to encrypt | +| arena | `Arena` | Memory arena | + +**Reply: ChangeFeedStreamReply** — `Optional acknowledgeToken`, `uint16_t sequence`, `VectorRef mutations`, `bool atLatestVersion`, `Version minStreamVersion`, `Version popVersion`, `Arena arena`. + +### Other Storage Server Messages +- **GetShardStateRequest**: `KeyRange keys`, `int32_t mode`, `bool includePhysicalShard`, `reply` → **GetShardStateReply** {`Version first`, `Version second`, `std::vector shards`} +- **WaitMetricsRequest**: `KeyRangeRef keys`, `StorageMetrics min`, `StorageMetrics max`, `Version legacyVersion`, `Arena arena`, `reply` → `StorageMetrics` +- **SplitMetricsRequest**: `KeyRangeRef keys`, `StorageMetrics limits/used/estimated`, `bool isLastShard`, `Optional minSplitBytes`, `Arena arena`, `reply` → **SplitMetricsReply** {`Standalone> splits`, `StorageMetrics used`, `bool more`} +- **GetStorageMetricsRequest**: `reply` → **GetStorageMetricsReply** {`StorageMetrics load/available/capacity`, `double bytesInputRate`, `int64_t versionLag`, `double lastUpdate`, `int64_t bytesDurable`, `int64_t bytesInput`, `int ongoingBulkLoadTaskCount`} +- **StorageQueuingMetricsRequest**: `reply` → **StorageQueuingMetricsReply** {`double localTime`, `int64_t instanceID/bytesDurable/bytesInput`, `StorageBytes storageBytes`, `Version version/durableVersion`, `double cpuUsage/diskUsage/localRateLimit`, `std::vector busiestTags`} +- **OverlappingChangeFeedsRequest**: `KeyRange range`, `Version minVersion`, `reply` → **OverlappingChangeFeedsReply** {`VectorRef feeds`, `Version feedMetadataVersion`, `Arena arena`} +- **ChangeFeedPopRequest**: `Key rangeID`, `Version version`, `KeyRange range`, `reply` → `Void` +- **ChangeFeedVersionUpdateRequest**: `Version minVersion`, `reply` → **ChangeFeedVersionUpdateReply** {`Version version`} +- **GetCheckpointRequest**: `Version version`, `std::vector ranges`, `int16_t format`, `Optional actionId`, `reply` → `CheckpointMetaData` +- **FetchCheckpointRequest**: `UID checkpointID`, `Standalone token`, `reply` (stream) → **FetchCheckpointReply** {`Optional acknowledgeToken`, `uint16_t sequence`, `Standalone token`, `Standalone data`} +- **FetchCheckpointKeyValuesRequest**: `UID checkpointID`, `KeyRange range`, `reply` (stream) → **FetchCheckpointKeyValuesStreamReply** {`Optional acknowledgeToken`, `uint16_t sequence`, `VectorRef data`, `Arena arena`} +- **ReadHotSubRangeRequest**: `KeyRangeRef keys`, `uint8_t type`, `int chunkCount`, `Arena arena`, `reply` → **ReadHotSubRangeReply** {`Standalone> readHotRanges`} +- **SplitRangeRequest**: `KeyRangeRef keys`, `int64_t chunkSize`, `Arena arena`, `reply` → **SplitRangeReply** {`Standalone> splitPoints`} +- **UpdateCommitCostRequest**: `UID ratekeeperID`, `double postTime/elapsed`, `TransactionTag busiestTag`, `int opsSum`, `uint64_t costSum/totalWriteCosts`, `bool reported`, `reply` → `Void` +- **GetHotShardsRequest**: `reply` → **GetHotShardsReply** {`std::vector hotShards`} +- **GetStorageCheckSumRequest**: `std::vector>> ranges`, `Optional actionId`, `uint8_t checkSumMethod`, `reply` → **GetStorageCheckSumReply** {`std::vector checkSums`, `uint8_t checkSumMethod`} +- **BulkDumpRequest**: `std::vector checksumServers`, `BulkDumpState bulkDumpState`, `reply` → `BulkDumpState` +- **AuditStorageRequest**: `UID id/ddId`, `KeyRange range`, `uint8_t type`, `std::vector targetServers`, `reply` → `Void` + +--- + +## 11. Data Distributor Protocol + +**Source:** `fdbserver/core/include/fdbserver/core/DataDistributorInterface.h` + +### Interface: DataDistributorInterface +Serializes all endpoints directly: `waitFailure`, `haltDataDistributor`, +`locality` (`LocalityData`), `myId` (`UID`), and 7 additional `RequestStream`s. + +- **HaltDataDistributorRequest**: `UID requesterID`, `reply` → `Void` +- **GetDataDistributorMetricsRequest**: `KeyRange keys`, `int shardLimit`, `bool midOnly`, `reply` → **GetDataDistributorMetricsReply** {`Standalone> storageMetricsList`, `Optional midShardSize`} +- **DistributorSnapRequest**: `StringRef snapPayload`, `UID snapUID`, `Optional debugID`, `Arena arena`, `reply` → `Void` +- **DistributorExclusionSafetyCheckRequest**: `std::vector exclusions`, `reply` → **DistributorExclusionSafetyCheckReply** {`bool safe`} +- **DistributorSplitRangeRequest**: `std::vector splitPoints`, `reply` → **SplitShardReply** {`std::vector shards`} +- **GetStorageWigglerStateRequest**: `reply` → **GetStorageWigglerStateReply** {`uint8_t primary`, `uint8_t remote`} +- **PrepareBlobRestoreRequest**: `UID requesterID`, `StorageServerInterface ssi`, `KeyRange keys`, `reply` → **PrepareBlobRestoreReply** {`int8_t res`} + +--- + +## 12. Ratekeeper Protocol + +**Source:** `fdbserver/core/include/fdbserver/core/RatekeeperInterface.h` + +### Interface: RatekeeperInterface +Serializes all endpoints directly: `waitFailure`, `getRateInfo`, `haltRatekeeper`, +`reportCommitCostEstimation`, `getSSVersionLag`, `locality` (`LocalityData`), `myId` (`UID`). + +### GetRateInfoRequest + +| Field | Type | Purpose | +|-------|------|---------| +| requesterID | `UID` | Proxy ID | +| totalReleasedTransactions | `int64_t` | Total released | +| batchReleasedTransactions | `int64_t` | Batch released | +| version | `Version` → `int64_t` | Current version | +| throttledTagCounts | `TransactionTagMap` | Throttled tag counts | +| detailed | `bool` | Include detailed metrics | +| reply | `ReplyPromise` | Rate limits | + +**Reply: GetRateInfoReply** + +| Field | Type | Purpose | +|-------|------|---------| +| transactionRate | `double` | Default TPS limit | +| batchTransactionRate | `double` | Batch TPS limit | +| leaseDuration | `double` | Rate lease duration | +| healthMetrics | `HealthMetrics` | Cluster health | +| clientThrottledTags | `Optional>` | Client throttles | + +`PrioritizedTransactionTagMap` is `std::map>` where `TransactionPriority` is a `uint8_t` enum. + +### Other Ratekeeper Messages +- **HaltRatekeeperRequest**: `UID requesterID`, `reply` → `Void` +- **ReportCommitCostEstimationRequest**: `UIDTransactionTagMap ssTrTagCommitCost`, `reply` → `Void` +- **GetSSVersionLagRequest**: `reply` → **GetSSVersionLagReply** {`Version maxPrimarySSVersion`, `Version maxRemoteSSVersion`} + +--- + +## 13. Worker Protocol + +**Source:** `fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h` + +Workers host server roles. The cluster controller sends initialization requests. + +### Interface: WorkerInterface +Serializes all 24 `RequestStream` endpoints and `TesterInterface` directly. + +### RegisterWorkerRequest + +| Field | Type | Purpose | +|-------|------|---------| +| wi | `WorkerInterface` | Worker's interface | +| initialClass | `ProcessClass` | Initial class | +| processClass | `ProcessClass` | Current class | +| priorityInfo | `ClusterControllerPriorityInfo` | Priority info | +| generation | `Generation` → `int64_t` | Cluster generation | +| distributorInterf | `Optional` | DD interface | +| ratekeeperInterf | `Optional` | RK interface | +| consistencyScanInterf | `Optional` | CS interface | +| issues | `Standalone>` | Worker issues | +| incompatiblePeers | `std::vector` | Incompatible peers | +| degraded | `bool` | Worker degraded | +| recoveredDiskFiles | `bool` | Disk files recovered | +| clusterId | `Optional` | Cluster ID | +| reply | `ReplyPromise` | Registration result | + +**Reply: RegisterWorkerReply** — `ProcessClass processClass`, `ClusterControllerPriorityInfo priorityInfo`. + +### Role Initialization Requests + +All follow the pattern: fields describing the role configuration + `ReplyPromise`. + +- **RecruitMasterRequest**: `LifetimeToken lifetime`, `bool forceRecovery`, `reply` → `MasterInterface` +- **InitializeTLogRequest**: `UID recruitmentID`, `LogSystemConfig recoverFrom`, `Version recoverAt/knownCommittedVersion`, `LogEpoch epoch`, `std::vector recoverTags/allTags`, `TLogVersion logVersion`, `KeyValueStoreType storeType`, `TLogSpillType spillType`, `Tag remoteTag`, `int8_t locality`, `bool isPrimary`, `Version startVersion`, `int logRouterTags/txsTags`, `Version recoveryTransactionVersion`, `std::vector oldGenerationRecoverAtVersions`, `reply` → `TLogInterface` +- **InitializeCommitProxyRequest**: `MasterInterface master`, `LifetimeToken masterLifetime`, `uint64_t recoveryCount`, `Version recoveryTransactionVersion`, `bool firstProxy`, `EncryptionAtRestModeDeprecated encryptMode`, `uint16_t commitProxyIndex`, `reply` → `CommitProxyInterface` +- **InitializeGrvProxyRequest**: `MasterInterface master`, `LifetimeToken masterLifetime`, `uint64_t recoveryCount`, `reply` → `GrvProxyInterface` +- **InitializeResolverRequest**: `LifetimeToken masterLifetime`, `uint64_t recoveryCount`, `int commitProxyCount/resolverCount`, `UID masterId`, `EncryptionAtRestModeDeprecated encryptMode`, `reply` → `ResolverInterface` +- **InitializeStorageRequest**: `Tag seedTag`, `UID reqId/interfaceId`, `KeyValueStoreType storeType`, `Optional> tssPairIDAndVersion`, `Version initialClusterVersion`, `EncryptionAtRestModeDeprecated encryptMode`, `reply` → **InitializeStorageReply** {`StorageServerInterface interf`, `Version addedVersion`} +- **InitializeDataDistributorRequest**: `UID reqId`, `reply` → `DataDistributorInterface` +- **InitializeRatekeeperRequest**: `UID reqId`, `reply` → `RatekeeperInterface` +- **InitializeConsistencyScanRequest**: `UID reqId`, `reply` → `ConsistencyScanInterface` +- **InitializeLogRouterRequest**: `uint64_t recoveryCount`, `Tag routerTag`, `Version startVersion`, `std::vector tLogLocalities`, `Reference tLogPolicy`, `int8_t locality`, `Optional recoverAt`, `Optional>> knownLockedTLogIds`, `bool allowDropInSim/isReplacement`, `UID reqId`, `reply` → `TLogInterface` +- **InitializeBackupRequest**: `UID reqId`, `LogEpoch recruitedEpoch/backupEpoch`, `Tag routerTag`, `int totalTags`, `Version startVersion`, `Optional endVersion`, `reply` → **InitializeBackupReply** {`BackupInterface interf`, `LogEpoch backupEpoch`} + +### Recruitment Requests +- **RecruitFromConfigurationRequest**: `DatabaseConfiguration configuration`, `bool recruitSeedServers`, `int maxOldLogRouters`, `reply` → **RecruitFromConfigurationReply** {`std::vector` for: tLogs, satelliteTLogs, commitProxies, grvProxies, resolvers, storageServers, oldLogRouters, backupWorkers; `Optional dcId`, `bool satelliteFallback`} +- **RecruitRemoteFromConfigurationRequest**: `DatabaseConfiguration configuration`, `Optional dcId`, `int logRouterCount`, `std::vector exclusionWorkerIds`, `Optional dbgId`, `reply` → **RecruitRemoteFromConfigurationReply** {`std::vector remoteTLogs/logRouters`, `Optional dbgId`} +- **RecruitStorageRequest**: `std::vector>> excludeMachines/includeDCs`, `std::vector excludeAddresses`, `bool criticalRecruitment`, `reply` → **RecruitStorageReply** {`WorkerInterface worker`, `ProcessClass processClass`} + +### Other Worker Messages +- **RegisterMasterRequest**: `UID id`, `LocalityData mi`, `LogSystemConfig logSystemConfig`, `std::vector commitProxies`, `std::vector grvProxies`, `std::vector resolvers`, `DBRecoveryCount recoveryCount`, `int64_t registrationCount`, `Optional configuration`, `std::vector priorCommittedLogServers`, `RecoveryState recoveryState`, `bool recoveryStalled` (no reply) +- **GetWorkersRequest**: `int flags`, `reply` → `std::vector` where **WorkerDetails** = {`WorkerInterface interf`, `ProcessClass processClass`, `bool degraded`, `bool recoveredDiskFiles`} +- **TLogRejoinRequest**: `TLogInterface myInterface`, `reply` → **TLogRejoinReply** {`bool masterIsRecovered`} +- **BackupWorkerDoneRequest**: `UID workerUID`, `LogEpoch backupEpoch`, `reply` → `Void` +- **GetEncryptionAtRestModeRequest**: `UID tlogId`, `reply` → **GetEncryptionAtRestModeResponse** {`uint32_t mode`} +- **CoordinationPingMessage** (fire-and-forget): `UID clusterControllerId`, `int64_t timeStep` +- **SetMetricsLogRateRequest** (fire-and-forget): `uint32_t metricsLogsPerSecond` +- **UpdateWorkerHealthRequest** (fire-and-forget): `NetworkAddress address`, `std::vector degradedPeers/disconnectedPeers/recoveredPeers` +- **LoadedPingRequest**: `UID id`, `bool loadReply`, `Standalone payload`, `reply` → **LoadedReply** {`Standalone payload`, `UID id`} +- **EventLogRequest**: `bool getLastError`, `Standalone eventName`, `reply` → `TraceEventFields` +- **TraceBatchDumpRequest**: `reply` → `Void` +- **ExecuteRequest**: `StringRef execPayload`, `Arena arena`, `reply` → `Void` +- **WorkerSnapRequest**: `StringRef snapPayload`, `UID snapUID`, `StringRef role`, `Arena arena`, `reply` → `Void` +- **DiskStoreRequest**: `bool includePartialStores`, `reply` → `Standalone>` + +--- + +## 14. Backup / Consistency Scan / Test Protocols + +### BackupInterface +`RequestStream> waitFailure`, `LocalityData locality`. + +### ConsistencyScanInterface +`RequestStream> waitFailure`, `RequestStream haltConsistencyScan`, `LocalityData locality`, `UID myId`. + +**HaltConsistencyScanRequest**: `UID requesterID`, `reply` → `Void`. + +### TesterInterface +`RequestStream recruitments`. + +**WorkloadRequest**: `StringRef title`, `int timeout`, `double databasePingDelay`, `int64_t sharedRandomNumber`, `bool useDatabase/runFailureWorkloads`, `VectorRef> options`, `std::vector rangesToCheck`, `int clientId/clientCount`, `std::vector disabledFailureInjectionWorkloads`, `Arena arena`, `reply` → **WorkloadInterface** {`RequestStream` endpoints for setup/start/check/metrics/stop}. + +### NetworkTestRequest +`Key key`, `uint32_t replySize`, `reply` → **NetworkTestReply** {`Value value`}. + +### NetworkTestStreamingRequest +`reply` (stream) → **NetworkTestStreamingReply** {`Optional acknowledgeToken`, `uint16_t sequence`, `int index`}. + +--- + +## 15. Client Worker / Debug / Process Protocols + +### ClientWorkerInterface +`RequestStream reboot`, `RequestStream profiler`, `RequestStream setFailureInjection`, `Optional grpcAddress`. + +- **RebootRequest** (fire-and-forget): `bool deleteData`, `bool checkData`, `uint32_t waitForDuration` +- **ProfilerRequest**: `Type` (enum/int), `Action` (enum/int), `int duration`, `Standalone outputFile`, `reply` → `Void` +- **SetFailureInjection**: `Optional diskFailure`, `Optional flipBits`, `reply` → `Void` + +### ProcessInterface +`RequestStream getInterface`, `RequestStream actorLineage`. + +- **GetProcessInterfaceRequest**: `reply` → `ProcessInterface` +- **ActorLineageRequest**: `WaitState waitStateStart/waitStateEnd` (enum/int), `time_t timeStart/timeEnd` (int64_t), `reply` → **ActorLineageReply** {`std::vector samples`} + +--- + +## Appendix A: Log System Configuration Types + +**Source:** `fdbserver/core/include/fdbserver/core/LogSystemConfig.h`, +`fdbserver/core/include/fdbserver/core/DBCoreState.h`, +`fdbserver/core/include/fdbserver/core/ServerDBInfo.h` + +### OptionalInterface\ +Conditionally serializes a full interface or just its UID. + +| Field | Type | Purpose | +|-------|------|---------| +| iface | `Optional` | Full interface (if present) | +| ident | `UID` | Interface ID (serialized only if iface is absent) | + +### TLogSet + +| Field | Type | Purpose | +|-------|------|---------| +| tLogs | `std::vector>` | TLog interfaces | +| logRouters | `std::vector>` | Log router interfaces | +| backupWorkers | `std::vector>` | Backup worker interfaces | +| tLogWriteAntiQuorum | `int32_t` | Write anti-quorum | +| tLogReplicationFactor | `int32_t` | Replication factor | +| tLogPolicy | `Reference` | Replication policy (custom serialization) | +| tLogLocalities | `std::vector` | TLog localities | +| isLocal | `bool` | Whether this is the local DC | +| locality | `int8_t` | DC locality | +| startVersion | `Version` → `int64_t` | Start version | +| satelliteTagLocations | `std::vector>` | Satellite tag locations | +| tLogVersion | `TLogVersion` → `uint32_t` | TLog version | + +### OldTLogConf + +| Field | Type | Purpose | +|-------|------|---------| +| tLogs | `std::vector` | TLog sets | +| epochBegin | `Version` → `int64_t` | Epoch begin version | +| epochEnd | `Version` → `int64_t` | Epoch end version | +| recoverAt | `Version` → `int64_t` | Recovery version | +| logRouterTags | `int32_t` | Number of log router tags | +| txsTags | `int32_t` | Number of txs tags | +| pseudoLocalities | `std::set` | Pseudo-localities | +| epoch | `LogEpoch` → `uint64_t` | Epoch number | + +### LogSystemConfig + +| Field | Type | Purpose | +|-------|------|---------| +| logSystemType | `LogSystemType` → enum (int) | Always `tagPartitioned` (2) | +| tLogs | `std::vector` | Current TLog sets | +| logRouterTags | `int32_t` | Log router tag count | +| txsTags | `int32_t` | Txs tag count | +| oldTLogs | `std::vector` | Previous generation configs | +| expectedLogSets | `int32_t` | Expected log set count | +| recruitmentID | `UID` | Recruitment ID | +| stopped | `bool` | Whether stopped | +| recoveredAt | `Optional` | Recovery version | +| pseudoLocalities | `std::set` | Pseudo-localities | +| epoch | `LogEpoch` → `uint64_t` | Current epoch | +| oldestBackupEpoch | `LogEpoch` → `uint64_t` | Oldest backup epoch | +| knownLockedTLogIds | `std::map>` | Known locked TLog IDs | + +### CoreTLogSet +Persisted on coordinators (UIDs only, no full interfaces). + +| Field | Type | Purpose | +|-------|------|---------| +| tLogs | `std::vector` | TLog UIDs | +| tLogWriteAntiQuorum | `int32_t` | Write anti-quorum | +| tLogReplicationFactor | `int32_t` | Replication factor | +| tLogPolicy | `Reference` | Replication policy | +| tLogLocalities | `std::vector` | TLog localities | +| isLocal | `bool` | Local DC | +| locality | `int8_t` | DC locality | +| startVersion | `Version` → `int64_t` | Start version | +| satelliteTagLocations | `std::vector>` | Satellite locations | +| tLogVersion | `TLogVersion` → `uint32_t` | TLog version | + +### OldTLogCoreData + +| Field | Type | Purpose | +|-------|------|---------| +| tLogs | `std::vector` | Core TLog sets | +| logRouterTags | `int32_t` | Log router tags | +| txsTags | `int32_t` | Txs tags | +| epochBegin | `Version` → `int64_t` | Epoch begin | +| epochEnd | `Version` → `int64_t` | Epoch end | +| recoverAt | `Version` → `int64_t` | Recovery version (conditional on protocol version) | +| pseudoLocalities | `std::set` | Pseudo-localities | +| epoch | `LogEpoch` → `uint64_t` | Epoch | + +### DBCoreState +Persisted on coordinators — the ground truth for recovery. + +| Field | Type | Purpose | +|-------|------|---------| +| tLogs | `std::vector` | Current TLog sets | +| logRouterTags | `int32_t` | Log router tags | +| txsTags | `int32_t` | Txs tags | +| oldTLogData | `std::vector` | Previous generations | +| recoveryCount | `DBRecoveryCount` → `uint64_t` | Recovery counter | +| logSystemType | `LogSystemType` → enum (int) | Log system type | +| pseudoLocalities | `std::set` | Pseudo-localities | +| newestProtocolVersion | `ProtocolVersion` → `uint64_t` | Newest protocol (conditional) | +| lowestCompatibleProtocolVersion | `ProtocolVersion` → `uint64_t` | Lowest compatible (conditional) | +| encryptionAtRestModeDeprecated | `EncryptionAtRestModeDeprecated` → `uint32_t` | Encryption mode (conditional) | + +### ServerDBInfo +Distributed to all server processes. + +| Field | Type | Purpose | +|-------|------|---------| +| id | `UID` | Info snapshot ID | +| clusterInterface | `ClusterControllerFullInterface` | CC interface | +| client | `ClientDBInfo` | Client-facing info | +| distributor | `Optional` | DD interface | +| master | `MasterInterface` | Master interface | +| ratekeeper | `Optional` | RK interface | +| consistencyScan | `Optional` | CS interface | +| resolvers | `std::vector` | Resolver interfaces | +| recoveryCount | `DBRecoveryCount` → `uint64_t` | Recovery counter | +| recoveryState | `RecoveryState` → enum (int) | Recovery state | +| masterLifetime | `LifetimeToken` | Master lifetime token | +| logSystemConfig | `LogSystemConfig` | Log system config | +| priorCommittedLogServers | `std::vector` | Prior committed log servers | +| latencyBandConfig | `Optional` | Latency band config | +| infoGeneration | `int64_t` | Info generation counter | + +### LatencyBandConfig + +| Field | Type | Purpose | +|-------|------|---------| +| grvConfig | `GrvConfig` | GRV latency bands | +| readConfig | `ReadConfig` | Read latency bands | +| commitConfig | `CommitConfig` | Commit latency bands | + +Each config inherits from **RequestConfig** {`std::set bands`} and adds: +- **ReadConfig**: `Optional maxReadBytes`, `Optional maxKeySelectorOffset` +- **CommitConfig**: `Optional maxCommitBytes` + +### UpdateServerDBInfoRequest + +| Field | Type | Purpose | +|-------|------|---------| +| serializedDbInfo | `Standalone` | Serialized `ServerDBInfo` | +| broadcastInfo | `std::vector` | Endpoints to broadcast to | +| reply | `ReplyPromise>` | Updated endpoints | + +### GetServerDBInfoRequest + +| Field | Type | Purpose | +|-------|------|---------| +| knownServerInfoID | `UID` | Last known info ID (long-poll) | +| reply | `ReplyPromise` | Server DB info | + +--- + +## Appendix B: In-Band Log Messages + +**Source:** `fdbserver/core/include/fdbserver/core/LogProtocolMessage.h`, +`fdbserver/core/include/fdbserver/core/SpanContextMessage.h`, +`fdbserver/core/include/fdbserver/core/OTELSpanContextMessage.h` + +These messages are embedded in the TLog mutation stream (`messages` field of +`TLogCommitRequest` / `TLogPeekReply`). They use reserved `MutationRef` type codes. + +### LogProtocolMessage +Marks a protocol version change in the log stream. + +| Field | Type | Purpose | +|-------|------|---------| +| (type code) | `uint8_t` | `MutationRef::Reserved_For_LogProtocolMessage` | +| (version) | `ProtocolVersion` → `uint64_t` | New protocol version | + +### SpanContextMessage + +| Field | Type | Purpose | +|-------|------|---------| +| (type code) | `uint8_t` | `MutationRef::Reserved_For_SpanContextMessage` | +| spanContext | `SpanID` → `UID` (two `uint64_t`) | Tracing span context | + +### OTELSpanContextMessage + +| Field | Type | Purpose | +|-------|------|---------| +| (type code) | `uint8_t` | `MutationRef::Reserved_For_OTELSpanContextMessage` | +| spanContext | `SpanContext` | OpenTelemetry span context | + +--- + +## Appendix C: Backpressure Protocol + +**Source:** `fdbrpc/include/fdbrpc/fdbrpc.h` + +For streaming replies (`ReplyPromiseStream`), FDB implements flow control via +acknowledgments: + +### AcknowledgementReply +Sent from client to server to acknowledge received bytes. + +| Field | Type | Purpose | +|-------|------|---------| +| bytes | `int64_t` | Total bytes acknowledged | + +The server tracks `bytesSent` vs `bytesAcknowledged` and pauses when the +difference exceeds `bytesLimit`. On cancellation, the client sends +`operation_obsolete` on the acknowledgment stream. + +### ReplyPromiseStreamReply (base for all streaming replies) + +| Field | Type | Purpose | +|-------|------|---------| +| acknowledgeToken | `Optional` | Ack endpoint token (sent in first reply) | +| sequence | `uint16_t` | Monotonic sequence number | + +--- + +## Appendix D: Client Library Protocol Flows + +**Source:** `fdbclient/NativeAPI.actor.cpp`, `fdbclient/MonitorLeader.cpp`, +`fdbclient/GlobalConfig.cpp` + +This appendix describes the exact sequence of protocol messages the FDB client +library sends during each operation, as implemented in the NativeAPI layer. + +### D.1 Cluster Discovery and Connection + +The client's first task is to discover the cluster controller and obtain the +current proxy list. This happens continuously in the background. + +**Step 1: Parse connection file.** +The client reads the cluster file (e.g., `fdb.cluster`) containing +`description:id@coordinator1:port,coordinator2:port,...`. This yields a +`ClusterConnectionString`. + +**Step 2: Contact coordinators (monitorProxies loop).** +The client enters an infinite loop that monitors proxy changes. For each +coordinator, it creates a `ClientLeaderRegInterface` and sends: + +``` +Client → Coordinator: OpenDatabaseCoordRequest + clusterKey: Key (from connection string) + hostnames: std::vector + coordinators: std::vector + knownClientInfoID: UID (last known ClientDBInfo.id, for long-poll) + supportedVersions: std::vector + traceLogGroup: Key + internal: bool +``` + +The coordinator responds with `CachedSerialization`, which the +client deserializes to obtain: + +``` +Coordinator → Client: ClientDBInfo (serialization order) + grvProxies: std::vector + commitProxies: std::vector + id: UID + forward: Optional (if set, reconnect to new coordinators) + history: std::vector + clusterId: UID + clusterType: ClusterType +``` + +Note: `firstCommitProxy` is a struct member but is NOT serialized — it is +reconstructed locally. + +If `forward` is set, the client switches to the new connection string and +restarts discovery. Otherwise, it stores the proxy lists and long-polls +(blocks until `id` changes). + +**Step 3: Proxy list management.** +The client randomly shuffles the proxy lists and truncates them to +`MAX_COMMIT_PROXY_CONNECTIONS` and `MAX_GRV_PROXY_CONNECTIONS` to limit +connection fan-out. It selects `firstCommitProxy` for operations that must +go to a single proxy. + +### D.2 Get Read Version (GRV) + +Every transaction begins by acquiring a read version. + +**Step 1: Check GRV cache.** +If GRV caching is enabled and a cached version is recent enough (within +`MAX_VERSION_CACHE_LAG`), the client returns the cached version immediately +with no network round-trip. + +**Step 2: Batch GRV requests.** +Multiple concurrent transactions' GRV needs are batched together by the +`readVersionBatcher` actor. It collects requests until `MAX_BATCH_SIZE` is +reached or `GRV_BATCH_TIMEOUT` expires, then sends a single request. + +**Step 3: Send batched request to GRV proxy.** + +``` +Client → GrvProxy: GetReadVersionRequest + spanContext: SpanContext + transactionCount: uint32_t (number of transactions in batch) + flags: uint32_t (PRIORITY_SYSTEM_IMMEDIATE | PRIORITY_DEFAULT | PRIORITY_BATCH) + tags: TransactionTagMap (tag counts for throttling) + debugID: Optional + maxVersion: Version (max version in client's version vector cache) +``` + +Proxy selection: `basicLoadBalance()` across `grvProxies[]` with round-robin. + +**Step 4: Process reply.** + +``` +GrvProxy → Client: GetReadVersionReply + processBusyTime: double + version: Version (the assigned read version) + locked: bool + metadataVersion: Optional + midShardSize: int64_t + rkDefaultThrottled: bool + rkBatchThrottled: bool + tagThrottleInfo: TransactionTagMap + ssVersionVectorDelta: VersionVector + proxyId: UID +``` + +The client caches the version, updates its tag throttle table, and applies +the storage server version vector delta for read-your-writes consistency. + +### D.3 Key Location Discovery + +Before reading, the client must know which storage servers hold the target keys. + +**Step 1: Check location cache.** +The client maintains a `KeyRangeMap` mapping key ranges → storage server +interfaces. If the target range is cached, skip to the read. + +**Step 2: Request locations from commit proxy.** + +``` +Client → CommitProxy: GetKeyServerLocationsRequest + spanContext: SpanContext + begin: KeyRef (start of range) + end: Optional (end of range, if range query) + limit: int (max shard locations, typically 100) + reverse: bool + legacyVersion: Version + arena: Arena +``` + +Proxy selection: `commitProxyLoadBalance()` across `commitProxies[]`. + +**Step 3: Cache response.** + +``` +CommitProxy → Client: GetKeyServerLocationsReply + results: std::vector>> + resultsTssMapping: std::vector> + resultsTagMapping: std::vector> + arena: Arena +``` + +Each result maps a key range to the set of storage server replicas hosting it. +The client caches these mappings. TSS (Testing Storage Server) pairs are stored +separately for read validation. + +### D.4 Reading Data + +#### Single Key Read (getValue) + +``` +Client → StorageServer: GetValueRequest (serialization order) + key: Key + version: Version (transaction's read version) + tags: Optional + reply: ReplyPromise + spanContext: SpanContext + options: Optional + ssLatestCommitVersions: VersionVector +``` + +Server selection: `loadBalance()` across replicas from the location cache, +with locality-aware preference for same-DC servers. + +``` +StorageServer → Client: GetValueReply + penalty: double (load balancing hint) + error: Optional + value: Optional (None if key not found) + cached: bool +``` + +#### Range Read (getRange / getKeyValues) + +``` +Client → StorageServer: GetKeyValuesRequest (serialization order) + begin: KeySelectorRef + end: KeySelectorRef + version: Version + limit: int (row limit; negative for reverse) + limitBytes: int (byte limit) + tags: Optional + reply: ReplyPromise + spanContext: SpanContext + options: Optional + ssLatestCommitVersions: VersionVector + taskID: Optional + arena: Arena +``` + +``` +StorageServer → Client: GetKeyValuesReply + penalty: double + error: Optional + data: VectorRef (key-value pairs) + version: Version + more: bool (true if more data available) + cached: bool + arena: Arena +``` + +If `more` is true, the client adjusts the key selectors and sends another +request. If the range spans multiple shards, the client sends parallel +requests to different storage servers and merges results. + +**Error handling during reads:** +- `wrong_shard_server`: Shard moved; invalidate location cache, re-lookup, retry. +- `all_alternatives_failed`: All replicas failed; same as wrong_shard_server. +- `future_version`: Requested version not yet available; wait and retry. +- `transaction_too_old`: Version expired; restart transaction with new GRV. + +#### Streaming Range Read (getKeyValuesStream) + +For large range reads, the client can use the streaming variant: + +``` +Client → StorageServer: GetKeyValuesStreamRequest + (same fields as GetKeyValuesRequest, without taskID) + reply: ReplyPromiseStream +``` + +This returns a stream of `GetKeyValuesStreamReply` messages with backpressure +(see Appendix C). Each reply carries `acknowledgeToken` and `sequence` for +flow control. + +### D.5 Committing a Transaction + +**Step 1: Build commit request.** +The client accumulates mutations (set, clear, atomic operations) and conflict +ranges during the transaction. When `commit()` is called: + +``` +Client → CommitProxy: CommitTransactionRequest + spanContext: SpanContext + transaction: CommitTransactionRef + read_conflict_ranges: VectorRef + write_conflict_ranges: VectorRef + mutations: VectorRef + read_snapshot: Version (the read version) + report_conflicting_keys: bool + lock_aware: bool + spanContext: Optional + tenantIds: Optional> + flags: uint32_t (LOCK_AWARE, FIRST_IN_BATCH, etc.) + debugID: Optional + commitCostEstimation: Optional + tagSet: Optional + idempotencyId: IdempotencyIdRef + arena: Arena +``` + +Proxy selection: `basicLoadBalance()` across `commitProxies[]`, or +`firstCommitProxy` if `COMMIT_ON_FIRST_PROXY` option is set. + +**Step 2: Wait for reply.** + +``` +CommitProxy → Client: CommitID + version: Version (committed version, or invalidVersion on failure) + txnBatchId: uint16_t (for constructing versionstamps) + metadataVersion: Optional + conflictingKRIndices: Optional>> +``` + +**Step 3: Handle result.** +- **Success** (`version != invalidVersion`): Transaction is committed. The + client constructs a `Versionstamp` from `version` and `txnBatchId`, updates + its committed version, and sends `ExpireIdempotencyIdRequest` if using + idempotency. +- **Conflict** (`not_committed`): Transaction conflicted; throw to caller. +- **Unknown** (`commit_unknown_result`): Proxy may have crashed after committing. + The client can retry with idempotency to determine the outcome. + +### D.6 Watching a Key + +``` +Client → StorageServer: WatchValueRequest + spanContext: SpanContext + key: Key + value: Optional (expected current value) + version: Version (version at which value was read) + tags: Optional + debugID: Optional +``` + +This is a long-poll: the storage server holds the request until the key's value +changes, then replies: + +``` +StorageServer → Client: WatchValueReply + version: Version (version at which value changed) + cached: bool +``` + +The client then waits for `waitForCommittedVersion(version)` to ensure the +change is durable before triggering the watch future. If the watch is cancelled +by a `watch_cancelled` error (too many concurrent watches), the client retries +with backoff. + +### D.7 Change Feed Streaming + +``` +Client → StorageServer: ChangeFeedStreamRequest + spanContext: SpanContext + rangeID: Key (change feed identifier) + begin: Version (start version) + end: Version (end version) + range: KeyRange + replyBufferSize: int + canReadPopped: bool + id: UID + options: Optional + encrypted: bool + arena: Arena + reply: ReplyPromiseStream +``` + +Returns a stream of `ChangeFeedStreamReply` messages, each containing: +``` + mutations: VectorRef + atLatestVersion: bool + minStreamVersion: Version + popVersion: Version +``` + +### D.8 Global Configuration Refresh + +Runs periodically in the background: + +``` +Client → GrvProxy: GlobalConfigRefreshRequest + lastKnown: Version (last known config version) +``` + +``` +GrvProxy → Client: GlobalConfigRefreshReply + version: Version + result: RangeResultRef (config key-value pairs) + arena: Arena +``` + +### D.9 Error Handling and Retry Strategy + +| Error | Meaning | Client Action | +|-------|---------|---------------| +| `wrong_shard_server` | Key range relocated | Invalidate cache, retry | +| `all_alternatives_failed` | All replicas unreachable | Same as wrong_shard_server | +| `transaction_too_old` | Read version expired | Get new GRV, restart transaction | +| `future_version` | Version not yet available | Wait `FUTURE_VERSION_RETRY_DELAY`, retry | +| `not_committed` | Transaction conflicted | Throw conflict error to caller | +| `commit_unknown_result` | Proxy crash mid-commit | Verify via idempotency or retry | +| `commit_proxy_memory_limit_exceeded` | Proxy overloaded | Exponential backoff | +| `batch_transaction_throttled` | Batch priority throttled | Retry after delay | + +Backoff: starts at `CLIENT_KNOBS->BACKOFF_DELAY`, multiplied by +`BACKOFF_GROWTH_RATE` on each failure, capped at +`RESOURCE_CONSTRAINED_MAX_BACKOFF`. + +--- + +## Appendix E: fdbcli Protocol Usage + +**Source:** `fdbcli/*.actor.cpp`, `fdbclient/include/fdbclient/ManagementAPI.h` + +The `fdbcli` command-line tool uses three protocol layers: +1. **Normal transactions** via the NativeAPI (same as any client). +2. **Special Key Space writes** — transactions that write to `\xff\xff/management/` + keys, which the commit proxy intercepts and translates into cluster operations. +3. **Direct RPC** to ClusterInterface or worker endpoints. + +### E.1 Direct RPC Commands + +These commands send protocol messages directly to cluster endpoints, bypassing +the transaction layer. + +#### `status` Command +``` +fdbcli → ClusterController: StatusRequest + statusField: std::string (optional filter like "json") + reply: ReplyPromise +``` +Reply: `StatusReply` containing `statusStr` (JSON string). + +#### `force_recovery_with_data_loss ` +``` +fdbcli → ClusterController: ForceRecoveryRequest + dcId: Key (datacenter ID) + reply: ReplyPromise +``` + +#### `kill` / `suspend` / `expensive_data_check` +First discovers workers via: +``` +fdbcli → ClusterController: GetClientWorkersRequest + reply: ReplyPromise> +``` + +Then sends to each target worker: +``` +fdbcli → Worker: RebootRequest (fire-and-forget) + deleteData: bool (false for kill/suspend) + checkData: bool (true for expensive_data_check) + waitForDuration: uint32_t (0 for kill, >0 for suspend seconds) +``` + +#### `profile` Command (flow/heap profiling) +``` +fdbcli → Worker: ProfilerRequest + type: Type enum (GPROF, FLOW, GPROF_HEAP) + action: Action enum (ENABLE, DISABLE, RUN) + duration: int + outputFile: Standalone + reply: ReplyPromise +``` + +#### `audit_storage` Command +``` +fdbcli → ClusterController: TriggerAuditRequest + type: uint8_t (ValidateHA, ValidateReplica, ValidateLocationMetadata, ...) + range: KeyRange + id: UID + cancel: bool + engineType: KeyValueStoreType (uint32_t) + reply: ReplyPromise +``` + +#### `snapshot` Command +Uses `IDatabase::createSnapshot()` which sends `ProxySnapRequest` to a commit +proxy. The proxy then forwards a `DistributorSnapRequest` to the data distributor, +which orchestrates the actual snapshot across TLogs and storage servers. + +### E.2 Special Key Space Commands + +These commands write to `\xff\xff/management/` keys within a transaction with +`SPECIAL_KEY_SPACE_ENABLE_WRITES` option. The commit proxy intercepts these +writes and performs the corresponding cluster operations. + +#### `exclude` / `include` Commands +``` +Write to: \xff\xff/management/excluded/
= "" +Write to: \xff\xff/management/failed/
= "" +Write to: \xff\xff/management/excluded_locality/ = "" +Write to: \xff\xff/management/failed_locality/ = "" + +Options: +Write to: \xff\xff/management/options/excluded/force = "" (skip safety check) + +Read from: \xff\xff/management/in_progress_exclusion/
(check progress) +``` + +#### `configure` Command +Uses `ManagementAPI::changeConfig()` which: +1. Reads current configuration from `\xff/conf/` system keys. +2. Writes updated configuration via a normal transaction. +3. The commit proxy detects system key changes and triggers reconfiguration. + +#### `coordinators` Command +``` +Write to: \xff\xff/configuration/coordinators/processes = ",,..." +Write to: \xff\xff/configuration/coordinators/cluster_description = "" + +Read from: \xff\xff/management/auto_coordinators (get auto-selected coordinators) +``` + +#### `lock` / `unlock` Commands +``` +Write to: \xff\xff/management/db_locked = (lock) +Clear: \xff\xff/management/db_locked (unlock, requires matching UID) +``` + +#### `maintenance` Command +``` +Write to: \xff\xff/management/maintenance/ = +Write to: \xff\xff/management/maintenance/IgnoreSSFailures = "" (disable DD for SS failures) +Clear: \xff\xff/management/maintenance/ (end maintenance) +``` + +#### `datadistribution` Command +``` +Write to: \xff\xff/management/data_distribution/mode = "0" or "1" (off/on) +Write to: \xff\xff/management/data_distribution/rebalance_ignored = +``` + +#### `advanceversion` Command +``` +Write to: \xff\xff/management/min_required_commit_version = +``` + +#### `throttle` Command +Uses `ThrottleApi` functions which read/write: +``` +\xff\x02/throttledTags/tag/// = TagThrottleValue +\xff\x02/throttledTags/autoThrottledTags/// = TagThrottleValue +``` + +#### `setclass` Command +Uses `ManagementAPI::setClass()` which writes to system keys to change a +worker's process class assignment. + +### E.3 Backup / Restore CLI Commands + +Backup and restore commands use the `FileBackupAgent` and `DatabaseBackupAgent` +classes, which operate through normal transactions reading/writing system keys: + +``` +\xff\x02/fdbbackup/ — backup task state +\xff\x02/fdbrestore/ — restore task state +\xff\x02/backupProgress/ — backup progress markers +\xff\x02/backupStarted/ — backup start markers +``` + +These are standard key-value operations — the backup agents are implemented as +Taskbucket workflows that run as normal transactions within the cluster. + +### E.4 Transaction Options Used by fdbcli + +| Option | Purpose | +|--------|---------| +| `SPECIAL_KEY_SPACE_ENABLE_WRITES` | Enable writes to `\xff\xff/` keys | +| `PRIORITY_SYSTEM_IMMEDIATE` | Highest priority (bypass throttling) | +| `READ_SYSTEM_KEYS` | Allow reading `\xff/` system keys | +| `LOCK_AWARE` | Operate while database is locked | +| `ACCESS_SYSTEM_KEYS` | Full access to system keyspace | + +--- + +## Appendix F: Server-Side Protocol Flows + +**Source:** `fdbserver/commitproxy/CommitProxyServer.actor.cpp`, +`fdbserver/clustercontroller/ClusterRecovery.cpp`, +`fdbserver/storageserver/storageserver.actor.cpp`, +`fdbserver/sequencer/masterserver.cpp` + +### F.1 Transaction Commit Path + +This is the critical write path: how a client's `CommitTransactionRequest` +becomes durable. The commit proxy orchestrates a multi-phase protocol. + +``` +┌────────┐ ┌──────────────┐ ┌────────┐ ┌──────────┐ ┌───────┐ +│ Client │────▶│ Commit Proxy │────▶│ Master │ │ Resolver │ │ TLog │ +└────────┘ └──────────────┘ └────────┘ └──────────┘ └───────┘ + │ │ │ │ + │ GetCommitVersion │ │ │ + │─────────────────▶│ │ │ + │ Reply(version) │ │ │ + │◀─────────────────│ │ │ + │ │ │ + │ ResolveTransactionBatchRequest │ │ + │─────────────────────────────────▶│ │ + │ Reply(committed[], conflicts) │ │ + │◀─────────────────────────────────│ │ + │ │ + │ TLogCommitRequest │ + │─────────────────────────────────────────────────▶│ + │ TLogCommitReply │ + │◀─────────────────────────────────────────────────│ + │ │ │ + │ ReportRawCommittedVersion │ + │─────────────────▶│ │ +``` + +**Phase 1: Get Commit Version** + +The commit proxy batches incoming `CommitTransactionRequest` messages and +requests a commit version from the master: + +``` +CommitProxy → Master: GetCommitVersionRequest + spanContext: SpanContext + requestNum: uint64_t (monotonic per-proxy) + mostRecentProcessedRequestNum: uint64_t + requestingProxy: UID +``` + +``` +Master → CommitProxy: GetCommitVersionReply + resolverChanges: Standalone> + resolverChangesVersion: Version + version: Version (assigned commit version) + prevVersion: Version + requestNum: uint64_t +``` + +The master assigns monotonically increasing versions. If resolver topology has +changed, it includes `resolverChanges` so the proxy knows which resolver +handles which key ranges. + +**Phase 2: Conflict Resolution** + +The proxy sends the transaction batch to each resolver. Each resolver is +responsible for a range of keys. Transactions whose read/write conflict ranges +span multiple resolvers are sent to all relevant resolvers. + +``` +CommitProxy → Resolver[i]: ResolveTransactionBatchRequest + spanContext: SpanContext + prevVersion: Version + version: Version + lastReceivedVersion: Version + transactions: VectorRef + txnStateTransactions: VectorRef + debugID: Optional + writtenTags: std::set + lastShardMove: Version + arena: Arena +``` + +``` +Resolver[i] → CommitProxy: ResolveTransactionBatchReply + committed: VectorRef (per-txn: 0=ok, 1=conflict) + stateMutations: VectorRef> + conflictingKeyRangeMap: std::map> + privateMutations: VectorRef + privateMutationCount: uint32_t + tpcvMap: std::unordered_map + writtenTags: std::set + lastShardMove: Version + arena: Arena +``` + +The proxy merges results from all resolvers. A transaction is committed only +if all resolvers agree it has no conflicts. + +**Phase 3: TLog Push** + +For committed transactions, the proxy serializes mutations into a `LogPushData` +object. Each mutation is tagged with the `Tag` of the storage server(s) that +should receive it. The proxy then pushes to all TLog replicas: + +``` +CommitProxy → TLog[j]: TLogCommitRequest + spanContext: SpanContext + prevVersion: Version + version: Version + knownCommittedVersion: Version + minKnownCommittedVersion: Version + seqPrevVersion: Version + messages: StringRef (packed, tagged mutations) + tLogCount: uint16_t + tLogLocIds: std::vector + debugID: Optional + arena: Arena +``` + +The commit is durable when a quorum of TLog replicas acknowledge: + +``` +TLog[j] → CommitProxy: TLogCommitReply + version: Version (durable version) +``` + +**Phase 4: Report Committed Version** + +``` +CommitProxy → Master: ReportRawCommittedVersionRequest + version: Version + locked: bool + metadataVersion: Optional + minKnownCommittedVersion: Version + prevVersion: Optional + writtenTags: Optional> +``` + +The master updates its committed version. The proxy then replies to the +original client with `CommitID`. + +### F.2 Recovery Protocol + +Recovery happens when the cluster controller detects that the master has +failed, or during initial cluster startup. It is a complex multi-phase +protocol that establishes a new transaction system generation. + +``` +Phase 0: CC recruits new roles +Phase 1: Lock old TLogs to freeze previous generation +Phase 2: Determine recovery point (last committed version) +Phase 3: Recruit and initialize new TLogs +Phase 4: Initialize resolvers and proxies with transaction state +Phase 5: Write new DBCoreState to coordinators +Phase 6: Begin accepting commits +``` + +**Phase 0: Role Recruitment** + +``` +ClusterController → Workers: RecruitFromConfigurationRequest + configuration: DatabaseConfiguration + recruitSeedServers: bool + maxOldLogRouters: int +``` + +``` +Workers → ClusterController: RecruitFromConfigurationReply + tLogs: std::vector + satelliteTLogs: std::vector + commitProxies: std::vector + grvProxies: std::vector + resolvers: std::vector + storageServers: std::vector + oldLogRouters: std::vector + backupWorkers: std::vector + dcId: Optional + satelliteFallback: bool +``` + +**Phase 1: Lock Old TLogs** + +The master sends a lock request to every TLog from the previous generation: + +``` +Master → OldTLog[i]: TLogInterface::lock (ReplyPromise) +``` + +``` +OldTLog[i] → Master: TLogLockResult + end: Version (last version on this TLog) + knownCommittedVersion: Version + unknownCommittedVersions: std::deque + id: UID + logId: UID +``` + +The master gathers all lock results to determine the recovery point: the +highest version that reached a quorum of TLogs. + +**Phase 2: Initialize New TLogs** + +``` +ClusterController → NewTLog[j]: InitializeTLogRequest + recruitmentID: UID + recoverFrom: LogSystemConfig (previous generation's config) + recoverAt: Version + knownCommittedVersion: Version + epoch: LogEpoch + recoverTags: std::vector + allTags: std::vector + logVersion: TLogVersion + storeType: KeyValueStoreType + spillType: TLogSpillType + remoteTag: Tag + locality: int8_t + isPrimary: bool + startVersion: Version + logRouterTags: int + txsTags: int + recoveryTransactionVersion: Version + oldGenerationRecoverAtVersions: std::vector +``` + +``` +NewTLog[j] → ClusterController: TLogInterface + (the new TLog's interface, with all its RequestStream endpoints) +``` + +**Phase 3: Initialize Proxies and Resolvers** + +``` +CC → NewProxy: InitializeCommitProxyRequest + master: MasterInterface + masterLifetime: LifetimeToken + recoveryCount: uint64_t + recoveryTransactionVersion: Version + firstProxy: bool + encryptMode: EncryptionAtRestModeDeprecated + commitProxyIndex: uint16_t +``` + +``` +CC → NewResolver: InitializeResolverRequest + masterLifetime: LifetimeToken + recoveryCount: uint64_t + commitProxyCount: int + resolverCount: int + masterId: UID + encryptMode: EncryptionAtRestModeDeprecated +``` + +**Phase 4: Broadcast Transaction State** + +The master broadcasts the transaction system state (metadata about shards, +server assignments, etc.) to all resolvers: + +``` +Master → Resolver[i]: TxnStateRequest (sent in chunks) + data: VectorRef (transaction state KV pairs) + sequence: Sequence (chunk sequence number) + last: bool (true for final chunk) + broadcastInfo: std::vector +``` + +**Phase 5: Write to Coordinators** + +The master writes the new `DBCoreState` to the coordination service using +`GenerationRegWriteRequest`. This is the atomic commit point — once this +write succeeds, the new generation is official. + +**Phase 6: Accept Commits** + +The master updates `ServerDBInfo` and broadcasts it to all workers: + +``` +CC → Workers: UpdateServerDBInfoRequest + serializedDbInfo: Standalone (serialized ServerDBInfo) + broadcastInfo: std::vector +``` + +The master then sets `recoveryReadyForCommits`, allowing commit proxies to +begin processing client transactions. + +### F.3 Storage Server Data Flow + +Storage servers pull mutations from TLogs using a continuous peek cursor. +The storage server calls `logSystem->peekSingle()` which creates a cursor +for its assigned tag. By default (knob `PEEK_USING_STREAMING = false`), this +uses regular `TLogPeekRequest` messages. If `PEEK_USING_STREAMING` is enabled, +it uses the streaming `TLogPeekStreamRequest` variant instead. + +**Steady-State Mutation Pull (default, non-streaming):** + +``` +StorageServer → TLog: TLogPeekRequest + begin: Version + tag: Tag (this SS's tag) + returnIfBlocked: bool + onlySpilled: bool + sequence: Optional> + end: Optional + returnEmptyIfStopped: Optional +``` + +``` +TLog → StorageServer: TLogPeekReply + messages: StringRef (packed mutations) + end: Version + popped: Optional + maxKnownVersion: Version + minKnownCommittedVersion: Version + begin: Optional + onlySpilled: bool + arena: Arena +``` + +**Streaming variant (when `PEEK_USING_STREAMING = true`):** + +``` +StorageServer → TLog: TLogPeekStreamRequest + begin: Version + tag: Tag + returnIfBlocked: bool + limitBytes: int + end: Optional + returnEmptyIfStopped: Optional + reply: ReplyPromiseStream +``` + +The storage server deserializes mutations from `messages`, applies them to its +in-memory versioned data structure, and periodically makes them durable. + +**Pop (Garbage Collection):** + +Once a storage server has durably written mutations, it tells the TLog to +discard them: + +``` +StorageServer → TLog: TLogPopRequest + to: Version (pop up to this version) + durableKnownCommittedVersion: Version + tag: Tag +``` + +**Shard Movement (Data Distribution):** + +When data distribution moves a shard to a new storage server: + +1. The new SS begins peeking the TLog for the shard's tag at the transfer version. +2. The new SS uses `Transaction::getRange()` (or `getRangeStream()` if + `FETCH_USING_STREAMING` is enabled) to read the shard's data. This goes through + the normal client path: commit proxy for location lookup, then storage server + reads via `GetKeyValuesRequest` (or `GetKeyValuesStreamRequest`). The reads + target the old storage servers that still hold the shard. +3. Once caught up, the new SS transitions the shard to `ReadWrite` state. +4. Data distribution updates the shard map (via system key transactions through + the commit proxy), and future `GetKeyServerLocationsReply` messages will + direct clients to the new SS. diff --git a/design/AI-generated/README.md b/design/AI-generated/README.md new file mode 100644 index 00000000000..d7e35407c0a --- /dev/null +++ b/design/AI-generated/README.md @@ -0,0 +1,63 @@ +# AI-generated FDB code overview + +This directory contains an AI-generated code overview of the current `main` code base as of April, 2026. + +This was generated using a commercially available coding assistant in widespread use. We are not disclosing +the specific model used. It took about an hour and cost about $20 in API usage. + +Prompts: + +*Ok I want you to study the foundationdb code in the current + directory. It should be unmodified main. I am not interested in + specific diffs or PRs. I am interested in understanding the entire + system. What are the major modules? Do not rely too much on + filenames. Instead try to figure out the flow of data through the + system and the dynamic relationships more than just the static + arrangements. I would like some assessment of the code in terms of + a partition of subsystems that covers materially the entire code + base, but not too granular. I am thinking of 10-15 subsystems at + most (or fewer if that is justified by the structure of the code). + Tell me the major subsystems, the principle files in each, what they + are responsible for, and so on. Then we can do deep dives on each + one. But for now I want an overview of the whole system.* + +This generated the file [`foundationdb_subsystem_map.md`](foundationdb_subsystem_map.md). + +Subsequently: + +*for each of the 12 subsystems identified above, please study the code + in depth and generate a module-specific + architecture/design/implementation overview, focusing on the key data + structures, methods, and data flow in the system. Please save each + subsystem-specific overview to its own .md file* + +This generated the 12 files whose names have a `subsystem_` prefix. + +## Rationale and relationship to existing documentation + +Basically we did this on a Friday afternoon to see what the AI would +come up with. On cursory examination it seems pretty reasonable. We +were pleased to note that the first thing mentioned in the first file +we reviewed is `SAV`, which is of central importance to the system but +is buried in `flow.h` and is hardly mentioned in ../*.md. + +In any case, we hope this is usable as a starting point for people new +to the system or new to specific areas. It is meant to supplement, +not replace, the existing documentation in the parent directory. + +Note that the industry standard for developer-maintained documentation +of software systems (not just FDB) is generally considered to be +*widely uneven*. Some areas have in-depth, well-written deep dives. +Some areas are out of date. Some areas are not covered at all. This +is just the nature of things. AI-generated documentation, by +contrast, will be more comprehensive in coverage, but not quite as +in-depth. It might have random errors and omissions, but +human-maintained documentation certainly suffers from that too. + +## Caveats + +We have not reviewed this in detail. Errors/omissions will be +incrementally addressed. Feel free to send PRs to make corrections. +At some point in a few quarters it may be appropriate to completely +regenerate this documentation with more advanced AI models or +more creative prompts. diff --git a/design/AI-generated/diagram_01_flow_runtime.md b/design/AI-generated/diagram_01_flow_runtime.md new file mode 100644 index 00000000000..3fc47498cc6 --- /dev/null +++ b/design/AI-generated/diagram_01_flow_runtime.md @@ -0,0 +1,76 @@ +# Flow Runtime — Internal Architecture + +```mermaid +graph TB + subgraph ActorSystem["Actor System"] + Actor["ACTOR function\n(.actor.cpp source)"] + Compiler["Actor Compiler\n(Python preprocessor)"] + StateMachine["Generated State Machine\n(.actor.g.h)"] + Coro["C++20 Coroutine\n(co_await migration)"] + end + + subgraph AsyncPrimitives["Async Primitives"] + Promise["Promise<T>"] + Future["Future<T>"] + SAV["SAV\n(SingleAssignmentVar)"] + PS["PromiseStream<T>"] + FS["FutureStream<T>"] + end + + subgraph EventLoop["Event Loop (Net2)"] + Scheduler["Priority Task Scheduler"] + ASIO["Boost.ASIO\n(I/O multiplex)"] + Timers["Timer Queue"] + Yields["Yield Points"] + end + + subgraph Memory["Memory Management"] + Arena["Arena"] + StringRef["StringRef / KeyRef\n(non-owning view)"] + Standalone["Standalone<T>\n(owning)"] + RefCounted["Reference<T>\n(intrusive refcount)"] + end + + subgraph Diagnostics["Diagnostics"] + Trace["TraceEvent\n(.detail() chains)"] + DRandom["deterministicRandom()\n(seedable PRNG)"] + CodeProbe["CODE_PROBE\n(coverage marks)"] + Buggify["BUGGIFY\n(fault injection)"] + end + + Actor --> Compiler --> StateMachine + Actor -.->|migrating to| Coro + StateMachine --> Future + Promise --> SAV --> Future + PS --> FS + + Future --> Scheduler + Timers --> Scheduler + Scheduler --> ASIO + + Arena --> StringRef + Arena --> Standalone + + style ActorSystem fill:#e1f0ff,stroke:#4a90d9 + style AsyncPrimitives fill:#fff3e0,stroke:#f5a623 + style EventLoop fill:#e8f5e9,stroke:#4caf50 + style Memory fill:#fce4ec,stroke:#e91e63 + style Diagnostics fill:#f3e5f5,stroke:#9c27b0 +``` +## Future/Promise Lifecycle + +```mermaid +sequenceDiagram + participant Producer as Producer Actor + participant SAV as SAV (SharedState) + participant Consumer as Consumer Actor + participant Scheduler as Net2 Scheduler + + Producer->>SAV: Promise created (SAV allocated) + Consumer->>SAV: Future obtained (refcount++) + Consumer->>SAV: wait(future) — register callback + Note over Consumer: Suspended + Producer->>SAV: promise.send(value) + SAV->>Scheduler: enqueue callback + Scheduler->>Consumer: Resume with value +``` diff --git a/design/AI-generated/diagram_02_rpc_transport.md b/design/AI-generated/diagram_02_rpc_transport.md new file mode 100644 index 00000000000..17371aa7fcc --- /dev/null +++ b/design/AI-generated/diagram_02_rpc_transport.md @@ -0,0 +1,109 @@ +# RPC & Transport — Internal Architecture + +```mermaid +graph TB + subgraph Transport["FlowTransport (singleton)"] + SendR["sendReliable()"] + SendU["sendUnreliable()"] + Listen["listen()"] + end + + subgraph Peers["Peer Management"] + Peer["Peer\n(per-destination)"] + UnsentQ["Unsent Queue"] + ReliableQ["Reliable Packet List"] + ConnFuture["Connection Future\n(lazy connect)"] + end + + subgraph Addressing["Addressing"] + Endpoint["Endpoint\n= (NetworkAddressList, Token)"] + EndpointMap["EndpointMap\n(Token → Receiver)"] + WellKnown["Well-Known Tokens\n(system services)"] + end + + subgraph RequestReply["Request/Reply Pattern"] + ReqStream["RequestStream<Req>"] + RP["ReplyPromise<Rep>\n(self-addressed envelope)"] + Interface["Server Interface\n(struct of RequestStreams)"] + end + + subgraph Monitoring["Failure Monitoring"] + FM["FailureMonitor"] + OnFailed["onFailed(Endpoint)\n→ Future<Void>"] + OnStateChange["onStateChanged(Endpoint)"] + end + + subgraph SimNetwork["Simulation Network (Sim2)"] + VirtProcess["Virtual Processes\n(in-thread)"] + VirtTime["Deterministic Time"] + Clogging["Network Clogging"] + Partitions["Network Partitions"] + FaultInj["Fault Injection\n(kill process/machine/DC)"] + end + + Interface --> ReqStream + ReqStream --> SendR + SendR --> Peer --> UnsentQ + Peer --> ReliableQ + Peer --> ConnFuture + + Listen --> EndpointMap --> Endpoint + RP --> SendR + + FM --> OnFailed + FM --> OnStateChange + + SimNetwork -.->|replaces| Transport + + style Transport fill:#e1f0ff,stroke:#4a90d9 + style Peers fill:#fff3e0,stroke:#f5a623 + style Addressing fill:#e8f5e9,stroke:#4caf50 + style RequestReply fill:#fce4ec,stroke:#e91e63 + style Monitoring fill:#f3e5f5,stroke:#9c27b0 + style SimNetwork fill:#fffde7,stroke:#fbc02d +``` + +## Message Flow Between Processes + +```mermaid +sequenceDiagram + participant A as Actor A (Process 1) + participant FT1 as FlowTransport (P1) + participant Net as Network / Sim2 + participant FT2 as FlowTransport (P2) + participant EM as EndpointMap (P2) + participant B as Actor B (Process 2) + + A->>FT1: send(Endpoint, Request{ReplyPromise}) + FT1->>FT1: serialize(Request) + FT1->>Net: TCP send / sim route + Net->>FT2: deliver packet + FT2->>EM: lookup(Token) + EM->>B: receive(Request) + B->>B: process request + B->>FT2: ReplyPromise.send(Reply) + FT2->>Net: TCP send / sim route + Net->>FT1: deliver reply + FT1->>A: Future resolved with Reply +``` + +## Locality & Placement + +```mermaid +graph LR + subgraph LocalityData + Zone["zoneId"] + Machine["machineId"] + DC["dcId"] + DataHall["data_hall"] + end + + subgraph ProcessClass + Fitness["Fitness for Role"] + ClassType["ClassType\n(storage, transaction,\nresolution, stateless)"] + end + + LocalityData --> ReplicationPolicy["Replication Policy\n(e.g. 3 zones)"] + ProcessClass --> Recruitment["CC Role Recruitment"] + ReplicationPolicy --> TeamBuilding["DD Team Building"] +``` diff --git a/design/AI-generated/diagram_03_client_library.md b/design/AI-generated/diagram_03_client_library.md new file mode 100644 index 00000000000..944e6f80b51 --- /dev/null +++ b/design/AI-generated/diagram_03_client_library.md @@ -0,0 +1,104 @@ +# Client Library — Internal Architecture + +```mermaid +graph TB + subgraph Layers["Transaction Layers (outside → in)"] + App["Application Code"] + CAPI["C API (fdb_c.h)\n/ Language Bindings"] + MV["MultiVersionTransaction\n(protocol negotiation)"] + RYW["ReadYourWritesTransaction\n(local write map + cache)"] + Native["NativeAPI / Transaction\n(raw operations)"] + end + + subgraph ReadPath["Read Operations"] + Get["get(key)"] + GetRange["getRange(begin, end)"] + LocCache["Location Cache\n(KeyRange → SS list)"] + SS["Storage Server"] + end + + subgraph WritePath["Write Operations"] + Set["set(key, value)"] + Clear["clear(key/range)"] + WriteMap["Local Write Map"] + Commit["commit()"] + CP["Commit Proxy"] + end + + subgraph VersionMgmt["Version Management"] + GRV["getReadVersion()"] + GRVProxy["GRV Proxy"] + Watch["watch(key)"] + end + + subgraph Context["DatabaseContext"] + ProxyLB["Proxy Load Balancer\n(round-robin + failover)"] + RetryLoop["Retry Loop\n(onError → backoff)"] + MonLeader["monitorLeader()\n(cluster file → CC)"] + end + + App --> CAPI --> MV --> RYW --> Native + + Native --> Get + Native --> GetRange + Get --> LocCache + GetRange --> LocCache + LocCache -->|cache hit| SS + LocCache -->|cache miss| CP + + Native --> Set --> WriteMap + Native --> Clear --> WriteMap + Native --> Commit --> CP + + RYW -->|reads check| WriteMap + RYW -->|then merges| SS + + Native --> GRV --> GRVProxy + Native --> Watch --> SS + + ProxyLB --> CP + ProxyLB --> GRVProxy + RetryLoop --> Native + + style Layers fill:#e1f0ff,stroke:#4a90d9 + style ReadPath fill:#e8f5e9,stroke:#4caf50 + style WritePath fill:#fff3e0,stroke:#f5a623 + style VersionMgmt fill:#f3e5f5,stroke:#9c27b0 + style Context fill:#fffde7,stroke:#fbc02d +``` + +## Read-Your-Writes Merge + +```mermaid +graph LR + subgraph RYW["ReadYourWritesTransaction"] + Read["read(key)"] + WM["Write Map\n(local mutations)"] + SC["Snapshot Cache\n(previous reads)"] + end + + Read --> WM + WM -->|hit: locally set| Result["Return local value"] + WM -->|hit: locally cleared| Cleared["Return not found"] + WM -->|miss| SC + SC -->|hit| Result2["Return cached"] + SC -->|miss| SS["Storage Server read"] + SS --> Merge["Merge SS result\nwith write map"] + Merge --> Result3["Return merged"] +``` + +## Transaction Retry Loop + +```mermaid +stateDiagram-v2 + [*] --> GetReadVersion + GetReadVersion --> Execute: version V + Execute --> Commit: mutations ready + Commit --> Success: committed + Commit --> OnError: not_committed / too_old + Execute --> OnError: error during read + OnError --> GetReadVersion: retryable (reset + backoff) + OnError --> Failed: non-retryable + Success --> [*] + Failed --> [*] +``` diff --git a/design/AI-generated/diagram_04_cluster_controller.md b/design/AI-generated/diagram_04_cluster_controller.md new file mode 100644 index 00000000000..bbaef2f44e7 --- /dev/null +++ b/design/AI-generated/diagram_04_cluster_controller.md @@ -0,0 +1,107 @@ +# Cluster Controller & Coordination — Internal Architecture + +```mermaid +graph TB + subgraph Election["Leader Election"] + Coord1["Coordinator 1"] + Coord2["Coordinator 2"] + Coord3["Coordinator 3"] + GenReg["Generation Register\n(monotonic gen numbers)"] + Cand["CC Candidates\n(CandidacyRequest)"] + end + + subgraph CC["Cluster Controller"] + WorkerPool["Worker Registry\n(ProcessClass + Locality)"] + Recruit["Role Recruitment"] + Monitor["Health Monitoring"] + ServerDBInfo["ServerDBInfo\n(cluster-wide broadcast)"] + end + + subgraph Roles["Recruited Roles"] + Master["Master / Sequencer"] + CPr["Commit Proxies"] + GRVPr["GRV Proxies"] + Resolvers["Resolvers"] + TLogs["TLogs"] + DDist["Data Distributor"] + RK["Ratekeeper"] + CS["Consistency Scan"] + end + + subgraph Workers["Worker Processes"] + W1["Worker 1\n(transaction class)"] + W2["Worker 2\n(storage class)"] + W3["Worker 3\n(stateless class)"] + Wn["Worker N"] + end + + Cand --> Coord1 + Cand --> Coord2 + Cand --> Coord3 + Coord1 --> GenReg + Coord2 --> GenReg + Coord3 --> GenReg + GenReg -->|majority agrees| CC + + W1 -->|registrationClient()| WorkerPool + W2 -->|registrationClient()| WorkerPool + W3 -->|registrationClient()| WorkerPool + Wn -->|registrationClient()| WorkerPool + WorkerPool --> Recruit + Recruit -->|fitness + locality| Roles + + Monitor -->|process failure| Recruit + CC --> ServerDBInfo -->|broadcast| Workers + + style Election fill:#f3e5f5,stroke:#9c27b0 + style CC fill:#e1f0ff,stroke:#4a90d9 + style Roles fill:#fff3e0,stroke:#f5a623 + style Workers fill:#e8f5e9,stroke:#4caf50 +``` + +## Leader Election Protocol + +```mermaid +sequenceDiagram + participant C1 as CC Candidate 1 (better fitness) + participant C2 as CC Candidate 2 + participant Coord as Coordinators (majority) + + C1->>Coord: CandidacyRequest (fitness in changeID) + C2->>Coord: CandidacyRequest (fitness in changeID) + Coord->>Coord: Compare fitness (top bits of changeID) + Coord-->>C1: Nominated as leader + Coord-->>C2: Not nominated + C1->>C1: Become CC + C2->>C2: monitorLeader() — defer to C1 + Note over C1: Begins recruiting roles +``` + +## Role Recruitment Decision + +```mermaid +graph TD + Need["Need to fill role\n(e.g., CommitProxy)"] + Need --> Filter["Filter workers by\nProcessClass fitness"] + Filter --> Locality["Apply locality constraints\n(DC, zone, machine)"] + Locality --> Exclude["Exclude failed /\nexcluded processes"] + Exclude --> Best["Select best candidate"] + Best --> Recruit["Send recruitment request"] + Recruit --> Running["Role running on worker"] +``` + +## ServerDBInfo Broadcast + +```mermaid +graph LR + CC["Cluster Controller"] -->|update| Info["ServerDBInfo"] + + Info --> |contains| M["Master interface"] + Info --> |contains| PL["Proxy list"] + Info --> |contains| LSC["Log system config"] + Info --> |contains| RS["Recovery state"] + Info --> |contains| LB["Latency band config"] + + Info -->|broadcast to all| W["All Workers"] + W -->|react to changes| Roles["Update connections\nto new proxies, etc."] +``` diff --git a/design/AI-generated/diagram_05_commit_pipeline.md b/design/AI-generated/diagram_05_commit_pipeline.md new file mode 100644 index 00000000000..b962555c469 --- /dev/null +++ b/design/AI-generated/diagram_05_commit_pipeline.md @@ -0,0 +1,107 @@ +# Transaction Commit Pipeline — Internal Architecture + +```mermaid +graph TB + subgraph GRVPath["GRV Path (Read Versions)"] + Client1["Client"] + GRVProxy["GRV Proxy"] + RK["Ratekeeper"] + end + + subgraph CommitPath["Commit Path (4 Phases)"] + Client2["Client"] + Batcher["commitBatcher\n(batches by byte size)"] + CP["Commit Proxy"] + Seq["Master / Sequencer"] + Res["Resolver(s)\n(sharded by key range)"] + TLog["TLog (quorum write)"] + end + + Client1 -- "GetReadVersionRequest" --> GRVProxy + RK -- "rate limit" --> GRVProxy + GRVProxy -- "read version V" --> Client1 + + Client2 -- "CommitTransactionRequest" --> Batcher --> CP + CP -- "① GetCommitVersion" --> Seq + Seq -- "version V + prevVersion" --> CP + CP -- "② ResolveTransactionBatch" --> Res + Res -- "conflict/no-conflict per txn" --> CP + CP -- "③ LogSystem::push()" --> TLog + TLog -- "quorum ack" --> CP + CP -- "④ CommitTransactionReply" --> Client2 + + style GRVPath fill:#e1f0ff,stroke:#4a90d9 + style CommitPath fill:#fff3e0,stroke:#f5a623 +``` + +## Commit Proxy: Batch Processing Pipeline + +```mermaid +graph LR + subgraph Intake["Intake"] + Requests["Incoming\nCommitRequests"] + Batcher["Batcher\n(group by size/count)"] + end + + subgraph Phase1["Phase 1: Version"] + GetVer["GetCommitVersion\nfrom Sequencer"] + end + + subgraph Phase2["Phase 2: Resolve"] + Split["Split batch by\nresolver key range"] + R1["Resolver 1"] + R2["Resolver 2"] + Merge["Merge conflict\nresults"] + end + + subgraph Phase3["Phase 3: Log"] + TagMut["Tag mutations\n(key → shard → SS tag)"] + Push["LogSystem::push()\nto TLog quorum"] + end + + subgraph Phase4["Phase 4: Reply"] + Reply["Send replies\n(committed / conflicted)"] + end + + Requests --> Batcher --> GetVer --> Split + Split --> R1 --> Merge + Split --> R2 --> Merge + Merge --> TagMut --> Push --> Reply +``` + +## Resolver: Conflict Detection + +```mermaid +graph TB + subgraph Window["Sliding Conflict Window"] + Committed["Committed write ranges\n(version-ordered)"] + Incoming["Incoming txn batch\n(read version + read ranges)"] + end + + Incoming --> Check{"Any read range overlaps\nwith writes at versions\n> txn's read version?"} + Check -->|yes| Conflict["CONFLICT\n(txn rejected)"] + Check -->|no| NoConflict["NO CONFLICT\n(txn proceeds)"] + NoConflict --> Update["Add txn's write ranges\nto committed window"] + Update --> Expire["Expire old entries\nbeyond window"] + + style Conflict fill:#fce4ec,stroke:#e91e63 + style NoConflict fill:#e8f5e9,stroke:#4caf50 +``` + +## Master/Sequencer: Version Assignment + +```mermaid +sequenceDiagram + participant CP1 as Commit Proxy 1 + participant CP2 as Commit Proxy 2 + participant Seq as Sequencer + + Note over Seq: Maintains: nextVersion, prevVersion + CP1->>Seq: GetCommitVersion(lastVersion=100) + Seq->>Seq: nextVersion = max(wall_clock, prev+1) + Seq-->>CP1: version=150, prevVersion=100 + CP2->>Seq: GetCommitVersion(lastVersion=120) + Seq->>Seq: nextVersion = max(wall_clock, prev+1) + Seq-->>CP2: version=200, prevVersion=150 + Note over Seq: Monotonic, roughly tracks real time +``` diff --git a/design/AI-generated/diagram_06_tlog_logsystem.md b/design/AI-generated/diagram_06_tlog_logsystem.md new file mode 100644 index 00000000000..1e6939b1351 --- /dev/null +++ b/design/AI-generated/diagram_06_tlog_logsystem.md @@ -0,0 +1,139 @@ +# TLog & Log System — Internal Architecture + +```mermaid +graph TB + subgraph LogSystem["Log System Abstraction"] + LogSystemType["LogSystem"] + Push["push(mutations, version)\n→ quorum write"] + Peek["peek(tag, fromVersion)\n→ IPeekCursor"] + end + + subgraph TLogInstance["TLog Server Instance"] + Receive["Receive push\nfrom Commit Proxy"] + DiskQ["DiskQueue\n(append-only log)"] + MemIdx["In-Memory Index\n(tag → version → data)"] + PeekServe["Serve peek requests\nfrom Storage Servers"] + end + + subgraph Replication["Replication"] + TLog1["TLog 1"] + TLog2["TLog 2"] + TLog3["TLog 3"] + Quorum["Quorum: f+1 of 2f+1"] + end + + subgraph MultiRegion["Multi-Region"] + PrimaryTLogs["Primary TLogs"] + LogRouter["Log Router"] + RemoteTLogs["Remote TLogs"] + SatTLogs["Satellite TLogs"] + end + + subgraph Consumers["Log Consumers"] + SS["Storage Servers\n(peek by tag)"] + BW["Backup Worker\n(peek mutations)"] + end + + LogSystemType --> Push + LogSystemType --> Peek + + Push --> TLog1 + Push --> TLog2 + Push --> TLog3 + TLog1 --> Quorum + TLog2 --> Quorum + TLog3 --> Quorum + + Receive --> DiskQ + Receive --> MemIdx + MemIdx --> PeekServe + + PrimaryTLogs --> LogRouter --> RemoteTLogs + PrimaryTLogs --> SatTLogs + + SS --> Peek + BW --> Peek + + style LogSystem fill:#e1f0ff,stroke:#4a90d9 + style TLogInstance fill:#fff3e0,stroke:#f5a623 + style Replication fill:#e8f5e9,stroke:#4caf50 + style MultiRegion fill:#f3e5f5,stroke:#9c27b0 + style Consumers fill:#fffde7,stroke:#fbc02d +``` + +## Tag-Partitioned Mutation Flow + +```mermaid +sequenceDiagram + participant CP as Commit Proxy + participant LS as LogSystem + participant T1 as TLog 1 + participant T2 as TLog 2 + participant T3 as TLog 3 + participant SS_A as Storage Server A (tag=2) + participant SS_B as Storage Server B (tag=5) + + CP->>CP: Tag mutations by destination SS + Note over CP: set("apple","1") → tag=2 (SS_A's shard)
set("zebra","2") → tag=5 (SS_B's shard) + CP->>LS: push(tagged mutations, version V) + LS->>T1: replicate (all tags) + LS->>T2: replicate (all tags) + LS->>T3: replicate (all tags) + T1-->>LS: ack + T2-->>LS: ack + Note over LS: Quorum (2 of 3) — committed + + SS_A->>T1: peek(tag=2, fromVersion) + T1-->>SS_A: set("apple","1") at V + SS_B->>T2: peek(tag=5, fromVersion) + T2-->>SS_B: set("zebra","2") at V +``` + +## Log Epochs and Recovery + +```mermaid +graph LR + subgraph Epoch1["Epoch 1 (old)"] + OT1["TLog A"] + OT2["TLog B"] + OT3["TLog C"] + end + + subgraph Epoch2["Epoch 2 (old)"] + OT4["TLog D"] + OT5["TLog E"] + OT6["TLog F"] + end + + subgraph Epoch3["Epoch 3 (current)"] + NT1["TLog G"] + NT2["TLog H"] + NT3["TLog I"] + end + + Epoch1 -->|"recovery: lock + peek"| Epoch2 + Epoch2 -->|"recovery: lock + peek"| Epoch3 + + SS["Storage Server"] -->|"peek across epochs\n(merged cursor)"| Epoch1 + SS -->|"peek across epochs\n(merged cursor)"| Epoch2 + SS -->|"peek across epochs\n(merged cursor)"| Epoch3 + + Note1["Old epochs kept until\nall SS have consumed\ntheir data, then GC'd"] + + style Epoch1 fill:#fffde7,stroke:#fbc02d + style Epoch2 fill:#fff3e0,stroke:#f5a623 + style Epoch3 fill:#e8f5e9,stroke:#4caf50 +``` + +## TLog Internal Write Path + +```mermaid +graph TD + Push["push(mutations, version)"] + Push --> Deser["Deserialize mutations"] + Deser --> TagIndex["Index by tag\nin memory"] + Deser --> DiskWrite["Append to DiskQueue\n(sequential I/O)"] + DiskWrite --> Fsync["fsync / fdatasync"] + Fsync --> Ack["Acknowledge to\nCommit Proxy"] + TagIndex --> ServePeek["Serve peek(tag)\nfrom memory or disk"] +``` diff --git a/design/AI-generated/diagram_07_storage_server.md b/design/AI-generated/diagram_07_storage_server.md new file mode 100644 index 00000000000..7a789480fe9 --- /dev/null +++ b/design/AI-generated/diagram_07_storage_server.md @@ -0,0 +1,119 @@ +# Storage Server & Engines — Internal Architecture + +```mermaid +graph TB + subgraph UpdateLoop["Update Loop (pull from log)"] + Peek["peek(tag, fromVersion)\nfrom TLog"] + Deser["Deserialize mutations"] + Apply["Apply to KVStore"] + AdvVer["Advance durable version"] + end + + subgraph ReadServing["Read Serving"] + GetVal["GetValueRequest"] + GetKey["GetKeyRequest"] + GetRange["GetKeyValuesRequest"] + VerCheck["Version check\n(oldest ≤ V ≤ current)"] + ShardCheck["Shard ownership check"] + end + + subgraph Versions["Version Management"] + StorageVer["storageVersion\n(latest applied)"] + DurableVer["durableVersion\n(latest fsynced)"] + OldestVer["desiredOldestVersion\n(GC boundary)"] + end + + subgraph Engine["KV Engine (pluggable)"] + IKV["IKeyValueStore interface"] + RocksDB["RocksDB\n(primary production)"] + ShardedRocks["Sharded RocksDB\n(per-shard column families)"] + SQLite["SQLite\n(legacy)"] + MemEngine["Memory Engine\n(testing / txnStateStore)"] + end + + subgraph ShardMgmt["Shard Management"] + Shards["Assigned Shards\n(key ranges)"] + AddShard["addShard\n(from data movement)"] + RemShard["removeShard"] + FetchKeys["fetchKeys\n(bulk copy from source SS)"] + end + + Peek --> Deser --> Apply --> AdvVer + Apply --> IKV + + GetVal --> VerCheck + GetKey --> VerCheck + GetRange --> VerCheck + VerCheck --> ShardCheck --> IKV + IKV --> RocksDB + IKV --> ShardedRocks + IKV --> SQLite + IKV --> MemEngine + + StorageVer --> VerCheck + DurableVer --> VerCheck + OldestVer --> VerCheck + + AddShard --> FetchKeys --> Shards + RemShard --> Shards + + style UpdateLoop fill:#e1f0ff,stroke:#4a90d9 + style ReadServing fill:#e8f5e9,stroke:#4caf50 + style Versions fill:#fff3e0,stroke:#f5a623 + style Engine fill:#fce4ec,stroke:#e91e63 + style ShardMgmt fill:#f3e5f5,stroke:#9c27b0 +``` + +## Storage Server Lifecycle + +```mermaid +stateDiagram-v2 + [*] --> Recruited: DD assigns shard + Recruited --> FetchingKeys: fetchKeys from source SS + FetchingKeys --> WaitingForLog: data fetched, start peeking log + WaitingForLog --> Serving: caught up to current version + Serving --> Serving: continuous peek + apply + serve reads + Serving --> Removed: DD moves shard away + Removed --> [*] +``` + +## Read Request Processing + +```mermaid +sequenceDiagram + participant Client as Client + participant SS as Storage Server + participant KV as KV Engine + + Client->>SS: GetKeyValuesRequest(range, version V) + SS->>SS: Check: oldest ≤ V ≤ storageVersion? + alt Version too old + SS-->>Client: Error: transaction_too_old + else Version too new + SS->>SS: Wait until storageVersion ≥ V + end + SS->>SS: Check: owns shard for range? + alt Shard not owned + SS-->>Client: Error: wrong_shard_server + end + SS->>KV: readRange(range, version V) + KV-->>SS: results + SS-->>Client: GetKeyValuesReply(results) +``` + +## Storage Engine Comparison + +```mermaid +graph LR + subgraph Engines + R["RocksDB"] + SR["Sharded RocksDB"] + SQ["SQLite"] + M["Memory"] + end + + R --- R_desc["LSM-tree, production default\nGood compression, range perf"] + SR --- SR_desc["Per-shard column families\nBetter isolation between shards"] + SQ --- SQ_desc["B-tree, legacy\nStill supported"] + M --- M_desc["In-memory only\nFor txnStateStore, testing"] +``` diff --git a/design/AI-generated/diagram_08_data_distribution.md b/design/AI-generated/diagram_08_data_distribution.md new file mode 100644 index 00000000000..776ee0aeb39 --- /dev/null +++ b/design/AI-generated/diagram_08_data_distribution.md @@ -0,0 +1,129 @@ +# Data Distribution — Internal Architecture + +```mermaid +graph TB + subgraph DD["Data Distributor (singleton)"] + DDMain["DD Main Loop"] + end + + subgraph Tracking["Shard Tracking"] + ShardTracker["DDShardTracker\n(monitors size + load)"] + Split["Split decision\n(shard too large)"] + Merge["Merge decision\n(adjacent shards too small)"] + end + + subgraph Teams["Team Management"] + TeamColl["DDTeamCollection"] + BuildTeams["buildTeams()\n(satisfy replication policy)"] + HealthyTeams["Healthy Teams"] + UnhealthyTeams["Unhealthy Teams\n(missing replicas)"] + SATF["ShardsAffectedByTeamFailure\n(tracks which shards need moves)"] + end + + subgraph Relocation["Relocation Queue"] + RelocQ["DDRelocationQueue\n(prioritized)"] + Priority["Priorities:\n• Emergency (replica lost)\n• High (rebalance unhealthy)\n• Normal (load balance)\n• Low (merge)"] + Parallel["Concurrent moves\n(parallelism limit)"] + end + + subgraph MoveKeys["MoveKeys Protocol"] + Lock["moveKeysLock\n(serialize concurrent moves)"] + Fetch["Dest SS: fetchKeys\nfrom source SS"] + LogPeek["Dest SS: start\npeeking TLog"] + AtomicXfer["Atomic ownership\ntransfer (system key txn)"] + end + + DDMain --> ShardTracker + DDMain --> TeamColl + DDMain --> RelocQ + + ShardTracker --> Split + ShardTracker --> Merge + Split --> RelocQ + Merge --> RelocQ + TeamColl --> BuildTeams --> HealthyTeams + BuildTeams --> UnhealthyTeams + UnhealthyTeams --> SATF --> RelocQ + + RelocQ --> Priority --> Parallel --> MoveKeys + Lock --> Fetch --> LogPeek --> AtomicXfer + + style DD fill:#e1f0ff,stroke:#4a90d9 + style Tracking fill:#e8f5e9,stroke:#4caf50 + style Teams fill:#fff3e0,stroke:#f5a623 + style Relocation fill:#fce4ec,stroke:#e91e63 + style MoveKeys fill:#f3e5f5,stroke:#9c27b0 +``` + +## Shard Move Sequence + +```mermaid +sequenceDiagram + participant DD as Data Distributor + participant RQ as Relocation Queue + participant MK as MoveKeys + participant Src as Source SS + participant Dst as Destination SS + participant TLog as TLog + participant SysKey as System Key Space + + DD->>RQ: RelocateShard(range, priority) + RQ->>MK: execute(range, srcTeam, dstTeam) + MK->>MK: Acquire moveKeysLock + + MK->>Dst: Assign shard (begin fetch) + Dst->>Src: fetchKeys(range) + Src-->>Dst: key-value data (bulk) + + Dst->>TLog: peek(tag, version) — start consuming + TLog-->>Dst: mutations (catch up) + Note over Dst: Catching up to current version... + + Dst-->>MK: Ready (caught up) + + MK->>SysKey: Transaction: update keyServers + serverKeys + Note over MK: Atomic ownership transfer + SysKey-->>MK: committed + + MK->>Src: Remove shard assignment + MK-->>RQ: Move complete + MK->>MK: Release moveKeysLock +``` + +## Team Building + +```mermaid +graph TD + Policy["Replication Policy\ne.g., triple redundancy\nacross 3 zones"] + Policy --> Candidates["Available Storage Servers\n(with LocalityData)"] + Candidates --> Build["Build teams of N servers\nsatisfying zone/DC/rack diversity"] + Build --> Team1["Team {SS1-zoneA, SS2-zoneB, SS3-zoneC}"] + Build --> Team2["Team {SS1-zoneA, SS4-zoneD, SS5-zoneE}"] + Build --> TeamN["..."] + + Team1 --> Health{"All members\nhealthy?"} + Health -->|yes| Healthy["Healthy Team List"] + Health -->|no| Unhealthy["Unhealthy Team List\n→ trigger re-replication"] +``` + +## Reasons for Data Movement + +```mermaid +graph LR + subgraph Triggers + T1["Shard too large\n→ split"] + T2["SS failure\n→ re-replicate"] + T3["Load imbalance\n→ rebalance"] + T4["Config change\n(replication factor)"] + T5["Server exclusion\n(maintenance)"] + T6["Storage wiggle\n(proactive)"] + end + + T1 --> RQ["Relocation Queue"] + T2 --> RQ + T3 --> RQ + T4 --> RQ + T5 --> RQ + T6 --> RQ + RQ --> Move["Execute MoveKeys"] +``` diff --git a/design/AI-generated/diagram_09_cluster_recovery.md b/design/AI-generated/diagram_09_cluster_recovery.md new file mode 100644 index 00000000000..6b86b6e7cae --- /dev/null +++ b/design/AI-generated/diagram_09_cluster_recovery.md @@ -0,0 +1,149 @@ +# Cluster Recovery — Internal Architecture + +```mermaid +stateDiagram-v2 + [*] --> READING_CSTATE: Master failure detected + + state READING_CSTATE { + [*] --> ReadCoord: Read from coordinators + ReadCoord --> ParseDBState: Parse DBCoreState + ParseDBState --> LearnOldEpoch: Learn previous epoch's TLogs + } + + READING_CSTATE --> LOCKING_CSTATE + + state LOCKING_CSTATE { + [*] --> LockOldTLogs: Prevent split-brain + LockOldTLogs --> EndOldEpoch: Old TLogs stop accepting writes + } + + LOCKING_CSTATE --> RECRUITING + + state RECRUITING { + [*] --> RecruitTLogs: New TLog set + RecruitTLogs --> RecruitProxies: CommitProxies + GrvProxies + RecruitProxies --> RecruitResolvers: Resolvers + } + + RECRUITING --> RECOVERY_TRANSACTION + + state RECOVERY_TRANSACTION { + [*] --> PeekOldLogs: Read uncommitted mutations + PeekOldLogs --> ReplayTxnState: Reconstruct txnStateStore + ReplayTxnState --> SetRecoveryVer: Establish recovery version + } + + RECOVERY_TRANSACTION --> WRITING_CSTATE + + state WRITING_CSTATE { + [*] --> WriteCoord: Write new epoch to coordinators + WriteCoord --> [*]: New epoch official + } + + WRITING_CSTATE --> ACCEPTING_COMMITS: Cluster is live + + ACCEPTING_COMMITS --> ALL_LOGS_RECRUITED: Remote TLogs recruited + + ALL_LOGS_RECRUITED --> FULLY_RECOVERED: Old generations consumed + GC'd +``` + +## Recovery Data Flow + +```mermaid +graph TB + subgraph OldEpoch["Old Epoch"] + OldTLogs["Old TLogs\n(locked)"] + OldCState["Old Coordinated State\n(stored on coordinators)"] + end + + subgraph Recovery["Recovery Process"] + ReadCState["① Read coordinated state"] + LockTLogs["② Lock old TLogs"] + PeekReplay["③ Peek + replay\ntxnStateStore"] + RecVer["④ Set recovery version"] + WriteCState["⑤ Write new coordinated\nstate to coordinators"] + end + + subgraph NewEpoch["New Epoch"] + NewTLogs["New TLogs"] + NewProxies["New Proxies"] + NewResolvers["New Resolvers"] + NewMaster["New Master/Sequencer"] + end + + OldCState --> ReadCState + ReadCState --> LockTLogs --> OldTLogs + OldTLogs --> PeekReplay + PeekReplay --> RecVer + RecVer --> WriteCState + + WriteCState --> NewTLogs + WriteCState --> NewProxies + WriteCState --> NewResolvers + WriteCState --> NewMaster + + NewTLogs -.->|"SS peek across epochs"| OldTLogs + + style OldEpoch fill:#fffde7,stroke:#fbc02d + style Recovery fill:#e1f0ff,stroke:#4a90d9 + style NewEpoch fill:#e8f5e9,stroke:#4caf50 +``` + +## Generation GC (trackTlogRecovery) + +```mermaid +sequenceDiagram + participant TR as trackTlogRecovery + participant TLogs as TLog replicas + participant CState as Coordinated State + + loop Poll TLog recovery progress + TR->>TLogs: trackRecovery.getReply() + TLogs-->>TR: oldestUnrecoveredStartVersion + TR->>TR: recoveredVersion = min(all TLogs) + end + + TR->>TR: For each old generation:
if recoverAt < recoveredVersion
AND epoch < oldestBackupEpoch + TR->>CState: Purge old generation data + Note over CState: oldTLogData.resize(i)
removes consumed generations + TR->>CState: Write updated state +``` + +## Multi-Region Recovery + +```mermaid +graph TB + subgraph Primary["Primary DC"] + PCC["Cluster Controller"] + PTLogs["Primary TLogs"] + PProxies["Proxies + Resolvers"] + end + + subgraph Satellite["Satellite DCs"] + STLogs1["Satellite TLogs (DC2)"] + STLogs2["Satellite TLogs (DC4)"] + end + + subgraph Remote["Remote DC"] + RTLogs["Remote TLogs"] + RLogRouter["Log Router"] + end + + PCC -->|recruit| PTLogs + PCC -->|recruit| PProxies + PCC -->|recruit| STLogs1 + PCC -->|recruit| STLogs2 + PCC -->|recruit| RTLogs + PCC -->|recruit| RLogRouter + + PProxies -->|push| PTLogs + PTLogs -->|replicate| STLogs1 + PTLogs -->|replicate| STLogs2 + PTLogs -->|route via| RLogRouter -->|push| RTLogs + + Note1["Recovery states:\nACCEPTING_COMMITS = primary ready\nALL_LOGS_RECRUITED = remote ready\nFULLY_RECOVERED = all generations GC'd"] + + style Primary fill:#e8f5e9,stroke:#4caf50 + style Satellite fill:#fff3e0,stroke:#f5a623 + style Remote fill:#e1f0ff,stroke:#4a90d9 +``` diff --git a/design/AI-generated/diagram_10_ratekeeper.md b/design/AI-generated/diagram_10_ratekeeper.md new file mode 100644 index 00000000000..6e8189d4533 --- /dev/null +++ b/design/AI-generated/diagram_10_ratekeeper.md @@ -0,0 +1,91 @@ +# Rate Keeping & Throttling — Internal Architecture + +```mermaid +graph TB + subgraph Monitoring["What Ratekeeper Monitors"] + SSQueue["Storage Server\nqueue depth"] + TLogQueue["TLog\nqueue depth"] + DiskIO["Storage Server\ndisk throughput / IOPS"] + TagRates["Per-tag\ntransaction rates"] + end + + subgraph RK["Ratekeeper"] + Calc["Rate Calculator"] + TT["TagThrottler\n(per-tag limits)"] + end + + subgraph Control["What Ratekeeper Controls"] + TxnRate["Transaction rate limit\n(to GRV Proxies)"] + BatchRate["Batch priority rate\n(lower priority)"] + TagThrottle["Per-tag throttle\n(slow/reject specific tags)"] + end + + subgraph Enforcement["Enforcement"] + GRVProxy["GRV Proxy"] + Delay["Delay GetReadVersion\nresponses"] + Reject["Reject when\noverthrottled"] + end + + SSQueue --> Calc + TLogQueue --> Calc + DiskIO --> Calc + TagRates --> Calc + Calc --> TxnRate + Calc --> BatchRate + TagRates --> TT --> TagThrottle + + TxnRate --> GRVProxy + BatchRate --> GRVProxy + TagThrottle --> GRVProxy + GRVProxy --> Delay + GRVProxy --> Reject + + style Monitoring fill:#e1f0ff,stroke:#4a90d9 + style RK fill:#fff3e0,stroke:#f5a623 + style Control fill:#e8f5e9,stroke:#4caf50 + style Enforcement fill:#fce4ec,stroke:#e91e63 +``` + +## Back-Pressure Feedback Loop + +```mermaid +sequenceDiagram + participant SS as Storage Servers + participant TLog as TLogs + participant RK as Ratekeeper + participant GRV as GRV Proxy + participant Client as Clients + + SS->>RK: Report queue depth, disk stats + TLog->>RK: Report queue depth + RK->>RK: Calculate safe transaction rate + RK->>GRV: UpdateRateInfo(txnRate, batchRate) + + Client->>GRV: GetReadVersion + alt Under limit + GRV-->>Client: version (immediately) + else Over limit + GRV->>GRV: Delay response + GRV-->>Client: version (delayed) + end + Note over RK: Continuous feedback loop:
SS falls behind → lower rate →
fewer commits → SS catches up →
raise rate +``` + +## Tag-Based Throttling + +```mermaid +graph TD + Txn["Transaction with tag='team-A'"] + Txn --> GRV["GRV Proxy"] + GRV --> CheckTag{"Tag throttled?"} + CheckTag -->|no| Proceed["Assign read version"] + CheckTag -->|yes| CheckRate{"Under tag's\nrate limit?"} + CheckRate -->|yes| Proceed + CheckRate -->|no| ThrottleResp["Reject or delay\n(tag_throttled error)"] + + TT["TagThrottler"] -->|"per-tag limits"| GRV + RK["Ratekeeper"] -->|"tag usage stats"| TT + + style ThrottleResp fill:#fce4ec,stroke:#e91e63 + style Proceed fill:#e8f5e9,stroke:#4caf50 +``` diff --git a/design/AI-generated/diagram_11_backup_dr.md b/design/AI-generated/diagram_11_backup_dr.md new file mode 100644 index 00000000000..b98a49a0df1 --- /dev/null +++ b/design/AI-generated/diagram_11_backup_dr.md @@ -0,0 +1,114 @@ +# Backup, Restore & DR — Internal Architecture + +```mermaid +graph TB + subgraph BackupPipeline["Backup Pipeline"] + BW["Backup Worker\n(server-side role)"] + TLog["TLog"] + FBA["FileBackupAgent\n(orchestrator)"] + TaskBucket["TaskBucket\n(reliable task queue)"] + end + + subgraph FileTypes["Backup File Types"] + RangeFile["Range Files\n(KV snapshot at version)"] + LogFile["Log Files\n(mutation stream\nbetween versions)"] + end + + subgraph Storage["Backup Storage"] + BC["BackupContainer\n(abstract interface)"] + LocalDir["Local Directory"] + S3["S3 Blob Store"] + end + + subgraph Restore["Restore"] + RA["Restore Agent"] + ReadRange["Read range files\n(base state)"] + ApplyLog["Apply log files\n(to target version)"] + PrefixXform["Key prefix\ntransformation"] + end + + subgraph DR["Disaster Recovery"] + DBA["DatabaseBackupAgent"] + SrcCluster["Source Cluster"] + DstCluster["Standby Cluster"] + end + + BW -->|"peek mutations"| TLog + FBA --> TaskBucket + BW --> RangeFile + BW --> LogFile + RangeFile --> BC + LogFile --> BC + BC --> LocalDir + BC --> S3 + + RA --> ReadRange --> BC + RA --> ApplyLog --> BC + RA --> PrefixXform + + DBA -->|"stream mutations"| SrcCluster + DBA -->|"apply mutations"| DstCluster + + style BackupPipeline fill:#e1f0ff,stroke:#4a90d9 + style FileTypes fill:#fff3e0,stroke:#f5a623 + style Storage fill:#e8f5e9,stroke:#4caf50 + style Restore fill:#fce4ec,stroke:#e91e63 + style DR fill:#f3e5f5,stroke:#9c27b0 +``` + +## Backup Lifecycle + +```mermaid +sequenceDiagram + participant User as User / fdbbackup + participant FBA as FileBackupAgent + participant TB as TaskBucket + participant BW as Backup Worker + participant TLog as TLogs + participant BC as BackupContainer (S3) + + User->>FBA: Start backup + FBA->>TB: Create backup tasks + + loop Continuous + TB->>BW: Snapshot task + BW->>BW: Read key ranges from SS + BW->>BC: Write range files + + TB->>BW: Log task + BW->>TLog: peek(logRouterTag) + TLog-->>BW: mutation stream + BW->>BC: Write log files + end + + User->>FBA: Stop backup + FBA->>TB: Finalize + Note over BC: Backup complete:
range files + log files
= point-in-time restorable +``` + +## Point-in-Time Restore + +```mermaid +graph LR + subgraph BackupData["Backup Data"] + R1["Range File @ V100\n(full snapshot)"] + L1["Log File V100→V200"] + L2["Log File V200→V300"] + L3["Log File V300→V400"] + end + + subgraph Restore["Restore to V350"] + Step1["① Load range file @ V100\n(base state)"] + Step2["② Apply logs V100→V200"] + Step3["③ Apply logs V200→V300"] + Step4["④ Apply logs V300→V350\n(partial)"] + end + + R1 --> Step1 + L1 --> Step2 + L2 --> Step3 + L3 --> Step4 + Step4 --> Result["Restored cluster\nat version 350"] + + style Result fill:#e8f5e9,stroke:#4caf50 +``` diff --git a/design/AI-generated/diagram_12_simulation_testing.md b/design/AI-generated/diagram_12_simulation_testing.md new file mode 100644 index 00000000000..b40a6ef7e18 --- /dev/null +++ b/design/AI-generated/diagram_12_simulation_testing.md @@ -0,0 +1,155 @@ +# Simulation & Testing — Internal Architecture + +```mermaid +graph TB + subgraph Sim2["Sim2 (replaces Net2)"] + VirtProcs["Virtual Processes\n(many in one OS thread)"] + DetTime["Deterministic Time\n(virtual clock)"] + DetRand["deterministicRandom()\n(seedable PRNG)"] + SimNet["Simulated Network"] + end + + subgraph FaultInjection["Fault Injection"] + Buggify["BUGGIFY\n(probabilistic code paths)"] + NetFaults["Network Faults"] + ProcFaults["Process Faults"] + DiskFaults["Disk Faults"] + end + + subgraph NetFaultTypes["Network Fault Types"] + Clog["Clogging\n(delay packets)"] + Partition["Partitions\n(drop packets)"] + Latency["Latency injection"] + end + + subgraph ProcFaultTypes["Process Fault Types"] + KillProc["Kill process"] + KillMachine["Kill machine\n(all processes)"] + KillZone["Kill zone"] + KillDC["Kill datacenter"] + Reboot["Reboot process"] + end + + subgraph TestFramework["Test Framework"] + TOML["Test Config\n(.toml files)"] + SimCluster["SimulatedCluster\n(virtual topology)"] + Tester["Test Runner\n(fdbserver -r simulation)"] + end + + subgraph Workloads["Workloads (~160)"] + WBase["TestWorkload base class"] + Setup["setup()"] + Start["start()"] + Check["check()"] + Metrics["getMetrics()"] + end + + Tester --> TOML --> SimCluster --> VirtProcs + VirtProcs --> DetTime + VirtProcs --> DetRand + VirtProcs --> SimNet + SimNet --> NetFaults --> NetFaultTypes + ProcFaults --> ProcFaultTypes + DiskFaults --> SimNet + + Buggify --> VirtProcs + + Tester --> WBase + WBase --> Setup --> Start --> Check --> Metrics + + style Sim2 fill:#e1f0ff,stroke:#4a90d9 + style FaultInjection fill:#fce4ec,stroke:#e91e63 + style NetFaultTypes fill:#fff3e0,stroke:#f5a623 + style ProcFaultTypes fill:#fff3e0,stroke:#f5a623 + style TestFramework fill:#e8f5e9,stroke:#4caf50 + style Workloads fill:#f3e5f5,stroke:#9c27b0 +``` + +## Deterministic Simulation Flow + +```mermaid +sequenceDiagram + participant User as User + participant Tester as fdbserver -r simulation + participant Sim as Sim2 + participant Cluster as Virtual Cluster + + User->>Tester: seed=12345, test=Fast/Cycle.toml + Tester->>Sim: Initialize with seed + Sim->>Sim: deterministicRandom(12345) + Sim->>Cluster: Create virtual topology
(machines, DCs, processes) + + loop Simulation Loop + Sim->>Sim: Advance virtual clock
to next event + Sim->>Cluster: Deliver network messages + Sim->>Cluster: Fire timers + Sim->>Cluster: Inject faults (BUGGIFY) + Cluster->>Cluster: Execute actors + end + + Tester->>Tester: Run workload check() + Tester->>User: Pass/Fail + Unseed + Note over User: Same seed → same unseed
(on identical binary) +``` + +## BUGGIFY System + +```mermaid +graph TD + subgraph BuggifyDecision["At Startup (per BUGGIFY site)"] + Site["BUGGIFY at file:line"] + Coin["deterministicRandom()\ncoin flip"] + Coin -->|"~10% chance"| Enabled["Site ENABLED\nfor this run"] + Coin -->|"~90% chance"| Disabled["Site DISABLED\nfor this run"] + end + + subgraph Runtime["At Runtime"] + Enabled --> Hit{"Code reaches\nBUGGIFY site"} + Hit -->|"second coin flip\n(~10%)"| Fire["Fault injected"] + Hit -->|"~90%"| Skip["Normal path"] + end + + subgraph Examples["Example BUGGIFY Effects"] + E1["Short timeouts\n(trigger timeouts)"] + E2["Extra delays\n(expose races)"] + E3["Small buffer sizes\n(exercise edge cases)"] + E4["Wrong shard responses\n(test retry logic)"] + E5["Knob value changes\n(test parameter sensitivity)"] + end + + Fire --> E1 + Fire --> E2 + Fire --> E3 + Fire --> E4 + Fire --> E5 + + style BuggifyDecision fill:#e1f0ff,stroke:#4a90d9 + style Runtime fill:#fff3e0,stroke:#f5a623 + style Examples fill:#fce4ec,stroke:#e91e63 +``` + +## Test Organization + +```mermaid +graph LR + subgraph Tests["tests/"] + Fast["fast/\n(quick correctness)"] + Rare["rare/\n(failure scenarios:\nrollbacks, clogging,\nattrition)"] + Slow["slow/\n(longer, complex\nscenarios)"] + Negative["negative/\n(error conditions)"] + NoSim["noSim/\n(live cluster\nintegration tests)"] + end + + subgraph WorkloadTypes["Workload Categories"] + Correctness["Correctness\n(Cycle, AtomicOps,\nWriteDuringRead)"] + Config["Configuration\n(ConfigureDatabase,\nChangeConfig)"] + Failure["Failure\n(Attrition, Rollback,\nMachineAttrition)"] + Perf["Performance\n(ReadWrite,\nThroughput)"] + Feature["Feature-specific\n(Backup, BulkLoad,\nGcGenerations)"] + end + + Fast --> TOML[".toml config\n(workload list +\nparameters)"] + Rare --> TOML + Slow --> TOML + TOML --> WorkloadTypes +``` diff --git a/design/AI-generated/diagram_overview.md b/design/AI-generated/diagram_overview.md new file mode 100644 index 00000000000..b2118616e83 --- /dev/null +++ b/design/AI-generated/diagram_overview.md @@ -0,0 +1,163 @@ +# FoundationDB Architecture Overview + +## System-Level Architecture + +```mermaid +graph TB + subgraph Client["Client Layer"] + App["Application"] + CAPI["C API / Bindings"] + MV["MultiVersionTransaction"] + RYW["ReadYourWritesTransaction"] + Native["NativeAPI / Transaction"] + end + + subgraph Coordination["Coordination"] + Coord["Coordinators"] + CC["Cluster Controller"] + end + + subgraph CommitPipeline["Transaction Commit Pipeline"] + GRV["GRV Proxy"] + CP["Commit Proxy"] + Seq["Master / Sequencer"] + Res["Resolver"] + end + + subgraph LogSystem["Log System"] + TLog["TLog Replicas"] + LR["Log Router"] + TLogRemote["Remote TLogs"] + end + + subgraph Storage["Storage Layer"] + SS["Storage Server"] + KV["KV Engine\n(RocksDB / SQLite / Memory)"] + end + + subgraph Background["Background Services"] + DD["Data Distributor"] + RK["Ratekeeper"] + BW["Backup Worker"] + end + + App --> CAPI --> MV --> RYW --> Native + + Native -->|GetReadVersion| GRV + Native -->|"get / getRange"| SS + Native -->|commit| CP + + GRV -.->|rate limit check| RK + CP -->|GetCommitVersion| Seq + CP -->|resolve conflicts| Res + CP -->|push mutations| TLog + + SS -->|"peek (pull)"| TLog + SS --> KV + + TLog -->|route to remote| LR --> TLogRemote + + CC -->|recruit| CP + CC -->|recruit| GRV + CC -->|recruit| Res + CC -->|recruit| Seq + CC -->|recruit| TLog + CC -->|recruit| SS + CC -->|recruit| DD + CC -->|recruit| RK + CC -->|leader election| Coord + + DD -->|move shards| SS + RK -->|throttle signals| GRV + BW -->|peek mutations| TLog + + style Client fill:#e1f0ff,stroke:#4a90d9 + style CommitPipeline fill:#fff3e0,stroke:#f5a623 + style LogSystem fill:#e8f5e9,stroke:#4caf50 + style Storage fill:#fce4ec,stroke:#e91e63 + style Coordination fill:#f3e5f5,stroke:#9c27b0 + style Background fill:#fffde7,stroke:#fbc02d +``` + +## Write Path + +```mermaid +sequenceDiagram + participant App as Application + participant CP as Commit Proxy + participant Seq as Master/Sequencer + participant Res as Resolver + participant TLog as TLog (quorum) + participant SS as Storage Server + + App->>CP: CommitTransactionRequest + CP->>Seq: GetCommitVersion + Seq-->>CP: version V (monotonic) + CP->>Res: ResolveTransactionBatch + Res-->>CP: conflict/no-conflict + Note over CP: Drop conflicting txns + CP->>TLog: push(mutations, version V) + TLog-->>CP: ack (quorum) + CP-->>App: committed at version V + Note over TLog,SS: Async (decoupled) + SS->>TLog: peek(tag, fromVersion) + TLog-->>SS: mutations + SS->>SS: apply to KVStore +``` + +## Read Path + +```mermaid +sequenceDiagram + participant App as Application + participant GRV as GRV Proxy + participant RK as Ratekeeper + participant SS as Storage Server + participant KV as KV Engine + + App->>GRV: GetReadVersion + Note over GRV,RK: Rate-limited + GRV-->>App: read version V + App->>SS: get/getRange at version V + SS->>KV: read(key, version V) + KV-->>SS: value + SS-->>App: result +``` + +## Recovery Path + +```mermaid +stateDiagram-v2 + [*] --> READING_CSTATE: CC detects failure + READING_CSTATE --> LOCKING_CSTATE: Read coordinated state + LOCKING_CSTATE --> RECRUITING: Lock old TLogs + RECRUITING --> RECOVERY_TRANSACTION: Recruit new roles + RECOVERY_TRANSACTION --> WRITING_CSTATE: Replay last epoch + WRITING_CSTATE --> ACCEPTING_COMMITS: Write new coordinated state + ACCEPTING_COMMITS --> ALL_LOGS_RECRUITED: Primary + remote TLogs ready + ALL_LOGS_RECRUITED --> FULLY_RECOVERED: Old TLog data consumed + FULLY_RECOVERED --> [*] +``` + +## Data Movement Path + +```mermaid +sequenceDiagram + participant DDT as DD Shard Tracker + participant DDQ as DD Relocation Queue + participant MK as MoveKeys Protocol + participant Src as Source SS + participant Dst as Destination SS + participant TLog as TLog + + DDT->>DDQ: RelocateShard (priority) + DDQ->>MK: execute move + MK->>Dst: fetch data from source + Dst->>Src: bulk fetch key range + Src-->>Dst: data + Dst->>TLog: start peeking tag + TLog-->>Dst: mutations (catch up) + Note over MK: Atomic ownership transfer + MK->>MK: update keyServers system key + MK-->>DDQ: move complete +``` diff --git a/design/AI-generated/foundationdb_subsystem_map.md b/design/AI-generated/foundationdb_subsystem_map.md new file mode 100644 index 00000000000..e714626692e --- /dev/null +++ b/design/AI-generated/foundationdb_subsystem_map.md @@ -0,0 +1,359 @@ +# FoundationDB Codebase: Major Subsystem Map + +This is a research document, not an implementation plan. It partitions the entire FoundationDB codebase into 12 major subsystems based on data flow and runtime relationships. + +--- + +## The 12 Subsystems at a Glance + +| # | Subsystem | Primary Location | ~LOC | One-line summary | +|---|-----------|-----------------|------|------------------| +| 1 | [**Flow Runtime**](subsystem_01_flow_runtime.md) | [`flow/`](https://github.com/apple/foundationdb/tree/main/flow) | 31K impl + headers | Async actor framework, event loop, deterministic time, memory arenas | +| 2 | [**RPC & Transport**](subsystem_02_rpc_transport.md) | [`fdbrpc/`](https://github.com/apple/foundationdb/tree/main/fdbrpc) | 28K impl + headers | Endpoint-addressed messaging, peer management, failure monitoring, simulation network | +| 3 | [**Client Library**](subsystem_03_client_library.md) | [`fdbclient/`](https://github.com/apple/foundationdb/tree/main/fdbclient) | 75K impl, 34K headers | Transaction API, read-your-writes, location caching, multi-version client | +| 4 | [**Cluster Controller & Coordination**](subsystem_04_cluster_controller.md) | [`fdbserver/clustercontroller/`](https://github.com/apple/foundationdb/tree/main/fdbserver/clustercontroller), [`fdbserver/coordinator/`](https://github.com/apple/foundationdb/tree/main/fdbserver/coordinator), [`fdbserver/core/LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp) | 13K + coordination | Leader election, role recruitment, process registration, cluster-wide decision making | +| 5 | [**Transaction Commit Pipeline**](subsystem_05_commit_pipeline.md) | [`fdbserver/commitproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/commitproxy), [`fdbserver/grvproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/grvproxy), [`fdbserver/resolver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/resolver), [`fdbserver/sequencer/`](https://github.com/apple/foundationdb/tree/main/fdbserver/sequencer) | ~12K | Version assignment, conflict detection, commit batching — the write-path orchestration | +| 6 | [**Transaction Log (TLog) & Log System**](subsystem_06_tlog_logsystem.md) | [`fdbserver/tlog/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tlog), [`fdbserver/logsystem/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logsystem), [`fdbserver/logrouter/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logrouter) | 17K | Durable mutation logging, tag-partitioned replication, peek cursors for storage servers | +| 7 | [**Storage Server & Engines**](subsystem_07_storage_server.md) | [`fdbserver/storageserver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/storageserver), [`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore) | 63K | Serves reads, applies mutations from log, pluggable storage backends (RocksDB, SQLite, memory) | +| 8 | [**Data Distribution**](subsystem_08_data_distribution.md) | [`fdbserver/datadistributor/`](https://github.com/apple/foundationdb/tree/main/fdbserver/datadistributor) | 22K | Shard management, team building, rebalancing, MoveKeys protocol | +| 9 | [**Cluster Recovery**](subsystem_09_cluster_recovery.md) | [`fdbserver/clustercontroller/ClusterRecovery*`](https://github.com/apple/foundationdb/tree/main/fdbserver/clustercontroller), [`fdbserver/core/RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h) | (part of #4 files) | 10-state recovery state machine to reconstitute the transaction system after failures | +| 10 | [**Rate Keeping & Throttling**](subsystem_10_ratekeeper.md) | [`fdbserver/ratekeeper/`](https://github.com/apple/foundationdb/tree/main/fdbserver/ratekeeper) | 4K | Back-pressure on commits and reads, tag-based throttling, throughput tracking | +| 11 | [**Backup, Restore & DR**](subsystem_11_backup_dr.md) | [`fdbserver/backupworker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/backupworker), `fdbclient/FileBackupAgent*`, `fdbclient/BackupContainer*`, [`fdbbackup/`](https://github.com/apple/foundationdb/tree/main/fdbbackup) | ~10K server, ~20K client | Continuous backup to external storage, point-in-time restore, cross-cluster DR | +| 12 | [**Simulation & Testing**](subsystem_12_simulation_testing.md) | [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), [`fdbserver/tester/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tester), [`tests/`](https://github.com/apple/foundationdb/tree/main/tests) | 51K workloads + sim | Deterministic simulation (Sim2), fault injection (Buggify), workload-based integration tests | + +Plus supporting code: [`fdbserver/worker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/worker) (process startup, ~4K), [`fdbserver/core/`](https://github.com/apple/foundationdb/tree/main/fdbserver/core) (shared interfaces/utilities, ~14K), [`fdbcli/`](https://github.com/apple/foundationdb/tree/main/fdbcli) (CLI, ~8K), [`bindings/`](https://github.com/apple/foundationdb/tree/main/bindings) (language FFI), [`fdbserver/consistencyscan/`](https://github.com/apple/foundationdb/tree/main/fdbserver/consistencyscan) (~2K). + +--- + +## Detailed Subsystem Descriptions + +### 1. [Flow Runtime](subsystem_01_flow_runtime.md) ([`flow/`](https://github.com/apple/foundationdb/tree/main/flow)) + +**What it is:** A custom async programming framework that underpins every line of FDB code. It is *not* a library you can opt out of — it *is* the execution model. + +**Key abstractions:** +- **Future / Promise** — single-assignment async values. A Promise delivers once; a Future receives. Connected via reference-counted `SingleAssignmentVar` (SAV). +- **FutureStream / PromiseStream** — multi-value async channels. The primary request/reply pattern: send a struct containing a `ReplyPromise` into a `RequestStream`, and the receiver sends the reply back through it (self-addressed envelope). +- **ACTOR functions** — compiled by a custom C# actor compiler (`actorcompiler.exe`) into state machines. `wait(expr)` suspends; `state` variables persist across suspensions. Being migrated to C++20 coroutines (`co_await`). +- **Arena / StringRef / Standalone** — zero-copy memory management. `StringRef` is a non-owning pointer+length; `Arena` owns the backing memory; `Standalone` bundles a T with its Arena. +- **Net2** ([`Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp)) — the real event loop. Single-threaded, priority-based task scheduling over Boost.ASIO. Handles timers, yields, I/O multiplexing. +- **Deterministic random** ([`IRandom.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/IRandom.h)) — `deterministicRandom()` provides a seedable PRNG used everywhere, enabling reproducible simulation. +- **Tracing** ([`Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp)) — structured event logging with `.detail()` chains, used for diagnostics and simulation analysis. + +**Key dynamic behavior:** Every server-side component is a collection of ACTOR coroutines scheduled on Net2's run loop. There is no threading within a process (except for disk I/O thread pools). All concurrency is cooperative, driven by Future resolution. + +**Principal files:** [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h), [`Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h), [`Error.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Error.h), [`genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h), [`Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp), [`Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp), [`serialize.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/serialize.h), [`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h) + +--- + +### 2. [RPC & Transport](subsystem_02_rpc_transport.md) ([`fdbrpc/`](https://github.com/apple/foundationdb/tree/main/fdbrpc)) + +**What it is:** The layer that makes Flow's Future/Promise work *across process boundaries*. It maps the actor model onto a network. + +**Key abstractions:** +- **Endpoint** = `(NetworkAddressList, Token)` — a globally-unique address for a message receiver. Tokens are 128-bit UIDs. Well-known tokens exist for system services. +- **FlowTransport** — singleton that owns all peer connections. `sendReliable()` guarantees delivery (retransmit on reconnect); `sendUnreliable()` is fire-and-forget. +- **Peer** — per-destination state: unsent queue, reliable packet list, connection future, latency histogram. Connection is lazy (established on first send). +- **EndpointMap** — maps Token → `NetworkMessageReceiver*`. When a packet arrives, its token is looked up and the receiver's `receive()` method is called to deserialize and deliver. +- **FailureMonitor** — tracks endpoint/process liveness. Components register interest via `onFailed(Endpoint)` futures. +- **Locality** ([`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h)) — `LocalityData` (zone, machine, datacenter, data hall) and `ProcessClass` (fitness for roles). Drives replication policy enforcement. +- **Sim2** ([`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp)) — *replaces* Net2 during simulation. Runs many virtual processes in one real thread with deterministic time, simulated network (clogging, latency, partitions), and fault injection (kill process/machine/zone/datacenter). + +**Key dynamic behavior:** When Actor A on Process 1 sends a request to Actor B on Process 2, FlowTransport serializes the message, TCP delivers it, the receiver's EndpointMap dispatches it, and the reply flows back the same way. In simulation, this all happens in-process with virtual routing. The abstraction is seamless — actors don't know whether they're simulated or real. + +**Principal files:** [`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h), [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h), [`FlowTransport.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/FlowTransport.cpp), [`FailureMonitor.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FailureMonitor.h), [`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h), [`ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h), [`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h) + +--- + +### 3. [Client Library](subsystem_03_client_library.md) ([`fdbclient/`](https://github.com/apple/foundationdb/tree/main/fdbclient)) + +**What it is:** Everything an application (or internal FDB component acting as a client) uses to read and write data. + +**Transaction layers (outside-in):** +1. **C API / Language Bindings** — FFI boundary. All external languages go through [`fdb_c.h`](https://github.com/apple/foundationdb/blob/main/bindings/c/foundationdb/fdb_c.h). Multi-version support loads different client `.so` files for protocol compatibility. +2. **MultiVersionTransaction** — version-negotiating wrapper. +3. **ReadYourWritesTransaction** (RYW) — the layer most internal code uses. Maintains a local write map and snapshot read cache. Reads check the write map first, then merge with storage server results. This provides read-your-own-writes semantics within a transaction. +4. **NativeAPI / Transaction** — the raw transaction. `getReadVersion()` contacts a GRV proxy; `get()`/`getRange()` contact storage servers; `commit()` sends mutations + conflict ranges to a commit proxy. + +**Key dynamic behavior:** +- **Location cache**: `DatabaseContext` caches key-range → storage-server mappings. On cache miss or stale entry, re-resolves via a commit proxy's `getKeyServerLocations`. +- **Proxy load balancing**: `commitProxyLoadBalance()` / `grvProxyLoadBalance()` — round-robin with failover and automatic retry on proxy set changes. +- **Retry loop**: `Transaction::onError()` handles `not_committed`, `transaction_too_old`, etc. by resetting and retrying with backoff. The canonical usage pattern is `loop { try { ... tr.commit(); break; } catch(Error& e) { wait(tr.onError(e)); } }`. +- **Watches**: `tr.watch(key)` registers interest; the storage server notifies when the key changes. + +**Principal files:** [`NativeAPI.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/NativeAPI.actor.cpp), [`ReadYourWrites.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/ReadYourWrites.actor.cpp), [`DatabaseContext.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseContext.cpp), [`MultiVersionTransaction.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/MultiVersionTransaction.cpp), [`CommitTransaction.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/CommitTransaction.h), [`SystemData.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/SystemData.h), [`MonitorLeader.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/MonitorLeader.h) + +--- + +### 4. [Cluster Controller & Coordination](subsystem_04_cluster_controller.md) + +**What it is:** The "brain" that decides who does what. One process is elected Cluster Controller (CC); it recruits and monitors all server roles. + +**Leader election:** +- Coordinators (typically 3 or 5, addresses in the cluster file) run a `leaderRegister` actor. +- CC candidates send `CandidacyRequest`s. Coordinators track candidates and nominate based on process class fitness (encoded in top bits of `LeaderInfo.changeID`). +- The winner becomes CC; losers detect this via `monitorLeader()` and defer. +- Uses a generation register (`localGenerationReg`) for safe state transitions — reads and writes carry generation numbers to prevent stale updates. This protocol is isomorphic to single-decree Paxos (generation = ballot, read = prepare, write = accept), with quorum voting in `CoordinatedState`. +- Coordinator state is persisted via `KeyValueStoreMemory` backed by a `DiskQueue` (`.fdq` files) -- the same engine used by TLogs. + +**Role recruitment:** +- Worker processes register with CC via `registrationClient()` ([`worker.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/worker/worker.actor.cpp)). +- CC maintains a pool of available workers with their `ProcessClass` and `LocalityData`. +- When the transaction system needs to be (re)constituted, CC recruits: Master/Sequencer, CommitProxies, GrvProxies, Resolvers, TLogs. +- Recruitment considers fitness (class match), locality (datacenter placement), and excludes failed/excluded processes. +- CC also recruits singleton roles: DataDistributor, Ratekeeper, ConsistencyScan. + +**Cluster-wide state:** +- `ServerDBInfo` is the cluster-wide configuration broadcast. Contains: master interface, proxy lists, log system config, recovery state, latency band config. +- Updated by CC and distributed to all workers. Workers react to changes (e.g., new proxy set). + +**Principal files:** [`ClusterController.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterController.actor.cpp), `ClusterController.h`, [`Coordination.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/Coordination.cpp), [`LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp), [`CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp), [`WorkerInterface.actor.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h) + +--- + +### 5. [Transaction Commit Pipeline](subsystem_05_commit_pipeline.md) + +**What it is:** The assembly line that takes a client's commit request and makes it durable. Four server roles collaborate in a strict pipeline. + +**The pipeline (a single commit's journey):** + +``` +Client ──CommitTransactionRequest──▶ CommitProxy + │ +Phase 1: GET VERSION │ GetCommitVersionRequest + ▼ + Master/Sequencer + │ assigns monotonic version + │ (tracks prevVersion for ordering) + ▼ +Phase 2: RESOLVE CommitProxy + │ ResolveTransactionBatchRequest + ▼ + Resolver(s) [parallel, sharded by key range] + │ checks read-ranges vs committed write-ranges + │ returns conflict/no-conflict per txn + ▼ +Phase 3: LOG CommitProxy + │ pushes non-conflicting mutations + │ via LogSystem::push() + ▼ + TLog replicas (quorum write) + │ +Phase 4: REPLY CommitProxy + │ CommitTransactionReply + ▼ + Client (committed!) +``` + +**Master/Sequencer** ([`masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp)): Assigns commit versions via `getVersion()`. Ensures versions are monotonically increasing and roughly track wall-clock time. Also tracks the "live committed version" for GRV. + +**GRV Proxy** ([`GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp)): Handles `GetReadVersionRequest` from clients. Assigns read versions that are guaranteed to see all previously committed transactions. Enforces cluster-wide flow control: receives TPS limits from Ratekeeper and gates transaction starts using a smoothed token bucket (`GrvTransactionRateInfo`). Batch-priority transactions are throttled more aggressively and dropped first under pressure. + +**Commit Proxy** ([`CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp)): The workhorse. Batches commits (`commitBatcher`), drives the 4-phase pipeline, handles metadata mutations (system key writes), and updates the key-server location map. + +**Resolver** ([`Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp), [`ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp)): Maintains a sliding window of committed write ranges. For each incoming batch, checks if any transaction's read ranges overlap with committed writes at versions newer than the transaction's read version. Pure conflict detection — no side effects. + +**Principal files:** [`CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp), [`GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp), [`masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp), [`Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp), [`ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp), [`LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h) + +--- + +### 6. [Transaction Log (TLog) & Log System](subsystem_06_tlog_logsystem.md) + +**What it is:** The durable write-ahead log. All committed mutations flow through TLogs before being applied to storage servers. This is FDB's durability guarantee. + +**TLog** ([`TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp)): +- Receives mutation batches from commit proxies via `push()`. +- Writes to a `DiskQueue` (append-only on-disk log) and an in-memory index. +- Mutations are **tagged** — each mutation carries one or more `Tag`s indicating which storage server(s) it's relevant to. Tags are assigned by the commit proxy based on the key-to-shard mapping. +- Provides `peek()` interface: storage servers pull mutations by tag and version range. +- Quorum replication: a mutation is committed when `f+1` of `2f+1` TLog replicas acknowledge. + +**Log System** ([`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp)): +- Abstraction over the set of TLog replicas. +- `LogSystem::push()` — sends mutations to all relevant TLog replicas, waits for quorum. +- `LogSystem::peek()` → `IPeekCursor` — merges results from multiple TLogs for a given tag, handling multi-generation log sets during recovery. +- Manages log epochs: each recovery creates a new epoch of TLogs. Old epochs are kept until storage servers have consumed all their data. + +**Log Router** ([`LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp)): +- In multi-region configurations, routes mutation data from the primary's TLogs to the remote region's TLogs. +- Bridges the tag-partitioned log system across datacenters. + +**Key dynamic behavior:** Storage servers are *consumers* of the log. They continuously peek their assigned tag, pulling committed mutations and applying them to local storage. This decoupling means commits don't wait for storage servers — only for TLog quorum. + +**Principal files:** [`TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp), [`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp), [`LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp), [`LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h), [`LogSystemConfig.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/LogSystemConfig.h), [`IDiskQueue.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/IDiskQueue.h) + +--- + +### 7. [Storage Server & Engines](subsystem_07_storage_server.md) + +**What it is:** The read path and durable key-value store. Storage servers serve client reads and maintain the materialized state of their assigned key ranges (shards). + +**Storage Server** ([`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)): +- Each SS owns a set of key ranges (shards) and a version. +- **Update loop**: continuously peeks its tag from the log system, deserializes mutations, applies them to the local KVStore, and advances its durable version. +- **Read serving**: handles `GetKeyValuesRequest`, `GetKeyRequest`, `GetValueRequest`. Checks that the requested version is available (between oldest readable version and current version), that the shard is owned, and reads from the KVStore. +- **Version management**: tracks `storageVersion` (latest durable), `durableVersion`, and `desiredOldestVersion`. Advertises its version to the cluster for GRV decisions. + +**Storage Engines** ([`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore)): +- **IKeyValueStore** interface: `readValue()`, `readRange()`, `set()`, `clear()`, `commit()`. All operations are on a versioned key-value store. +- **RocksDB** ([`KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp)) — primary production engine. Also a sharded variant ([`KeyValueStoreShardedRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreShardedRocksDB.actor.cpp)) that uses per-shard column families. +- **SQLite** ([`KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp)) — legacy engine, still supported. +- **Memory** ([`KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp)) — for testing and special uses (e.g., txnStateStore during recovery). + +**Key dynamic behavior:** The storage server is *pull-based*. It pulls from the log system at its own pace. If a storage server falls behind, it catches up by reading more from TLogs. If it falls too far behind, it may be removed from its team and re-replicated. Reads at a given version are served from a snapshot — the SS maintains enough history to serve reads at any version between its oldest and current. + +**Principal files:** [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp), [`KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp), [`KeyValueStoreShardedRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreShardedRocksDB.actor.cpp), [`KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp), [`KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp), [`IKeyValueStore.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h), [`StorageMetrics.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/StorageMetrics.cpp) + +--- + +### 8. [Data Distribution](subsystem_08_data_distribution.md) + +**What it is:** The subsystem that decides which storage servers hold which key ranges, and moves data when the assignment needs to change. + +**Why data moves:** +- A shard grows too large → split +- A storage server fails → re-replicate its shards to healthy servers +- Load imbalance → move shards from hot to cold servers +- Configuration change (e.g., replication factor increase) +- Exclusion of a server/machine/datacenter + +**Components:** +- **DDShardTracker** ([`DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp)) — monitors shard sizes and read/write rates. Triggers splits, merges, and moves. +- **DDTeamCollection** ([`DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp)) — builds "teams" of storage servers that satisfy the replication policy (e.g., 3 servers in 3 different zones). Maintains healthy/unhealthy team lists. +- **DDRelocationQueue** ([`DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp)) — prioritized queue of shard movements. Executes moves concurrently up to a parallelism limit. +- **MoveKeys** ([`fdbserver/core/MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp)) — the protocol for atomically transferring shard ownership. Uses a "move keys lock" to serialize concurrent moves. Updates the `keyServers` and `serverKeys` system key spaces. +- **DataMovement** ([`fdbserver/core/DataMovement.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/DataMovement.cpp)) — data move metadata and physical shard ID tracking. + +**Key dynamic behavior:** Data distribution is a continuous background process. The DD singleton (recruited by CC) watches for changes in shard metrics, server health, and configuration. It produces `RelocateShard` requests that flow through the relocation queue, which executes the MoveKeys protocol. During a move, the destination SS fetches data from the source SS and begins consuming from the log system. When caught up, ownership transfers atomically via a system key transaction. + +**Principal files:** [`DataDistribution.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DataDistribution.cpp), [`DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp), [`DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp), [`DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp), [`MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp), [`DataMovement.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/DataMovement.cpp), [`ShardsAffectedByTeamFailure.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp) + +--- + +### 9. [Cluster Recovery](subsystem_09_cluster_recovery.md) + +**What it is:** The state machine that reconstitutes the entire transaction system after a failure (master crash, network partition, coordinator change, etc.). + +**The 10 recovery states** (from [`RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h)): +0. **UNINITIALIZED** — starting +1. **READING_CSTATE** — read coordinated state from coordinators (learn about previous epoch) +2. **LOCKING_CSTATE** — lock old TLogs to prevent split-brain +3. **RECRUITING** — recruit new TLogs, CommitProxies, GrvProxies, Resolvers +4. **RECOVERY_TRANSACTION** — replay the last epoch's committed-but-unapplied mutations; establish the recovery version +5. **WRITING_CSTATE** — write the new epoch's coordinated state to coordinators +6. **ACCEPTING_COMMITS** — new transaction system is live, accepting commits +7. **ALL_LOGS_RECRUITED** — all TLog sets (including remote region) are populated +8. **STORAGE_RECOVERED** — storage servers have recovered their data from the new TLogs +9. **FULLY_RECOVERED** — old TLog data fully consumed; old generations can be discarded + +**Key operations during recovery:** +- `readTransactionSystemState()` — peeks old TLogs to reconstruct the txnStateStore (key-server mappings, configuration, etc.) +- `recruitEverything()` — recruits the full transaction system in parallel +- `trackTlogRecovery()` — monitors TLog set completeness and writes coordinated state updates +- `provisionalMaster()` — handles the gap between old and new transaction systems + +**Principal files:** [`ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp), [`ClusterRecovery.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.h), [`RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h), [`DBCoreState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/DBCoreState.h), [`CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp), [`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp) (newEpoch) + +--- + +### 10. [Rate Keeping & Throttling](subsystem_10_ratekeeper.md) ([`fdbserver/ratekeeper/`](https://github.com/apple/foundationdb/tree/main/fdbserver/ratekeeper)) + +**What it is:** Back-pressure mechanism that prevents the cluster from being overwhelmed. + +**What it monitors:** +- Storage server queue depths (how far behind are they from the log?) +- TLog queue depths +- Storage server disk throughput and IOPS +- Per-tag transaction rates (for tenant/workload isolation) + +**What it controls:** +- Transaction rate limits (`tpsLimit`) computed separately for normal (SYSTEM+DEFAULT) and batch priorities, sent to GRV proxies via `GetRateInfoReply`. Each proxy receives `tpsLimit / numProxies`. See [Subsystem 5](subsystem_05_commit_pipeline.md) for how GRV proxies enforce these limits. +- Per-tag throttling via `TagThrottler` — can slow down or reject transactions from specific tags +- Batch priority vs. default priority differentiation (batch uses more aggressive queue depth thresholds) + +**Principal files:** [`Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp), [`Ratekeeper.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.h), [`TagThrottler.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/TagThrottler.cpp), [`RkTagThrottleCollection.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/RkTagThrottleCollection.cpp) + +--- + +### 11. [Backup, Restore & DR](subsystem_11_backup_dr.md) + +**What it is:** Continuous backup of the mutation stream to external storage, with point-in-time restore capability. + +**Backup architecture:** +- **BackupWorker** ([`fdbserver/backupworker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/backupworker)) — server-side role that reads mutations from TLogs (like a storage server, but writes to backup storage instead of a local KVStore). Tag: `tagLocalityLogRouter`. +- **FileBackupAgent** ([`FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp)) — orchestrates backup lifecycle. Uses a TaskBucket pattern for reliable, resumable tasks. +- **BackupContainer** (`fdbclient/BackupContainer*`) — abstraction over backup storage destinations. Implementations: local directory, S3 blob store. +- **Two file types**: Range files (snapshot of key-value pairs at a version) and Log files (mutation stream between versions). Together they enable point-in-time restore. + +**Restore:** +- Restore agent reads range files to reconstruct base state, then applies log files to reach target version. +- Supports key prefix transformation during restore (add/remove prefix). + +**DR (Disaster Recovery):** +- `DatabaseBackupAgent` — streams mutations from one cluster to another in near-real-time. +- Enables warm standby clusters. + +**Principal files:** [`BackupWorker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/backupworker/BackupWorker.cpp), [`FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp), [`DatabaseBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseBackupAgent.cpp), [`BackupContainer.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupContainer.h), [`BackupContainerFileSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/BackupContainerFileSystem.cpp), [`BackupAgent.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupAgent.h), [`backup.cpp`](https://github.com/apple/foundationdb/blob/main/fdbbackup/backup.cpp) + +--- + +### 12. [Simulation & Testing](subsystem_12_simulation_testing.md) + +**What it is:** FDB's most distinctive engineering practice — deterministic simulation testing that can find bugs that would take years to manifest in production. + +**Sim2** ([`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp)): +- Replaces Net2 as the network implementation during tests. +- Runs the entire cluster (multiple "processes") in a single OS thread with virtual time. +- Deterministic: given the same random seed, produces identical execution. +- Simulates: network latency/clogging/partitions, disk I/O errors, process crashes, machine failures, datacenter failures. + +**Buggify** (throughout the codebase): +- `BUGGIFY` / `BUGGIFY_WITH_PROB(p)` — randomly enables rare code paths in simulation. +- Examples: short timeouts, extra delays, buffer size changes, early returns. +- Dramatically increases code path coverage during testing. +- Disabled in production builds. + +**Workloads** ([`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), ~51K LOC): +- `TestWorkload` base class with `setup()`, `start()`, `check()`, `getMetrics()`. +- ~60+ workload types covering: API correctness, backup/restore, bulk operations, configuration changes, failure injection, performance benchmarks. +- Configured via TOML files in [`tests/`](https://github.com/apple/foundationdb/tree/main/tests). +- Multiple workloads can run concurrently in a single simulation. + +**Test runner** ([`fdbserver/tester/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tester)): +- [`SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp) — sets up simulated cluster topology (machines, processes, datacenter layout). +- `TestConfig` — reads TOML test configuration. +- Orchestrates workload execution and result checking. + +**Principal files:** [`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h), [`SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp), [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), [`tests/`](https://github.com/apple/foundationdb/tree/main/tests) + +--- + +## Data Flow Summary + +### Write Path (end to end) +``` +App → C API → NativeAPI.commit() → CommitProxy → Master (version) → Resolver (conflicts) + → CommitProxy → LogSystem.push() → TLog quorum (durable) → reply to client + ~~async~~> StorageServer.peek() → apply mutations to KVStore +``` + +### Read Path (end to end) +``` +App → C API → NativeAPI.getReadVersion() → GrvProxy (rate-limited by Ratekeeper TPS budget) +App → C API → NativeAPI.get()/getRange() → [location cache lookup] → StorageServer + → KVStore.read() at requested version → reply to client +``` + +### Recovery Path +``` +CC detects master failure → elect new CC (if needed) → read coordinated state + → lock old TLogs → recruit new {Master, Proxies, Resolvers, TLogs} + → replay old epoch → write new coordinated state → accept commits +``` + +### Data Movement Path +``` +DDShardTracker detects imbalance → RelocateShard request → DDRelocationQueue + → MoveKeys protocol: dest SS fetches data + starts consuming log + → atomic ownership transfer via system key transaction +``` diff --git a/design/AI-generated/subsystem_01_flow_runtime.md b/design/AI-generated/subsystem_01_flow_runtime.md new file mode 100644 index 00000000000..60688cee2e1 --- /dev/null +++ b/design/AI-generated/subsystem_01_flow_runtime.md @@ -0,0 +1,378 @@ +# Subsystem 1: Flow Runtime + +**[Diagrams](diagram_01_flow_runtime.md)** + +**Location:** [`flow/`](https://github.com/apple/foundationdb/tree/main/flow) +**Size:** ~31K implementation + headers +**Role:** The custom async programming framework that is the execution model for all FoundationDB code. + +--- + +## Overview + +Flow is not a library you can opt out of -- it **is** the execution model. Every server component, every client operation, every test runs as a collection of cooperatively-scheduled coroutines (called "actors") on a single-threaded event loop. There are no application-level threads; all concurrency comes from Future resolution and callback dispatch. + +--- + +## Key Data Structures + +### SAV (SingleAssignmentVar) -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:730-914` + +The core primitive underlying all async communication. A reference-counted cell that holds either a value or an error, set exactly once. + +``` +struct SAV : Callback, FastAllocated> { + int promises; // Promise reference count + int futures; // Future reference count + aligned_storage value_storage; + Error error_state; // UNSET_ERROR_CODE (-3), SET_ERROR_CODE, NEVER_ERROR_CODE, or actual error +}; +``` + +**Key methods:** +- `send(U&& value)` -- sets value, fires all callbacks in the chain +- `sendError(Error)` -- sets error, fires error callbacks +- `isSet()` / `canBeSet()` / `isError()` -- state queries +- `addCallbackAndDelFutureRef()` -- registers a continuation +- `delPromiseRef()` / `delFutureRef()` -- reference counting; destroys when both reach 0 + +**Callback chain:** SAV inherits from `Callback`, which forms a **doubly-linked circular list**. Multiple waiters can register on one SAV. When the SAV fires, it walks the chain calling `fire(value)` or `error(err)` on each callback, then unlinks them. + +### Future -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:973-1068` + +Read-only handle to a SAV. Cannot set the value, only observe it. + +``` +Future f; // default: points to nothing +Future f(value); // immediately ready +Future f(Never()); // never completes +Future f(error); // immediately errored +``` + +**Key methods:** +- `get()` -- returns value or throws error (must be ready) +- `isReady()` -- true if set (value or error) +- `canGet()` -- ready and not error +- `cancel()` -- calls `sav->cancel()`, used by actor system +- `addCallbackAndClear(cb)` -- transfers ownership of callback to SAV + +### Promise -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:1091-1157` + +Write-only handle to a SAV. Delivers the value exactly once. + +**Key methods:** +- `send(value)` -- delivers value +- `sendError(error)` -- delivers error +- `getFuture()` -- returns the associated Future +- `isSet()` / `canBeSet()` -- state queries + +**Lifecycle:** Creating a Promise creates a SAV with `promises=1, futures=0`. Calling `getFuture()` increments `futures`. When the Promise is destroyed without sending, it sends `broken_promise()` error. + +### FutureStream / PromiseStream -- [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:1159-1456` + +Multi-value async channels for request/reply patterns. + +**NotifiedQueue** (backing structure): +- `std::queue> queue` -- buffered values +- Single callback chain (not doubly-linked) +- `send(value)` -- if callback waiting, delivers directly; else queues +- `pop()` -- dequeues or throws error + +**PromiseStream** key pattern -- self-addressed envelope: +```cpp +struct MyRequest { + int param; + ReplyPromise reply; // sender embeds a reply channel +}; +PromiseStream stream; +stream.send(MyRequest{42, ReplyPromise()}); +// receiver calls req.reply.send(result) +``` + +No explicit backpressure -- values accumulate in the queue. Implicit pressure: if the consumer doesn't drain, memory grows. + +### Error / ErrorOr -- [`Error.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Error.h)`:44-81`, [`flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h)`:124-317` + +**Error class:** +- `uint16_t error_code` + `uint16_t flags` (includes `FLAG_INJECTED_FAULT`) +- Propagates through Futures: `SAV::sendError()` stores the error; `Future::get()` throws it +- Defined via macros in `error_definitions.h`: `ERROR(name, number, description)` + +**ErrorOr:** +- `std::variant` -- holds either an error or a value +- `present()` / `get()` / `getError()` -- access methods +- Used by `tryGetReply()` to return errors without throwing + +--- + +## The Actor System + +### ACTOR Compiler + +FDB uses a custom **C# actor compiler** (`actorcompiler.exe`) that transforms `.actor.cpp` files into standard C++. The `ACTOR` keyword marks a function as a coroutine: + +```cpp +ACTOR Future myActor(Future input) { + state int x = 42; // persists across waits + int val = wait(input); // suspension point + return val + x; +} +``` + +The compiler: +1. Scans for `wait()` and `waitNext()` calls +2. Breaks the function into callback states at each suspension point +3. Generates a C++ class inheriting from `Actor` with a SAV +4. Each `wait()` becomes a callback registration + return +5. `state` variables are promoted to class members (persist across suspensions) + +**Key macros** (`actorcompiler.h`): +- `ACTOR` -- marks coroutine function +- `state` -- marks variables that survive across `wait()` +- `wait(Future)` -- suspend until Future ready, return T +- `waitNext(FutureStream)` -- suspend until next stream value +- `choose { when(wait(a)) {...} when(wait(b)) {...} }` -- race two futures +- `loop` -- `while(true)` alias + +### C++20 Coroutine Integration -- [`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h) + +The codebase is being migrated from the custom actor compiler to C++20 native coroutines: + +**CoroPromise** ([`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h)`:737-844`): +- The `promise_type` for `Future` coroutines +- Embeds a `CoroActor` which inherits from `Actor` (which inherits from SAV) +- `get_return_object()` returns `Future` backed by the coroutine's SAV +- `initial_suspend()` returns `suspend_never` (eager start) +- `await_transform()` overloads convert `Future` into `AwaitableFuture` + +**AwaitableFuture** ([`CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h)`:453-534`): +- The awaiter type for `co_await Future` +- `await_ready()` -- checks if future is ready or coroutine cancelled +- `await_suspend()` -- registers callback with the future's SAV +- `await_resume()` -- extracts value or throws error +- Inherits from `Callback` so it can be inserted into SAV's callback chain + +**Cancellation:** `CoroActor::cancel()` sets `ACTOR_WAIT_STATE_CANCELLED` and resumes the coroutine, which checks the flag in `await_ready()` and throws `actor_cancelled()`. + +--- + +## Net2 Event Loop -- [`Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp) + +The real (non-simulated) event loop. Single-threaded, priority-based task scheduling over Boost.ASIO. + +### Net2 Class (line 139) + +``` +class Net2 : public INetwork, public INetworkConnections { + ASIOReactor reactor; // Boost.ASIO wrapper + TaskQueue taskQueue; // Priority queue of pending tasks + std::atomic currentTime; + std::atomic stopped; +}; +``` + +### Run Loop + +The `run()` method: +1. Polls ASIO for I/O events (socket readable/writable, timers) +2. Processes ready tasks from the priority queue (highest priority first) +3. Tracks starvation metrics per priority level +4. Yields back to ASIO periodically + +### Task Priority + +`TaskPriority` enum defines ~30 priority levels from `Min` to `Max`. Examples: +- `RunLoop` (highest) -- event loop housekeeping +- `ReadSocket` -- incoming network data +- `DefaultPromiseEndpoint` -- RPC message delivery +- `DefaultYield` -- cooperative yielding +- `DefaultDelay` -- timer-based delays +- `DiskWrite` / `DiskRead` -- I/O completions + +### Key Methods + +- `delay(seconds, taskId)` -- returns `Future` that fires after `seconds` at priority `taskId` +- `yield(taskId)` -- suspends current actor, re-enqueues at `taskId` priority +- `check_yield(taskId)` -- returns true if the current task has been running too long +- `onMainThread(promise, taskId)` -- schedules work from a thread pool thread back onto the main thread + +### Boost.ASIO Integration + +- TCP/TLS connections via `boost::asio::ip::tcp::socket` +- UDP sockets for connectionless messaging +- Timer operations for `delay()` implementation +- Thread pool integration for async file I/O + +--- + +## Arena Memory System -- [`Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h) + +### Arena + +A bump-pointer allocator. Memory is allocated in blocks (`ArenaBlock`), never freed individually -- the entire Arena is freed at once. + +``` +class Arena { + Reference impl; // first block in chain + + Arena(); // empty + Arena(size_t reservedSize); // pre-allocate + void dependsOn(const Arena&); // link arenas (for reference counting) + void* operator new(size_t, Arena&); // allocate in arena +}; +``` + +**ArenaBlock** ([`Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h)`:171-212`): +- Fixed block sizes: `SMALL=64`, `LARGE=8193` +- Bump pointer allocation: `bigUsed` tracks next free offset +- Blocks chain together; all freed when Arena destroyed +- `secure` flag for zero-on-dealloc (sensitive data) + +### StringRef / KeyRef / ValueRef + +Non-owning pointer+length references. Do not own memory. + +``` +struct StringRef { + const uint8_t* data; + int length; + + StringRef(Arena& a, const StringRef& toCopy); // copy into arena + StringRef substr(int start, int size); + int compare(StringRef const& other); // lexicographic +}; + +using KeyRef = StringRef; +using ValueRef = StringRef; +``` + +### Standalone + +Bundles a T with its own Arena, creating an owning wrapper: + +``` +template +class Standalone : private Arena, public T { + Arena& arena(); // access the arena + T& contents(); // access the value +}; +// Example: Standalone owns the bytes it points to +``` + +### VectorRef + +Non-owning reference to an array of T. Like StringRef but for structured data. + +--- + +## FastAlloc -- `FastAlloc.h` + +Thread-local pool allocator for common sizes (16, 32, 64, 96, 128, 256 bytes). + +- `FastAllocator::allocate()` -- O(1) from thread-local free list +- `FastAllocator::release()` -- O(1) return to free list +- Magazine system: when free list empty, swap with a full magazine from global pool +- `FastAllocated` trait: overloads `operator new/delete` to use FastAllocator +- No locks for allocation when free list has items +- 128KB magazines exchanged between thread-local and global pools + +--- + +## Serialization -- [`serialize.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/serialize.h) + +Unified framework for binary serialization: + +- `is_binary_serializable` trait for fixed-size types (int, double, UID, etc.) +- `Serializer` template: calls `t.serialize(ar)` by default +- Specializations for containers: `std::vector`, `std::map`, `std::variant`, etc. +- `BinaryWriter` / `BinaryReader` archives +- Protocol version support: `_IncludeVersion`, `_AssumeVersion`, `_Unversioned` +- Flatbuffers integration via `ObjectSerializer` for network messages + +--- + +## Tracing -- `Trace.h`, [`Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp) + +Structured event logging used everywhere: + +```cpp +TraceEvent("MyEvent", self->dbgid) + .detail("Key", key) + .detail("Version", version) + .detail("Duration", timer() - startTime); +``` + +- Fluent `.detail()` chain builds key-value fields +- Severity levels: `SevVerbose(0)`, `SevDebug(5)`, `SevInfo(10)`, `SevWarn(20)`, `SevError(40)` +- `AuditedEvent` for security-critical events that bypass suppression +- `.suppressFor(duration)` -- rate-limit repeated events +- `.trackLatest(key)` -- keep latest value of a recurring event + +--- + +## Generic Actors -- [`genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h) + +Key utility actors: + +| Actor | Purpose | +|-------|---------| +| `timeout(Future, seconds, defaultValue)` | Race future against timer | +| `timeoutError(Future, seconds)` | Timeout throws `timed_out()` | +| `delayed(Future, seconds)` | Add delay after completion | +| `errorOr(Future)` | Catch error into `ErrorOr` | +| `store(X& out, Future)` | Store result in variable | +| `waitForAll(vector>)` | Wait for all futures | +| `waitForAllReady(vector>)` | Wait ignoring errors | +| `map(Future, func)` | Transform value | +| `mapAsync(Future, actorFunc)` | Async transform | +| `holdWhile(object, Future)` | Keep object alive until future resolves | +| `uncancellable(Future)` | Prevent cancellation | + +**AsyncVar** ([`genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h)`:753-785`): +- Reactive variable: holds a value, notifies on change +- `get()` -- current value +- `onChange()` -- returns `Future` that fires on next change +- `set(value)` -- updates value and triggers onChange if different +- Used extensively for cluster state (e.g., `AsyncVar`) + +--- + +## Data Flow Summary + +### Actor Execution Cycle + +``` +1. Actor created → SAV allocated, initial code runs until first wait() +2. wait(future) → registers callback on future's SAV, returns to event loop +3. Event fires (network data, timer, another actor sends) → SAV::send(value) +4. Callback chain walked → actor's callback called +5. Actor resumes from wait point, runs until next wait() or return +6. return value → SAV::send(value) on actor's own SAV → caller's callback fires +``` + +### Memory Flow + +``` +1. Arena allocated (bump pointer in ArenaBlock) +2. StringRef/KeyRef/ValueRef created referencing arena memory +3. Standalone bundles T + Arena for ownership transfer +4. When Standalone destroyed, Arena freed, all StringRefs invalidated +5. No per-object deallocation -- whole Arena freed at once +``` + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`flow/include/flow/flow.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/flow.h) | Master include: Future, Promise, SAV, Callback, ErrorOr | +| [`flow/include/flow/Arena.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Arena.h) | Arena, StringRef, Standalone, VectorRef | +| [`flow/include/flow/Error.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/Error.h) | Error class and error code definitions | +| [`flow/include/flow/CoroutinesImpl.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/CoroutinesImpl.h) | C++20 coroutine integration (CoroPromise, AwaitableFuture) | +| [`flow/include/flow/genericactors.actor.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/genericactors.actor.h) | Utility actors (delay, timeout, waitForAll, AsyncVar) | +| [`flow/include/flow/serialize.h`](https://github.com/apple/foundationdb/blob/main/flow/include/flow/serialize.h) | Serialization framework | +| `flow/include/flow/FastAlloc.h` | Thread-local pool allocator | +| [`flow/Net2.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Net2.cpp) | Real event loop (Boost.ASIO based) | +| [`flow/Trace.cpp`](https://github.com/apple/foundationdb/blob/main/flow/Trace.cpp) | Structured event tracing | +| `flow/include/flow/actorcompiler.h` | ACTOR/wait/state macro definitions | diff --git a/design/AI-generated/subsystem_02_rpc_transport.md b/design/AI-generated/subsystem_02_rpc_transport.md new file mode 100644 index 00000000000..f00fcf71315 --- /dev/null +++ b/design/AI-generated/subsystem_02_rpc_transport.md @@ -0,0 +1,340 @@ +# Subsystem 2: RPC & Transport + +**[Diagrams](diagram_02_rpc_transport.md)** + +**Location:** [`fdbrpc/`](https://github.com/apple/foundationdb/tree/main/fdbrpc) +**Size:** ~28K implementation + headers +**Role:** Maps Flow's actor model onto a network -- makes Future/Promise work across process boundaries. + +--- + +## Overview + +FlowTransport is the layer that lets actors on different processes (or simulated processes) communicate transparently. It provides endpoint-addressed messaging with both reliable (at-least-once) and unreliable (at-most-once) delivery semantics. In simulation, the same code runs in-process with virtual routing via Sim2. + +--- + +## Key Data Structures + +### Endpoint -- [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h)`:43-120` + +A globally unique address for a message receiver: + +``` +class Endpoint { + NetworkAddressList addresses; // primary + optional secondary (TLS/non-TLS) + Token token; // UID: 128-bit unique identifier +}; +``` + +- **Token** = `UID` (pair of `uint64_t`). Lower 32 bits encode an index into the EndpointMap; upper 32 bits encode task priority. +- **Well-known tokens**: `wellKnownToken(int id)` returns `UID(-1, id)`. Reserved IDs: `WLTOKEN_ENDPOINT_NOT_FOUND(0)`, `WLTOKEN_PING_PACKET`, `WLTOKEN_UNAUTHORIZED_ENDPOINT`, plus system services (leader election, config transactions, etc.) +- **Address selection**: `choosePrimaryAddress()` swaps primary/secondary based on local TLS preference. + +### Peer -- [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h)`:149-193` + +Per-destination connection state: + +``` +struct Peer : ReferenceCounted { + NetworkAddress destination; + UnsentPacketQueue unsent; // buffered outgoing packets + ReliablePacketList reliable; // retransmit list (circular linked) + AsyncTrigger dataToSend; // signal: unsent queue non-empty + Future connect; // connectionKeeper actor + bool compatible, connected; + double reconnectionDelay; // exponential backoff + int peerReferences; // stream reference count (-1 = no streams) + int outstandingReplies; + DDSketch pingLatencies; // latency distribution + AsyncVar> protocolVersion; + Promise disconnect; // notified on connection loss +}; +``` + +**Connection lifecycle:** +1. First send to address creates Peer and starts `connectionKeeper()` actor +2. `connectionKeeper()` calls `INetworkConnections::connect()`, performs TLS handshake +3. Sends `ConnectPacket` (protocol version, local address, connection ID) +4. Spawns `connectionWriter()` (async write loop) and `connectionReader()` (async read loop) +5. `connectionMonitor()` sends periodic pings, detects timeouts +6. On failure: exponential backoff (INITIAL_RECONNECTION_TIME to MAX_RECONNECTION_TIME, growth factor 1.5x) +7. `discardUnreliablePackets()` on disconnect; reliable packets resent after reconnect + +### EndpointMap -- [`FlowTransport.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/FlowTransport.cpp)`:90-230` + +Maps tokens to local message receivers: + +``` +struct Entry { + union { uint64_t uid[2]; uint32_t nextFree; }; + NetworkMessageReceiver* receiver; +}; +``` + +- Pre-allocates slots for well-known endpoints (indices 0 to wellKnownEndpointCount-1) +- Dynamic endpoints allocated from free list; doubles table size when full +- `get(token)` -- O(1) lookup by token's lower 32 bits +- `insert()` -- allocates from free list, encodes priority in upper 32 bits +- `remove()` -- returns slot to free list + +### FlowTransport -- [`FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h)`:199-315` + +Singleton managing all network communication: + +**Key methods:** +- `bind(publicAddress, listenAddress)` -- start listening +- `sendReliable(data, endpoint)` -- guaranteed delivery (retransmit on reconnect) +- `sendUnreliable(data, endpoint, openConnection)` -- fire-and-forget +- `addEndpoint(endpoint, receiver, priority)` -- register local endpoint +- `addWellKnownEndpoint(endpoint, receiver, priority)` -- register system service +- `removeEndpoint(endpoint, receiver)` -- unregister +- `resetConnection(address)` -- force reconnect +- `getDegraded()` -- `AsyncVar` for network degradation status + +--- + +## Request/Reply Pattern -- [`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h) + +The fundamental communication pattern in FDB: + +### RequestStream ([`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h)`:728-949`) + +A typed channel for sending requests to a service. Wraps a `NetNotifiedQueue`: + +```cpp +RequestStream getValue; // on StorageServerInterface + +// Sending a request: +GetValueRequest req; +req.key = myKey; +req.reply = ReplyPromise(); // self-addressed envelope +stream.send(req); +GetValueReply reply = wait(req.reply.getFuture()); +``` + +### ReplyPromise ([`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h)`:131-204`) + +Single-value reply channel embedded in requests: + +- Wraps a `NetSAV` (SAV + FlowReceiver + FastAllocated) +- Serialization only stores the endpoint token (16 bytes on wire) +- On deserialize at receiver: creates local endpoint to receive reply + +### Complete Request-Reply Lifecycle + +``` +CLIENT: +1. Create request with embedded ReplyPromise +2. stream.send(request) → serialize → FlowTransport.sendReliable() +3. Transport finds/creates Peer → queues in unsent buffer +4. connectionWriter() sends packet: [length][checksum][token][serialized data] +5. Client waits on reply.getFuture() + +SERVER: +6. connectionReader() receives packet → scanPackets() +7. Extract token → EndpointMap lookup → find receiver +8. deliver() actor: yield to correct TaskPriority, call receiver.receive() +9. NetNotifiedQueue deserializes request, queues for application +10. Application calls request.reply.send(response) +11. sendUnreliable() back to client's reply endpoint + +CLIENT: +12. connectionReader() receives reply packet +13. Token lookup → NetSAV::receive() → deserialize → SAV::send(value) +14. Client's wait() resolves +``` + +### ReplyPromiseStream ([`fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h)`:460-640`) + +Stream of replies with flow control: +- Server sends multiple responses +- Client acknowledges consumed bytes +- `setByteLimit()` configures backpressure threshold +- `onReady()` blocks server when too much data in-flight + +--- + +## Wire Protocol + +### Packet Format + +``` +┌─────────────────┬────────────┬───────────────┬──────────────┐ +│ Packet Length │ Checksum │ Token (UID) │ Message Data │ +│ (4 bytes) │ (8 bytes)* │ (16 bytes) │ (variable) │ +└─────────────────┴────────────┴───────────────┴──────────────┘ +* Checksum only for non-TLS connections (XXH3_64bits) +``` + +### ConnectPacket (sent at connection start) + +```cpp +struct ConnectPacket { + uint32_t connectPacketLength; + ProtocolVersion protocolVersion; + uint16_t canonicalRemotePort; + uint64_t connectionId; // multi-version client ID + uint32_t canonicalRemoteIp4; // or 0 for IPv6 + uint16_t flags; // FLAG_IPV6 + uint8_t canonicalRemoteIp6[16]; +}; +``` + +--- + +## Failure Monitoring -- [`FailureMonitor.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FailureMonitor.h) + +Tracks endpoint/process availability: + +``` +struct FailureStatus { bool failed; }; + +class SimpleFailureMonitor : public IFailureMonitor { + std::unordered_map addressStatus; + YieldedAsyncMap endpointKnownFailed; + AsyncMap disconnectTriggers; +}; +``` + +**Key methods:** +- `getState(endpoint)` -- current failure status +- `onStateChanged(endpoint)` -- future that fires on status change +- `onDisconnectOrFailure(endpoint)` -- wait for either event +- `onlyEndpointFailed(endpoint)` -- endpoint failed but address still up +- `setStatus(address, status)` -- update status + +**Detection mechanism** (`connectionMonitor()` actor): +1. Sends periodic ping to `WLTOKEN_PING_PACKET` +2. Waits for reply with `CONNECTION_MONITOR_TIMEOUT` (1s default) +3. On timeout: increment count, eventually throw `connection_failed()` +4. Records ping latency in `DDSketch` histogram +5. After `FAILURE_DETECTION_DELAY` (5s): mark address as failed + +--- + +## Locality & Replication -- [`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h), [`ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h) + +### LocalityData + +Key-value map of location attributes for each process: + +| Key | Example | Purpose | +|-----|---------|---------| +| `keyProcessId` | UUID | Unique process identifier | +| `keyZoneId` | "zone-a" | Failure zone | +| `keyMachineId` | "machine-1" | Physical machine | +| `keyDcId` | "dc-east" | Datacenter | +| `keyDataHallId` | "hall-1" | Data hall within DC | + +### ProcessClass ([`Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h)`:27-154`) + +Classifies process suitability for roles: +- **ClassType**: `StorageClass`, `TransactionClass`, `LogClass`, `CommitProxyClass`, `GrvProxyClass`, `MasterClass`, `ResolverClass`, etc. +- **Fitness**: `BestFit` > `GoodFit` > `UnsetFit` > `OkayFit` > `WorstFit` > `ExcludeFit` > `NeverAssign` + +### Replication Policies ([`ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h)) + +Composable constraint system: +- **PolicyOne** -- select 1 replica anywhere +- **PolicyAcross(count, attribute, subpolicy)** -- "N replicas across distinct values of attribute" + - Example: `PolicyAcross(3, "dcId", PolicyOne())` = 3 datacenters +- **PolicyAnd(policies...)** -- intersection of multiple policies + +Used by team building (Data Distribution) to ensure failure domain separation. + +### Load Balance Distance + +``` +enum LBDistance { SAME_MACHINE=0, SAME_DC=1, DISTANT=2 }; +``` + +Computed from locality attributes; used for preferring nearby replicas. + +--- + +## Sim2 -- Deterministic Simulator -- [`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h) + +Replaces Net2 during testing. Runs the entire cluster in a single OS thread with virtual time. + +### Key Properties + +- **Deterministic**: Given same random seed → identical execution +- **Virtual time**: No wall-clock dependency; time advances via task scheduling +- **Multi-process**: Many virtual processes in one real thread via `ProcessInfo` +- **Fault injection**: Kill processes, machines, zones, datacenters; inject network clogging, disconnection, disk errors + +### ProcessInfo (`SimulatorProcessInfo.h:42-170`) + +State per simulated process: +- `name`, `dataFolder`, `coordinationFolder` +- `machine` -- `MachineInfo*` for shared machine state +- `addresses` -- virtual network addresses +- `locality` -- zone/machine/DC placement +- `failed`, `excluded`, `rebooting`, `failedDisk` -- fault state +- `fault_injection_p1`, `fault_injection_p2` -- injection probabilities +- `globals` -- per-process global variables +- `shutdownSignal` -- Promise for clean shutdown + +### SimClogging ([`sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp)`:198-297`) + +Simulates network impairment: +- `getSendDelay()` / `getRecvDelay()` -- adds latency between addresses +- `clogPairFor(addr1, addr2, seconds)` -- delay packets between pair +- `disconnectPairFor(addr1, addr2, seconds)` -- sever connection between pair +- Latency model: 99.9% fast (`FAST_NETWORK_LATENCY`), 0.1% slow (`SLOW_NETWORK_LATENCY`) + +### Kill Operations + +| Method | Scope | Effect | +|--------|-------|--------| +| `killProcess(pid, kt)` | Single process | Fail/fault/reboot process | +| `killMachine(machineId, kt)` | All processes on machine | Checks replication safety first | +| `killZone(zoneId, kt)` | All machines in zone | | +| `killDataCenter(dcId, kt)` | All machines in DC | | + +**KillTypes**: `KillInstantly`, `InjectFaults`, `FailDisk`, `Reboot`, `RebootProcess`, `RebootAndDelete`, `RebootProcessAndDelete` + +--- + +## Load Balancing -- `LoadBalance.actor.h` + +Client-side request distribution: + +- **QueueModel**: Estimates response latency based on pending requests per server +- **Round-robin**: Distributes across replicas +- **Failure routing**: Routes around failed/degraded servers +- **Locality-aware**: Prefers geographically proximate servers (SAME_MACHINE > SAME_DC > DISTANT) +- **TSS comparison**: Can shadow requests to Testing Storage Servers for correctness validation + +--- + +## Key Constants + +| Knob | Default | Purpose | +|------|---------|---------| +| `INITIAL_RECONNECTION_TIME` | 0.2s | First reconnect delay | +| `MAX_RECONNECTION_TIME` | 30s | Max backoff | +| `RECONNECTION_TIME_GROWTH_RATE` | 1.5x | Backoff multiplier | +| `CONNECTION_MONITOR_TIMEOUT` | 1s | Ping timeout | +| `FAILURE_DETECTION_DELAY` | 5s | Before marking failed | +| `PACKET_LIMIT` | 512MB | Max packet size | +| `MIN_COALESCE_DELAY` | 0.5ms | Min buffer wait | +| `MAX_COALESCE_DELAY` | 2ms | Max buffer wait | + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbrpc/include/fdbrpc/fdbrpc.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/fdbrpc.h) | RequestStream, ReplyPromise, NetSAV, request/reply pattern | +| [`fdbrpc/include/fdbrpc/FlowTransport.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FlowTransport.h) | Endpoint, Peer, FlowTransport API | +| [`fdbrpc/FlowTransport.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/FlowTransport.cpp) | Transport implementation, EndpointMap, connection actors | +| [`fdbrpc/include/fdbrpc/FailureMonitor.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/FailureMonitor.h) | Failure detection interface and implementation | +| [`fdbrpc/include/fdbrpc/Locality.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/Locality.h) | LocalityData, ProcessClass, LBDistance | +| [`fdbrpc/include/fdbrpc/ReplicationPolicy.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/ReplicationPolicy.h) | PolicyOne, PolicyAcross, PolicyAnd | +| `fdbrpc/include/fdbrpc/LoadBalance.actor.h` | Client-side load balancing | +| [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp) | Sim2 deterministic simulator | +| [`fdbrpc/include/fdbrpc/simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h) | ISimulator interface, ProcessInfo | +| `flow/include/flow/IConnection.h` | IConnection, IListener interfaces | diff --git a/design/AI-generated/subsystem_03_client_library.md b/design/AI-generated/subsystem_03_client_library.md new file mode 100644 index 00000000000..e3bb2071b82 --- /dev/null +++ b/design/AI-generated/subsystem_03_client_library.md @@ -0,0 +1,288 @@ +# Subsystem 3: Client Library + +**[Diagrams](diagram_03_client_library.md)** + +**Location:** [`fdbclient/`](https://github.com/apple/foundationdb/tree/main/fdbclient) +**Size:** ~75K implementation, ~34K headers +**Role:** Everything an application (or internal FDB component) uses to read and write data. + +--- + +## Overview + +The client library provides a layered transaction API. From outside in: C API / language bindings -> MultiVersionTransaction -> ReadYourWritesTransaction -> NativeAPI Transaction. Most internal FDB code uses ReadYourWritesTransaction. External applications go through the C API. + +--- + +## Transaction Layers + +### Layer 1: NativeAPI / Transaction -- `NativeAPI.actor.h:311-525` + +The raw transaction. Directly communicates with proxies and storage servers. + +**TransactionState** (`NativeAPI.actor.h:255-309`): +``` +- Database cx // DatabaseContext reference +- Future readVersionFuture +- TransactionOptions options +- int64_t totalCost // accumulated operation cost for throttling +- Version committedVersion // set after successful commit +``` + +**Key methods:** +- `getReadVersion()` -- fetches version from GRV proxy (batched via versionBatcher) +- `get(key, Snapshot)` -- point read from storage server, adds read conflict range if !snapshot +- `getRange(begin, end, limits, Snapshot, Reverse)` -- range scan from storage server(s) +- `set(key, value)` -- appends `MutationRef(SetValue, key, value)` to mutations vector +- `clear(range)` -- appends `MutationRef(ClearRange, begin, end)` +- `commit()` -- sends `CommitTransactionRequest` to commit proxy via `tryCommit()` +- `onError(error)` -- handles retryable errors, resets transaction, implements backoff + +### Layer 2: ReadYourWritesTransaction -- `ReadYourWrites.h:69-268` + +Adds read-your-own-writes semantics on top of NativeAPI: + +``` +class ReadYourWritesTransaction { + Arena arena; + Transaction tr; // underlying native transaction + SnapshotCache cache; // cached reads from storage servers + WriteMap writes; // pending local writes + CoalescedKeyRefRangeMap readConflicts; +}; +``` + +**RYW read flow:** +1. Check `writes` (local write map) for the key +2. If found: return local value (or "cleared" indicator) +3. If not found: check `cache` (snapshot cache) for previously-read value +4. If not cached: issue read to storage server, cache result +5. Merge results using `RYWIterator` which walks both maps in parallel + +**RYW write flow:** +- `set()`/`clear()` buffer in local `WriteMap` +- On `commit()`: flush all writes to underlying `Transaction`, then commit + +### Layer 3: MultiVersionTransaction -- [`MultiVersionTransaction.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/MultiVersionTransaction.cpp) + +Supports running against clusters with different protocol versions: + +- **DLTransaction**: Wraps C API function pointers from dynamically-loaded client library +- **MultiVersionApi**: Loads multiple client `.so` files for protocol compatibility +- Converts between `ThreadFuture` (cross-thread) and internal `Future` +- Enables rolling upgrades with mixed-version clusters + +### Layer 4: C API / Language Bindings -- [`fdb_c.h`](https://github.com/apple/foundationdb/blob/main/bindings/c/foundationdb/fdb_c.h) + +FFI boundary for external languages: +- `fdb_setup_network()` / `fdb_run_network()` / `fdb_stop_network()` -- lifecycle +- `fdb_database_create_transaction()` -- create transaction +- `fdb_transaction_get()` / `fdb_transaction_set()` / `fdb_transaction_commit()` -- operations +- All async operations return `FDBFuture*` handles + +--- + +## DatabaseContext -- `DatabaseContext.h`, [`DatabaseContext.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseContext.cpp) + +Central state for a database connection. Shared by all transactions on this connection. + +### Location Cache (`DatabaseContext.h:410`) + +``` +CoalescedKeyRangeMap> locationCache; +``` + +Maps key ranges to storage server locations. Used to route reads directly to the right SS without asking a proxy every time. + +- `getCachedLocation(key)` -- O(log n) lookup in range map +- `setCachedLocation(range, servers)` -- insert/update, evicts old entries when cache full +- `invalidateCache(key)` / `invalidateCache(range)` -- clear on shard movement + +### Proxy Management + +``` +Reference commitProxies; // for commits and location queries +Reference grvProxies; // for read version assignment +AsyncTrigger onProxiesChanged; // fires when proxy set updates +``` + +Updated by `monitorProxies()` actor which watches for cluster configuration changes. + +### GRV Cache + +``` +double lastGrvTime; +Version cachedReadVersion; +``` + +Caches recent read versions to reduce GRV proxy load. Background updater via `backgroundGrvUpdater()`. + +--- + +## Commit Flow -- [`NativeAPI.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/NativeAPI.actor.cpp)`:4369+` + +### tryCommit() Actor + +1. **Estimate cost**: If tagged transaction, calculate cost for throttling +2. **Build CommitTransactionRequest**: + - `read_snapshot` = read version + - `mutations[]` = all set/clear/atomic operations + - `read_conflict_ranges[]` = keys that were read + - `write_conflict_ranges[]` = keys that were written +3. **Select proxy**: `basicLoadBalance()` across commit proxies, or first proxy if `commitOnFirstProxy` +4. **Send and wait**: Retry on proxy set change +5. **Handle reply**: + - **Success** (version != invalidVersion): update `committedVersion`, send versionstamp, update metadata cache + - **Conflict** (version == invalidVersion): extract conflicting key info, throw `not_committed()` + - **Unknown result**: use idempotency ID to determine final status + +### CommitTransactionRequest structure + +``` +struct CommitTransactionRequest { + SpanContext spanContext; + TransactionRef transaction { + VectorRef mutations; + VectorRef read_conflict_ranges; + VectorRef write_conflict_ranges; + Version read_snapshot; + }; + bool report_conflicting_keys; + // ... +}; +``` + +### MutationRef -- [`CommitTransaction.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/CommitTransaction.h)`:64-150` + +``` +struct MutationRef { + enum Type : uint8_t { + SetValue=0, ClearRange, AddValue, CompareAndClear, + SetVersionstampedKey, SetVersionstampedValue, + // atomic ops: And, Or, Xor, AppendIfFits, Max, Min, ByteMin, ByteMax, ... + }; + uint8_t type; + StringRef param1; // key (or begin key for ClearRange) + StringRef param2; // value (or end key for ClearRange) +}; +``` + +--- + +## Read Flow + +### Point Read -- `get(key)` + +``` +1. getReadVersion() → GRV proxy (batched) +2. getKeyLocation(key) → location cache or commit proxy +3. getValue(key, version) → storage server via loadBalance() +4. Return value (or Optional() if not found) +``` + +### Range Read -- `getRange(begin, end, limits)` + +``` +1. getReadVersion() +2. Resolve KeySelectors to concrete keys +3. For each shard in range: + a. getKeyLocation() → find storage server + b. Send GetKeyValuesRequest to SS + c. Accumulate results respecting row/byte limits + d. If more data and limits not reached, continue to next shard +4. Return RangeResult +``` + +### KeySelector Resolution + +``` +struct KeySelectorRef { + KeyRef key; + bool orEqual; + int offset; +}; + +firstGreaterOrEqual(k) = KeySelector(k, false, +1) +lastLessOrEqual(k) = KeySelector(k, true, 0) +lastLessThan(k) = KeySelector(k, false, 0) +firstGreaterThan(k) = KeySelector(k, true, +1) +``` + +### Proxy Load Balancing -- `ProxyLoadBalance.h` + +```cpp +// Pattern: send request, race against proxy set change +while (true) { + auto reply = basicLoadBalance(cx->getCommitProxies(), channel, req); + auto result = co_await race(reply, cx->onProxiesChanged()); + if (result.index() == 0) co_return std::get<0>(result); + // proxy set changed, retry with new proxies +} +``` + +--- + +## Cluster Discovery -- [`MonitorLeader.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/MonitorLeader.h) + +### Connection Flow + +1. Parse cluster file: `description:id@coord1,coord2,coord3` +2. `monitorLeader()` contacts coordinators, tracks current cluster controller +3. `monitorProxies()` fetches commit/GRV proxy lists from cluster controller +4. `DatabaseContext` updated with new proxy interfaces +5. Transactions use proxies for all operations + +### Watch Operations + +```cpp +Future watch = tr.watch(key); // register interest +wait(tr.commit()); // commit first +wait(watch); // then wait for change +``` + +- Watch metadata tracked in `DatabaseContext::watchMap` +- Storage server monitors key, notifies on value change +- Watch survives transaction commit + +--- + +## System Keys -- [`SystemData.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/SystemData.h) + +Special key ranges for cluster metadata (prefix `\xff`): + +| Range | Purpose | +|-------|---------| +| `keyServersKeys` (`\xff/keyServers/`) | key range → storage server mapping | +| `serverKeysRange` (`\xff/serverKeys/`) | storage server → owned key ranges (reverse map) | +| `serverListKeys` (`\xff/serverList/`) | storage server interfaces | +| `configKeys` (`\xff/conf/`) | database configuration | +| `tagLocalityListKeys` | datacenter locality assignments | +| `serverTagKeys` | storage server tag assignments | +| `moveKeysLockOwnerKey` | data distribution lock | +| `backupStartedKey` | active backup state | + +--- + +## Special Key Space (`\xff\xff` prefix) + +Management API exposed as key-value operations: + +- Read `\xff\xff/status/json` for cluster status +- Write to `\xff\xff/management/` for configuration changes +- `SpecialKeyRangeReadImpl` / `SpecialKeyRangeRWImpl` abstract handlers + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbclient/NativeAPI.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/NativeAPI.actor.cpp) | Core transaction implementation, tryCommit, getValue, getRange | +| `fdbclient/include/fdbclient/NativeAPI.actor.h` | Transaction class, TransactionState | +| [`fdbclient/ReadYourWrites.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/ReadYourWrites.actor.cpp) | RYW layer, write map, snapshot cache merging | +| `fdbclient/include/fdbclient/DatabaseContext.h` | DatabaseContext: location cache, proxy tracking, watches | +| [`fdbclient/include/fdbclient/CommitTransaction.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/CommitTransaction.h) | MutationRef, CommitTransactionRef | +| [`fdbclient/MultiVersionTransaction.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/MultiVersionTransaction.cpp) | Multi-version client, DLTransaction | +| `fdbclient/MonitorLeader.cpp` | Cluster discovery, leader tracking | +| [`fdbclient/include/fdbclient/SystemData.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/SystemData.h) | System key range definitions | +| `fdbclient/ProxyLoadBalance.h` | Proxy load balancing templates | diff --git a/design/AI-generated/subsystem_04_cluster_controller.md b/design/AI-generated/subsystem_04_cluster_controller.md new file mode 100644 index 00000000000..8d29c2e8d5b --- /dev/null +++ b/design/AI-generated/subsystem_04_cluster_controller.md @@ -0,0 +1,319 @@ +# Subsystem 4: Cluster Controller & Coordination + +**[Diagrams](diagram_04_cluster_controller.md)** + +**Location:** [`fdbserver/clustercontroller/`](https://github.com/apple/foundationdb/tree/main/fdbserver/clustercontroller), [`fdbserver/coordinator/`](https://github.com/apple/foundationdb/tree/main/fdbserver/coordinator), [`fdbserver/core/LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp) +**Size:** ~13K + coordination code +**Role:** Leader election, role recruitment, process registration, cluster-wide decision making. + +--- + +## Overview + +One process is elected Cluster Controller (CC). It is the "brain" of the cluster: it recruits all server roles, monitors health, broadcasts configuration, and initiates recovery. Leadership is determined by a coordination quorum running on designated coordinator processes. + +--- + +## Leader Election + +### Coordinators + +Typically 3 or 5 processes (addresses hardcoded in cluster file) that run a `leaderRegister` actor. They don't store user data -- only coordination metadata. Coordinators are not involved in committing transactions; they hold ~1KB of state that changes only during recovery. + +### Generation Register (Paxos Acceptor) -- [`Coordination.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/Coordination.cpp)`:121-173` + +The fundamental primitive: a register with generation numbers that prevents stale writes. Each coordinator runs a `localGenerationReg()` actor that implements a single Paxos acceptor. The generation register protocol is isomorphic to single-decree Paxos: + +| Paxos concept | FDB equivalent | +|---|---| +| Ballot number | `UniqueGeneration` (uint64 generation + UID tiebreaker) | +| Prepare(n) | `GenerationRegReadRequest(key, gen)` | +| Promise(n, v) | `GenerationRegReadReply(value, writeGen, readGen)` | +| Accept(n, v) | `GenerationRegWriteRequest(kv, gen)` | +| Quorum of acceptors | Majority of coordinators (n/2 + 1) | + +``` +struct GenerationRegVal { + UniqueGeneration readGen; // highest generation promised (Paxos: highest prepare seen) + UniqueGeneration writeGen; // highest generation accepted (Paxos: highest accept seen) + Optional val; // the accepted value +}; +``` + +**Read protocol** (`GenerationRegReadRequest`) -- equivalent to Paxos **prepare**: +- If client's generation > stored readGen: update readGen (promise not to accept lower) +- Persist to disk (`store->commit()`) +- Return (value, writeGen, readGen) + +**Write protocol** (`GenerationRegWriteRequest`) -- equivalent to Paxos **accept**: +- Check: `readGen <= requestGen AND writeGen < requestGen` +- If valid: update writeGen and value, persist, return writeGen +- If invalid: return `max(readGen, writeGen)` so client can retry with a higher generation + +The core consensus logic in `localGenerationReg()` is ~50 lines of code. + +### Election Algorithm -- [`LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp) + +**`tryBecomeLeaderInternal()`** (lines 107-300): + +1. **Nomination phase** (lines 136-227): + - Generate `changeID` and `LeaderInfo` (encodes process class fitness in top bits) + - Submit `CandidacyRequest` to all coordinators + - Each coordinator nominates the candidate with best fitness + - Loop: check if quorum nominates us + - If quorum: become leader. If not: retry with updated info + +2. **Heartbeat phase** (lines 233-294): + - Send `LeaderHeartbeatRequest` to all coordinators at `HEARTBEAT_FREQUENCY` + - Expect quorum to respond `true` + - If quorum responds `false`: leadership lost, restart nomination + - Update `changeID` when priority info changes + +**LeaderInfo structure:** +- `serializedInfo` -- serialized cluster interface +- `changeID` -- unique ID for this leadership claim (top 7 bits = process class fitness) +- `forward` flag -- signals connection string update + +### Coordinated State (Paxos Proposer) -- [`CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp)`:71-210` + +Higher-level abstraction that implements the Paxos proposer role over generation registers. This is where the quorum logic lives -- individual coordinators are acceptors that know nothing about quorums; `CoordinatedStateImpl` drives the two-phase protocol across the quorum. + +``` +struct CoordinatedStateImpl { + ServerCoordinators coordinators; + UniqueGeneration gen; + uint64_t conflictGen; + bool doomed; + bool initial; // true if no prior value has been written +}; +``` + +**`read()`** -- Paxos prepare phase (two rounds): +1. **Round 1**: Send `GenerationRegReadRequest(key, UniqueGeneration())` to all coordinators. Wait for majority. Learns the highest existing generation. +2. **Compute ballot**: `conflictGen = max(all seen generations) + 1`. Create `gen = UniqueGeneration(conflictGen, randomUID)`. +3. **Round 2**: Send `GenerationRegReadRequest(key, gen)` to all coordinators. Wait for majority. Each acceptor promises not to accept writes below `gen`. Returns the value with the highest `writeGen`. + +**`setExclusive()`** -- Paxos accept phase: +- Send `GenerationRegWriteRequest(key, value, gen)` to all coordinators. +- For initial state (no prior writes): wait for **all** replicas. Otherwise: wait for **majority** (n/2 + 1). +- If any coordinator returns a generation > `gen`, throw `coordinated_state_conflict` (another proposer won). + +**Quorum sizes** (from `replicatedRead` / `replicatedWrite`): +- Read quorum: `replicas.size() / 2 + 1` (majority) +- Write quorum: `replicas.size() / 2 + 1` (majority), except initial writes require all replicas + +**`replicatedRead()`** has a subtlety: it races two sets of futures -- replies with a value vs. replies without. If a majority report empty (no value), it returns the empty reply with the highest `rgen`. Otherwise it returns the non-empty reply with the highest `writeGen`. This handles the bootstrap case where some coordinators have been written and others haven't. + +Used by recovery to store/update `DBCoreState` (log system configuration). This is the only mutable state managed by coordinators. + +### Coordinator Storage -- [`OnDemandStore`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/OnDemandStore.cpp), [`DiskQueue`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DiskQueue.cpp) + +Each coordinator's Paxos state (generation numbers and the coordinated value) is persisted via `KeyValueStoreMemory` -- an in-memory key-value store backed by a `DiskQueue` for durability. + +**On-disk format:** +- Two files per coordinator: `coordination-0.fdq` and `coordination-1.fdq` in the process's data directory +- `.fdq` = **F**oundation**D**B **Q**ueue -- an append-only circular log that alternates between the two files +- Entries are checksummed (xxhash3 in DiskQueueVersion::V2) +- `KeyValueStoreMemory` periodically snapshots the full in-memory state into the log, after which older entries are reclaimed + +**Access pattern:** +- `OnDemandStore` is a lazy wrapper: opens the `KeyValueStoreMemory` only on first access (coordinator processes that are never contacted don't create files) +- `localGenerationReg()` calls `store->readValue(key)` on each request and `store->set()` + `store->commit()` to persist generation updates +- `store->commit()` flushes to the `.fdq` log, making Paxos promises and accepts crash-safe +- The memory budget is 500MB (`keyValueStoreMemory(path, id, 500e6)`) -- vastly more than needed for the ~1KB of coordination state, but it's the standard `KeyValueStoreMemory` interface + +This is the same storage engine used by transaction logs for their mutation queues and by `txnStateStore` during recovery. + +--- + +## ClusterControllerData -- `ClusterController.h:122-3412` + +The state maintained by the elected CC. + +### DBInfo (lines 124-215) + +``` +struct DBInfo { + Reference> clientInfo; // proxy list for clients + Reference> serverInfo; // full cluster config for servers + AsyncTrigger forceMasterFailure; + DatabaseConfiguration config, fullyRecoveredConfig; + int unfinishedRecoveries; + Database db; +}; +``` + +### Worker Pool + +``` +std::map>, WorkerInfo> id_worker; // processId → WorkerInfo +std::map>, ProcessClass> id_class; // processId → configured class +``` + +### Recruitment State + +``` +std::vector> outstandingRecruitmentRequests; +std::vector> outstandingRemoteRecruitmentRequests; +std::vector> outstandingStorageRequests; + +// Singleton role tracking +AsyncVar recruitDistributor, recruitRatekeeper, recruitConsistencyScan; +Optional recruitingDistributorID, recruitingRatekeeperID, recruitingConsistencyScanID; +``` + +### Health Monitoring + +``` +std::unordered_map workerHealth; +struct WorkerHealth { + std::unordered_map degradedPeers; + std::unordered_map disconnectedPeers; +}; +``` + +--- + +## Worker Registration -- [`ClusterController.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterController.actor.cpp)`:1252-1434` + +### RegisterWorkerRequest + +``` +struct RegisterWorkerRequest { + WorkerInterface wi; + ProcessClass initialClass, processClass; + ClusterControllerPriorityInfo priorityInfo; + Generation generation; + Optional distributorInterf; + Optional ratekeeperInterf; + Optional consistencyScanInterf; + bool degraded, recoveredDiskFiles; +}; +``` + +### Registration Flow + +1. **Cluster ID validation** -- reject workers from different clusters +2. **Priority calculation** -- compute CC fitness and DC fitness for this worker +3. **Process class resolution** -- use DB-configured class or initial class +4. **Worker add/update** -- create `WorkerInfo`, start watcher actor, add to `id_worker` +5. **Singleton handling** -- if worker reports DD/Ratekeeper/ConsistencyScan interface, update DBInfo +6. **Trigger recruitment check** -- call `checkOutstandingRequests()` to re-evaluate pending recruitment + +--- + +## Role Recruitment + +### Fitness Calculation -- `ClusterController.h:1443-1643` + +**RoleFitness structure:** +``` +struct RoleFitness { + ProcessClass::Fitness bestFit, worstFit; + ProcessClass::ClusterRole role; + int count, worstUsed; + bool degraded; +}; +``` + +**Comparison priority** (for `operator<`): +1. Worst fitness (lower is better) +2. Worst used count (fewer concurrent roles is better) +3. Total count (more is better) +4. Prefer non-degraded +5. For non-TLog: compare best fitness + +**Worker selection methods:** +- `getWorkerForRoleInDatacenter()` -- single best worker for role in DC +- `getWorkersForRoleInDatacenter()` -- multiple workers, prioritized by fitness +- `getWorkersForTlogsSimple()` -- TLog recruitment respecting zone distribution +- `getWorkersForTlogsComplex()` -- TLog recruitment for cross-zone policies + +### Recruitment Functions -- `ClusterRecovery.cpp` + +| Function | Role Recruited | +|----------|---------------| +| `recruitNewMaster()` | Master/Sequencer (must be same DC as CC) | +| `newCommitProxies()` | Commit proxies (via `InitializeCommitProxyRequest`) | +| `newGrvProxies()` | GRV proxies (via `InitializeGrvProxyRequest`) | +| `newResolvers()` | Resolvers (via `InitializeResolverRequest`) | +| `newTLogServers()` | TLogs, including remote region and log routers | +| `newSeedServers()` | Initial storage servers (version 0 only) | + +--- + +## ServerDBInfo Broadcasting -- `ServerDBInfo.h:34-88` + +### ServerDBInfo Contents + +``` +struct ServerDBInfo { + UID id; + ClusterControllerFullInterface clusterInterface; + ClientDBInfo client; // proxy lists + MasterInterface master; + Optional distributor; + Optional ratekeeper; + std::vector resolvers; + DBRecoveryCount recoveryCount; + RecoveryState recoveryState; + LifetimeToken masterLifetime; + LogSystemConfig logSystemConfig; + int64_t infoGeneration; // increment on each change +}; +``` + +### Broadcasting + +`dbInfoUpdater()` actor: +1. Waits for `updateDBInfo` trigger or `serverInfo` change +2. Batches changes with `DBINFO_BATCH_DELAY` +3. Serializes and sends `UpdateServerDBInfoRequest` to all workers +4. Workers store in their local `AsyncVar` and react to changes + +--- + +## ClusterController Main Loop -- `clusterControllerCore()` + +**Core actors spawned:** +- `clusterWatchDatabase()` -- monitors master, triggers recovery +- `statusServer()` -- handles status requests from fdbcli +- `timeKeeper()` -- periodic version tracking +- `monitorProcessClasses()` -- watches DB for class changes +- `monitorDataDistributor()` / `monitorRatekeeper()` / `monitorConsistencyScan()` -- singleton lifecycles +- `dbInfoUpdater()` -- broadcasts ServerDBInfo +- `workerHealthMonitor()` -- optional health tracking + +**Event loop handles:** +- `RegisterWorkerRequest` -- worker self-registration +- `RecruitStorageRequest` -- on-demand storage server recruitment +- `OpenDatabaseRequest` -- client database access +- `GetWorkersRequest` / `GetClientWorkersRequest` -- worker queries +- Coordination pings at `WORKER_COORDINATION_PING_DELAY` intervals +- Leader failure detection + +--- + +## WorkerInterface -- [`WorkerInterface.actor.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h)`:45-126` + +RPCs exposed by every worker process: + +**Initialization:** +`tLog`, `master`, `commitProxy`, `grvProxy`, `dataDistributor`, `ratekeeper`, `consistencyScan`, `resolver`, `storage`, `logRouter`, `backup` + +**Monitoring:** +`debugPing`, `coordinationPing`, `waitFailure`, `setMetricsRate`, `eventLogRequest`, `updateServerDBInfo` + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/clustercontroller/ClusterController.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterController.actor.cpp) | CC main loop, worker registration, event handling | +| `fdbserver/clustercontroller/ClusterController.h` | ClusterControllerData, fitness calculation, recruitment | +| [`fdbserver/coordinator/Coordination.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/coordinator/Coordination.cpp) | leaderRegister, generation register, coordination | +| [`fdbserver/core/LeaderElection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/LeaderElection.actor.cpp) | tryBecomeLeaderInternal, candidacy, heartbeat | +| [`fdbserver/core/CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp) | Replicated read/write over generation registers | +| [`fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/WorkerInterface.actor.h) | WorkerInterface, ClusterControllerFullInterface | +| `fdbserver/core/include/fdbserver/core/ServerDBInfo.h` | ServerDBInfo structure and broadcasting | diff --git a/design/AI-generated/subsystem_05_commit_pipeline.md b/design/AI-generated/subsystem_05_commit_pipeline.md new file mode 100644 index 00000000000..93d62016b7c --- /dev/null +++ b/design/AI-generated/subsystem_05_commit_pipeline.md @@ -0,0 +1,305 @@ +# Subsystem 5: Transaction Commit Pipeline + +**[Diagrams](diagram_05_commit_pipeline.md)** + +**Location:** [`fdbserver/commitproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/commitproxy), [`fdbserver/grvproxy/`](https://github.com/apple/foundationdb/tree/main/fdbserver/grvproxy), [`fdbserver/resolver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/resolver), [`fdbserver/sequencer/`](https://github.com/apple/foundationdb/tree/main/fdbserver/sequencer) +**Size:** ~12K +**Role:** Version assignment, conflict detection, commit batching -- the write-path orchestration. + +--- + +## Overview + +Four server roles collaborate in a strict pipeline to commit transactions: + +``` +Client ──CommitTransactionRequest──▶ CommitProxy + │ +Phase 1: GET VERSION ▼ + Master/Sequencer (monotonic version) +Phase 2: RESOLVE ▼ + Resolver(s) (conflict detection) +Phase 3: LOG ▼ + TLog replicas (durable write) +Phase 4: REPLY ▼ + Client (committed!) +``` + +--- + +## Commit Proxy -- [`CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp) + +The workhorse of the commit pipeline. Batches client commits and drives them through all phases. + +### ProxyCommitData -- `ProxyCommitData.h:183` + +``` +struct ProxyCommitData { + MasterInterface master; // for version assignment + std::vector resolvers; // for conflict detection + Reference logSystem; // for TLog push + IKeyValueStore* txnStateStore; // persistent metadata + + NotifiedVersion committedVersion; // largest durable version + NotifiedVersion version; // current proxy version + + KeyRangeMap keyInfo; // key range → storage servers + tags + + const std::vector& tagsForKey(StringRef key); // tag lookup for mutations +}; +``` + +### commitBatch() -- 5-Phase Pipeline + +**CommitBatchContext** (`CommitProxyServer.actor.cpp:491`): +``` +struct CommitBatchContext { + std::vector trs; // batch of transactions + LogPushData toCommit; // serialized mutations for TLogs + Version commitVersion, prevVersion; + std::vector resolution; // conflict results + std::set writtenTags; +}; +``` + +**Phase 1: Pre-Resolution** (`preresolutionProcessing`): +- Validate transaction sizes +- Compute read/write conflict ranges +- Determine storage server tags via `tagsForKey()` for each mutation +- Check hot shard throttling + +**Phase 2: Resolution** (`getResolution`): +- Send `ResolveTransactionBatchRequest` to all resolvers in parallel +- Each resolver handles a partition of the key space +- Wait for all responses + +**Phase 3: Post-Resolution** (`postResolution`): +- Process conflict results: mark conflicting transactions +- Apply metadata mutations (`applyMetadataToCommittedTransactions()`) +- Update storage server mappings +- Process idempotency keys + +**Phase 4: Transaction Logging** (`transactionLogging`): +- Call `logSystem->push()` with tagged mutations +- Block until mutations are durable (TLog quorum ack) +- Update `queueCommittedVersion` + +**Phase 5: Reply** (`reply`): +- Send `CommitTransactionReply` to each client +- Conflicting transactions get `not_committed` error +- Successful transactions get their commit version + +--- + +## GRV Proxy -- [`GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp) + +Assigns read versions to client transactions. The GRV proxy is the primary enforcement point for cluster-wide flow control: it is where ratekeeper's back-pressure decisions become concrete delays and rejections applied to client requests. + +### GrvProxyData (line 202) + +``` +struct GrvProxyData { + MasterInterface master; // version source + VersionVector ssVersionVectorCache; // storage server version tracking + Version version; +}; +``` + +### End-to-End Flow Control: Ratekeeper → GRV Proxy → Client + +Flow control is the mechanism that prevents the cluster from being overwhelmed when storage servers or TLogs fall behind. It works as a feedback loop: + +**1. Ratekeeper computes global TPS limits** (`Ratekeeper::updateRate()`, `Ratekeeper.cpp:642`): +- Runs periodically (every `METRIC_UPDATE_RATE` seconds, default 0.1s). +- Polls every storage server and TLog for queue depths, durable bytes rates, and free disk space. +- Computes a `tpsLimit` for both `normalLimits` (DEFAULT + SYSTEM priority) and `batchLimits` (BATCH priority) independently. +- The core formula scales `actualTps` (smoothed observed transaction rate) by the ratio of durable-bytes throughput to input-bytes rate, further modulated by a `targetRateRatio` derived from how full the storage/TLog queue is relative to target and spring thresholds. The result is: if queues are within target, `tpsLimit ≈ infinity`; as queues grow past the target, the limit ramps down toward zero. +- Batch limits use more aggressive thresholds (larger `storageTargetBytes`, `logTargetBytes`) so batch transactions are throttled first. + +**2. Ratekeeper sends per-proxy rate limits** (`handleGetRateInfoReqs()`, `Ratekeeper.cpp:347`): +- Each GRV proxy periodically sends a `GetRateInfoRequest` to ratekeeper (every `leaseDuration/2` seconds, jittered). +- Ratekeeper replies with `transactionRate = normalLimits.tpsLimit / numProxies` and `batchTransactionRate = batchLimits.tpsLimit / numProxies` — the global limit divided evenly across all GRV proxies. +- The reply also includes a `leaseDuration` (default `METRIC_UPDATE_RATE` = 0.1s). If the proxy doesn't hear back within the lease, it disables rate limiting (sets rate to 0 via `GrvTransactionRateInfo::disable()`), which smoothly ramps the allowed rate to zero and stops releasing transactions. + +**3. GRV proxy enforces the rate via a token bucket** (`GrvTransactionRateInfo`, `GrvTransactionRateInfo.h`): +- Each proxy maintains two `GrvTransactionRateInfo` objects: `normalRateInfo` (for SYSTEM + DEFAULT) and `batchRateInfo` (for BATCH). +- Each object is a smoothed token bucket: `setRate()` feeds the ratekeeper-provided rate through a `Smoother`, and `startReleaseWindow()` computes a `limit` from the smoothed difference between the allowed rate and the actual release rate, times the rate window. +- `canStart(numAlreadyStarted, count)` checks if `numAlreadyStarted + count <= limit + budget`. The `budget` accumulates unused capacity across release windows, bounded by `maxEmptyQueueBudget` when queues are empty (to avoid stale budget causing bursts). +- The smoothing prevents sudden rate changes from causing oscillation. + +**4. The `transactionStarter` loop releases batches** (`transactionStarter()`, line 877): +- On each GRVTimer tick, it drains the three priority queues in strict order: SYSTEM first, then DEFAULT, then BATCH. +- SYSTEM transactions bypass rate limiting entirely (`canStart` is never checked for them). +- DEFAULT transactions are gated by `normalRateInfo.canStart()`. +- BATCH transactions are gated by `batchRateInfo.canStart()`, which uses the more aggressive batch rate limit. +- Transactions that pass rate-limiting are batched into a single `getLiveCommittedVersion()` call to the master, and replies are sent to all clients in the batch simultaneously. + +### Dynamic Batching Based on Reply Latency + +The GRV batch interval is adaptively tuned based on observed round-trip latency (`queueGetReadVersionRequests`, line 518): + +- After each batch of non-risky-read requests completes, `timeReply()` measures the round-trip time and feeds it into a `normalGRVLatency` stream. +- The `queueGetReadVersionRequests` actor receives this latency and computes: + ``` + target_latency = reply_latency * START_TRANSACTION_BATCH_INTERVAL_LATENCY_FRACTION + GRVBatchTime = α * target_latency + (1 - α) * GRVBatchTime + ``` + clamped to `[START_TRANSACTION_BATCH_INTERVAL_MIN, START_TRANSACTION_BATCH_INTERVAL_MAX]`. +- When the first request arrives into empty queues, a GRVTimer is scheduled with delay `max(0, GRVBatchTime - timeSinceLastGRV)`. This means: when the system is fast, batching intervals shrink (lower latency for clients); when the system is slow, batches grow larger (better amortization of the master round-trip). + +### Lower Priorities Dropped Under Pressure + +When the total GRV queue depth exceeds `START_TRANSACTION_MAX_QUEUE_SIZE`, the proxy drops lower-priority requests to make room (`queueGetReadVersionRequests`, line 541): + +- A new **BATCH** request is rejected immediately with `grv_proxy_memory_limit_exceeded`. +- A new **DEFAULT** request evicts one BATCH request from the front of its queue (if any); if none exist, the DEFAULT request itself is rejected. +- A new **SYSTEM** request evicts one BATCH request, or failing that one DEFAULT request; only if both queues are empty is the SYSTEM request itself rejected. +- Additionally, when the batch rate from ratekeeper drops to effectively zero (`batchRateInfo.getRate() <= 1/numProxies`), BATCH requests are immediately rejected with `batch_transaction_throttled` before even being queued (line 605). + +### GetReadVersion Flow + +1. **Queuing** (`queueGetReadVersionRequests`, line 518): + - Three priority queues: SYSTEM, DEFAULT, BATCH + - Dynamic batching as described above + +2. **Version fetch** (`getLiveCommittedVersion`, line 670): + - Request to master: `master.getLiveCommittedVersion.getReply()` + - Causal read check (unless `CAUSAL_READ_RISKY`): `updateLastCommit()` confirms epoch is still live by calling `logSystem->confirmEpochLive()`. This ensures the version returned actually represents committed state. + - Version vector integration if enabled: applies delta from master to local SS version cache + +3. **Reply** (`sendGrvReplies`, line 747): + - Sends the version to all requests in the batch + - Attaches per-tag throttle information so clients can self-throttle + - Sets `rkBatchThrottled` / `rkDefaultThrottled` flags if sustained throttling detected (sustained for `GRV_SUSTAINED_THROTTLING_THRESHOLD` seconds), signaling clients to back off + +--- + +## Master/Sequencer -- [`masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp) + +Assigns monotonically increasing commit versions. + +### MasterData -- `MasterData.h:50` + +``` +struct MasterData { + Version lastEpochEnd; // last version from prior epoch + Version recoveryTransactionVersion; // first version in this epoch + NotifiedVersionValue liveCommittedVersion; // largest live-committed version + Version version; // last assigned version + double lastVersionTime; // timestamp of last version + Optional referenceVersion; // for wall-clock alignment + + std::map lastCommitProxyVersionReplies; + VersionVector ssVersionVector; // per-SS commit versions + ResolutionBalancer resolutionBalancer; +}; +``` + +### Version Assignment -- `getVersion()` (lines 74-178) + +1. **Proxy validation**: Check proxy is known +2. **Request ordering**: Wait for prior request (`latestRequestNum.whenAtLeast(requestNum - 1)`) +3. **Deduplication**: Return cached reply if already processed +4. **Version calculation**: + ``` + toAdd = max(1, min(MAX_READ_TRANSACTION_LIFE_VERSIONS, + VERSIONS_PER_SECOND * (now - lastVersionTime))) + ``` + If `referenceVersion` set, align to wall clock via `figureVersion()` +5. **Reply**: `GetCommitVersionReply` with `prevVersion` and `version` + +### figureVersion() -- Wall-Clock Alignment (lines 43-72) + +``` +expectedVersion = now * VERSIONS_PER_SECOND - referenceVersion +version = clamp(version + toAdd, expectedVersion ± scaled_bounds) +``` + +Keeps versions roughly aligned with wall-clock microseconds while maintaining monotonicity. + +### Live Committed Version Tracking + +- `serveLiveCommittedVersion()`: Responds to GRV proxy and proxy reports +- `updateLiveCommittedVersion()`: Updates when proxy reports new committed version +- If Version Vectors enabled: tracks per-tag commit info + +--- + +## Resolver -- [`Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp), [`ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp) + +Pure conflict detection. Maintains a sliding window of committed write ranges. + +### Resolver Structure (line 128) + +``` +struct Resolver { + int commitProxyCount, resolverCount; + NotifiedVersion version; + ConflictSet* conflictSet; // SkipList-based conflict tracking + IKeyValueStore* txnStateStore; // metadata KVS + std::map proxyInfoMap; + KeyRangeMap keyInfo; +}; +``` + +### ConflictSet + +``` +struct ConflictSet { + SkipList versionHistory; // version → conflict data + Version oldestVersion; +}; +``` + +### resolveBatch() (line 261) + +1. **Memory check**: Block if `totalStateBytes > RESOLVER_STATE_MEMORY_LIMIT` +2. **Version ordering** (`versionReady()`): Wait for resolver to reach `prevVersion` +3. **Conflict detection**: + ```cpp + ConflictBatch conflictBatch(self->conflictSet, &reply.conflictingKeyRangeMap); + for (auto& txn : req.transactions) + conflictBatch.addTransaction(txn, newOldestVersion); + conflictBatch.detectConflicts(req.version, newOldestVersion, commitList, &tooOldList); + ``` + - For each transaction: check if read ranges overlap with committed writes at newer versions + - Populates `reply.committed[]` with per-transaction conflict status +4. **State transaction processing**: Apply metadata mutations if enabled +5. **Version cleanup**: Erase old state transactions to bound memory + +### ConflictBatch::detectConflicts() + +``` +void detectConflicts(Version now, Version newOldestVersion, + std::vector& nonConflicting, + std::vector* tooOldTransactions) +``` + +Algorithm: For each transaction's read ranges, check overlap with committed write ranges in the SkipList at versions newer than the transaction's read version. + +--- + +## Tag Assignment + +Tags connect mutations to storage servers: + +1. **During pre-resolution**: For each mutation, `tagsForKey(mutation.key)` returns tags +2. **Tag source**: `KeyRangeMap keyInfo` maps key ranges → storage servers → tags +3. **Tags added to LogPushData**: `toCommit.addTags(tags)` for each mutation +4. **In TLog**: Messages routed to `TagData[tag.locality][tag.id]` queues +5. **Storage servers**: Peek their assigned tag to pull relevant mutations + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/commitproxy/CommitProxyServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/commitproxy/CommitProxyServer.actor.cpp) | 5-phase commit batch pipeline | +| `fdbserver/commitproxy/ProxyCommitData.h` | ProxyCommitData, tag lookup | +| [`fdbserver/grvproxy/GrvProxyServer.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/grvproxy/GrvProxyServer.cpp) | Read version assignment, rate limiting enforcement | +| `fdbserver/grvproxy/GrvTransactionRateInfo.h` | Token-bucket rate limiter driven by ratekeeper | +| [`fdbserver/sequencer/masterserver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/sequencer/masterserver.cpp) | Version assignment, wall-clock alignment | +| `fdbserver/sequencer/MasterData.h` | MasterData, version tracking | +| [`fdbserver/resolver/Resolver.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/Resolver.cpp) | Conflict detection, batch resolution | +| [`fdbserver/resolver/ConflictSet.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/resolver/ConflictSet.cpp) | SkipList-based conflict range tracking | diff --git a/design/AI-generated/subsystem_06_tlog_logsystem.md b/design/AI-generated/subsystem_06_tlog_logsystem.md new file mode 100644 index 00000000000..429c613b510 --- /dev/null +++ b/design/AI-generated/subsystem_06_tlog_logsystem.md @@ -0,0 +1,243 @@ +# Subsystem 6: Transaction Log (TLog) & Log System + +**[Diagrams](diagram_06_tlog_logsystem.md)** + +**Location:** [`fdbserver/tlog/`](https://github.com/apple/foundationdb/tree/main/fdbserver/tlog), [`fdbserver/logsystem/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logsystem), [`fdbserver/logrouter/`](https://github.com/apple/foundationdb/tree/main/fdbserver/logrouter) +**Size:** ~17K +**Role:** Durable mutation logging, tag-partitioned replication, peek cursors for storage servers. + +--- + +## Overview + +The TLog subsystem is FDB's durability guarantee. All committed mutations flow through TLogs before being applied to storage servers. A mutation is committed when a quorum of TLog replicas acknowledge. Storage servers then pull (peek) mutations asynchronously. + +--- + +## TLog Server -- [`TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp) + +### TLogData (lines 295-420) + +Top-level container for all TLog data in a process: + +``` +struct TLogData { + Deque popOrder; // pop ordering across generations + Deque spillOrder; // spill ordering + std::map> id_data; // all log generations by UID + + IKeyValueStore* persistentData; // spilled data on disk (KVStore) + IDiskQueue* rawPersistentQueue; // physical disk queue + TLogQueue* persistentQueue; // logical queue wrapper + + int64_t diskQueueCommitBytes; + int64_t bytesInput, bytesDurable; + int64_t targetVolatileBytes; // spill threshold + + NotifiedVersion queueCommitEnd; // durable commit location + Version queueCommitBegin; +}; +``` + +### LogData (lines 422-810) + +Per-epoch/generation log container: + +``` +struct LogData { + struct TagData : ReferenceCounted { + std::deque> versionMessages; + bool nothingPersistent; + Version popped; // consumed up to this version + Version persistentPopped; // durable pop point + Tag tag; + }; + + Map> versionLocation; + Deque>>> messageBlocks; + std::vector>> tag_data; // [locality][id] + + VersionMetricHandle persistentDataVersion; + VersionMetricHandle persistentDataDurableVersion; + NotifiedVersion version; // current log version + NotifiedVersion queueCommittedVersion; // disk queue durability point + Version knownCommittedVersion; +}; +``` + +### Push Flow (`tLogPush`) + +1. Receive mutation batch from commit proxy +2. Deserialize `TLogQueueEntry` containing tagged mutations +3. Route each message by tag to appropriate `TagData` queue +4. Append to `versionMessages` deque (in-memory) +5. Write to `persistentQueue` (DiskQueue) for durability +6. Update `versionLocation` map +7. Reply when disk write confirmed + +### Peek Flow (`tLogPeekStream`) + +1. Storage server requests mutations for tag starting at version +2. Iterate `TagData::versionMessages` from requested version +3. For versions not in memory: read from `persistentData` (KVStore) +4. Send results in `TLogPeekStreamReply` chunks +5. Respect `limitBytes` and `peekMemoryLimiter` for backpressure +6. Track popped version and cursor position + +### DiskQueue -- `TLogQueue` (lines 112-285) + +Append-only durable queue: +- `push(entry)` -- add entry with tag metadata +- `pop(location)` -- remove committed entries up to location +- Entries prefixed with length + protocol version +- Recovery-safe: CRC checks for partial writes +- Linked to version boundaries for efficient cleanup + +### Spilling + +When in-memory data exceeds `targetVolatileBytes`: +- Oldest data spilled from memory to `persistentData` (KVStore on disk) +- `spillOrder` determines which generation spills first +- Peek reads transparently merge in-memory and spilled data + +--- + +## Log System -- [`LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp) + +Abstraction over the set of TLog replicas. + +### LogSystem Interface -- [`LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h)`:260-601` + +**Core methods:** + +| Method | Purpose | +|--------|---------| +| `push(versionSet, data, spanContext)` | Send mutations to TLog replicas, wait for quorum | +| `peek(dbgid, begin, end, tag)` | Create cursor for reading mutations by tag | +| `peekSingle(dbgid, begin, tag, history)` | Peek from preferred TLog | +| `peekLogRouter(dbgid, begin, tag)` | Peek from same-generation logs only | +| `peekTxs(dbgid, begin, tag, locality, isLong)` | Peek transaction state mutations | +| `pop(upTo, tag, knownCommittedVersion)` | Allow TLog to discard messages before version | +| `onCoreStateChanged()` | Future that fires when log set changes | +| `newEpoch(recr, fRemoteWorkers, config, ...)` | Create new log epoch during recovery | + +### IPeekCursor Interface (lines 60-141) + +Lazy iterator over log messages: + +``` +struct IPeekCursor { + bool hasMessage() const; // current message exists? + VectorRef getTags() const; // tags for current message + Arena& arena(); // arena for message data + ArenaReader* reader(); // deserialize current message + void nextMessage(); // advance to next + Future getMore(TaskPriority); // wait for more messages + LogMessageVersion version(); // current message version + Version popped(); // earliest available version + bool isActive() / isExhausted(); // cursor status +}; +``` + +### Push Implementation + +1. **Tag distribution**: Partition mutations by tag (from commit proxy's `tagsForKey()`) +2. **TLog selection**: `getPushLocations()` determines which TLogs handle each tag +3. **Message formatting**: Build `LogPushData` with serialized mutations +4. **Parallel push**: Send to all relevant TLog replicas +5. **Quorum wait**: Block until `f+1` of `2f+1` TLogs acknowledge +6. **Return**: Promise resolves to next pushable version + +### Epoch Management + +Each recovery creates a new "epoch" of TLogs: +- `newEpoch()` recruits new TLogs and creates new log generation +- Old epochs kept until all storage servers have consumed their data +- `purgeOldRecoveredGenerations()` cleans up fully-consumed epochs +- Peek cursors can span multiple epochs during recovery + +--- + +## Log Router -- [`LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp) + +Routes mutations across regions in multi-datacenter deployments: + +- Reads mutations from primary TLogs via peek cursor +- Forwards to remote region's TLogs +- Reduces cross-region traffic by batching +- Uses `tagLocalityLogRouter` tags +- Bridges tag-partitioned log system across datacenters + +--- + +## Tag System + +### Tag Structure + +``` +struct Tag { + int8_t locality; // identifies the shard location + uint16_t id; // identifies the specific storage server +}; +``` + +### Special Tag Localities + +| Locality | Purpose | +|----------|---------| +| `tagLocalitySpecial` | Special/system tags | +| `tagLocalityLogRouter` | Log router tags for cross-region | +| `tagLocalityRemoteLog` | Remote region log tags | +| `tagLocalityUpgraded` | Tags from older versions | +| `tagLocalitySS` | Standard storage server tags | + +### Tag Assignment Flow + +1. **Cluster controller** assigns tags to storage servers during recruitment +2. **Tags stored** in `serverTagKeys` system key range +3. **Commit proxy** maintains `keyInfo` map: key range → storage servers → tags +4. **During commit**: `tagsForKey(key)` looks up tags for each mutation's key +5. **TLog routing**: Tags attached to mutations; TLog stores per-tag queues +6. **Storage server**: Peeks its assigned tag to pull only relevant mutations + +### Version Vector + +Optional per-storage-server version tracking: +- Master maintains `VersionVector ssVersionVector` +- GRV proxy caches and sends delta in `GetReadVersionReply` +- Enables per-shard causal consistency without global ordering + +--- + +## Data Flow: Mutation Lifecycle + +``` +1. Client commits transaction with mutations +2. CommitProxy assigns tags: mutation.key → tagsForKey() → [Tag1, Tag2, ...] +3. CommitProxy calls logSystem->push(mutations_with_tags) +4. LogSystem sends to all relevant TLog replicas +5. Each TLog: + a. Routes mutation to TagData[tag.locality][tag.id].versionMessages + b. Writes to DiskQueue for durability + c. Acks to LogSystem +6. LogSystem waits for quorum → commit confirmed +7. Storage server (background): + a. logSystem->peek(myTag, myVersion) → IPeekCursor + b. cursor.getMore() → new mutations + c. Apply mutations to local KVStore + d. Advance version + e. logSystem->pop(myVersion, myTag) → allow TLog cleanup +``` + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/tlog/TLogServer.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/tlog/TLogServer.actor.cpp) | TLog server: push, peek, spill, DiskQueue | +| [`fdbserver/logsystem/LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp) | LogSystem: push/peek/pop across TLog replicas | +| [`fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/include/fdbserver/logsystem/LogSystem.h) | LogSystem, IPeekCursor interfaces | +| [`fdbserver/core/include/fdbserver/core/LogSystemConfig.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/LogSystemConfig.h) | Log system configuration and epoch tracking | +| [`fdbserver/core/include/fdbserver/core/IDiskQueue.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/IDiskQueue.h) | IDiskQueue interface for TLog persistence | +| [`fdbserver/logrouter/LogRouter.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logrouter/LogRouter.cpp) | Cross-region mutation routing | diff --git a/design/AI-generated/subsystem_07_storage_server.md b/design/AI-generated/subsystem_07_storage_server.md new file mode 100644 index 00000000000..8df3bc998ff --- /dev/null +++ b/design/AI-generated/subsystem_07_storage_server.md @@ -0,0 +1,766 @@ +# Subsystem 7: Storage Server & Engines + +**[Diagrams](diagram_07_storage_server.md)** + +**Location:** [`fdbserver/storageserver/`](https://github.com/apple/foundationdb/tree/main/fdbserver/storageserver), [`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore) +**Size:** ~63K +**Role:** Serves reads, applies mutations from log, pluggable storage backends. + +--- + +## Overview + +Storage servers are the read path and the materialized state of the database. Each SS owns a set of key ranges (shards), continuously pulls committed mutations from the TLog via tagged peek cursors, applies them to a local key-value store, and serves client reads at specific versions. + +--- + +## StorageServer Structure -- [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)`:850-1528` + +``` +struct StorageServer : IStorageMetricsService { + // MVCC data + VersionedData versionedData; // versioned sets/clears + std::map> mutationLog; // (durableVersion, version] + + // Shard ownership + KeyRangeMap> shards; // key range → shard state + KeyRangeMap newestAvailableVersion; // key availability per version + + // Version tracking + NotifiedVersion version; // current SS version + NotifiedVersion desiredOldestVersion; // target for advancing durable + NotifiedVersion oldestVersion; // minimum readable version + NotifiedVersion durableVersion; // committed to disk + + // Log system connection + Reference logSystem; + Reference logCursor; + + // Storage engine + StorageServerDisk storage; // wraps IKeyValueStore + + // Move-in tracking + std::unordered_map> moveInShards; +}; +``` + +--- + +## StorageServerDisk Wrapper -- [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)`:602-719` + +`StorageServerDisk` is the thin layer between the storage server logic and the `IKeyValueStore` interface. It delegates all I/O to the underlying engine and adds metrics counters (`kvGets`, `kvScans`, `kvCommits`, `kvCommitLogicalBytes`, `kvClearRanges`, `kvClearSingleKey`). + +Key methods: +- **`readValue(key)`** / **`readValuePrefix(key, maxLength)`** -- point reads delegated to `storage->readValue()`, incrementing `kvGets` +- **`readRange(keys, rowLimit, byteLimit)`** -- range reads delegated to `storage->readRange()`, incrementing `kvScans` +- **`readNextKeyInclusive(key)`** -- helper that does a `readRange(key, allKeys.end, rowLimit=1)` and returns the first key found (or `allKeys.end`) +- **`makeVersionMutationsDurable(prevVersion, newVersion, bytesLeft, ...)`** -- iterates the in-memory `mutationLog` and writes `set()`/`clear()` calls to the storage engine for each mutation +- **`commit()`** -- calls `storage->commit()` to make buffered writes durable +- **Checkpoint operations** -- `checkpoint()`, `restore()`, `deleteCheckpoint()` -- forwarded for physical shard movement + +--- + +## Update Loop -- [`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp)`:9562+` + +The `update()` function (a coroutine) pulls mutations from the log system and applies them locally. + +### Flow + +1. **Emergency brake** (lines 9576-9610): + - If `queueSize >= STORAGE_HARD_LIMIT_BYTES` and `durableVersion < desiredOldestVersion`: block + - Prevents OOM from unbounded mutation accumulation + - Tracks `lastBytesInputEBrake` and `lastDurableVersionEBrake` to allow progress between braking periods + +2. **Log cursor read** (lines 9636-9651): + - `cursor->getMore()` -- wait for new mutations from TLog + - Validate cursor hasn't been popped (would mean SS removed via `worker_removed()`) + +3. **Eager read collection** (lines 9674+): + - Clone cursor, extract mutations to identify keys needing eager reads + - `UpdateEagerReadInfo` collects keys that need pre-reading (for atomic ops like `AppendIfFits`, `ByteMin`/`ByteMax`, `CompareAndClear`, and `ClearRange` endpoint resolution) + - `doEagerReads()` prefetches values from the storage engine + - Handles shard changes during read, retries if needed + +4. **Mutation application** (version loop): + - Create `StorageUpdater` for version transitions + - For each mutation at each version: + - `applyMutation(data, mutation, version)` -- update MVCC data + - Advance `data->version` on new version boundary + - Yield if accumulated bytes exceed `DESIRED_UPDATE_BYTES` + +### Version Management + +| Version | Meaning | +|---------|---------| +| `version` | Latest version SS knows about (from TLog stream) | +| `durableVersion` | Latest version committed to disk | +| `oldestVersion` | Minimum version readable from storage | +| `desiredOldestVersion` | Target set by data distributor for MVCC cleanup | + +**Gap `(durableVersion, version]`**: Held in MVCC `mutationLog` (memory). +**Gap `(oldestVersion, durableVersion]`**: On disk but not yet MVCC-freeable. + +--- + +## Durability Loop -- `updateStorage()` (line 10274) + +The `updateStorage` actor runs continuously and flushes in-memory mutations to the storage engine. + +### Flow + +1. **Wait** until `desiredOldestVersion > storageVersion` (i.e., the data distributor has indicated it's safe to advance) +2. **Check pending operations** -- inspect `pendingCheckpoints` and clamp `desiredVersion` accordingly +3. **Write mutations** via `storage.makeVersionMutationsDurable()` which iterates the `mutationLog` from `storageVersion` to `desiredVersion`, calling `storage.set()` and `storage.clear()` for each mutation. This is bounded by `STORAGE_COMMIT_BYTES` (bytes limit) and for RocksDB also by `ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT` +4. **Advance `oldestVersion`** and asynchronously free MVCC data for versions before `newOldestVersion` +5. **Checkpoint handling** -- if a pending checkpoint version was reached, create the checkpoint via `storage.checkpoint()` +6. **`storage.commit()`** -- atomically make all the above mutations durable +7. **Advance `durableVersion`** and signal `durableInProgress` to unblock the update loop +8. **Pop** old TLog data via `data->logSystem->pop()` now that mutations are durable + +### Persistent Version Tracking + +Each storage engine stores a special key `\xff\xffVersion` (`persistVersion`) that records the durable version. This is written by `makeVersionDurable()` on every commit and read during `restoreDurableState()` on startup. On recovery, the storage server reads `persistVersion` from the engine to learn how far it had committed, then resumes pulling mutations from the TLog starting from that version. If `persistVersion` is absent, the database was never initialized and the storage server self-destructs. + +### RocksDB-Specific Tuning + +For RocksDB backends, `updateStorage()` has additional controls: +- **`ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT`** -- limits the number of `DeleteRange` operations per commit to avoid excessive compaction pressure +- **`ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS`** -- enables pre-reading clear range endpoints for optimization +- **`canCommit()` backpressure** -- before committing, the RocksDB engine checks pending compaction bytes and immutable memtable count; if overloaded, it delays the commit + +--- + +## Read Serving + +### Point Read -- `getValueQ()` (line 2148) + +1. **Priority management**: Acquire `PriorityMultiLock` with request priority +2. **Version wait**: `waitForVersion(data, commitVersion, req.version)` -- block until SS version >= requested +3. **Shard check**: `data->shards[key]->isReadable()` -- throw `wrong_shard_server()` if not owned +4. **Value read**: + - Try MVCC data at version: `data->data().at(version).lastLessOrEqual(key)` + - If not in MVCC: read from storage engine: `data->storage.readValue(key)` + - Validate version hasn't become too old, shard ownership hasn't changed +5. **Response**: Return value with penalty metric + +### Range Read -- `getKeyValuesQ()` (line 3314) + +1. **Shard validation**: Ensure begin and end resolve within single shard +2. **Key selector resolution**: Resolve begin/end `KeySelector` to concrete keys via `findKey()` +3. **Range read**: `readRange(data, version, range, limit, &remainingBytes)` +4. **Shard change detection**: Double-check ownership via change counters before returning + +--- + +## Shard Management + +### ShardInfo (lines 448-600) + +Each shard is in one of four states: + +| State | Meaning | +|-------|---------| +| `readWrite` | Assigned, readable at storageVersion and later | +| `adding` | Being fetched from another SS (fetchKeys in progress) | +| `moveInShard` | Physical shard being moved via checkpoint | +| `notAssigned` | SS has no data for this range | + +### AddingShard (lines 386-446) + +Active data transfer state: +``` +struct AddingShard { + KeyRange keys; + Future fetchClient; // fetchKeys() actor + Promise fetchComplete, readWrite; + std::deque> updates; // mutations during fetch + enum Phase { WaitPrevious, Fetching, FetchingCF, Waiting }; + Version transferredVersion, fetchVersion; +}; +``` + +### fetchKeys() Actor (line 6850) + +Transfers shard data from source to destination SS: + +1. **Setup**: Wait for `coreStarted`, version advancement, overlapping ranges durable +2. **Fetch preparation**: Acquire `fetchKeysParallelismLock`, set phase to `Fetching` +3. **Data fetch loop**: + - Create transaction at `fetchVersion` + - Read key-value blocks from source SS via `tryGetRange()` + - Apply deferred mutations from `updates` deque + - Write to storage: `data->storage.set()` for each KV pair +4. **Completion**: Set phase to `Waiting`, fire `fetchComplete`, update `newestAvailableVersion` + +### changeServerKeys + +Modifies shard map to move ranges between states: +- Updates `serverKeys` system namespace reflecting ownership +- Contexts: `CSK_UPDATE` (normal), `CSK_RESTORE` (recovery), `CSK_ASSIGN_EMPTY` (new), `CSK_FALL_BACK` (fallback) + +--- + +## Storage Engines -- [`fdbserver/kvstore/`](https://github.com/apple/foundationdb/tree/main/fdbserver/kvstore) + +### IKeyValueStore Interface -- [`IKeyValueStore.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h) + +All storage engines implement the `IKeyValueStore` interface (which extends `IClosable`). This is the contract between the storage server and the underlying storage mechanism. + +``` +class IKeyValueStore : public IClosable { +public: + // === Core mutation operations (buffered, not immediately durable) === + virtual void set(KeyValueRef keyValue, const Arena* arena = nullptr) = 0; + virtual void clear(KeyRangeRef range, const Arena* arena = nullptr) = 0; + + // === Durability === + virtual Future canCommit() { return Void(); } // backpressure hook + virtual Future commit(bool sequential = false) = 0; // atomically make prior set/clear durable + + // === Read operations (async) === + virtual Future> readValue(KeyRef key, Optional) = 0; + virtual Future> readValuePrefix(KeyRef key, int maxLength, Optional) = 0; + virtual Future readRange(KeyRangeRef keys, int rowLimit, int byteLimit, Optional) = 0; + + // === Metadata === + virtual KeyValueStoreType getType() const = 0; + virtual StorageBytes getStorageBytes() const = 0; + + // === Checkpoints (physical shard movement) === + virtual Future checkpoint(const CheckpointRequest& request) { throw not_implemented(); } + virtual Future restore(const std::vector& checkpoints) { throw not_implemented(); } + virtual Future deleteCheckpoint(const CheckpointMetaData& checkpoint) { throw not_implemented(); } + + // === Bulk operations === + virtual Future compactRange(KeyRangeRef range) { throw not_implemented(); } + virtual Future replaceRange(KeyRange range, Standalone> data); + virtual Future ingestSSTFiles(std::shared_ptr localFileSets) { throw not_implemented(); } + + // === Lifecycle === + virtual Future init() { return Void(); } // MUST be idempotent (may be called again on rollback) +}; +``` + +#### Concurrency Contract + +From the header comment: + +> **Causal consistency**: A read which begins after a commit ends sees the effects of the commit. A read which ends before a commit begins does not see the effects of the commit. + +In practice: reads return data as of a version established by the most recent `commit()` that completed before the read ends, such that no subsequent commit also completed before the read began. This means reads and writes can overlap, but reads always see a consistent committed snapshot. + +#### ReadOptions + +``` +struct ReadOptions { + ReadType type = ReadType::NORMAL; // NORMAL or FETCH + Optional debugID; +}; +``` + +`ReadType::FETCH` is used during shard data movement (fetchKeys). The engine may apply different throttling or priority to fetch reads vs. normal client reads. + +#### Factory -- [`IKeyValueStore.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/IKeyValueStore.cpp) + +``` +IKeyValueStore* openKVStore(KeyValueStoreType storeType, std::string filename, UID logID, ...) { + switch (storeType) { + case SSD_BTREE_V1: return keyValueStoreSQLite(...); + case SSD_BTREE_V2: return keyValueStoreSQLite(...); + case MEMORY: return keyValueStoreMemory(...); + case SSD_REDWOOD_V1: return keyValueStoreRedwoodV1(...); + case SSD_ROCKSDB_V1: return keyValueStoreRocksDB(...); + case MEMORY_RADIXTREE: return keyValueStoreMemory(..., "fdr", MEMORY_RADIXTREE); + } +} +``` + +--- + +### RocksDB Engine -- [`KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp) (~3K lines) + +The primary production storage engine. Wraps Facebook's RocksDB LSM-tree engine, adapting it to the `IKeyValueStore` interface via thread pools for non-blocking I/O. + +#### Architecture + +``` +RocksDBKeyValueStore (IKeyValueStore) +├── Writer (IThreadPoolReceiver, single thread) +│ ├── OpenAction -- opens/creates the RocksDB database +│ ├── CommitAction -- writes accumulated WriteBatch to RocksDB +│ ├── CloseAction -- closes the database +│ ├── CheckpointAction -- creates a RocksDB checkpoint +│ ├── RestoreAction -- restores from checkpoint +│ ├── CompactRangeAction +│ └── IngestSSTFilesAction +├── Reader (IThreadPoolReceiver, N parallel threads) +│ ├── ReadValueAction +│ ├── ReadValuePrefixAction +│ └── ReadRangeAction +├── SharedRocksDBState -- holds RocksDB options configured from knobs +├── ReadIteratorPool -- recycles rocksdb::Iterator objects across reads +└── FlowLock semaphores -- readSemaphore, fetchSemaphore for backpressure +``` + +In simulation, the reader/writer threads use `CoroThreadPool` (running on the network thread) to avoid time-advancement issues. In production, they use real OS threads. + +#### Write Path + +**`set(KeyValueRef kv)`** (line 2183): +1. Lazy-initializes a `rocksdb::WriteBatch` (with configurable per-key protection bytes) +2. Calls `writeBatch->Put(columnFamily, key, value)` +3. If `ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE` is enabled, tracks the key in `keysSet` for later delete optimization + +**`clear(KeyRangeRef range)`** (line 2202): +Three strategies depending on knobs and range shape: +1. **Single-key delete** -- if `range.singleKeyRange()` and not `ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE`: `writeBatch->Delete(key)`. This is preferred because RocksDB handles point tombstones more efficiently than range tombstones. +2. **Converted deletes** -- if `ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE` is enabled and the delete budget (`maxDeletes`) hasn't been exhausted: iterate the range using a RocksDB iterator and emit individual `Delete()` calls for each key found. Also deletes any keys in `keysSet` and `previousCommitKeysSet` that fall within the range. Falls back to `DeleteRange()` if iteration fails or budget exhausted. +3. **Native `DeleteRange()`** -- `writeBatch->DeleteRange(begin, end)`. Used as the default when conversion is disabled. + +**`commit()`** (line 2308, via `commitInRocksDB`): +1. If `writeBatch` is null (no mutations), returns immediately +2. Creates a `Writer::CommitAction` and transfers ownership of the `WriteBatch` +3. Records delete histograms, moves `keysSet` to `previousCommitKeysSet` +4. Posts the action to `writeThread` +5. The writer thread calls `rocksdb::DB::Write(writeOptions, writeBatch)` -- this atomically persists all buffered mutations + +**`canCommit()` backpressure** (line 2282): +Before each commit, the storage server calls `canCommit()` which invokes `checkRocksdbState()`. This checks: +- `EstimatePendingCompactionBytes` > `ROCKSDB_CAN_COMMIT_COMPACT_BYTES_LIMIT` +- `NumImmutableMemTable` >= `ROCKSDB_CAN_COMMIT_IMMUTABLE_MEMTABLES_LIMIT` + +If overloaded, it delays up to `ROCKSDB_CAN_COMMIT_DELAY_TIMES_ON_OVERLOAD * ROCKSDB_CAN_COMMIT_DELAY_ON_OVERLOAD` seconds, rechecking each iteration. + +#### Read Path + +**`readValue(KeyRef key, ReadOptions)`** (line 2365): +1. Extract `ReadType` (NORMAL or FETCH) from options +2. If the key is a system key or an eager read, skip throttling and post directly to `readThreads` +3. Otherwise, check `FlowLock` waiter count; throw `server_overloaded()` if too many waiters +4. Acquire `FlowLock` semaphore with timeout (`ROCKSDB_READ_QUEUE_WAIT`) +5. Post `ReadValueAction` to `readThreads`; the reader thread calls `rocksdb::DB::Get()` +6. Return the result future + +**`readRange(KeyRangeRef keys, rowLimit, byteLimit, ReadOptions)`** (line 2433): +Similar throttling path. The reader thread: +1. Gets an iterator from `ReadIteratorPool` (reuse if configured, or create new with bounded key range) +2. For forward reads: `Seek(begin)`, iterate with `Next()` until `end`/limit +3. For reverse reads: `SeekForPrev(end)`, iterate with `Prev()` until `begin`/limit +4. Enforces read timeouts (`ROCKSDB_SET_READ_TIMEOUT`) and returns `transaction_too_old` if exceeded +5. Returns the iterator to the pool for potential reuse + +#### ReadIteratorPool (line 539) + +Manages a pool of `rocksdb::Iterator` objects to avoid the cost of creating/destroying them for every read: +- **On read**: provides an unused iterator if available (matching key range for bounded iterators), or creates a new one +- **On commit**: marks all iterators as stale (increments `deletedUptoIndex`), since the underlying data has changed +- **Periodic refresh**: a background actor (`refreshReadIteratorPool`) periodically removes stale and expired iterators +- **Bounded iterators**: when `ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS` is enabled, iterators are created with `iterate_lower_bound`/`iterate_upper_bound` set, and can only be reused for sub-ranges +- **Pool size limit**: `ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT` caps the number of cached iterators to avoid memory pressure + +#### Error Handling + +`RocksDBErrorListener` (an `rocksdb::EventListener`) catches background errors from compaction and flush: +- IO errors → `io_error()` +- Corruption → `file_corrupt()` +- Other → `unknown_error()` + +These errors are forwarded to the storage server via a promise, causing the SS to shut down and be re-recruited. + +#### Configuration (Knobs) + +`SharedRocksDBState` constructs RocksDB options from server knobs. Key settings: + +| Category | Knobs | +|----------|-------| +| **Write buffers** | `ROCKSDB_MEMTABLE_BYTES`, `ROCKSDB_WRITE_BUFFER_SIZE`, `ROCKSDB_MAX_WRITE_BUFFER_NUMBER`, `ROCKSDB_MIN_WRITE_BUFFER_NUMBER_TO_MERGE` | +| **Compaction** | `ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER`, `ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER`, `ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER`, `ROCKSDB_PERIODIC_COMPACTION_SECONDS` (with random 2-day jitter), `ROCKSDB_MAX_COMPACTION_BYTES`, `ROCKSDB_MAX_BYTES_FOR_LEVEL_MULTIPLIER`, `ROCKSDB_COMPACTION_PRI` | +| **Block cache** | `ROCKSDB_BLOCK_CACHE_SIZE` (LRU cache), `ROCKSDB_CACHE_HIGH_PRI_POOL_RATIO`, `ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS` | +| **Bloom filter** | `ROCKSDB_BLOOM_BITS_PER_KEY` (typically 10, ~1% false positive rate), `ROCKSDB_PREFIX_LEN` (fixed prefix for prefix bloom), `ROCKSDB_BLOOM_WHOLE_KEY_FILTERING` | +| **I/O** | `ROCKSDB_USE_DIRECT_READS`, `ROCKSDB_USE_DIRECT_IO_FLUSH_COMPACTION`, `ROCKSDB_BACKGROUND_PARALLELISM`, `ROCKSDB_MAX_SUBCOMPACTIONS`, `ROCKSDB_MAX_OPEN_FILES` | +| **Protection** | `ROCKSDB_MEMTABLE_PROTECTION_BYTES_PER_KEY`, `ROCKSDB_BLOCK_PROTECTION_BYTES_PER_KEY`, `ROCKSDB_PARANOID_FILE_CHECKS`, `ROCKSDB_FULLFILE_CHECKSUM` (CRC32c) | +| **Write rate** | `ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC`, `ROCKSDB_WRITE_RATE_LIMITER_FAIRNESS`, `ROCKSDB_WRITE_RATE_LIMITER_AUTO_TUNE` | +| **Deletion optimization** | `ROCKSDB_ENABLE_COMPACT_ON_DELETION` (triggers compaction when tombstone density is high in a sliding window: `ROCKSDB_CDCF_SLIDING_WINDOW_SIZE`, `ROCKSDB_CDCF_DELETION_TRIGGER`, `ROCKSDB_CDCF_DELETION_RATIO`), `ROCKSDB_SINGLEKEY_DELETES_ON_CLEARRANGE`, `ROCKSDB_SINGLEKEY_DELETES_MAX` | +| **Read throttling** | `ROCKSDB_READ_QUEUE_SOFT_MAX`, `ROCKSDB_READ_QUEUE_HARD_MAX`, `ROCKSDB_FETCH_QUEUE_SOFT_MAX`, `ROCKSDB_FETCH_QUEUE_HARD_MAX`, `ROCKSDB_READ_QUEUE_WAIT` | +| **Read parallelism** | `ROCKSDB_READ_PARALLELISM` (number of reader threads), `ROCKSDB_WRITER_THREAD_PRIORITY`, `ROCKSDB_READER_THREAD_PRIORITY` | + +#### Observability + +Comprehensive metrics are emitted via `rocksDBMetricLogger` (line 1008): +- **RocksDB ticker stats**: block cache hits/misses, bloom filter stats, bytes read/written, compaction bytes, WAL syncs, memtable hits, iterator counts +- **RocksDB histogram stats** (when `ROCKSDB_STATS_LEVEL > kExceptHistogramOrTimers`): compaction time, compression time, write stall duration +- **RocksDB property stats**: live SST size, pending compaction bytes, memtable counts, snapshot counts, estimated key count +- **FDB-side histograms**: commit latency, read value/range latency, queue wait times, deletes per commit +- **PerfContext metrics** (when `ROCKSDB_PERFCONTEXT_ENABLE`): per-thread breakdown of block reads, seeks, memtable operations, environment operations + +--- + +### Redwood (VersionedBTree) Engine -- [`VersionedBTree.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/VersionedBTree.actor.cpp) (~11K lines) + +A custom versioned copy-on-write B-tree storage engine built from scratch for FoundationDB. Named "Redwood," it is designed for high space efficiency through delta compression and for tight integration with FDB's versioned storage model. + +#### Architecture + +``` +KeyValueStoreRedwood (IKeyValueStore) +└── VersionedBTree + ├── IPager2* m_pager -- DWALPager instance (physical page I/O) + ├── MutationBuffer* m_pBuffer -- in-memory pending mutations (ART-based) + ├── BTreeCommitHeader m_header -- persistent root pointer + metadata + ├── LazyClearQueue -- deferred page deletions (FIFO) + └── BTreeCursor -- read cursor for tree traversal + +DWALPager (IPager2) +├── PageCacheT pageCache -- LRU page cache (ObjectCache) +├── PageToVersionedMapT remappedPages -- logical→physical page remapping for COW +├── DelayedFreePageQueueT -- pages freed at a version (deferred reclamation) +├── RemapQueueT -- remap cleanup queue +├── PriorityMultiLock ioLock -- I/O prioritization +└── IAsyncFile* pageFile -- the underlying data file + +BTreePage (in-memory representation of a disk page) +├── height (1 = leaf, >1 = internal) +├── kvBytes (total uncompressed KV bytes) +└── DeltaTree2 tree -- delta-compressed sorted record storage +``` + +#### KeyValueStoreRedwood (line 7609) + +The `IKeyValueStore` adapter. Constructor creates a `DWALPager` and wraps it in a `VersionedBTree`. + +Note: Although `VersionedBTree` supports multi-version reads via the pager's snapshot mechanism, `KeyValueStoreRedwood` currently only keeps one version -- each `commit()` immediately sets `oldestReadableVersion` to the commit version, discarding all prior versions. All reads go against `getLastCommittedVersion()`. This means the storage server's MVCC layer (in-memory `versionedData`) provides version history, not the storage engine itself. + +```cpp +KeyValueStoreRedwood(filename, logID, db, encodingType, pageCacheBytes) { + IPager2* pager = new DWALPager(pageSize, extentSize, filename, pageCacheBytes, + remapCleanupWindowBytes, concurrentExtentReads, false); + m_tree = new VersionedBTree(pager, filename, logID, db); +} +``` + +Key parameters: +- **`pageSize`**: `REDWOOD_DEFAULT_PAGE_SIZE` (default ~4096), BUGGIFied in simulation to random values 1000-16384 +- **`extentSize`**: `REDWOOD_DEFAULT_EXTENT_SIZE` (multiple pages per I/O for large sequential reads) +- **`pageCacheBytes`**: configurable via `PAGE_CACHE_4K` knob (or `SIM_PAGE_CACHE_4K` in simulation) +- **`remapCleanupWindowBytes`**: `REDWOOD_REMAP_CLEANUP_WINDOW_BYTES` (100MB default), controls how much remap history to keep before cleaning up; BUGGIFied in simulation + +#### Write Path + +**`set(KeyValueRef kv)`** / **`clear(KeyRangeRef range)`** -- delegated directly to `VersionedBTree`: + +```cpp +// VersionedBTree::set() (line 4916) +void set(KeyValueRef keyValue) { + ++m_mutationCount; + m_pBuffer->insert(keyValue.key) + .mutation().setBoundaryValue(m_pBuffer->copyToArena(keyValue.value)); +} + +// VersionedBTree::clear() (line 4924) +void clear(KeyRangeRef clearedRange) { + ++m_mutationCount; + if (clearedRange.singleKeyRange()) { + m_pBuffer->insert(clearedRange.begin).mutation().clearBoundary(); + } else { + auto iBegin = m_pBuffer->insert(clearedRange.begin); + auto iEnd = m_pBuffer->insert(clearedRange.end); + iBegin.mutation().clearAll(); + ++iBegin; + m_pBuffer->erase(iBegin, iEnd); + } +} +``` + +Mutations are buffered in `MutationBuffer` (either an ART (Adaptive Radix Tree) implementation for performance, or `std::map`-based fallback). The buffer represents pending mutations as a sorted map from key boundaries to `RangeMutation` structures, which track whether the boundary is set/cleared and whether the range after it is cleared. + +**`commit(Version v)`** (line 5168): +1. Swaps the current `MutationBuffer` with a fresh one (so new mutations can arrive during commit) +2. Creates a `CommitBatch` with the mutations, read version (from pager), and write version +3. Takes a snapshot for reading the current tree state +4. Walks the B-tree, merging buffered mutations with existing pages: + - Reads existing leaf pages + - Applies mutations (sets, clears) to produce updated records + - Rebuilds modified pages using `PageToBuild` to calculate optimal page splits + - Writes modified pages via `m_pager->updatePage()` or `atomicUpdatePage()` (copy-on-write) + - Propagates changes up through internal pages as needed +5. Runs lazy clear to free pages from deleted subtrees +6. Updates `BTreeCommitHeader` with new root pointer and lazy delete queue state +7. Calls `m_pager->commit(version, commitRecord)` to make everything durable + +#### Read Path + +**`readValue(key)`** (line 7879): +1. Initialize a `BTreeCursor` at the last committed version +2. `cursor.seekGTE(key)` -- traverse the B-tree from root to leaf +3. If the cursor lands on the exact key, return the value (with arena dependency on the page buffer) +4. Otherwise return `Optional()` + +**`readRange(keys, rowLimit, byteLimit)`** (line 7738): +1. Initialize `BTreeCursor` at last committed version +2. For forward reads: `seekGTE(keys.begin)`, optionally `prefetch(keys.end, ...)` +3. Walk through leaf pages using the `BTreePage::BinaryTree::Cursor` directly (no async waits needed within a leaf page) +4. For each leaf: check whether the entire leaf is within the query range to skip per-key bounds checking +5. Accumulate results until `rowLimit` or `byteLimit` is reached +6. When a leaf is exhausted, `popPath()` and `moveNext()` to traverse to the next leaf (this may require async page reads) +7. For reverse reads: `seekLT(keys.end)`, traverse with `movePrev()` + +The `prefetch()` mechanism pre-reads pages along the expected traversal path, reducing latency for sequential scans. + +#### DWALPager (Double Write-Ahead Log Pager) (line 1934) + +The physical page storage layer for Redwood. Manages pages on disk with copy-on-write versioning. + +**Key concepts:** +- **Logical vs. Physical pages**: The B-tree operates on logical page IDs. The pager maintains a mapping (`PageToVersionedMapT`) from logical to physical page IDs, which changes when a page is updated (copy-on-write). Old physical pages are freed at the version they were superseded. +- **Page cache**: An LRU `ObjectCache` that caches pages in memory. Each entry tracks read and write futures and is evictable when both are complete. +- **Header pages**: Two header pages (primary at physical page 0, backup at page 1) store the pager's metadata. They are alternately updated for crash safety. +- **Recovery**: On startup, `recover()` reads header pages, validates checksums, and reconstructs the page mapping from the remap queue. + +**Key operations:** +- `updatePage(logicalPageIDs, data)` -- writes a page, potentially to a new physical location (copy-on-write) +- `atomicUpdatePage(logicalPageID, data, version)` -- creates a new physical page and remaps the logical ID to it at the given version +- `readPage(physicalPageID)` -- reads a page from cache or disk +- `newPageID()` -- allocates a new logical page ID from the free list +- `freePage(logicalPageID, version)` -- schedules a page for deferred deletion at the given version +- `commit(version, commitRecord)` -- flushes all pending writes, updates the header, and syncs to disk + +**Queues:** +- `DelayedFreePageQueueT` -- pages whose physical storage can be reclaimed after `oldestReadableVersion` advances past them +- `RemapQueueT` -- tracks logical→physical remappings that need cleanup; entries have types: `REMAP` (logical moved to new physical), `FREE` (logical page freed), `DETACH` (logical page detached from remap tracking) +- `LazyClearQueueT` (in VersionedBTree) -- subtrees that need their pages freed; processed incrementally between commits by `incrementalLazyClear()` + +#### BTreePage and DeltaTree Compression (line 4566) + +Each B-tree page stores its records in a `DeltaTree2` -- a sorted, delta-compressed structure optimized for space efficiency. + +**RedwoodRecordRef** (line 4106): +``` +struct RedwoodRecordRef { + KeyRef key; + Optional value; // absent = tombstone (cleared key) + // For internal pages, value = child page IDs (BTreeNodeLinkRef) +}; +``` + +**Delta encoding**: Each record is stored as a delta relative to a reference record (typically the previous sibling in the tree). The `Delta` structure uses a flags byte and variable-length fields: +- **Prefix borrowing**: Keys share a common prefix with a reference record (left or right neighbor). The delta stores only the differing suffix. +- **4 length formats** (encoded in 2 bits of the flags byte): 3, 4, 6, or 7 bytes of length fields, supporting keys and values from small (<256 byte) to large (multi-GB) +- **Flags**: `PREFIX_SOURCE_PREV` (borrow from left vs. right ancestor), `IS_DELETED` (tombstone), `HAS_VALUE` (value present) + +This typically achieves 50-70% space savings over raw key-value storage for workloads with key locality. + +**Page structure:** +- `height`: 1 = leaf (stores user key-value pairs), >1 = internal (stores separator keys → child page ID pointers) +- `kvBytes`: total uncompressed KV bytes on the page (for metrics) +- The rest of the page is the `DeltaTree2` binary data + +**Page building** (`PageToBuild`, line 5469): During commits, records are packed into pages with awareness of delta sizes, block boundaries, and page utilization. The `shiftItem()` method rebalances records between adjacent pages to improve space efficiency. + +#### Lazy Clear Queue + +When a subtree is deleted (e.g., via a range clear that removes an entire internal page's worth of data), the internal page is pushed onto the `LazyClearQueue` rather than immediately freeing all descendant pages. The `incrementalLazyClear()` actor (line 4974) processes this queue between commits: +1. Pop entries from the queue in batches of `REDWOOD_LAZY_CLEAR_BATCH_SIZE_PAGES` +2. For each entry, read the page and iterate its children: + - Height 2 pages: free leaf children directly (`freeBTreePage`) + - Height >2 pages: re-queue children for later processing +3. Free the page itself +4. Stop when the queue is exhausted or `REDWOOD_LAZY_CLEAR_MAX_PAGES` have been freed + +This amortizes the cost of large deletions across multiple commits. + +#### BTreeCommitHeader (line 4862) + +Persistent metadata stored in the pager's commit record: +``` +struct BTreeCommitHeader { + uint32_t formatVersion; // currently FORMAT_VERSION = 17 + EncodingType encodingType; // page encoding (XXHash64, etc.) + uint8_t height; // tree height + LazyClearQueueT::QueueState lazyDeleteQueue; // lazy clear queue state + BTreeNodeLink root; // root page ID(s) -- may be multi-page for oversized root +}; +``` + +--- + +### SQLite Engine -- [`KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp) (~2.4K lines) + +The original FDB storage engine, using a modified embedded SQLite B-tree. Available as `SSD_BTREE_V1` (`.fdb` files) and `SSD_BTREE_V2` (`.sqlite` files, with checksums and an improved page format). This is a legacy engine; RocksDB and Redwood are the active production engines. + +#### Architecture + +``` +KeyValueStoreSQLite (IKeyValueStore) +├── Writer (IThreadPoolReceiver, single CoroThreadPool thread) +│ ├── InitAction -- opens SQLite DB, runs checkpoint, optional integrity check +│ ├── SetAction -- cursor->set(kv) +│ ├── ClearAction -- cursor->fastClear(range) + cursor->clear(range) +│ ├── CommitAction -- cursor->commit(), fullCheckpoint() +│ └── SpringCleaningAction -- lazy page deletion + vacuuming +├── Reader (IThreadPoolReceiver, SQLITE_READER_THREADS CoroThreadPool threads) +│ ├── ReadValueAction +│ ├── ReadValuePrefixAction +│ └── ReadRangeAction +├── ReadCursor[] -- per-reader cursor, recycled up to SQLITE_CURSOR_MAX_LIFETIME_BYTES +├── PageChecksumCodec -- per-page CRC32c checksums (V2 only) +└── VFSAsync -- async VFS layer bridging SQLite to FDB's IAsyncFile +``` + +Like RocksDB, all I/O runs on `CoroThreadPool` threads (coroutine-based, running on the network thread). The writer thread handles all mutations and commits; multiple reader threads handle reads in parallel. + +#### Write Path + +**`set(KeyValueRef kv)`** (line 2231): +Posts a `Writer::SetAction` to the write thread. The writer calls `cursor->set(kv)` on the SQLite B-tree cursor. Before each set, `checkFreePages()` runs lazy deletion if the free list is below `CHECK_FREE_PAGE_AMOUNT`. + +**`clear(KeyRangeRef range)`** (line 2235): +Posts a `Writer::ClearAction`. The writer calls both: +1. `cursor->fastClear(range)` -- marks B-tree pages as pending lazy deletion (added to the "free table") without immediately reclaiming them +2. `cursor->clear(range)` -- removes the key range from the B-tree index + +**`commit()`** (line 2239): +Posts a `Writer::CommitAction`. The writer: +1. Calls `cursor->commit()` to finalize the SQLite transaction +2. Destroys the cursor (releasing the read transaction) +3. Calls `fullCheckpoint()` -- resets all reader cursors, then runs `conn.checkpoint()` twice (once to flush WAL to the main DB file, once to reset the WAL). This ensures the WAL file is truncated after every commit. +4. Creates a new write cursor for the next batch of mutations + +#### Read Path + +Reads are posted directly to the reader thread pool without throttling (unlike RocksDB): +- **`readValue(key)`** -- `getCursor()->get(key)` on the SQLite B-tree +- **`readValuePrefix(key, maxLength)`** -- `getCursor()->getPrefix(key, maxLength)` +- **`readRange(keys, rowLimit, byteLimit)`** -- `getCursor()->getRange(keys, rowLimit, byteLimit)` + +Each reader thread holds a `ReadCursor` that is recycled after `SQLITE_CURSOR_MAX_LIFETIME_BYTES` of data read. All reader cursors are invalidated on each commit via `resetReaders()` in `fullCheckpoint()`. + +#### Spring Cleaning + +A background `cleanPeriodically()` actor runs `SpringCleaningAction` on the writer thread to reclaim space: +1. **Lazy deletion** -- processes the "free table" (B-tree pages marked for deletion by `fastClear`), freeing them in batches of `SPRING_CLEANING_LAZY_DELETE_BATCH_SIZE`. Bounded by time (`SPRING_CLEANING_LAZY_DELETE_TIME_ESTIMATE`) and page count (`SPRING_CLEANING_MIN/MAX_LAZY_DELETE_PAGES`). +2. **Vacuuming** -- calls `conn.vacuum()` to compact the database file by moving pages from the end to free slots, enabling file truncation. Bounded by `SPRING_CLEANING_MIN/MAX_VACUUM_PAGES` and `SPRING_CLEANING_VACUUM_TIME_ESTIMATE`. +3. The relative frequency of lazy deletion vs. vacuuming is controlled by `SPRING_CLEANING_VACUUMS_PER_LAZY_DELETE_PAGE`. + +The cleaning interval adapts: `SPRING_CLEANING_LAZY_DELETE_INTERVAL` if there's more work to do, `SPRING_CLEANING_NO_ACTION_INTERVAL` if idle. + +#### Page Checksums (V2) + +`PageChecksumCodec` computes an 8-byte checksum (two 32-bit halves: `part1` and `part2`) stored in the reserved bytes at the end of each SQLite page. + +**Writes** always use xxHash3 (with `part1`'s top 8 bits zeroed as a format marker). + +**Reads** try three algorithms in order to handle pages written by older code: +1. **CRC32c** -- tried first if `part1 == 0` (legacy format) +2. **xxHash3** -- tried next if `part1`'s top 8 bits are zero (current format) +3. **hashlittle2** -- fallback for the oldest format (seeded with page number) + +If all three fail, the page is corrupt. The V1 format (`SSD_BTREE_V1`) does not use page checksums. + +#### Integrity Checking + +On open, optional integrity checks can be performed: +- **`checkAllChecksumsOnOpen`** -- scans every page and validates checksums; throws `file_corrupt()` on mismatch +- **`checkIntegrityOnOpen`** -- runs a full B-tree structural integrity check via `conn.check()` + +A standalone `KVFileCheck()` utility function (line 2311) can validate `.fdb`/`.sqlite` files offline, and `KVFileDump()` can dump all key-value pairs. + +--- + +### Memory Engine -- [`KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp) (~960 lines) + +In-memory key-value store backed by a [`DiskQueue`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DiskQueue.cpp) (append-only circular log) for durability: +- All mutations are logged to `.fdq` files (a pair of alternating files, checksummed with xxhash3) +- On recovery, the DiskQueue log is replayed to reconstruct the in-memory state +- Periodic snapshots write the full state to the log so older entries can be reclaimed +- Two backing containers: `IKeyValueContainer` (std::map-like, default) or `radix_tree` (`MEMORY_RADIXTREE` type) + +**Users:** +- **Coordinator Paxos state** -- `OnDemandStore` creates a `KeyValueStoreMemory` per coordinator to persist generation register state (see [Cluster Controller & Coordination](subsystem_04_cluster_controller.md#coordinator-storage)) +- **Transaction log mutation queues** -- TLogs use `KeyValueStoreMemory` with a DiskQueue for their durable mutation log +- **`txnStateStore`** during recovery -- ephemeral metadata reconstructed from TLog data +- **Testing** -- in-memory storage engine for simulation tests + +--- + +## Data Flow Summary + +``` +TLog stream ──peek(myTag)──▶ StorageServer update() loop + │ + ▼ + Parse mutations from cursor + │ + ▼ + Eager reads (prefetch for atomic ops) + │ + ▼ + applyMutation() → MVCC versionedData + │ + ▼ + Advance version + │ + ▼ + updateStorage() → makeVersionMutationsDurable() + │ + ▼ + storage.set()/clear() → engine-specific write batch + │ + ▼ + canCommit() (RocksDB backpressure check) + │ + ▼ + storage.commit() → durableVersion advances + │ + ▼ + pop(durableVersion, myTag) → TLog can discard old data + + +Client ──GetValueRequest──▶ StorageServer getValueQ() + │ + ▼ + waitForVersion(requestedVersion) + │ + ▼ + Check shard ownership + │ + ▼ + Read from MVCC or KVStore at version + │ + ▼ + Return value to client +``` + +--- + +## Engine Comparison + +| Aspect | RocksDB | Redwood | SQLite | Memory | +|--------|---------|---------|--------|--------| +| **Type** | LSM-tree | Copy-on-write B-tree | Modified SQLite B-tree | In-memory + WAL | +| **Store types** | `SSD_ROCKSDB_V1` | `SSD_REDWOOD_V1` | `SSD_BTREE_V1`, `SSD_BTREE_V2` | `MEMORY`, `MEMORY_RADIXTREE` | +| **Write amplification** | Higher (compaction) | Lower (in-place COW) | Low (WAL + checkpoint) | None (append-only log) | +| **Space amplification** | Moderate (dead data in LSM levels) | Low (delta compression) | Moderate (free-list fragmentation) | N/A (RAM-bound) | +| **Read amplification** | Moderate (bloom filter helps) | Low (direct B-tree traversal) | Low (direct B-tree lookup) | None (direct map lookup) | +| **Compression** | RocksDB block compression | DeltaTree prefix/suffix sharing | None | None | +| **Checkpoints** | Native RocksDB checkpoints | Not implemented | Not implemented | Not implemented | +| **Page checksums** | Block protection bytes | XXHash64 page encoding | xxHash3 per-page (V2 only; reads also accept CRC32c and hashlittle2 legacy formats) | N/A | +| **Threading** | Dedicated read/write thread pools | Flow event loop (async page I/O) | CoroThreadPool (read + write) | Single-threaded | +| **Space reclamation** | Background compaction | Lazy clear queue + remap cleanup | Spring cleaning (lazy delete + vacuum) | DiskQueue log reclamation | +| **Primary use** | Production storage servers | Alternative storage engine | Legacy (original engine) | Coordinators, TLogs, testing | + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/storageserver/storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/storageserver/storageserver.actor.cpp) | SS main loop, update, read serving, shard management | +| [`fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/include/fdbserver/kvstore/IKeyValueStore.h) | IKeyValueStore interface definition | +| [`fdbserver/kvstore/IKeyValueStore.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/IKeyValueStore.cpp) | Factory function `openKVStore()` | +| [`fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreRocksDB.actor.cpp) | RocksDB engine implementation | +| [`fdbserver/kvstore/VersionedBTree.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/VersionedBTree.actor.cpp) | Redwood engine (VersionedBTree + DWALPager) | +| [`fdbserver/kvstore/IPager.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/IPager.h) | Pager interface (IPager2) for Redwood | +| [`fdbserver/kvstore/DeltaTree.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DeltaTree.h) | DeltaTree2 delta-compressed sorted structure | +| [`fdbserver/kvstore/KeyValueStoreSQLite.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreSQLite.cpp) | SQLite engine (V1/V2) | +| [`fdbserver/kvstore/KeyValueStoreMemory.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/KeyValueStoreMemory.cpp) | Memory engine | +| [`fdbserver/kvstore/DiskQueue.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/kvstore/DiskQueue.cpp) | Append-only circular log (for Memory engine durability) | diff --git a/design/AI-generated/subsystem_08_data_distribution.md b/design/AI-generated/subsystem_08_data_distribution.md new file mode 100644 index 00000000000..8e29afad6bb --- /dev/null +++ b/design/AI-generated/subsystem_08_data_distribution.md @@ -0,0 +1,238 @@ +# Subsystem 8: Data Distribution + +**[Diagrams](diagram_08_data_distribution.md)** + +**Location:** [`fdbserver/datadistributor/`](https://github.com/apple/foundationdb/tree/main/fdbserver/datadistributor) +**Size:** ~22K +**Role:** Shard management, team building, rebalancing, MoveKeys protocol. + +--- + +## Overview + +Data Distribution (DD) is a continuous background process that decides which storage servers hold which key ranges, and moves data when the assignment needs to change. The DD singleton is recruited by the Cluster Controller and runs for the lifetime of the cluster (re-recruited on failure). + +--- + +## Main Actor -- [`DataDistribution.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DataDistribution.cpp)`:2602-2888` + +The `dataDistribution()` actor spawns four major components: + +1. **DataDistributionTracker** -- monitors shard metrics, triggers splits/merges/moves +2. **DDQueue** -- prioritized queue of shard relocations, executes moves +3. **DDTeamCollection (Primary)** -- builds teams, monitors health, recruits storage servers +4. **DDTeamCollection (Remote)** -- same for remote datacenter (if `usableRegions > 1`) + +Plus optional bulk load/dump cores. + +### Initialization Sequence + +1. `initDDConfigWatch()` -- watch database configuration for changes +2. `DataDistributor::init()` -- load metadata from system keys +3. Resume relocations from previous run +4. Spawn tracker, queue, and team collections +5. `waitForAll(actors)` -- run until fatal error + +--- + +## DDTeamCollection -- [`DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp) + +### What is a Team? + +A "team" is a set of storage servers that together hold replicas of a shard. For `triple` replication, a team has 3 servers in 3 different zones. + +### Key Members (`DDTeamCollection.h:209+`) + +``` +std::vector> badTeams; +std::map> server_and_tss_info; +std::map lagging_zones; +Reference storageWiggler; + +int healthyTeamCount, optimalTeamCount; +AsyncVar zeroHealthyTeams; +Future teamBuilder; +PromiseStream output; // relocation requests +``` + +### Team Building + +`buildTeams()` creates new teams from healthy servers: +- Each team satisfies the replication policy (e.g., 3 replicas across 3 zones) +- All teams have exactly `teamSize` members +- Respects excluded/failed server lists +- **Machine teams** enforce datacenter locality constraints + +### Health Monitoring + +**`teamTracker()`** -- per-team actor: +- Watches for server failures (team < replication factor) +- Watches for server exclusion +- Emits `RelocateShard` when team health degrades + +**`storageServerTracker()`** -- per-server actor: +- Monitors interface responsiveness, disk space, excluded status +- Marks failed on timeout or storage errors + +**`storageRecruiter()`** -- continuous recruitment: +- Recruits new servers when existing ones fail +- Sends `RecruitStorageRequest` to cluster controller + +--- + +## DDShardTracker -- [`DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp) + +Monitors shard sizes and operation rates, triggers splits/merges. + +### Bandwidth Status (lines 42-58) + +| Status | Condition | +|--------|-----------| +| `BandwidthStatusLow` | `bytesWrittenPerKSec < SHARD_MIN_BYTES_PER_KSEC` | +| `BandwidthStatusNormal` | Between min and max | +| `BandwidthStatusHigh` | `bytesWrittenPerKSec > SHARD_MAX_BYTES_PER_KSEC` | + +### Shard Size Bounds (lines 85-151) + +`calculateShardSizeBounds()` returns bounds that trigger splits/merges: + +- **Byte bounds**: `max.bytes = max(currentBytes * 1.1, MIN_SHARD_BYTES)`, permits 10% deviation +- **Write bandwidth bounds**: Based on `SHARD_MIN/MAX_BYTES_PER_KSEC` +- **Read bandwidth bounds**: Based on `SHARD_MAX_READ_DENSITY_RATIO * bytes` +- **Read operation bounds**: Track `opsReadPerKSecond`, detect hot shards + +### Tracking Logic (`trackShardMetrics`) + +Continuously monitors each shard. When metrics exceed bounds: +- **Split** (high bytes/ops): Create two smaller shards, emit `RelocateShard` with `SPLIT_SHARD` +- **Merge** (low bytes/ops): Merge with adjacent shard, emit `RelocateShard` with `MERGE_SHARD` + +--- + +## DDQueue / DDRelocationQueue -- [`DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp) + +### Key Members (`DDRelocationQueue.h:125+`) + +``` +std::set> fetchingSourcesQueue; +std::map> queue; // per-server queue +KeyRangeMap inFlight; // currently executing +KeyRangeActorMap inFlightActors; // actor per in-flight move +KeyRangeMap dataMoves; // move metadata +std::map busymap, destBusymap; // server utilization +``` + +### RelocateData (lines 74-122) + +``` +struct RelocateData { + int priority; // urgency level + RelocateReason reason; + DataMovementReason dmReason; + KeyRange keys; // range to move + std::vector src; // source server IDs + int workFactor; // estimated cost + UID dataMoveId; + double startTime; +}; +``` + +### Relocation Priorities + +| Priority | Trigger | +|----------|---------| +| `PRIORITY_TEAM_REDUNDANT` | Team replicated on same machine | +| `PRIORITY_POPULATE_REGION` | Populate new region | +| `PRIORITY_TEAM_UNHEALTHY` | Unhealthy team | +| `PRIORITY_REBALANCE_*` | Disk/read balancing | +| `PRIORITY_SPLIT_SHARD` | Shard too large/hot | +| `PRIORITY_MERGE_SHARD` | Shard too small/cold | + +Ordering: `(priority desc) → (startTime asc) → (randomId desc)` + +### Core Operations + +- **`queueRelocation()`**: Enqueue shard relocation, search for source team +- **`launchQueuedWork()`**: Check for overlapping in-flight moves, cancel lower priority, start MoveKeys +- **`completeSourceFetch()`**: Source servers identified, move to ready-to-launch + +--- + +## MoveKeys Protocol -- [`fdbserver/core/MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp) + +Atomic shard ownership transfer using system key transactions. + +### Overall Flow + +1. Destination SS starts `fetchKeys()` -- pulls data from source SS +2. During fetch, mutations forwarded to dest via TLog stream (same tag assignment) +3. Once dest reaches `fetchVersion`: ownership transfers atomically +4. Source removes shard + +### System Keys Modified + +| Key Range | Change | +|-----------|--------| +| `serverKeysPrefixFor(sourceId)` | Remove key range | +| `serverKeysPrefixFor(destId)` | Add key range | +| `keyServersKey(range)` | Update server list for range | +| `moveKeysLockOwnerKey` | DD process ownership | + +### Key Functions + +- **`takeMoveKeysLock()`**: Acquire exclusive lock (prevents concurrent moves by different DD instances) +- **`readMoveKeysLock()`**: Validate current DD still holds lock +- **`unassignServerKeys()`**: Remove server from key range in system keys, with coalescing +- **`deleteCheckpoints()`**: Clean up checkpoint metadata after physical shard move + +### Conflict Resolution + +- Read conflicts added at range boundaries to detect concurrent assignments +- Ensures no mutations lost during ownership transfer +- Transactions retry if conflicts detected + +--- + +## Why Data Moves + +| Trigger | Action | +|---------|--------| +| Shard grows too large | Split into smaller shards | +| Storage server fails | Re-replicate shards to healthy servers | +| Load imbalance | Move shards from hot to cold servers | +| Configuration change | Adjust replication factor | +| Server exclusion | Move all shards off excluded server | +| Shard too small | Merge with adjacent shard | + +--- + +## Data Flow + +``` +DDShardTracker ──metrics exceed bounds──▶ RelocateShard request + │ + ▼ +DDQueue ──prioritize, dedup, find dest team──▶ launchQueuedWork() + │ + ▼ +MoveKeys protocol: + 1. Dest SS: fetchKeys() from source SS + 2. Dest SS: start consuming TLog for shard's tag + 3. Dest SS: catches up to current version + 4. Atomic system key transaction: transfer ownership + 5. Source SS: remove shard +``` + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/datadistributor/DataDistribution.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DataDistribution.cpp) | Main DD actor, initialization | +| [`fdbserver/datadistributor/DDTeamCollection.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDTeamCollection.actor.cpp) | Team building, health monitoring, SS recruitment | +| [`fdbserver/datadistributor/DDShardTracker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDShardTracker.cpp) | Shard metrics, split/merge decisions | +| [`fdbserver/datadistributor/DDRelocationQueue.actor.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/DDRelocationQueue.actor.cpp) | Relocation queue, prioritization, execution | +| [`fdbserver/core/MoveKeys.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/MoveKeys.cpp) | Atomic shard transfer protocol | +| [`fdbserver/core/DataMovement.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/DataMovement.cpp) | Data move metadata | +| [`fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/datadistributor/ShardsAffectedByTeamFailure.cpp) | Impact analysis for team failures | diff --git a/design/AI-generated/subsystem_09_cluster_recovery.md b/design/AI-generated/subsystem_09_cluster_recovery.md new file mode 100644 index 00000000000..2316caa1703 --- /dev/null +++ b/design/AI-generated/subsystem_09_cluster_recovery.md @@ -0,0 +1,215 @@ +# Subsystem 9: Cluster Recovery + +**[Diagrams](diagram_09_cluster_recovery.md)** + +**Location:** [`fdbserver/clustercontroller/ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp), [`fdbserver/core/RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h) +**Size:** Part of ClusterController files +**Role:** 10-state recovery state machine to reconstitute the transaction system after failures. + +--- + +## Overview + +Recovery is triggered when the master (sequencer) fails, a network partition heals, or coordinators change. The Cluster Controller drives a 10-state state machine that locks old TLogs, recruits a new transaction system, replays uncommitted data, and resumes accepting commits. + +--- + +## ClusterRecoveryData -- [`ClusterRecovery.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.h)`:178-318` + +``` +struct ClusterRecoveryData { + UID dbgid; + Version lastEpochEnd; // last committed version in old epoch + Version recoveryTransactionVersion; // first version in new epoch + Optional versionEpoch; + + DatabaseConfiguration configuration; + std::vector> primaryDcId, remoteDcIds; + + Reference logSystem; + IKeyValueStore* txnStateStore; // in-memory metadata store + LogSystemDiskQueueAdapter* txnStateLogAdapter; + + std::vector commitProxies; + std::vector grvProxies; + std::vector resolvers; + + ReusableCoordinatedState cstate; // coordinated cluster state + MasterInterface masterInterface; + + // Recovery signaling + Promise recoveryReadyForCommits; + Promise cstateUpdated; +}; +``` + +--- + +## The 10 Recovery States -- [`RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h)`:31-42` + +``` +enum class RecoveryState { + UNINITIALIZED = 0, + READING_CSTATE = 1, + LOCKING_CSTATE = 2, + RECRUITING = 3, + RECOVERY_TRANSACTION = 4, + WRITING_CSTATE = 5, + ACCEPTING_COMMITS = 6, + ALL_LOGS_RECRUITED = 7, + STORAGE_RECOVERED = 8, + FULLY_RECOVERED = 9 +}; +``` + +--- + +## clusterRecoveryCore() Walk-Through + +### State 1: READING_CSTATE (line ~1504) + +- Read coordinated state from coordinators +- Learn about previous epoch: `DBCoreState` with TLog configuration +- Validate protocol versions + +### State 2: LOCKING_CSTATE (line ~1517) + +- Lock coordinated state (increment recovery count) +- Prevents split-brain: old master cannot write new state +- Uses generation register for safe state transitions + +### State 3: RECRUITING (line ~1584) + +**`recruitEverything()`** (lines 999-1117): +1. Send `RecruitFromConfigurationRequest` to cluster controller +2. CC selects workers based on fitness, locality, exclusions +3. Initialize in parallel: + - `newCommitProxies()` -- `InitializeCommitProxyRequest` to workers + - `newGrvProxies()` -- `InitializeGrvProxyRequest` + - `newResolvers()` -- `InitializeResolverRequest` + - `newTLogServers()` -- creates new log epoch via `oldLogSystem->newEpoch()` + - `newSeedServers()` -- seed storage servers (only at version 0) +4. `monitorInitializingTxnSystem()` provides timeout with exponential backoff + +**`provisionalMaster()`** (line ~858): +- Provides minimal proxy implementation during recovery +- Serves clients with provisional (limited) responses +- Accepts emergency transactions (configuration mutations) + +### State 4: RECOVERY_TRANSACTION (line ~1635) + +**`readTransactionSystemState()`** (lines 1139-1270): +Peeks old TLogs to reconstruct metadata: +- `versionEpochKey` -- version epoch +- `minRequiredCommitVersionKey` -- for versionstamped operations +- Configuration keys -- database config +- Tag locality mappings -- datacenter localities +- Server tags and tag history + +Calculates: +- `lastEpochEnd` -- last committed version in old epoch +- `recoveryTransactionVersion` -- first version in new epoch + +Executes the recovery transaction: writes initial state to the new transaction system. + +### State 5: WRITING_CSTATE (line ~1767) + +- Write new epoch's `DBCoreState` to coordinators via coordinated state +- Contains: new TLog configuration, recovery count, log system state +- Must succeed at quorum of coordinators + +### State 6: ACCEPTING_COMMITS (line ~1802) + +- New transaction system is live +- CommitProxies, GrvProxies, Resolvers, TLogs all operational +- Client commits now flow through the new system +- Old TLog data still being consumed by storage servers + +### State 7: ALL_LOGS_RECRUITED + +Transition when all TLog sets (including remote region) are fully populated. + +### State 8: STORAGE_RECOVERED + +All old generation TLogs have been fully recruited into the system. + +### State 9: FULLY_RECOVERED + +- Old TLog data fully consumed by storage servers +- Old generations can be discarded +- `oldLogSystems->get()->stopRejoins()` +- Normal operation + +--- + +## trackTlogRecovery() -- [`ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp)`:449-533` + +Monitors TLog set completeness and drives state transitions: + +``` +loop { + // Convert current logSystem to DBCoreState + self->logSystem->toCoreState(newState); + + // Check completeness + allLogs = newState.tLogs.size() == configuration.expectedLogSets(primaryDcId); + finalUpdate = !newState.oldTLogData.size() && allLogs; + + // Write state durably + co_await self->cstate.write(newState, finalUpdate); + + // Signal readiness + if (self->recoveryReadyForCommits.canBeSet()) + self->recoveryReadyForCommits.send(Void()); + + // Broadcast updated recovery state + self->logSystem->coreStateWritten(newState); + + if (finalUpdate) { + // FULLY_RECOVERED: stop rejoins, start rejoin handler + co_return; + } + + co_await changed; // wait for next logSystem state change +} +``` + +--- + +## CoordinatedState Protocol + +### Read + +1. Replicated read from quorum of coordinators +2. Returns highest-generation value +3. If client's generation > stored readGen: updates readGen + +### Write + +1. Replicated write with generation check +2. `readGen <= requestGen AND writeGen < requestGen` must hold +3. If valid: update value and writeGen +4. If conflict: throws `coordinated_state_conflict` + +--- + +## Recovery Timing + +| Knob | Default | Purpose | +|------|---------|---------| +| `CC_RECOVERY_INIT_REQ_TIMEOUT` | 30s | Base timeout for recruitment | +| `CC_RECOVERY_INIT_REQ_TIMEOUT_GROWTH_FACTOR` | 2x | Exponential backoff per unfinished recovery | +| `CC_RECOVERY_INIT_REQ_MAX_TIMEOUT` | 300s | Maximum timeout cap | + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/clustercontroller/ClusterRecovery.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.cpp) | clusterRecoveryCore, recruitment, state machine | +| [`fdbserver/clustercontroller/ClusterRecovery.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.h) | ClusterRecoveryData struct | +| [`fdbserver/core/include/fdbserver/core/RecoveryState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/RecoveryState.h) | RecoveryState enum | +| [`fdbserver/core/include/fdbserver/core/DBCoreState.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/include/fdbserver/core/DBCoreState.h) | DBCoreState (coordinated TLog config) | +| [`fdbserver/core/CoordinatedState.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/core/CoordinatedState.cpp) | Replicated read/write over generation registers | +| [`fdbserver/logsystem/LogSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp) | newEpoch() for creating new TLog generations | diff --git a/design/AI-generated/subsystem_10_ratekeeper.md b/design/AI-generated/subsystem_10_ratekeeper.md new file mode 100644 index 00000000000..a55ea688194 --- /dev/null +++ b/design/AI-generated/subsystem_10_ratekeeper.md @@ -0,0 +1,203 @@ +# Subsystem 10: Rate Keeping & Throttling + +**[Diagrams](diagram_10_ratekeeper.md)** + +**Location:** [`fdbserver/ratekeeper/`](https://github.com/apple/foundationdb/tree/main/fdbserver/ratekeeper) +**Size:** ~4K +**Role:** Back-pressure on commits and reads, tag-based throttling, throughput tracking. + +--- + +## Overview + +The Ratekeeper is a singleton role (recruited by Cluster Controller) that monitors cluster health metrics and calculates transaction rate limits. These limits are sent to GRV proxies, which enforce them by delaying `GetReadVersion` responses. This prevents the cluster from being overwhelmed when storage servers or TLogs fall behind. + +--- + +## Ratekeeper Structure -- [`Ratekeeper.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.h)`:129-232` + +``` +class Ratekeeper { + Map storageQueueInfo; // per-SS metrics + Map tlogQueueInfo; // per-TLog metrics + std::map grvProxyInfo; + + Smoother smoothReleasedTransactions; // actual TPS + Smoother smoothBatchReleasedTransactions; + Smoother smoothTotalDurableBytes; + + std::unique_ptr tagThrottler; + + RatekeeperLimits normalLimits; // default priority limits + RatekeeperLimits batchLimits; // batch priority limits +}; +``` + +### StorageQueueInfo (lines 37-73) + +``` +class StorageQueueInfo { + Smoother smoothFreeSpace, smoothTotalSpace; + Smoother smoothDurableBytes, smoothInputBytes; + Smoother smoothDurableVersion, smoothLatestVersion; + StorageQueuingMetricsReply lastReply; + limitReason_t limitReason; + std::vector busiestReadTags, busiestWriteTags; +}; +``` + +### TLogQueueInfo (lines 75-94) + +``` +class TLogQueueInfo { + Smoother smoothDurableBytes, smoothInputBytes; + Smoother smoothFreeSpace, smoothTotalSpace; + TLogQueuingMetricsReply lastReply; +}; +``` + +### RatekeeperLimits (lines 96-127) + +``` +struct RatekeeperLimits { + double tpsLimit; + int64_t storageTargetBytes, storageSpringBytes; + int64_t logTargetBytes, logSpringBytes; + double maxVersionDifference; + int64_t durabilityLagTargetVersions; + double durabilityLagLimit; + TransactionPriority priority; +}; +``` + +--- + +## Rate Calculation -- [`Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp)`:642-850` + +### `updateRate(RatekeeperLimits* limits)` + +Evaluates multiple metrics and takes the minimum rate: + +### 1. Storage Queue Depth + +``` +storageQueue = ss.getStorageQueueBytes() +``` + +If queue between `springBytes` and `targetBytes`: +``` +targetRateRatio = (queue - target + spring) / spring +tpsLimit = min(actualTps * maxBytesPerSecond / inputRate, + maxBytesPerSecond * MAX_TRANSACTIONS_PER_BYTE) +``` + +### 2. Durability Lag + +``` +storageDurabilityLag = ss.getDurabilityLag() // version gap between latest and durable +``` + +If lag exceeds `durabilityLagTargetVersions`: reduce rate proportionally. + +### 3. Free Space + +- Minimum free space enforcement with ratio factor +- Hard limit: `STORAGE_HARD_LIMIT_BYTES` +- Below hard limit: rate drops to 0 + +### 4. TLog Queue Depth + +Similar to storage queue depth but for TLog queues: +- `logTargetBytes` / `logSpringBytes` thresholds +- Limits rate when TLogs fall behind + +### 5. Write Bandwidth MVCC + +``` +maxBytesPerSecond = (targetBytes - springBytes) / mvcc_window +``` + +Prevents storage server from filling its MVCC queue faster than it can drain. + +### Rate Selection + +The final rate is the **minimum** across all metrics, with the `limitReason` tracking which factor is most constraining. + +--- + +## Communication with GRV Proxies -- [`Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp)`:345-410` + +### GetRateInfoRequest + +``` +struct GetRateInfoRequest { + UID requesterID; + int64_t totalReleasedTransactions; + int64_t batchReleasedTransactions; + Version version; + TransactionTagMap throttledTagCounts; + bool detailed; +}; +``` + +### GetRateInfoReply + +``` +struct GetRateInfoReply { + double transactionRate; // normal priority TPS limit + double batchTransactionRate; // batch priority TPS limit + double leaseDuration; // METRIC_UPDATE_RATE + HealthMetrics healthMetrics; + Optional> clientThrottledTags; +}; +``` + +### Flow + +1. GRV proxy sends `GetRateInfoRequest` periodically +2. Ratekeeper updates released transaction counts +3. Calculates per-proxy limits (total limit / num proxies) +4. Includes per-tag throttle info if tags changed +5. Returns rate limits + health metrics + +GRV proxy enforces limits by delaying `GetReadVersion` responses when rate exceeded. + +--- + +## Tag Throttling -- [`TagThrottler.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/TagThrottler.cpp) + +Per-tag (per-tenant/workload) transaction throttling. + +### Concepts + +| Term | Meaning | +|------|---------| +| **Manual throttle** | Operator-defined tag rate limit | +| **Automatic throttle** | Ratekeeper-generated tag limit for a busy tag | +| **Priority** | Maximum transaction priority affected by a throttle | +| **Expiration** | Time at which a throttle is removed | + +### Key Functions + +- `tryUpdateAutoThrottling()` -- update automatic throttles from busy-tag signals +- `getClientRates()` -- collect active limits to send to clients +- `monitorThrottlingChanges()` -- watch persisted throttle state + +### Throttle Enforcement + +1. Ratekeeper calculates per-tag rate limits +2. Sends to GRV proxies in `GetRateInfoReply.clientThrottledTags` +3. Clients enforce tag-specific limits locally +4. Clients see `tag_throttled` if the rate is exceeded + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/ratekeeper/Ratekeeper.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.cpp) | Rate calculation, proxy communication | +| [`fdbserver/ratekeeper/Ratekeeper.h`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/Ratekeeper.h) | Ratekeeper, StorageQueueInfo, TLogQueueInfo, RatekeeperLimits | +| [`fdbserver/ratekeeper/TagThrottler.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/TagThrottler.cpp) | Per-tag throttling implementation | +| [`fdbserver/ratekeeper/RkTagThrottleCollection.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/ratekeeper/RkTagThrottleCollection.cpp) | Tag throttle collection management | +| `fdbserver/ratekeeper/RatekeeperLimits.cpp` | Limit calculation utilities | diff --git a/design/AI-generated/subsystem_11_backup_dr.md b/design/AI-generated/subsystem_11_backup_dr.md new file mode 100644 index 00000000000..94e3404ef03 --- /dev/null +++ b/design/AI-generated/subsystem_11_backup_dr.md @@ -0,0 +1,290 @@ +# Subsystem 11: Backup, Restore & DR + +**[Diagrams](diagram_11_backup_dr.md)** + +**Location:** [`fdbserver/backupworker/`](https://github.com/apple/foundationdb/tree/main/fdbserver/backupworker), `fdbclient/FileBackupAgent*`, `fdbclient/BackupContainer*`, [`fdbbackup/`](https://github.com/apple/foundationdb/tree/main/fdbbackup) +**Size:** ~10K server, ~20K client +**Role:** Continuous backup to external storage, point-in-time restore, cross-cluster DR. + +--- + +## Overview + +FDB supports continuous backup of the mutation stream to external storage, enabling point-in-time restore. The backup system has two main components: server-side BackupWorkers that pull mutations from TLogs, and client-side BackupAgents that orchestrate the backup lifecycle and manage backup containers. + +--- + +## Backup Architecture + +### Two Backup File Types + +**Log files** -- continuous mutation stream: +``` +struct LogFile { + Version beginVersion, endVersion; + uint32_t blockSize; + std::string fileName; + int64_t fileSize; + int tagId, totalTags; // for partitioned logs +}; +``` + +**Range files** -- snapshot of key-value pairs at a version: +``` +struct RangeFile { + Version version; + uint32_t blockSize; + std::string fileName; + int64_t fileSize; +}; +``` + +Together, range files + log files enable point-in-time restore: restore the range snapshot, then replay log files to reach any target version. + +--- + +## BackupWorker -- [`fdbserver/backupworker/BackupWorker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/backupworker/BackupWorker.cpp) + +Server-side role that pulls mutations from TLogs (like a storage server, but writes to backup storage). + +### BackupData (lines 105-287) + +``` +struct BackupData { + const UID myId; + const Tag tag; // log router tag (-2, i) + const int totalTags; + const Version startVersion; + const Optional endVersion; // old epoch end + const LogEpoch recruitedEpoch, backupEpoch; + + Version savedVersion; // largest version saved to storage + AsyncVar> logSystem; + std::vector messages; // in-memory mutation buffer + std::map backups; // per-backup configurations +}; +``` + +### PerBackupInfo (lines 125-258) + +``` +struct PerBackupInfo { + Version startVersion, lastSavedVersion; + Future>> container; + Future>> ranges; + bool stopped; +}; +``` + +### Mutation Pulling + +1. BackupWorker peeks TLog via its assigned `tagLocalityLogRouter` tag +2. Filters mutations based on backup key ranges +3. Buffers in `messages` vector +4. Writes to backup container when buffer reaches threshold +5. Tracks progress in system keys + +--- + +## BackupAgent -- [`fdbclient/BackupAgent.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupAgent.h), [`fdbclient/FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp) + +Client-side orchestration of backup lifecycle. + +### Backup States + +``` +enum class EnumState { + STATE_ERRORED = 0, + STATE_SUBMITTED = 1, + STATE_RUNNING = 2, + STATE_RUNNING_DIFFERENTIAL = 3, + STATE_COMPLETED = 4, + STATE_NEVERRAN = 5, + STATE_ABORTED = 6 +}; +``` + +### FileBackupAgent Methods + +| Method | Purpose | +|--------|---------| +| `submitBackup()` | Queue a new backup task | +| `restore()` | Start point-in-time restore | +| `atomicRestore()` | Atomic restore (single transaction) | +| `abortRestore()` | Cancel in-progress restore | +| `waitRestore()` | Wait for restore completion | +| `restoreStatus()` | Query restore progress | + +### TaskBucket Pattern + +Backup/restore operations use a distributed task execution pattern: +- Tasks stored as key-value entries in the database +- `TaskBucket` polls for ready tasks at `pollDelay` intervals +- Tasks can spawn sub-tasks for parallelism +- `FutureBucket` tracks task completion +- Provides reliability: tasks survive process failures + +``` +Future run(Database cx, double pollDelay, int maxConcurrentTasks); +``` + +--- + +## Backup Container -- [`fdbclient/BackupContainer.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupContainer.h) + +### IBackupContainer Interface (lines 249-366) + +``` +class IBackupContainer { + // Write operations + Future> writeLogFile(beginVersion, endVersion, blockSize); + Future> writeRangeFile(version, snapshotFileCount, ...); + Future> writeTaggedLogFile(...); + + // Read/describe + Future> readFile(name); + Future describeBackup(bool deepScan); + Future> getRestoreSet(Version targetVersion); + + // Lifecycle + Future create(); + Future exists(); + Future expireData(Version expireEndVersion, bool force); + Future deleteContainer(int* pNumDeleted); +}; +``` + +### Implementations + +- **BackupContainerLocalDirectory** -- local filesystem +- **BackupContainerS3BlobStore** -- S3-compatible object storage + +### File Format Versions + +| Version | Format | +|---------|--------| +| `BACKUP_AGENT_MLOG_VERSION = 2001` | Old FileBackupAgent mutation logs | +| `PARTITIONED_MLOG_VERSION = 4110` | BackupWorker partitioned logs | +| `RANGE_PARTITIONED_MLOG_VERSION = 5001` | Range-partitioned logs | +| `BACKUP_AGENT_SNAPSHOT_FILE_VERSION = 1001` | Range file snapshots | +| `BACKUP_AGENT_ENCRYPTED_SNAPSHOT_FILE_VERSION = 1002` | Encrypted snapshots | + +### RestorableFileSet (BackupContainer.h:221-234) + +``` +struct RestorableFileSet { + Version targetVersion; + std::vector ranges; // snapshot files + std::vector logs; // mutation log files + Version continuousBeginVersion; + Version continuousEndVersion; + std::vector snapshots; +}; +``` + +--- + +## Restore Flow + +### Restore Modes + +| Mode | Description | +|------|-------------| +| `RANGEFILE` | Traditional: read range files + apply log files | +| `BULKLOAD` | Efficient: SST file ingestion via BulkLoad | + +### Traditional Restore + +1. Read range files to reconstruct base key-value state at snapshot version +2. Apply log files from snapshot version to target version +3. Each log file contains mutations that are replayed in order +4. Supports key prefix transformation (add/remove prefix during restore) + +### RestoreConfig + +``` +class RestoreConfig : public KeyBackedTaskConfig { + KeyBackedProperty stateEnum(); + KeyBackedProperty addPrefix(), removePrefix(); + KeyBackedProperty onlyApplyMutationLogs(); + KeyBackedProperty bulkDumpJobId(); + KeyBackedProperty firstConsistentVersion(); +}; +``` + +--- + +## Disaster Recovery (DR) -- [`fdbclient/DatabaseBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseBackupAgent.cpp) + +Streams mutations from one cluster to another in near-real-time. + +### DatabaseBackupAgent + +``` +class DatabaseBackupAgent { + static const Key keyAddPrefix, keyRemovePrefix; + static const Key keyRangeVersions; + static const Key keyCopyStop, keyDatabasesInSync; +}; +``` + +### DR Flow + +1. Source cluster runs backup agent that reads committed mutations +2. Mutations streamed to destination cluster in near-real-time +3. Destination cluster applies mutations with optional prefix transformation +4. Enables warm standby cluster for disaster recovery + +### BackupRangeTaskFunc + +- Executes range-based copying for DR +- Finds shard boundaries, creates sub-tasks for parallelism +- Uses FlowLock for memory management (`CLIENT_KNOBS->BACKUP_LOCK_BYTES`) + +--- + +## Data Flow + +### Backup +``` +TLog ──peek(logRouterTag)──▶ BackupWorker + │ + ▼ + Filter by backup key ranges + │ + ▼ + Buffer mutations + │ + ▼ + Write to IBackupContainer (log files) + │ + + periodic range file snapshots +``` + +### Restore +``` +IBackupContainer ──read range files──▶ Restore Agent + │ + ▼ + Write base state to empty cluster + │ + ▼ + ──read log files──▶ Replay mutations to target version + │ + ▼ + Cluster at target version state +``` + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbserver/backupworker/BackupWorker.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/backupworker/BackupWorker.cpp) | Server-side mutation pulling and writing | +| [`fdbclient/include/fdbclient/BackupAgent.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupAgent.h) | BackupAgentBase, FileBackupAgent API | +| [`fdbclient/FileBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/FileBackupAgent.cpp) | Backup/restore orchestration, TaskBucket | +| [`fdbclient/DatabaseBackupAgent.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/DatabaseBackupAgent.cpp) | DR agent, cross-cluster streaming | +| [`fdbclient/include/fdbclient/BackupContainer.h`](https://github.com/apple/foundationdb/blob/main/fdbclient/include/fdbclient/BackupContainer.h) | IBackupContainer, file format definitions | +| [`fdbclient/BackupContainerFileSystem.cpp`](https://github.com/apple/foundationdb/blob/main/fdbclient/BackupContainerFileSystem.cpp) | Filesystem-based container implementation | +| [`fdbbackup/backup.cpp`](https://github.com/apple/foundationdb/blob/main/fdbbackup/backup.cpp) | Backup CLI tool | diff --git a/design/AI-generated/subsystem_12_simulation_testing.md b/design/AI-generated/subsystem_12_simulation_testing.md new file mode 100644 index 00000000000..179b63198fb --- /dev/null +++ b/design/AI-generated/subsystem_12_simulation_testing.md @@ -0,0 +1,345 @@ +# Subsystem 12: Simulation & Testing + +**[Diagrams](diagram_12_simulation_testing.md)** + +**Location:** [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp), [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads), `fdbserver/tester/`, `tests/` +**Size:** ~51K workloads + simulator +**Role:** Deterministic simulation (Sim2), fault injection (Buggify), workload-based integration tests. + +--- + +## Overview + +FDB's most distinctive engineering practice. The entire cluster -- multiple processes, network, disk -- runs deterministically in a single OS thread with virtual time. Combined with aggressive fault injection (Buggify), this finds bugs that would take years to manifest in production. + +--- + +## Sim2 -- Deterministic Simulator -- [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp) + +### Sim2 Class (line 1025) + +``` +class Sim2 final : public ISimulator, public INetworkConnections +``` + +Implements `INetwork` (the event loop interface) and `ISimulator` (fault injection). + +### Key Properties + +- **Deterministic**: Same random seed → identical execution +- **Virtual time**: No wall-clock dependency; time advances via task scheduling +- **Single-threaded**: All virtual processes share one OS thread +- **Full cluster**: Multiple processes with separate globals, endpoints, listeners + +### Virtual Event Loop + +- `TaskQueue taskQueue` -- priority queue of pending tasks +- Time advances through task completion, not wall-clock +- `delay(seconds, taskId)` enqueues task at `currentTime + seconds` +- `timer()` can be up to 0.1s ahead of `now()` for timer jitter +- Buggified delays added via `FLOW_KNOBS->MAX_BUGGIFIED_DELAY` + +--- + +## ProcessInfo -- `SimulatorProcessInfo.h:42-170` + +State per simulated process: + +``` +struct ProcessInfo : NonCopyable { + std::string name, dataFolder, coordinationFolder; + MachineInfo* machine; + NetworkAddressList addresses; + LocalityData locality; + ProcessClass startingClass; + + bool failed, excluded, cleared, rebooting, failedDisk; + double fault_injection_p1, fault_injection_p2; + + std::vector globals; // per-process globals + INetworkConnections* network; + UID uid; + ProtocolVersion protocolVersion; + std::vector childs; + Promise shutdownSignal; + + bool isReliable(); // no faults injected + bool isAvailable(); // !excluded && isReliable() +}; +``` + +Each process has its own: +- Global variables (via `globals` vector) +- Endpoint registrations +- Listener map +- Fault injection parameters + +--- + +## Fault Injection + +### Kill Types -- `SimulatorKillType.h` + +| KillType | Effect | +|----------|--------| +| `KillInstantly` | Immediate failure, process gone | +| `InjectFaults` | Enable probabilistic fault injection | +| `FailDisk` | Simulate disk failure | +| `Reboot` | Machine reboot, keep data | +| `RebootProcess` | Process reboot, keep data | +| `RebootAndDelete` | Machine reboot, delete data | +| `RebootProcessAndDelete` | Process reboot, delete data | +| `RebootProcessAndSwitch` | Reboot with different cluster file | + +### Kill Hierarchy + +| Method | Scope | +|--------|-------| +| `killProcess(pid, kt)` | Single process | +| `killMachine(machineId, kt)` | All processes on machine | +| `killZone(zoneId, kt)` | All machines in zone | +| `killDataCenter(dcId, kt)` | All machines in datacenter | +| `killDataHall(hallId, kt)` | All machines in data hall | + +`killMachine()` checks replication policies before killing -- may upgrade kill type if processes are protected. + +### Network Clogging -- `SimClogging` (sim2.cpp:198-297) + +``` +struct SimClogging { + std::map clogSendUntil, clogRecvUntil; + std::map, double> clogPairUntil, clogPairLatency; + std::map, double> disconnectPairUntil; + + double getSendDelay(from, to); + double getRecvDelay(from, to); + bool disconnected(from, to); + void clogPairFor(from, to, seconds); + void disconnectPairFor(from, to, seconds); +}; +``` + +Latency model: +- 99.9% of packets: `MIN_NETWORK_LATENCY + FAST_NETWORK_LATENCY` +- 0.1% tail: `MIN_NETWORK_LATENCY + SLOW_NETWORK_LATENCY` +- Additional per-pair clogging latency +- Disconnection: throws `connection_failed()` + +### Disk Fault Injection -- `AsyncFileNonDurable.h` + +Simulates unreliable disk writes: + +``` +class AsyncFileNonDurable { + Reference file; // real underlying file + RangeMap> pendingModifications; + enum KillMode { NO_CORRUPTION, DROP_ONLY, FULL_CORRUPTION }; +}; +``` + +- Delays writes randomly (up to `NON_DURABLE_MAX_WRITE_DELAY`) +- On `kill()`: drops or corrupts pending writes based on `KillMode` +- `sync()` makes writes durable +- Simulates power failure: un-synced writes are lost + +### I/O Error Injection + +`simulator_should_inject_fault()` (sim2.cpp) -- called during I/O operations: +- Checks per-process `fault_injection_p1` and `fault_injection_p2` +- Probabilistically returns true → caller throws `io_timeout`, `io_error`, or `platform_error` + +--- + +## Buggify System -- `flow/include/flow/Buggify.h` + +Randomly enables rare code paths throughout the codebase. + +### Mechanism + +```cpp +#define BUGGIFY (getGeneralSBVar(__FILE__, __LINE__) && deterministicRandom()->random01() < P_FIRES) +#define BUGGIFY_WITH_PROB(x) (getGeneralSBVar(__FILE__, __LINE__) && deterministicRandom()->random01() < (x)) +``` + +Two-level randomization: +1. **Activation** (once per code location): `P_BUGGIFIED_SECTION_ACTIVATED = 0.25` -- 25% of BUGGIFY sites are active +2. **Firing** (each execution): `P_BUGGIFIED_SECTION_FIRES = 0.25` -- active sites fire 25% of the time + +Decisions cached in `SBVars` map keyed by `__FILE__ + __LINE__` for deterministic replay. + +### Variants + +| Macro | Scope | +|-------|-------| +| `BUGGIFY` | General server code | +| `BUGGIFY_WITH_PROB(x)` | Custom probability | +| `CLIENT_BUGGIFY` | Client-only code | +| `CLIENT_BUGGIFY_WITH_PROB(x)` | Client custom probability | + +### Examples in Codebase + +```cpp +if (BUGGIFY) delay(deterministicRandom()->random01()); // random extra delay +if (BUGGIFY) bufferSize = 1; // tiny buffer +if (BUGGIFY_WITH_PROB(0.001)) throw io_error(); // rare I/O error +``` + +--- + +## CODE_PROBE System -- `flow/include/flow/CodeProbe.h` + +Tracks code path coverage during testing: + +```cpp +CODE_PROBE(condition, "description of what happened"); +``` + +### Implementation + +``` +struct ICodeProbe { + const char* filePath(), line(), comment(), condition(); + bool wasHit() const; + unsigned hitCount() const; + void trace(bool condition) const; +}; +``` + +- Static singleton per source location +- Atomic hit counter +- Trace event logged on first hit +- Annotations: `context::Sim2`, `decoration::Rare`, `assert::SimOnly` +- Used to verify rare code paths are exercised during simulation + +--- + +## Workload System -- [`fdbserver/workloads/`](https://github.com/apple/foundationdb/tree/main/fdbserver/workloads) + +### TestWorkload Base Class (`workloads.h:65-98`) + +``` +struct TestWorkload : NonCopyable, WorkloadContext, ReferenceCounted { + int phases; // SETUP | EXECUTION | CHECK | METRICS + + virtual std::string description() const = 0; + virtual Future setup(Database const& cx); // optional + virtual Future start(Database const& cx) = 0; // main logic + virtual Future check(Database const& cx) = 0; // verify results + virtual Future> getMetrics(); +}; +``` + +### WorkloadContext (lines 38-50) + +- `clientId`, `clientCount` -- for distributed coordination +- `sharedRandomNumber` -- coordinated randomness across clients +- Reference to `ServerDBInfo` + +### Representative Workloads + +| Workload | Purpose | +|----------|---------| +| `ApiCorrectness` | Tests get/set/getRange/clear correctness | +| `ReadWrite` | Mixed read/write load with configurable patterns | +| `ConsistencyCheck` | Validates data consistency across replicas | +| `Cycle` | Tests cluster configuration changes | +| `BackupCorrectness` | Validates backup/restore integrity | +| `BulkLoad` / `BulkDumping` | Tests bulk operations | +| `AtomicOps` | Validates atomic operation correctness | +| `ChangeConfig` | Tests configuration changes under load | + +~60+ workload types covering all aspects of the system. + +--- + +## Test Configuration -- `tests/` directory + +### TOML Format + +```toml +[[knobs]] +enable_replica_consistency_check_on_reads = true + +[[test]] +testTitle = 'ApiCorrectnessWithConsistencyCheck' +clearAfterTest = true +timeout = 2100 + + [[test.workload]] + testName = 'ApiCorrectness' + numKeys = 5000 + numGets = 1000 + numGetRanges = 100 +``` + +### Parameters + +- `testTitle` -- test name +- `timeout` -- seconds before test times out +- `clearAfterTest` -- clear database after test +- Multiple `[[test.workload]]` sections for concurrent workloads +- `[[knobs]]` sections for server knob overrides + +--- + +## SimulatedCluster -- [`fdbserver/SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp) + +### TestConfig (lines 131-429) + +``` +class TestConfig : public BasicTestConfig { + int minimumReplication, minimumRegions; + Optional datacenters, processesPerMachine; + std::set storageEngineExcludeTypes; + Optional generateFearless, buggify, faultInjection; + bool simHTTPServerEnabled; + bool injectTargetedSSRestart, injectSSDelay; +}; +``` + +### Cluster Setup + +1. Load test configuration from TOML/INI file +2. Create machines with virtual IP addresses +3. Create processes with locality data (zone, machine, DC) +4. Configure replication policies +5. Start workload execution + +--- + +## Test Execution -- `fdbserver/tester/test.cpp` + +### `runWorkload()` (lines 67-187) + +Four phases: + +1. **Setup** -- call `setup()` on each tester, initialize state +2. **Execution** -- call `start()` on each tester, run workload logic +3. **Check** -- call `check()` on each tester, verify results +4. **Metrics** -- call `getMetrics()`, aggregate performance data + +### Distributed Coordination + +Each tester receives `WorkloadRequest`: +- `clientId` -- unique client identifier +- `clientCount` -- total clients +- `sharedRandomNumber` -- coordinated RNG seed +- Results aggregated via `aggregateMetrics()` + +--- + +## Principal Files + +| File | Purpose | +|------|---------| +| [`fdbrpc/sim2.cpp`](https://github.com/apple/foundationdb/blob/main/fdbrpc/sim2.cpp) | Sim2 deterministic simulator implementation | +| [`fdbrpc/include/fdbrpc/simulator.h`](https://github.com/apple/foundationdb/blob/main/fdbrpc/include/fdbrpc/simulator.h) | ISimulator interface | +| `fdbrpc/include/fdbrpc/SimulatorProcessInfo.h` | ProcessInfo, MachineInfo | +| `fdbrpc/include/fdbrpc/AsyncFileNonDurable.h` | Simulated unreliable disk | +| `flow/include/flow/Buggify.h` | BUGGIFY macros | +| `flow/include/flow/CodeProbe.h` | CODE_PROBE coverage tracking | +| [`fdbserver/SimulatedCluster.cpp`](https://github.com/apple/foundationdb/blob/main/fdbserver/SimulatedCluster.cpp) | Simulated cluster setup, TestConfig | +| `fdbserver/tester/include/fdbserver/tester/workloads.h` | TestWorkload base class | +| `fdbserver/tester/test.cpp` | Test execution orchestration | +| `fdbserver/workloads/*.cpp` | 60+ workload implementations | +| `tests/**/*.toml` | Test configuration files | diff --git a/design/AcAC.md b/design/AcAC.md index 3e2360e6200..00183100d46 100644 --- a/design/AcAC.md +++ b/design/AcAC.md @@ -54,9 +54,9 @@ The output is: 15624 /root/src/fdbserver/worker.actor.cpp:storageServerRollbackRebooter 5407 /root/src/fdbserver/worker.actor.cpp:workerServer 5140 /root/src/fdbserver/worker.actor.cpp:fdbd - 16 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedFDBDRebooter - 15 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedMachine - 10 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulationSetupAndRun + 16 /root/src/fdbserver/SimulatedCluster.cpp:simulatedFDBDRebooter + 15 /root/src/fdbserver/SimulatedCluster.cpp:simulatedMachine + 10 /root/src/fdbserver/SimulatedCluster.cpp:simulationSetupAndRun ``` which provides the actor ID, an analog of process ID in the actor context; the path of the source file, which contains the actor; the name of the actor; and a `` mark for the current running actor. @@ -102,15 +102,15 @@ AAAAAAAAAADwo2kloY1WAKDXp1f+BrgAAAAAAAAAAA== \ qDHU7CAAAAAAAADsIAAAAAAAAAD/+N1Ch/X3AEwORpyq68Y8AAAAAAAAADwAAAAAAAAAAEqD \ W3c/M9kAHCBUzBTlojsAAAAAAAAAOwAAAAAAAAAA+uRVDFla0gA5DXKp8yQbCgAAAAAAAAAK \ AAAAAAAAAADwo2kloY1WAKDXp1f+BrgAAAAAAAAAAA=="|bin/acac - 1304263 /root/src/fdbserver/workloads/FuzzApiCorrectness.actor.cpp:loadAndRun + 1304263 /root/src/fdbserver/workloads/FuzzApiCorrectness.cpp:loadAndRun 1269030 /root/src/fdbserver/tester.actor.cpp:runWorkloadAsync 1254091 /root/src/fdbserver/tester.actor.cpp:testerServerWorkload 8760 /root/src/fdbserver/tester.actor.cpp:testerServerCore 8746 /root/src/fdbserver/worker.actor.cpp:workerServer 8428 /root/src/fdbserver/worker.actor.cpp:fdbd - 60 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedFDBDRebooter - 59 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulatedMachine - 10 /root/src/fdbserver/SimulatedCluster.actor.cpp:simulationSetupAndRun + 60 /root/src/fdbserver/SimulatedCluster.cpp:simulatedFDBDRebooter + 59 /root/src/fdbserver/SimulatedCluster.cpp:simulatedMachine + 10 /root/src/fdbserver/SimulatedCluster.cpp:simulationSetupAndRun ``` The reason for not being able to generate the actor call backtrace inside the debugger directly is that the actor names are internally represented as UUIDs, and the mapping requires external files generated by the actor compiler. @@ -126,4 +126,3 @@ The reason for not being able to generate the actor call backtrace inside the de * All binding tests fail * Random failure on the `fdbserver` * It is possible for AcAC to report the line number of the source code where the actor is created. - diff --git a/design/Commit/commit.sequence b/design/Commit/commit.sequence index 502a572c222..a03727967d8 100644 --- a/design/Commit/commit.sequence +++ b/design/Commit/commit.sequence @@ -10,7 +10,7 @@ end participantgroup **CommitProxy** (CommitProxyServer.actor.cpp) participant "commitBatcher" as cB participant "commitBatch" as Batch - participant "TagPartitionedLogSystem" as TPLS + participant "LogSystem" as TPLS end participantgroup **Master** diff --git a/design/backup_v2_partitioned_logs.md b/design/backup_v2_partitioned_logs.md index 3e85e7e3363..1905601d7c7 100644 --- a/design/backup_v2_partitioned_logs.md +++ b/design/backup_v2_partitioned_logs.md @@ -70,7 +70,7 @@ A command line tool `fdbconvert` has been written to convert new backup logs int * How to start a new type backup: e.g., ``` - fdbbackup start -C fdb.cluster --partitioned-log-experimental -d blob_url + fdbbackup start -C fdb.cluster --mutation-log-type partitioned-log-experimental -d blob_url ``` ### KPI's and Health @@ -224,7 +224,7 @@ We strive to keep the operational interface the same as the old backup system. T By default, backup workers are not enabled in the system. When operators submit a new backup request for the first time, the database performs a configuration change (`backup_worker_enabled:=1`) that enables backup workers. -The operator’s backup request can indicate if an old backup or a new backup is used. This is a command line option (i.e., `--partitioned-log-experimental`) in the `fdbbackup` command. A backup request of the new type is started in the following steps: +The operator’s backup request can indicate if an old backup or a new backup is used. This is a command line option (i.e., `--mutation-log-type partitioned-log-experimental`) in the `fdbbackup` command. A backup request of the new type is started in the following steps: 1. Operators use `fdbbackup` tool to write the backup range to a system key, i.e., `\xff\x02/backupStarted`. 2. All backup workers monitor the key `\xff\x02/backupStarted`, see the change, and start logging mutations. diff --git a/design/cluster_health_metric.md b/design/cluster_health_metric.md new file mode 100644 index 00000000000..5a15a9f1d4e --- /dev/null +++ b/design/cluster_health_metric.md @@ -0,0 +1,171 @@ +# Cluster Health Metric + +`cluster_health::Monitor` is a cluster controller background task that periodically evaluates a fixed set of health factors and emits a `ClusterHealthMetric` trace event. + +The monitor is controlled by these server knobs: + +| Knob | Default | Meaning | +| --- | --- | --- | +| `CLUSTER_HEALTH_METRIC_ENABLE` | `false` | Enables the cluster controller monitor. When disabled, the monitor exits immediately and does not emit `ClusterHealthMetric`. Simulation buggification can set this to `true`. | +| `CLUSTER_HEALTH_METRIC_POLL_INTERVAL` | `5.0` seconds | Time between monitor evaluations after the first evaluation. | +| `CLUSTER_HEALTH_METRIC_STORAGE_INTERVENTION_THRESHOLD` | `0.20` | Storage server free-space warning threshold. This is a ratio, not a percentage: `0.20` means 20% free space. | +| `CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD` | `0.10` | Storage server free-space critical threshold. This should be less than or equal to the storage intervention threshold. | +| `CLUSTER_HEALTH_METRIC_TLOG_INTERVENTION_THRESHOLD` | `0.20` | TLog queue-disk free-space warning threshold. This is a ratio, not a percentage: `0.20` means 20% free space. | +| `CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD` | `0.10` | TLog queue-disk free-space critical threshold. This should be less than or equal to the TLog intervention threshold. | +| `CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD` | `1.2` | Ratekeeper critical throttling threshold for `TPSLimit / ReleasedTPS`. With the default, ratekeeper is critical when the current TPS limit is less than 120% of recently released TPS. | + +Threshold comparisons are strict: a value equal to a threshold does not trigger that threshold's health level. + +## Health Levels + +Each factor uses the following levels: + +- `HEALTHY`: No issue is currently indicated by the factor. +- `SELF_HEALING`: The cluster is degraded, but automatic recovery or repair is already in progress. +- `INTERVENTION_REQUIRED`: The cluster is still functioning, but an operator likely needs to act to restore full health. +- `CRITICAL_INTERVENTION_REQUIRED`: The cluster is functioning, but in a severe state that likely requires immediate operator action. +- `OUTAGE`: The factor indicates loss of availability or loss of a required safety property. +- `METRICS_MISSING`: The factor could not evaluate because the required trace-event data was unavailable or malformed. + +For aggregation, the implementation assigns an internal score to each level and keeps the lowest-scoring factor as the aggregate result: + +- `HEALTHY` = 100 +- `SELF_HEALING` = 80 +- `INTERVENTION_REQUIRED` = 60 +- `CRITICAL_INTERVENTION_REQUIRED` = 40 +- `METRICS_MISSING` = 20 +- `OUTAGE` = 0 + +These numeric values provide a scalar metric that can be emitted, and leave room for additional intermediate values to be added in the future. + +## Factors + +The monitor currently evaluates these factors: + +### `StorageSpace` + +Source event: +- `"/StorageMetrics"` + +Fields used: +- `KvstoreBytesAvailable` +- `KvstoreBytesTotal` + +Behavior: +- Computes the minimum `KvstoreBytesAvailable / KvstoreBytesTotal` ratio across storage servers. +- Returns `CRITICAL_INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD`. +- Returns `INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_STORAGE_INTERVENTION_THRESHOLD` and at or above `CLUSTER_HEALTH_METRIC_STORAGE_CRITICAL_THRESHOLD`. +- Returns `HEALTHY` otherwise. + +### `TLogSpace` + +Source event: +- `"/TLogMetrics"` + +Fields used: +- `QueueDiskBytesAvailable` +- `QueueDiskBytesTotal` + +Behavior: +- Computes the minimum `QueueDiskBytesAvailable / QueueDiskBytesTotal` ratio across tlogs. +- Returns `CRITICAL_INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD`. +- Returns `INTERVENTION_REQUIRED` when the ratio is below `CLUSTER_HEALTH_METRIC_TLOG_INTERVENTION_THRESHOLD` and at or above `CLUSTER_HEALTH_METRIC_TLOG_CRITICAL_THRESHOLD`. +- Returns `HEALTHY` otherwise. + +### `StorageReplication` + +Source event: +- `MovingData` + +Fields used: +- `InQueue` +- `InFlight` +- `PriorityTeamUnhealthy` +- `PriorityTeam2Left` +- `PriorityTeam1Left` +- `PriorityTeam0Left` + +Behavior: +- Returns `OUTAGE` if any `PriorityTeam0Left > 0`. +- Returns `CRITICAL_INTERVENTION_REQUIRED` if any `PriorityTeam1Left > 0` and the configured storage team size is 3. +- Returns `SELF_HEALING` if data movement is queued or in flight to restore replication. +- Returns `HEALTHY` otherwise. + +### `RecoveryState` + +Source: +- `ServerDBInfo::recoveryState` from the cluster controller + +Fields used: +- none; this factor reads the cluster controller's in-memory `RecoveryState` + +Behavior: +- Returns `OUTAGE` if recovery state is below `RecoveryState::ACCEPTING_COMMITS`. +- Returns `SELF_HEALING` if recovery is at or above `ACCEPTING_COMMITS` but below `FULLY_RECOVERED`. +- Returns `HEALTHY` at `FULLY_RECOVERED`. + +### `ProcessErrors` + +Source event: +- latest worker error event, fetched with `EventLogRequest()` + +Fields used: +- none are parsed directly; the factor only checks whether a non-empty latest-error trace event exists. + +Behavior: +- Returns `CRITICAL_INTERVENTION_REQUIRED` if any worker reports a non-empty latest error. +- Returns `HEALTHY` if all non-failed latest-error records are empty and at least one latest-error request succeeded. +- Returns `METRICS_MISSING` only if every latest-error request fails. + +### `RkThrottling` + +Source event: +- `RkUpdate` + +Fields used: +- `ReleasedTPS`: Ratekeeper's smoothed rate of transactions that GRV proxies have recently released from their start-transaction queues. A released transaction has been allowed to get a read version and begin; this is not commit throughput. +- `TPSLimit`: Ratekeeper's current transaction-per-second limit for normal transaction starts, computed from cluster pressure signals and sent back to GRV proxies to pace future releases. A value of `0` means ratekeeper is not allowing normal transaction starts. + +Behavior: +- Returns `OUTAGE` if any `TPSLimit == 0`. +- Computes the minimum `TPSLimit / ReleasedTPS` ratio across samples with nonzero `ReleasedTPS`. +- Ignores samples with `ReleasedTPS == 0` for the ratio calculation to avoid dividing by zero. +- Returns `CRITICAL_INTERVENTION_REQUIRED` when the minimum ratio is below `CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD`. +- Returns `HEALTHY` otherwise, including the case where every non-outage sample has `ReleasedTPS == 0`. + +For `CLUSTER_HEALTH_METRIC_RK_CRITICAL_RELEASED_TPS_RATIO_THRESHOLD`, lower ratios are worse: + +- `TPSLimit / ReleasedTPS == 2.0` means ratekeeper is allowing twice the recently released transaction rate, so this factor is healthy with the default threshold. +- `TPSLimit / ReleasedTPS == 1.0` means ratekeeper's current limit is equal to the recently released transaction rate, so this factor is critical with the default threshold of `1.2`. +- `TPSLimit / ReleasedTPS < 1.0` means ratekeeper's current limit is below the recently released transaction rate, which is also critical. + +## Missing Metrics Semantics + +Most factors treat missing or malformed inputs as `METRICS_MISSING`. + +This can happen when: + +- the underlying latest-event RPC fails +- no relevant worker emitted the requested trace event +- the trace event exists but does not contain the required fields +- a field cannot be parsed to the expected type + +The implementation filters out empty `TraceEventFields` before interpreting a factor, so workers that never emit a role-specific event do not automatically degrade the whole metric. + +## `ClusterHealthMetric` Trace Event Schema + +The monitor emits a periodic trace event named `ClusterHealthMetric`. + +Fields: + +- `FactorStorageSpace`: string enum, one of `Outage`, `CriticalInterventionRequired`, `InterventionRequired`, `SelfHealing`, `MetricsMissing`, `Healthy` +- `FactorTLogSpace`: same enum +- `FactorStorageReplication`: same enum +- `FactorRecoveryState`: same enum +- `FactorProcessErrors`: same enum +- `FactorRkThrottling`: same enum +- `Aggregate`: same enum, computed from the most limiting factor +- `AggregateValue`: numeric aggregate score, currently one of `0`, `20`, `40`, `60`, `80`, or `100` +- `LimitingFactor`: factor name string, present when not healthy + +The aggregate is determined by taking the lowest-scoring level across all factors. diff --git a/design/coroutines.md b/design/coroutines.md index 230408a7144..b35524043ef 100644 --- a/design/coroutines.md +++ b/design/coroutines.md @@ -6,6 +6,7 @@ * [Choose-When](#choose-when) * [Execution in when-expressions](#execution-in-when-expressions) * [Waiting in When-Blocks](#waiting-in-when-blocks) + * [Migrating `choose`-`when`](#migrating-choose-when) * [Generators](#generators) * [Generators and ranges](#generators-and-ranges) * [Eager vs Lazy Execution](#eager-vs-lazy-execution) @@ -18,6 +19,7 @@ * [Don't Wait in Error-Handlers](#dont-wait-in-error-handlers) * [Make Static Functions Class Members](#make-static-functions-class-members) * [Initialization of Locals](#initialization-of-locals) +* [Internals Reference](fdb-coroutines-internals.md) — deep dive into the coroutine runtime (SAV, Actor, awaiters, cancellation) ## Introduction @@ -98,9 +100,36 @@ choose { } ``` -Since this is a compiler functionality, we can't use this with C++ coroutines. We could -keep only this feature around, but only using standard C++ is desirable. So instead, we -introduce a new `class` called `Choose` to achieve something very similar: +Since this is a compiler functionality, we can't use this with C++ coroutines. For most +coroutine conversions, prefer `race(...)` and then branch on the returned `std::variant`. +This keeps the control flow explicit and usually maps more directly to what the coroutine +will do after the wait. `Choose` is still available for cases where the old `choose`-`when` +shape is the clearest fit, or where you need its ordered evaluation behavior described below. + +For example, this actor pattern: + +```c++ +choose { + when(R res = wait(f1)) { + return res; + } + when(wait(timeout(...))) { + throw io_timeout(); + } +} +``` + +should usually become: + +```c++ +auto result = co_await race(f1, timeout(...)); +if (result.index() == 0) { + co_return std::get<0>(result); +} +throw io_timeout(); +``` + +`Choose` remains useful when you specifically want callback-style handling of the winner: ```c++ co_await Choose() @@ -130,7 +159,7 @@ co_await Choose() }) .When([](){ return foo() }, [](Foo const& f) { // do something else - }).Run(); + }).run(); ``` The implementation of `When` will guarantee that this lambda will only be executed if all previous @@ -195,7 +224,40 @@ loop { } ``` -However, often using `choose`-`when` (or `Choose().When`) is overkill and other facilities like `quorum` and +### Migrating `choose`-`when` + +When porting actor code, use this rule of thumb: + +* Prefer `race(...)` when the `choose` picks one winner and then the code branches on which future completed. +* Use `Choose()` when you need ordered `when` evaluation, lazy creation of later futures, or the callback style is + genuinely clearer than branching on a `std::variant`. +* Use more specific helpers like `quorum`, `waitForAll`, `waitForAllReady`, or `operator||` when they express the + intent better than either `race` or `Choose`. + +`race(...)` is usually the best direct replacement for patterns like: + +```c++ +choose { + when(R res = wait(f1)) { + return res; + } + when(S res = wait(f2)) { + return use(res); + } +} +``` + +which becomes: + +```c++ +auto result = co_await race(f1, f2); +if (result.index() == 0) { + co_return std::get<0>(result); +} +co_return use(std::get<1>(result)); +``` + +However, often using `choose`-`when` is overkill and other facilities like `quorum` and `operator||` should be used instead. For example this: ```c++ @@ -490,6 +552,22 @@ But this one is: co_await bar(Uncancellable()); ``` +## NoThrowOnCancel + +`NoThrowOnCancel` is a marker for coroutines that should still cancel, but should not run cancellation through an +`actor_cancelled` exception inside the coroutine: + +```c++ +Future foo(NoThrowOnCancel = {}) { + Resource r; + co_await something(); +} +``` + +When a coroutine with this marker is cancelled, the coroutine frame is destroyed and normal C++ RAII cleanup runs for +locals in scope. `catch` blocks inside the coroutine do not observe `actor_cancelled` for cancellation. This differs from +`Uncancellable`, where cancelling the returned future is a no-op and the coroutine continues until completion. + ## Porting `ACTOR`'s to C++ Coroutines If you have an existing `ACTOR`, you can port it to a C++ coroutine by following these steps: @@ -500,7 +578,8 @@ If you have an existing `ACTOR`, you can port it to a C++ coroutine by following 3. Remove all `state` modifiers from local variables. 4. Replace all `wait(expr)` with `co_await expr`. 5. Remove all `waitNext(expr)` with `co_await expr`. -6. Rewrite existing `choose-when` statements using the `Choose` class. +6. Rewrite existing `choose-when` statements by preferring `race(...)` for first-ready branching; use `Choose` + only when you need ordered `when` semantics or callback-style handling. In addition, the following things should be looked out for: @@ -789,17 +868,26 @@ When a function is converted from `ACTOR` to a coroutine, any forward declaratio If you also write `const&` explicitly, the generated code will contain `const& const&`, which is a compile error. ```c++ -// workloads.actor.h — WRONG: ACTOR + const& = double const& +// workloads.h — WRONG: ACTOR + const& = double const& ACTOR Future foo(Database const& cx); -// workloads.actor.h — CORRECT: remove ACTOR since foo() is now a coroutine +// workloads.h — CORRECT: remove ACTOR since foo() is now a coroutine Future foo(Database const& cx); ``` +### `DESCR` Deprecation + +The older TDMetric `DESCR` shorthand is deprecated. When a coroutine conversion touches metric event types, do not add +new `DESCR(...)`-style declarations. Instead, define an explicit payload type with a +`...Descriptor` suffix and specialize `Descriptor` with `DescribeType<...>` and `DescribeField<...>` next to it. + +This keeps descriptor types unambiguous in coroutine-converted code and matches the old TDMetric pattern in files +such as `flow/EventTypes.h`, `fdbclient/EventTypes.h`, and the workload metric definitions. + ### File Naming Converted files should be renamed from `.actor.cpp` to `.cpp` (or `.actor.h` to `.h`) since they no longer need the -actor compiler. Both `fdbserver` and `flowbench` use `fdb_find_sources()` in their `CMakeLists.txt`, which +actor compiler. Both `fdbserver` and `flow_bench` use `fdb_find_sources()` in their `CMakeLists.txt`, which automatically picks up files by glob, so the rename is usually sufficient without any CMake changes. ### Conversion Checklist @@ -811,7 +899,8 @@ automatically picks up files by glob, so the rename is usually sufficient withou - **Check**: is the variable inside a block (`if`/`else`/`for`/`try`)? If so, move it to function scope. 5. Replace `wait(expr)` with `co_await expr`. Replace `waitNext(expr)` with `co_await expr`. 6. Replace `return expr` with `co_return expr`. Replace `return Void()` with `co_return`. -7. Rewrite `choose`/`when` using the `Choose` class. +7. Rewrite `choose`/`when` by preferring `race(...)`; use `Choose` only for patterns that do not map cleanly to + `race`. 8. Simplify: `wait(success(f))` → `co_await f`; `wait(store(v, f))` → `v = co_await f`. 9. For `const&` parameters: copy to a local before the first `co_await`. 10. Remove `ACTOR` from any forward declarations of the converted functions in `.actor.h` files. @@ -931,7 +1020,7 @@ python3 ../contrib/benchmark_comparison.py - OVERALL_GEOMEAN calculations for statistical analysis **Requirements**: -- Working flowbench binary with both actor and coroutine benchmarks +- Working flow_bench binary with both actor and coroutine benchmarks - Benchmark infrastructure must include: bench_net2, coroutine_net2, bench_delay, coroutine_delay_bench, coroutine_yield_bench, bench_callback, coroutine_callback **Usage**: Tool automatically runs benchmarks and generates comparison report in the format matching historical coroutine optimization reports. diff --git a/design/data-distributor-internals.md b/design/data-distributor-internals.md index 55ec8da3068..c18423a4a7a 100644 --- a/design/data-distributor-internals.md +++ b/design/data-distributor-internals.md @@ -88,7 +88,7 @@ There are roughly 4 class of movement priorities * Boundary change priority. The movement will change current shard boundaries. * Others. Like resuming a in-flight movement. -Each shard movement has a priority associating with the move attempt, The explanation of each priority knob (`PRIORITY_`) is in `ServerKnobs.h`. +Each shard movement has a priority associating with the move attempt, The explanation of each priority knob (`PRIORITY_`) is in `Knobs.h`. In `status json` output, please look at field `.data.team_tracker.state` for team priority state. @@ -186,3 +186,807 @@ CPU utilization. This metric is in a positive relationship with “FinishedQueri * First find the TraceId fields in `BgDDMountainChopper*`, `BgDDValleyFiller*`, which indicates a relocation is triggered. * (Only when enabled) Find the `QueuedRelocation` event with the same `BeginPair` and `EndPair` as the original `TraceId`. This means the relocation request is queued. * Find the `RelocateShard` event whose `BeginPair`, `EndPair` field is the same as `TraceId`. This event means the relocation is ongoing. + + +# Alternative View -- AI-generated for FDB 7.3 + +The following was generated by an AI assistant with access to the FDB source repository, including the documentation above in this file. + +## 1. Role in the Cluster + +The Data Distributor (DD) is a singleton role recruited by the Cluster Controller. It is +the only component authorized to mutate the shard-to-server mapping stored in the system +keyspace. Its responsibilities: + +1. **Shard lifecycle management** -- splitting shards that grow too large or too hot, merging + shards that shrink below thresholds. +2. **Team formation** -- building and maintaining server teams (groups of `k` storage servers, + where `k` = replication factor) and machine teams that satisfy the replication policy. +3. **Data movement** -- relocating shards to repair replication (after server failures), to + rebalance disk/read/write load, or to carry out operational directives (exclusions, wiggle). +4. **Audit & bulk operations** -- storage audits and bulk load/dump coordination (7.3 additions). + +DD is a single point of coordination -- if it crashes, the Cluster Controller recruits a new +one which recovers state from the system keyspace. During the gap, no new data movement occurs, +but existing storage servers continue to serve reads/writes normally. + +### Singleton Guarantee: MoveKeysLock + +Only one DD instance can modify the shard map at a time, enforced by `MoveKeysLock`: + +``` +moveKeysLockOwnerKey (\xff/moveKeysLock/Owner) -- which DD owns the lock +moveKeysLockWriteKey (\xff/moveKeysLock/Write) -- which DD is actively mutating +``` + +When a new DD starts, it sets its UID as the owner. If during a `moveKeys` operation it +discovers another DD has taken ownership, it throws `movekeys_conflict` and terminates. This +prevents split-brain corruption of the shard map. + +Source: [`MoveKeys.actor.h:37`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/MoveKeys.actor.h#L37) (`MoveKeysLock`), +[`DataDistribution.actor.cpp:511`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L511) (`takeMoveKeysLock`). + + +## 2. Top-Level Actor Topology + +``` +dataDistribution() [DataDistribution.actor.cpp:889] + ├─ DataDistributor::init() -- recover state from system keyspace + ├─ resumeRelocations() -- resume in-flight data moves + │ + ├─ DataDistributionTracker::run() -- shard health monitoring + │ ├─ shardTracker (per shard) -- monitors size/bandwidth/read metrics + │ ├─ shardSplitter -- splits oversized/overloaded shards + │ ├─ shardMerger -- merges underutilized neighbors + │ └─ readHotDetection -- identifies read-hot shards + │ + ├─ DDQueue::run() -- relocation scheduling + │ ├─ dataDistributionRelocator -- per-relocation actor + │ ├─ BgDDLoadRebalance (MC disk) -- MountainChopper for disk + │ ├─ BgDDLoadRebalance (VF disk) -- ValleyFiller for disk + │ ├─ BgDDLoadRebalance (MC read) -- MountainChopper for read + │ ├─ BgDDLoadRebalance (VF read) -- ValleyFiller for read + │ └─ periodicalRefreshCounter -- refresh DDQueueServerCounter metrics + │ + ├─ DDTeamCollection::run() [primary] -- primary DC team management + │ ├─ buildTeams -- create server/machine teams + │ ├─ storageServerTracker (per SS) -- monitor SS health + │ ├─ teamTracker (per team) -- monitor team health, emit RelocateShards + │ ├─ serverTeamRemover -- prune excess server teams + │ ├─ machineTeamRemover -- prune excess machine teams + │ ├─ storageRecruiter -- recruit new storage servers + │ ├─ monitorPerpetualStorageWiggle -- perpetual wiggle orchestration + │ │ ├─ perpetualStorageWiggleIterator + │ │ └─ perpetualStorageWiggler + │ ├─ trackExcludedServers -- process exclusion list + │ └─ monitorHealthyTeams -- track healthy team count + │ + └─ DDTeamCollection::run() [remote] -- remote DC (if usableRegions > 1) + └─ (same actors as primary) +``` + +Source: [`dataDistribution()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L889). + + +## 3. Core Data Structures + +### 3.1 Storage Server Info (`TCServerInfo`) + +Defined in [`TCInfo.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/TCInfo.h). One per storage server tracked by DD. Contains: +- **Locality**: processID (unique to ip:port), zoneId (rack), dcId (datacenter) +- **Teams**: list of `TCTeamInfo` references this server belongs to +- **Metrics**: disk usage, CPU, storage queue length +- **Status**: [`ServerStatus`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDTeamCollection.h#L139) in `DDTeamCollection.h` tracks `isFailed`, `isUndesired`, + `isWiggling`, `isWrongConfiguration` + +A server is "healthy" if its storage engine matches the configured engine AND it is marked +as desired by DD. See `ServerStatus::isUnhealthy()` at [`DDTeamCollection.h:152`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDTeamCollection.h#L152). + +### 3.2 Machine Info (`TCMachineInfo`) + +A "machine" in FDB represents a **rack** -- the assumption is one physical host per rack. +All servers sharing a `zoneId` locality belong to the same machine. Defined in [`TCInfo.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/TCInfo.h). + +### 3.3 Server Teams (`TCTeamInfo`) + +A group of `k` servers (where `k` = `storageTeamSize`, typically 3) that replicate the same +shards. Implements `IDataDistributionTeam`. A team is healthy iff every server is healthy +AND the team satisfies the replication policy. + +### 3.4 Machine Teams (`TCMachineTeamInfo`) + +A group of `k` machines. Every server team must map onto a machine team (one server per +machine in the team). This ensures that server teams span distinct failure domains. + +### 3.5 Team Collection (`DDTeamCollection`) + +Defined in [`DDTeamCollection.h:205`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDTeamCollection.h#L205). The global view of all servers, server teams, machines, +and machine teams for one datacenter. Key data members: + +```cpp +std::map> server_info; // all servers +std::vector> teams; // all server teams +std::map, Reference> machine_info; +std::vector> machineTeams; +``` + +There is one `DDTeamCollection` per DC (primary + optionally remote). + +### 3.6 Shards (`DDShardInfo`) + +A shard is a contiguous key range assigned to a server team. Defined in +[`DataDistribution.actor.h:483`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L483): + +```cpp +struct DDShardInfo { + Key key; // start key of this shard + std::vector primarySrc; // current source servers (primary DC) + std::vector remoteSrc; // current source servers (remote DC) + std::vector primaryDest; // destination servers during move + std::vector remoteDest; + bool hasDest; // true if a move is in progress + UID srcId, destId; // data move IDs +}; +``` + +Shard boundaries are stored in the system keyspace: +- `\xff/keyServers/[start_key]` → source and destination server IDs +- `\xff/serverKeys/[serverID]/[start_key]` → which shards a server owns + +### 3.7 Shard Size Bounds + +Shard size is **workload-driven**, not fixed. The maximum shard size scales with database size: + +```cpp +int64_t getMaxShardSize(double dbSizeEstimate) { + return min((MIN_SHARD_BYTES + sqrt(max(dbSizeEstimate, 0)) * SHARD_BYTES_PER_SQRT_BYTES) + * SHARD_BYTES_RATIO, MAX_SHARD_BYTES); +} +``` + +| DB Size Estimate | Max Shard Size | +|------------------|----------------| +| 0 | ~40 MB | +| 1 GB | ~46 MB | +| 100 GB | ~97 MB | +| 1 TB | ~220 MB | +| >=6.5 TB | 500 MB (cap) | + +Source: [`DataDistribution.actor.h:518`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L518) (`ShardSizeBounds`), documented in [`design/hotshard.md`](https://github.com/apple/foundationdb/blob/release-7.3/design/hotshard.md). + +### 3.8 RelocateShard + +The fundamental unit of work for data movement. Defined in [`DataDistribution.actor.h:147`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L147): + +```cpp +struct RelocateShard { + KeyRange keys; + int priority; + bool cancelled; + std::shared_ptr dataMove; // non-null if restoring a prior move + UID dataMoveId; + RelocateReason reason; + DataMovementReason moveReason; + UID traceId; // for lifecycle tracking +}; +``` + +### 3.9 DDSharedContext + +The shared state container connecting all DD sub-components. Defined in [`DDSharedContext.h:31`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDSharedContext.h#L31): + +```cpp +class DDSharedContext { + std::unique_ptr ddEnabledState; + DataDistributorInterface interface; + UID ddId; + MoveKeysLock lock; + DatabaseConfiguration configuration; + Reference shardsAffectedByTeamFailure; + Reference tracker; + Reference ddQueue; + Reference primaryTeamCollection, remoteTeamCollection; +}; +``` + + +## 4. Initialization Flow + +When a new DD is recruited ([`DataDistribution.actor.cpp:889`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L889)): + +1. **Open database connection** -- `openDBOnServer()` with `TaskPriority::DataDistributionLaunch` +2. **Initialize config watcher** -- `initDDConfigWatch()` establishes baseline +3. **`DataDistributor::init()`** ([line 931](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L931)) -- reads system keyspace to recover: + - Current shard boundaries from `\xff/keyServers/` + - Server-shard assignments from `\xff/serverKeys/` + - Active storage servers and their interfaces from `\xff/serverList/` + - Cluster configuration (replication mode, storage engine, etc.) + - Takes ownership of `moveKeysLock` +4. **Resume relocations** -- [`resumeRelocations()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L806) reads persisted `DataMoveMetaData` to + resume any in-flight data moves from the previous DD +5. **Start sub-components** in parallel: + - `DataDistributionTracker::run()` -- begins shard monitoring + - `DDQueue::run()` -- begins processing relocation requests + - `DDTeamCollection::run()` (primary) -- team management for primary DC + - `DDTeamCollection::run()` (remote) -- if `usableRegions > 1` + - `pollMoveKeysLock()` -- periodic lock validation + - Physical shard monitoring (if enabled) + + +## 5. Data Movement Priority System + +Every relocation request has a priority that determines scheduling order. Higher values +take precedence. The full priority table from [`ServerKnobs.cpp:155-173`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/ServerKnobs.cpp#L155): + +| Priority | Value | DataMovementReason | Category | +|----------|-------|---------------------------------------|---------------| +| `PRIORITY_RECOVER_MOVE` | 110 | `RECOVER_MOVE` | Recovery | +| `PRIORITY_REBALANCE_UNDERUTILIZED_TEAM`| 120 | `REBALANCE_UNDERUTILIZED_TEAM` | Load Balance | +| `PRIORITY_REBALANCE_READ_UNDERUTIL_TEAM`| 121 | `REBALANCE_READ_UNDERUTIL_TEAM`| Load Balance | +| `PRIORITY_REBALANCE_OVERUTILIZED_TEAM` | 122 | `REBALANCE_OVERUTILIZED_TEAM` | Load Balance | +| `PRIORITY_REBALANCE_READ_OVERUTIL_TEAM`| 123 | `REBALANCE_READ_OVERUTIL_TEAM` | Load Balance | +| `PRIORITY_REBALANCE_STORAGE_QUEUE` | 124 | `REBALANCE_STORAGE_QUEUE` | Load Balance | +| `PRIORITY_TEAM_HEALTHY` | 140 | `TEAM_HEALTHY` | Health | +| `PRIORITY_PERPETUAL_STORAGE_WIGGLE` | 141 | `PERPETUAL_STORAGE_WIGGLE` | Maintenance | +| `PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER`| 150 | `TEAM_CONTAINS_UNDESIRED_SERVER`| Health | +| `PRIORITY_TEAM_REDUNDANT` | 200 | `TEAM_REDUNDANT` | Health | +| `PRIORITY_MERGE_SHARD` | 340 | `MERGE_SHARD` | Boundary | +| `PRIORITY_POPULATE_REGION` | 600 | `POPULATE_REGION` | Health | +| `PRIORITY_TEAM_UNHEALTHY` | 700 | `TEAM_UNHEALTHY` | Health | +| `PRIORITY_TEAM_2_LEFT` | 709 | `TEAM_2_LEFT` | Health | +| `PRIORITY_TEAM_1_LEFT` | 800 | `TEAM_1_LEFT` | Health | +| `PRIORITY_TEAM_FAILED` | 805 | `TEAM_FAILED` | Health | +| `PRIORITY_TEAM_0_LEFT` | 809 | `TEAM_0_LEFT` | Health | +| `PRIORITY_SPLIT_SHARD` | 950 | `SPLIT_SHARD` | Boundary | +| `PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD` | 960 | Physical shard enforcement | Boundary | + +Priority categories from [`DDRelocationQueue.h:41-86`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L41) (`RelocateData` class): +- **Health priorities**: repairs to replication factor, team health +- **Boundary priorities**: shard splits and merges +- **Load balance priorities**: disk/read/write rebalancing (lowest urgency) + +Source: [`DDRelocationQueue.actor.cpp:77-101`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L77) for the complete mapping. + + +## 6. Shard Tracker + +The [`DataDistributionTracker`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDShardTracker.h#L54) (defined in `DDShardTracker.h`) monitors every shard's +metrics and initiates splits, merges, and relocations. + +### 6.1 Shard Splitting + +Triggered when a shard exceeds size or write bandwidth thresholds: +- **Size split**: `shard.bytes > maxShardSize` → `PRIORITY_SPLIT_SHARD (950)` +- **Write split**: `bytesWrittenPerKSecond > SHARD_MAX_BYTES_PER_KSEC (1 GB/s)` + +The split process: +1. Tracker calls `splitStorageMetrics` on the storage servers hosting the shard +2. Storage servers use their sampled byte/write histograms to compute split points +3. `getSplitKey()` adds jitter to avoid picking the same boundary repeatedly +4. Tracker calls `executeShardSplit` to update the shard map and issue relocations + +Split constraints: +- Each resulting piece must have bandwidth < `SHARD_SPLIT_BYTES_PER_KSEC (250 MB/s)` +- Minimum shard size enforced: `MIN_SHARD_BYTES (10 MB by default)` +- If the shard is too small to split (e.g., single hot key), no split occurs + +Source: [`shardSplitter()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDShardTracker.actor.cpp#L852), [`StorageMetrics.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/StorageMetrics.actor.cpp). + +### 6.2 Shard Merging + +Triggered when a shard's write bandwidth drops below `SHARD_MIN_BYTES_PER_KSEC (100 MB/s)` +and its size is small enough that merging with a neighbor stays under the max shard size. +Priority: `PRIORITY_MERGE_SHARD (340)`. + +### 6.3 Read-Hot Shard Detection + +When `READ_SAMPLING_ENABLED=true`, storage servers sample read operations. Shards exceeding +`SHARD_MAX_READ_OPS_PER_KSEC (45k ops/s)` or having high read density ratios are flagged. +The tracker emits `ReadHotRangeLog` trace events. Read-hot shards trigger **rebalance moves** +(not splits) -- DD moves the shard to a less-loaded team. + +Source: [`StorageMetrics.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/StorageMetrics.actor.cpp), [`design/hotshard.md`](https://github.com/apple/foundationdb/blob/release-7.3/design/hotshard.md). + + +## 7. Relocation Queue (DDQueue) + +[`DDQueue`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L113) (defined in `DDRelocationQueue.h`) is the scheduler for all data movements. +It receives `RelocateShard` requests from the tracker and team collection, prioritizes them, +selects destination teams, and launches the actual data transfers. + +### 7.1 Queue Data Structures + +```cpp +KeyRangeMap queueMap; // all queued relocations +std::set fetchingSourcesQueue; // fetching source info +std::set fetchKeysComplete; // ready to launch +std::map> queue; // per-server queue +KeyRangeMap inFlight; // actively moving +std::map busymap; // source server busyness +std::map destBusymap; // dest server busyness +std::map priority_relocations; // count by priority +``` + +Source: [`DDRelocationQueue.h:113-307`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L113). + +### 7.2 Throttling + +The [`Busyness`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DDRelocationQueue.h#L86) struct tracks per-server load across 10 priority +buckets. Before launching a relocation, DDQueue checks: +- Source server busyness: `RELOCATION_PARALLELISM_PER_SOURCE_SERVER` +- Dest server busyness: `RELOCATION_PARALLELISM_PER_DEST_SERVER` +- Total move keys parallelism: `DD_MOVE_KEYS_PARALLELISM` + +Inflight penalties increase apparent busyness based on relocation health status: +- `INFLIGHT_PENALTY_HEALTHY`, `INFLIGHT_PENALTY_REDUNDANT`, `INFLIGHT_PENALTY_UNHEALTHY`, + `INFLIGHT_PENALTY_ONE_LEFT` + +### 7.3 Relocation Lifecycle + +Each relocation goes through: +1. **Queued** → `queueRelocation()` -- overlapping in-flight moves are cancelled +2. **Fetch sources** → `fetchSourceServersComplete` -- identify current shard owners +3. **Team selection** → `getSrcDestTeams()` → `GetTeamRequest` to `DDTeamCollection` +4. **Launch** → [`dataDistributionRelocator()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L1302) +5. **Move keys** → `moveKeys()` in [`MoveKeys.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3180) -- the actual data transfer: + - Add destination as new shard owner (private mutation to `keyServers`/`serverKeys`) + - Destination SS reads data from sources via FDB transactions + - Remove source from shard ownership + - Update `ShardsAffectedByTeamFailure` +6. **Complete** → `relocationComplete` stream + + +## 8. Load Rebalancing: MountainChopper & ValleyFiller + +DD continuously rebalances load using two complementary algorithms implemented in a single +actor, [`BgDDLoadRebalance`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L2281): + +### MountainChopper (Overutilized → Underutilized) + +- **Source selection**: `TeamSelect::WANT_TRUE_BEST` -- picks the most loaded team +- **Dest selection**: `TeamSelect::ANY` with `PreferLowerDiskUtil::True` +- **DataMovementReason**: `REBALANCE_OVERUTILIZED_TEAM` (disk) or `REBALANCE_READ_OVERUTIL_TEAM` (read) +- For read rebalancing, calls `rebalanceReadLoad()` which selects top-K shards by read density + +### ValleyFiller (Fill Underutilized Teams) + +- **Source selection**: `TeamSelect::ANY` -- picks a random loaded team +- **Dest selection**: `TeamSelect::WANT_TRUE_BEST` with `PreferLowerDiskUtil::True` -- picks + the least loaded team +- **DataMovementReason**: `REBALANCE_UNDERUTILIZED_TEAM` (disk) or `REBALANCE_READ_UNDERUTIL_TEAM` (read) + +### Rebalancing Controls + +- Polling interval: `BG_REBALANCE_POLLING_INTERVAL` (adaptive, backed off on repeated no-ops) +- Parallelism cap: `DD_REBALANCE_PARALLELISM` -- max concurrent rebalance moves per priority +- Can be disabled at runtime via special database key (checked via `getSkipRebalanceValue()`) +- Read rebalance CPU threshold: `READ_REBALANCE_CPU_THRESHOLD = 0.15` (source team worst CPU) +- Read rebalance shard selection: top-K by `READ_DENSITY = READ_BYTES_PER_KSEC / SHARD_BYTES` + with constraint `LOAD(shard) < (LOAD(src) - LOAD(dest)) * READ_REBALANCE_MAX_SHARD_FRAC` + +Source: [`BgDDLoadRebalance`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L2281), +[`isDataMovementForMountainChopper`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L58), +[`isDataMovementForValleyFiller`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDRelocationQueue.actor.cpp#L69). + + +## 9. Team Building + +The team builder ([`buildTeams`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDTeamCollection.actor.cpp#L706) in `DDTeamCollection`) is invoked when: +- A new server is added +- A server is removed +- Zero teams exist + +The algorithm: +1. **Build machine teams** using `addBestMachineTeams()`: + - Uses `ReplicationPolicy::selectReplicas()` to find machine combinations + - Target count: `DESIRED_TEAMS_PER_SERVER * serverCount / teamSize` +2. **Build server teams** using `addTeamsBestOf()`: + - For each machine team, pick one server from each machine + - Prefer the server with the fewest existing teams (`findOneLeastUsedServer`) + - Verify the team satisfies the replication policy + +### Team Health Monitoring + +**[`teamTracker`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DDTeamCollection.actor.cpp#L860)** (per team): When a team's health changes, it evaluates: +- If all servers healthy → no action +- If a server is undesired → `PRIORITY_TEAM_CONTAINS_UNDESIRED_SERVER (150)` +- If team is redundant → `PRIORITY_TEAM_REDUNDANT (200)` +- If team is unhealthy → `PRIORITY_TEAM_UNHEALTHY (700)` +- If only 2 replicas left → `PRIORITY_TEAM_2_LEFT (709)` +- If only 1 replica left → `PRIORITY_TEAM_1_LEFT (800)` +- If team failed → `PRIORITY_TEAM_FAILED (805)` +- If 0 replicas left → `PRIORITY_TEAM_0_LEFT (809)` + +The priority escalation drives DDQueue to process the most critical relocations first. + +**`storageServerTracker`** (per SS): Monitors each server's health, storage engine type, +locality validity. When a server fails, it triggers data moves off all teams containing +that server. + +### Team Pruning + +- `serverTeamRemover`: periodically removes the server team whose members are on the most + teams when total team count > desired. Threshold: `TR_REDUNDANT_TEAM_PERCENTAGE_THRESHOLD`. +- `machineTeamRemover`: removes machine teams with the most overlapping members when count + exceeds desired. + + +## 10. Perpetual Storage Wiggle + +The "wiggle" gradually cycles data off each storage server in turn, enabling rolling +maintenance and ensuring no server holds stale data indefinitely. + +### Components + +- **[`StorageWiggler`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L563)**: Maintains a min-heap priority queue + of servers ordered by `StorageMetadataType` (creation time). Servers with the oldest metadata + are wiggled first. +- **`perpetualStorageWiggleIterator`**: Selects the next server to wiggle and writes its ID to + `perpetualStorageWiggleIDPrefix` in the system keyspace. +- **`perpetualStorageWiggler`**: Watches the wiggle ID key, excludes the target server (causing + DD to move data off it), waits for completion, then includes it again and signals the iterator. +- **`clusterHealthCheckForPerpetualWiggle`**: Pauses wiggling if the cluster becomes unhealthy. + +### Configuration + +- Enable: `configure perpetual_storage_wiggle=1` +- Target specific locality: `configure perpetual_storage_wiggle_locality=:` +- Wiggle pause threshold: `DD_STORAGE_WIGGLE_PAUSE_THRESHOLD` +- Wiggle stuck threshold: `DD_STORAGE_WIGGLE_STUCK_THRESHOLD` +- Min bytes balance ratio: `PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO` +- Min SS age before wiggle: `DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC` + +### Wiggle States + +From `StorageWiggler::State`: +- `INVALID (0)`: Not initialized +- `RUN (1)`: Actively wiggling +- `PAUSE (2)`: Paused due to cluster health + +The wiggler state and timestamp are queryable via `GetStorageWigglerStateRequest` RPC. + + +## 11. Physical Shard Collection (Experimental) + +[`PhysicalShardCollection`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistribution.actor.h#L277) introduces a higher-level grouping +where multiple logical shards (key ranges) share the same server team. This reduces metadata +overhead and enables more efficient bulk moves. + +Key concepts: +- A **physical shard** contains one or more key ranges (logical shards) on the same team +- Two-step team selection: (1) select a physical shard for the primary team, (2) find the + remote team sharing that physical shard +- Physical shards track aggregate metrics and can be oversized, triggering + `PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD (960)` moves + +Enabled via: `SHARD_ENCODE_LOCATION_METADATA=true` + `ENABLE_DD_PHYSICAL_SHARD=true` + +When DD restarts, all key ranges start in the "anonymous" physical shard and are gradually +transitioned out as data moves occur. + + +## 12. Multi-Datacenter Architecture + +With `usableRegions > 1`, DD maintains two `DDTeamCollection` instances: + +1. **Primary Team Collection** -- manages teams in `primaryDcId` +2. **Remote Team Collection** -- manages teams in `remoteDcIds` + +The remote collection only starts after `remoteRecovered()` signals that the remote region +has completed recovery. + +Every shard requires a team in both DCs. The DDQueue coordinates +relocations across both DCs -- when selecting destination teams, it requests teams from +both collections and ensures consistency. + +`ShardsAffectedByTeamFailure` tracks teams with a `primary` flag to distinguish: +```cpp +struct Team { + std::vector servers; + bool primary; +}; +``` + + +## 13. Storage Server Recruitment + +`storageRecruiter` in `DDTeamCollection` handles recruiting new storage servers: +- Sends `RecruitStorageRequest` to the Cluster Controller +- Validates the candidate's locality against the replication policy +- Optionally pairs with a Testing Storage Server (TSS) via `TSSPairState` +- Calls `initializeStorage()` to register the new SS + +The recruiter is debounced via `restartRecruiting` to avoid thundering herd during +rapid server changes. + + +## 14. Shard Map Metadata Lifecycle: `keyServers` and `serverKeys` + +The shard-to-server mapping is the most critical metadata in the cluster. It is stored as +two complementary indexes in the system keyspace and flows through every major component. + +### 14.1 Schema + +**`keyServers`** (`\xff/keyServers/` prefix, defined in [`SystemData.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/include/fdbclient/SystemData.h)): + +``` +\xff/keyServers/[begin_key] → encoded(src_UIDs[], dest_UIDs[]) +``` + +Maps each shard's start key to its current source servers (who own the data) and destination +servers (who are receiving the data during a move). When no move is in progress, `dest` is +empty. The encoding supports both UID-based and Tag-based formats, plus optional shard IDs +when `SHARD_ENCODE_LOCATION_METADATA` is enabled. + +Key encoding/decoding functions in [`SystemData.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/SystemData.cpp): +- `keyServersKey(k)` -- constructs the key +- `keyServersValue(UIDtoTagMap, src, dest)` -- encodes src/dest with tags +- `decodeKeyServersValue(serverTagMap, value, src, dest)` -- decodes + +**`serverKeys`** (`\xff/serverKeys/` prefix, defined in [`SystemData.h`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/include/fdbclient/SystemData.h)): + +``` +\xff/serverKeys/[serverID]/[begin_key] → serverKeysTrue | serverKeysFalse +``` + +The inverse index: for each storage server, records which shard ranges it owns. +`serverKeysTrue` means the server owns that range; `serverKeysFalse` means it doesn't. +When `SHARD_ENCODE_LOCATION_METADATA` is enabled, the value also encodes the `DataMoveType`, +`DataMoveId`, and `DataMovementReason`. + +Key functions in [`SystemData.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbclient/SystemData.cpp): +- `serverKeysPrefixFor(serverID)` -- prefix for all shards on a server +- `serverKeysKey(serverID, key)` -- specific shard assignment key +- `decodeServerKeysValue(value, nowAssigned, emptyRange, type, id, reason)` -- decodes + +### 14.2 Bootstrap: `seedShardServers` + +At initial database creation, the master calls [`seedShardServers()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3202) +to write the very first shard map: + +```cpp +// 1. Set keyServers: all initial servers own the entire keyspace +auto ksValue = keyServersValue(serverTags, serverSrcUID); +tr.set(arena, keyServersKey(allKeys.begin), ksValue); // \xff/keyServers/\x00 +tr.set(arena, keyServersKey(allKeys.end), ...); // \xff/keyServers/\xff + +// 2. Set serverKeys: each server owns everything +for (auto& s : servers) { + tr.set(arena, serverKeysKey(s.id(), allKeys.begin), serverKeysTrue); +} +``` + +This is committed as the very first transaction (`read_snapshot = 0`), establishing the +invariant that every key in the database is assigned to a team. + +### 14.3 The moveKeys Protocol in Detail + +The data transfer protocol is implemented in [`MoveKeys.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp) and consists of three +coordinated phases. The [`moveKeys()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3180) function orchestrates them: + +```cpp +wait(rawStartMovement(occ, params, tssMapping)); // Phase 1: startMoveKeys +wait(rawCheckFetchingState(occ, params, ...)); // Phase 2: monitor data copy +wait(rawFinishMovement(occ, params, tssMapping)); // Phase 3: finishMoveKeys +``` + +#### Phase 1: `startMoveKeys` ([`MoveKeys.actor.cpp:924`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L924)) + +This phase sets up the destination servers and tells them to begin fetching data. + +1. **Acquire lock**: `checkMoveKeysLock()` verifies this DD still owns the lock +2. **Read existing shard map**: `krmGetRanges(tr, keyServersPrefix, keys)` fetches all + current shard assignments overlapping the move range +3. **Validate destinations**: confirms each destination server exists in `serverList`; + throws `move_to_removed_server` if any are gone +4. **For each overlapping sub-shard**: + - Decode existing `src` and `dest` from `keyServers` + - **Write new `keyServers`**: [`krmSetPreviouslyEmptyRange`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1044) sets `dest` to the new destination team while + preserving `src` + - Track old destinations that need cleanup +5. **Remove stale destination entries**: for old `dest` servers no longer needed, + `removeOldDestinations()` sets their `serverKeys` to `serverKeysFalse` +6. **Write new `serverKeys` for destinations**: [`krmSetRangeCoalescing`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1082) for each destination server +7. **Commit transaction** + +This transaction may be split into multiple batches if there are many overlapping sub-shards +(`MOVE_KEYS_KRM_LIMIT` controls batch size). Each batch advances the `begin` cursor. + +**Important**: at this point, `keyServers[range]` has both `src` (old team) and `dest` +(new team). Both teams serve reads for this shard. The commit of this transaction triggers +the private mutation propagation described in section 14.4. + +#### Phase 2: Data Copy (Storage Server Fetch) + +After the `startMoveKeys` transaction commits, it propagates as a private mutation through +the transaction system (see 14.4). The destination storage servers receive the mutation and +begin `fetchKeys` -- reading the shard data from source servers via standard FDB read +transactions. DD monitors this via `rawCheckFetchingState()` which polls the destination +servers until they report the shard is ready. + +#### Phase 3: `finishMoveKeys` ([`MoveKeys.actor.cpp:1229`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1229)) + +Once all destinations have the data, DD atomically swaps ownership: + +1. **Read and validate**: reads `keyServers` for the range, decodes `src` and `dest`, + verifies `dest` matches expected servers +2. **Check readiness**: for each destination server, checks if the shard is fetched and ready +3. **Atomic swap** ([line 1519-1524](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1519)): + ```cpp + // Update keyServers: src = old_dest, dest = [] (move complete) + krmSetRangeCoalescing(&tr, keyServersPrefix, currentKeys, keys, + keyServersValue(UIDtoTagMap, dest)); // dest becomes new src + + // Update serverKeys for ALL involved servers + for (server in allServers) { + bool destHasServer = (server in dest); + krmSetRangeCoalescing(&tr, serverKeysPrefixFor(server), currentKeys, + allKeys, destHasServer ? serverKeysTrue : serverKeysFalse); + } + ``` +4. **Commit** + +After this commit, the old source servers are no longer owners. The private mutation +propagation tells them to drop the shard data. + +**Retry handling**: on transaction conflict, `finishMoveKeys` retries with +exponential backoff. After every 10 retries ([line 1551](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1551)) it emits a `RelocateShard_FinishMoveKeysRetrying` +warning; at 20 retries it escalates to `SevWarnAlways`. + +### 14.4 Private Mutation Propagation + +When `keyServers` or `serverKeys` mutations commit, they are classified as **private +mutations** and receive special treatment at every layer of the transaction system. + +#### At the Commit Proxy ([`ApplyMetadataMutation.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp)) + +**[`checkSetKeyServersPrefix`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L207)** (line 207): When the proxy sees a mutation to +`\xff/keyServers/`: +1. Stores the mutation in the local `txnStateStore` -- this is the proxy's + in-memory replica of the shard map +2. Decodes `src` and `dest` UIDs +3. Looks up each UID's `Tag` from the `storageCache` +4. Updates `keyInfo` (the `ServerCacheInfo` map) with the combined tag set +5. This updated `keyInfo` is used immediately for **routing subsequent mutations** -- any + user transaction touching this key range will now be tagged for both `src` and `dest` + servers + +**[`checkSetServerKeysPrefix`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L258)** (line 258): When the proxy sees a mutation to +`\xff/serverKeys/`: +1. Looks up the target server's `Tag` (line 263: [`if (toCommit)`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L263) guard) +2. **Privatizes** the mutation by prepending `systemKeys.begin` to `param1` +3. Adds the server's tag to the commit batch: `toCommit->addTag(tag)` ([line 277](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/ApplyMetadataMutation.cpp#L277)) +4. Writes the privatized mutation + +This ensures the `serverKeys` change is streamed only to the specific storage server it +affects (via the tag-based routing through tLogs). + +#### At the tLog + +The tLog receives the tagged mutations and stores them in the appropriate tag-specific +queue. Storage servers pull mutations by their tag, so each SS only sees `serverKeys` +mutations addressed to it. + +#### At the Storage Server ([`storageserver.actor.cpp`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp)) + +[`applyPrivateData()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp#L10435) processes incoming `serverKeys` mutations in pairs (begin, end): + +**First mutation**: Decodes the start key and metadata: +```cpp +decodeServerKeysValue(m.param2, nowAssigned, emptyRange, dataMoveType, dataMoveId, dataMoveReason); +``` +This extracts: whether the server is now assigned this range, whether it's an empty-range +optimization, the `DataMoveType` (LOGICAL, PHYSICAL, PHYSICAL_BULKLOAD), and the move reason. + +**Second mutation**: Forms the key range and applies the change: +- `setAssignedStatus(data, keys, nowAssigned)` -- updates the SS's in-memory shard map +- For shard-aware SS: [`changeServerKeysWithPhysicalShards()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp#L8852) -- handles physical shard moves +- For non-shard-aware SS: [`changeServerKeys()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/storageserver.actor.cpp#L9803) -- triggers `fetchKeys` actor if newly assigned, + or drops data if unassigned + +If the SS is **newly assigned** this range (`nowAssigned = true`), `changeServerKeys` spawns +a `fetchKeys` actor that reads all data from the source servers and writes it into local +storage. Once complete, the shard transitions from FETCHING to READABLE. + +If the SS is **unassigned** from this range (`nowAssigned = false`), the data is removed +from the SS's versioned data. + +### 14.5 Recovery + +During cluster recovery, the `txnStateStore` is reconstructed from tLog data before any +new transactions can proceed. This means `keyServers` and `serverKeys` are recovered to +their last committed state. The new commit proxies replay all metadata mutations, rebuilding +their `keyInfo` routing tables. When DD starts on the new cluster controller, `init()` reads +the recovered `keyServers` and `serverKeys` to reconstruct its in-memory shard map, teams, +and any in-flight data moves. + +### 14.6 Consistency Model + +The use of FDB's own transaction system for shard map mutations provides strong guarantees: + +1. **Serializability**: all shard map changes are serialized through the resolver, so + concurrent moves to overlapping ranges are detected and one is retried +2. **Atomicity**: `finishMoveKeys` atomically updates both `keyServers` and all affected + `serverKeys` entries in a single transaction +3. **Consistency across proxies**: because metadata mutations update `txnStateStore` on + every commit proxy in the same total order, all proxies have an identical view of which + servers own which shards +4. **Durability**: mutations are persisted in tLogs before acknowledgment + +The cost of this elegance is coupling: shard map mutations compete with user transactions +for commit proxy bandwidth and resolver cycles. During heavy data movement, this can +measurably impact user commit latencies. + +### 14.7 Commented-Out Trace Events in moveKeys + +Notably, `startMoveKeys` contains several **commented-out** trace events ([lines 1009, 1031, 1054, 1060](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L1009)) that would log per-sub-shard details during the move: + +```cpp +// TraceEvent("StartMoveKeysOldRange", relocationIntervalId) +// .detail("KeyBegin", rangeIntersectKeys.begin.toString()) +// .detail("KeyEnd", rangeIntersectKeys.end.toString()) +// .detail("OldSrc", describe(src)) +// .detail("OldDest", describe(dest)) +// .detail("ReadVersion", tr->getReadVersion().get()); +``` + +These were likely disabled due to volume concerns, but their absence creates an observability +gap that we address in the recommendations document. + + +## 15. The High-Level moveKeys Orchestration + +Wrapping the protocol above, [`moveKeys()`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/MoveKeys.actor.cpp#L3180) orchestrates +the three phases: + +```cpp +ACTOR Future moveKeys(Database occ, MoveKeysParams params) { + wait(rawStartMovement(occ, params, tssMapping)); // startMoveKeys + state Future completionSignaller = + rawCheckFetchingState(occ, params, tssMapping); // monitor fetch progress + wait(rawFinishMovement(occ, params, tssMapping)); // finishMoveKeys + completionSignaller.cancel(); + if (!params.dataMovementComplete.isSet()) + params.dataMovementComplete.send(Void()); + return Void(); +} +``` + +The `dataMovementComplete` promise signals the relocation queue that the move has finished, +allowing it to update `ShardsAffectedByTeamFailure` and complete the relocation lifecycle. + + +## 16. DD RPC Interface + +[`DataDistributorInterface`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/include/fdbserver/DataDistributorInterface.h#L30) exposes these RPCs: + +| RPC | Purpose | +|-----|---------| +| `waitFailure` | Cluster Controller liveness check | +| `haltDataDistributor` | Graceful shutdown | +| `distributorSnapReq` | Consistent snapshot coordination | +| `distributorExclCheckReq` | Safety check before server exclusion | +| `dataDistributorMetrics` | Query shard metrics | +| `distributorSplitRange` | External split point insertion | +| `storageWigglerState` | Query wiggle status | +| `triggerAudit` | Trigger/cancel storage audit | + + +## 17. Error Handling & Recovery + +Normal DD errors that trigger re-recruitment (not cluster-wide failure): +```cpp +{error_code_movekeys_conflict, error_code_broken_promise, + error_code_data_move_cancelled, error_code_data_move_dest_team_not_found} +``` + +Source: [`DataDistribution.actor.cpp:305`](https://github.com/apple/foundationdb/blob/release-7.3/fdbserver/DataDistribution.actor.cpp#L305). + +When DD encounters a `movekeys_conflict` (another DD took the lock), it terminates and the +Cluster Controller recruits a fresh instance. The new DD re-reads all state from the system +keyspace, so no data is lost -- only in-flight moves may need to be re-evaluated. + +`DDEnabledState` tracks the overall mode: +- `ENABLED` -- normal operation +- `SNAPSHOT` -- DD paused for consistent snapshot +- `BLOB_RESTORE_PREPARING` -- DD paused for blob restore preparation diff --git a/design/fdb-coroutines-internals.md b/design/fdb-coroutines-internals.md new file mode 100644 index 00000000000..83f1da78287 --- /dev/null +++ b/design/fdb-coroutines-internals.md @@ -0,0 +1,727 @@ +# How C++ Coroutines Work in FoundationDB + +*Caveat: work in progress. This mentions code not yet submitted. Accuracy is believed to be good, +but ongoing work to increase clarity is needed.* + +This document explains the coroutine infrastructure in FoundationDB, from the +C++20 standard machinery down to FDB-specific implementation details. + +## C++20 Coroutine Fundamentals + +Any function whose body contains `co_await`, `co_return`, or `co_yield` is a +coroutine. The compiler rewrites it into a state machine. The programmer does +**not** call the coroutine machinery directly — the compiler generates all the +calls. The programmer's job is to implement the **policy** types that the +compiler calls into. + +### What the compiler generates + +Given: + +```cpp +Future foo(int x) { + int y = co_await bar(); + co_return y + 1; +} +``` + +The compiler emits roughly: + +```cpp +Future foo(int x) { + // 1. Look up the promise type via coroutine_traits, int> + using promise_type = std::coroutine_traits, int>::promise_type; + + // 2. Allocate the coroutine frame on the heap + // (calls promise_type::operator new if defined) + // The frame holds: the promise object, all local variables, + // bookkeeping for each suspend point, and the function arguments. + + // 3. Construct the promise + promise_type promise; + + // 4. Get the return object — this is what the caller receives. + // Called BEFORE the body executes. + Future retval = promise.get_return_object(); + + // 5. co_await promise.initial_suspend() + // FDB returns suspend_never → body starts immediately. + + try { + // === user's body === + + // co_await bar() expands to: + auto&& awaitable = promise.await_transform(bar()); + if (!awaitable.await_ready()) { + awaitable.await_suspend(handle_to_this_coroutine); + // <-- coroutine suspends here, control returns to caller --> + // <-- later: something calls handle.resume() --> + } + int y = awaitable.await_resume(); + + // co_return y + 1 expands to: + promise.return_value(y + 1); + goto final_suspend; + + } catch (...) { + promise.unhandled_exception(); + } + +final_suspend: + // 6. co_await promise.final_suspend() + // The compiler generates the same await_ready / await_suspend / + // await_resume sequence here. + + return retval; // Already returned to caller at step 4/5 +} +``` + +Key insight: `retval` is returned to the caller when the coroutine first +suspends (or completes without suspending). The caller never sees the +`co_return` value directly — it goes through `promise.return_value()` into +whatever result cell the promise manages. + +### The coroutine frame + +The compiler heap-allocates a "coroutine frame" containing: + +- The promise object +- Copies of all function parameters +- All local variables (live across any suspend point) +- Internal state for tracking which suspend point the coroutine is at +- Awaiter objects for each `co_await` (since they must survive suspension) + +FDB overrides `operator new`/`operator delete` on promise types to use its +`allocateFast`/`freeFast` allocator. + +A typical Clang-generated frame layout: + +``` ++0 void (*__resume)(void*) // resume function pointer ++8 void (*__destroy)(void*) // destroy function pointer ++16 promise_type promise // in FDB, contains the CoroActor / SAV ++?? uint32_t __suspend_index // which suspend point we're at ++?? bool __initial_await_resume_called ++?? // copies of function arguments ++?? // variables live across suspend points ++?? // one per co_await, alive during suspension ++?? +``` + +The layout is compiler-generated and not standardized. Everything that is live +across a suspend point must be stored in the frame. Typical FDB coroutines +range from ~200 bytes to over 1KB depending on the number of locals and +`co_await` points. + +**Parameter storage caveat:** The frame stores copies of all function +parameters. For `const&` parameters, the frame stores the *reference*, not a +copy of the referent. If the caller passed a temporary, it will be destroyed +before the coroutine resumes — leaving a dangling reference in the frame. +Always copy `const&` parameters to a local before the first `co_await`. + +The compiler splits the original function into three pieces: + +1. **Ramp function** — the entry point the caller invokes. Allocates the frame, + constructs the promise, calls `get_return_object()` and + `initial_suspend()`, starts the body, and returns `retval` to the caller + when the coroutine first suspends. +2. **Resume function** — called via `handle.resume()`. Jumps to the current + suspend point (via a switch on the saved index) and continues execution. +3. **Destroy function** — called via `handle.destroy()`. Jumps to the current + suspend point and runs destructors for in-scope locals, then deallocates. + +### `coroutine_handle<>` + +A single pointer to the coroutine frame — **8 bytes**. The resume and destroy +function pointers are stored in the frame itself (see layout above), not in +the handle. Calling `handle.resume()` dereferences the frame pointer to find +the resume function pointer, then makes an indirect call. + +`coroutine_handle

` (typed) and `coroutine_handle` (erased) are the +same size. The typed version adds a `promise()` accessor that returns a +reference to the promise at a compiler-known offset in the frame. They are +freely interconvertible. + +The compiler passes the handle to `await_suspend`. Key operations: + +- `handle.resume()` — resume the coroutine at its current suspend point +- `handle.destroy()` — destroy the coroutine frame (runs destructors for + in-scope locals, deallocates) +- `handle.done()` — true if the coroutine is at its final suspend point + +### `await_suspend` return types + +| Return type | Meaning | +|---|---| +| `void` | Always suspend | +| `bool` | `true` → suspend; `false` → don't suspend (resume immediately) | +| `coroutine_handle` | Symmetric transfer: suspend this coroutine, resume that one | + +### `final_suspend` rules + +- The expression `co_await promise.final_suspend()` **must not throw** + (compiler-enforced). Mark `final_suspend()` and the awaiter's methods + `noexcept`. +- If `await_suspend` returns `void` or `true`: the coroutine is suspended at + the final suspend point. The frame stays alive until someone calls + `handle.destroy()`. Calling `handle.resume()` on a coroutine suspended at + its final suspend point is **undefined behavior**. +- If `await_suspend` returns `false`: the coroutine was never suspended. The + runtime auto-destroys the frame after `await_resume()` completes. This is + well-defined. + +### `promise_type` required members + +| Method | Required? | Purpose | +|---|---|---| +| `get_return_object()` | Yes | Create the value returned to the caller | +| `initial_suspend()` | Yes | Controls whether body starts immediately | +| `final_suspend()` | Yes, `noexcept` | Controls frame lifetime after body completes | +| `unhandled_exception()` | Yes | Called if an exception escapes the body | +| `return_value(T)` or `return_void()` | Exactly one | Stores the co_return value | +| `await_transform(U)` | Optional | Intercepts every `co_await` expression | +| `yield_value(T)` | Optional | Enables `co_yield` | +| `operator new` / `operator delete` | Optional | Custom frame allocation | + +--- + +## FDB's Type Hierarchy + +### `Callback` — intrusive linked list node + +``` +flow/include/flow/flow.h:468 +``` + +A doubly-linked list node with virtual `fire(T const&)`, `fire(T&&)`, +`error(Error)`, and `unwait()`. When a `SAV` has no value yet, it serves as +the sentinel of a circular list of waiting callbacks. + +When a callback is `remove()`d and it was the last one, `unwait()` is called on +the sentinel (the SAV), which decrements the future refcount. + +### `SAV` — Single Assignment Variable + +``` +flow/include/flow/flow.h:732 +``` + +The core result cell. Inherits privately from `Callback` (to serve as the +list sentinel) and `FastAllocated>`. + +**Two independent refcounts:** + +- `promises` — one per `Promise` pointing at it, plus one for an active + actor (the producer side) +- `futures` — one per `Future` pointing at it, plus one if any callbacks + are registered + +**State machine** via `error_state.code()`: + +| Code | Constant | Meaning | +|---|---|---| +| -3 | `UNSET_ERROR_CODE` | Not yet assigned. `canBeSet()` is true. | +| -2 | `NEVER_ERROR_CODE` | Will never complete (set to `Never`). | +| -1 | `SET_ERROR_CODE` | Successfully set with a value. | +| > 0 | (error code) | Set with an error. | + +**Lifecycle rules:** + +- `delPromiseRef()`: when the last promise ref is released (promises is 1) and + the SAV is still unset, sends `broken_promise()` to any waiting callbacks. + Then sets promises to 0. If futures is also 0, calls `destroy()`. +- `delFutureRef()`: if futures drops to 0 and promises > 0, calls `cancel()` + (this is how dropping the last Future cancels an actor). If promises is also + 0, calls `destroy()`. +- `destroy()` and `cancel()` are virtual — subclasses override them. + +**Callback management:** `addCallbackAndDelFutureRef(cb)` transfers a future +reference into the callback list. If this is the first callback, the future +count stays the same (the "+1 for having callbacks" replaces the future ref). +If callbacks already exist, the future count is decremented. + +### `Actor` — actor base class + +``` +flow/include/flow/flow.h:1500 +``` + +Inherits from `SAV`. Adds one field: + +```cpp +int8_t actor_wait_state; +``` + +| Value | Constant | Meaning | +|---|---|---| +| > 0 | `ACTOR_WAIT_STATE_WAITING` (1) | Coroutine is suspended at a `co_await`; a callback is registered on some Future's SAV that will call `handle.resume()` when fired | +| 0 | `ACTOR_WAIT_STATE_NOT_WAITING` | Coroutine is running (between `co_await` points) or has completed | +| -1 | `ACTOR_WAIT_STATE_CANCELLED` | Cancellation requested | +| -2 | `ACTOR_WAIT_STATE_CANCELLED_DURING_READY_CHECK` | Cancellation detected during `await_ready()` before the awaiter registered a callback | + +In actor-compiler-generated code, values >1 identify *which* callback group +fired (the generated state machine uses the value to jump to the right +continuation). For C++ coroutines this distinction is unnecessary — the +`coroutine_handle` already encodes the suspend point — so the value is +always 1. + +Constructed with `SAV(1, 1)`: starts with futures=1 and promises=1. The +futures=1 is consumed by the `Future` returned from `get_return_object()`. +The promises=1 represents the actor itself as the producer. + +`Actor` is a separate specialization with no SAV base — for +fire-and-forget actors. + +### `Promise` and `Future` + +``` +flow/include/flow/flow.h:1091, 982 +``` + +`Promise` wraps a `SAV*`. Constructor creates `SAV(0, 1)` — zero +futures, one promise. `getFuture()` calls `addFutureRef()` then returns +`Future(sav)`. + +`Future` wraps a `SAV*` — **8 bytes**. Copy constructor calls +`addFutureRef()`. Destructor calls `delFutureRef()`. The explicit constructor +`Future(SAV* sav)` does **not** increment the refcount — it assumes +the count is already correct (used by `get_return_object()`). + +--- + +## FDB's Coroutine Promise: `CoroPromise` + +``` +flow/include/flow/CoroutinesImpl.h +``` + +### `coroutine_traits` specialization + +``` +flow/include/flow/Coroutines.h +``` + +```cpp +template +struct coroutine_traits, Args...> { + using promise_type = coro::CoroPromise, + coro::hasExplicitVoid, + coro::hasNoThrowOnCancel>; +}; +``` + +The `Args...` includes all function parameter types. Marker types like +`Uncancellable`, `NoThrowOnCancel`, and `ExplicitVoid` are detected via +template metaprogramming on the parameter list. They are always defaulted +parameters (`NoThrowOnCancel = {}`) so callers don't pass them. + +### `CoroActor` + +``` +flow/include/flow/CoroutinesImpl.h:124 +``` + +Inherits from `Actor`. Embedded **inside** the coroutine frame (the promise +holds it as a member). This means the frame allocation also allocates the +SAV — a single allocation for both. + +Adds a `coroutine_handle<>` and overrides: + +- `cancel()`: sets `actor_wait_state = CANCELLED`, then resumes the coroutine + if it was waiting. The resumed coroutine sees the cancelled state in + `resumeImpl()` and throws `actor_cancelled()`, which unwinds through the + coroutine body (running destructors and catch blocks), reaches + `unhandled_exception()`, and the error is stored on the SAV. +- `destroy()`: calls `handle.destroy()` to free the coroutine frame (which + also frees the embedded CoroActor, since it's part of the frame). + +### `get_return_object()` + +```cpp +ReturnFutureType get_return_object() noexcept { + coroActor.handle = handle(); + return ReturnFutureType(&coroActor); +} +``` + +`ReturnFutureType` is `Future` — an 8-byte wrapper around a +`SAV*` pointer. This constructs a `Future` pointing at the actor's SAV. +The SAV was initialized with `futures=1` by the `Actor` constructor, and the +explicit `Future(SAV*)` constructor does not increment the refcount — so the +returned Future consumes that initial reference. + +The compiler calls `get_return_object()` before the body executes. The caller +receives this Future immediately (when the coroutine first suspends or +completes without suspending). + +### `initial_suspend()` + +Returns `suspend_never` — the coroutine body starts executing immediately when +called. The `Future` has already been returned to the caller via +`get_return_object()`. + +### `return_value()` / `return_void()` + +Implemented via the `CoroReturn` CRTP mixin. Calls +`actor()->set(value)` which placement-news the value into the SAV's storage +and sets `error_state = SET_ERROR_CODE`. + +### `final_suspend()` + +Returns a custom `FinalAwaitable`: + +- `await_ready()` returns `false` — always enter `await_suspend` +- `await_suspend()` calls `finishSendAndDelPromiseRef()` (or the error + variant), which fires all waiting callbacks and decrements the promise count. + Returns `void` — the coroutine is left suspended. The frame stays alive + until the SAV's `destroy()` virtual is called (when both refcounts hit 0), + which calls `handle.destroy()`. + +### `unhandled_exception()` + +Catches the exception, extracts the `Error`, and stores it on the SAV via +`setError()`. If the exception isn't an `Error`, stores `unknown_error()`. + +### `await_transform()` + +Intercepts every `co_await` expression and wraps it in an FDB-specific awaiter +type: + +| Awaited type | Awaiter created | +|---|---| +| `Future` | `AwaitableFuture<..., false>` | +| `FutureStream` | `AwaitableFuture<..., true>` | +| `AsyncResult` | `AwaitableAsyncResult` | +| `ThreadFutureStream` | `ThreadAwaitableFutureStream` | +| `coro::FutureIgnore` | `AwaitableFutureIgnore` | +| `coro::FutureErrorOr` | `AwaitableFutureErrorOr` | + +--- + +## Awaiters: How `co_await` Suspends and Resumes + +All three awaiter methods — `await_ready()`, `await_suspend()`, and +`await_resume()` — are called by the **compiler**, never by FDB code directly. +The programmer implements them on the awaiter types; the compiler generates +calls to them in the correct order at each `co_await` expression. + +### `AwaitableFuture` + +The most common awaiter. Used for `co_await someFuture` and +`co_await someStream`. + +Inherits from `Callback` (or `SingleCallback` for streams), which means +the awaiter itself is the callback node that gets inserted into the Future's +callback list. + +**`await_ready()`**: Returns `true` if the Future is already resolved (skip +suspension). Also checks if the actor has been cancelled — if so, sets +`CANCELLED_DURING_READY_CHECK` and returns `true` to proceed to +`await_resume()` where the cancellation is handled. + +**`await_suspend(handle)`**: +1. Stores the coroutine handle via `pt->setHandle(h)` +2. Sets `actor_wait_state = ACTOR_WAIT_STATE_WAITING` +3. Registers this awaiter as a callback on the Future via + `addCallbackAndClear(this)` — this transfers the Future's reference to + the callback list + +**`fire(value)` / `error(err)`** (inherited from Callback): Called when the +awaited Future is resolved — someone called `promise.send(value)` or +`promise.sendError(err)`, and `SAV::send()` walked its callback list calling +`fire()` on each node. This happens synchronously on whatever call stack +resolved the Future (FDB is cooperative single-threaded — there is no +scheduler). For streams, `fire()` first stores the value in a local +`AwaitableFutureStore` (since the stream may yield further values). The +awaiter's `fire()` then calls `pt->resume()` → `handle.resume()`, which +resumes the coroutine at its suspend point. From the coroutine's perspective, +control returns from the `co_await` expression. + +**`resumeImpl()`**: Called at the start of `await_resume()`. Checks +`actor_wait_state`: +- If cancelled: detaches from the callback list and throws + `actor_cancelled()` +- If still waiting (future wasn't ready yet when we entered await_resume via + the non-ready-check path): detaches from callback list, resets wait state +- Returns whether the future was already ready (affects how the value is + retrieved for streams) + +**`await_resume()`**: Implemented via the `AwaitableResume` CRTP mixin. +Calls `resumeImpl()`, then returns the value from the resolved Future. +Throws if the Future completed with an error. + +### The full `co_await` sequence, step by step + +```cpp +Future example() { + Promise p; + Future f = p.getFuture(); + + // ... elsewhere, someone will call p.send(42) + + int val = co_await f; + // ^^^^^^^^^ + // 1. Compiler calls promise.await_transform(f) + // → creates AwaitableFuture<..., int, false>{f, &promise} + // + // 2. Compiler calls awaiter.await_ready() + // → f is not resolved yet → returns false + // + // 3. Compiler calls awaiter.await_suspend(handle) + // → stores handle in actor + // → sets actor_wait_state = WAITING + // → registers awaiter as callback on f's SAV + // → coroutine suspends, control returns to caller + // + // ... time passes, someone calls p.send(42) ... + // (The coroutine is suspended. Execution has returned to whoever + // called example(). That caller — or another coroutine that runs + // later on the same thread — eventually calls p.send(42). + // The send fires callbacks synchronously on that call stack.) + // + // 4. SAV::send() fires all callbacks → awaiter.fire(42) + // → calls promise.resume() → handle.resume() + // → coroutine resumes + // + // 5. Compiler calls awaiter.await_resume() + // → resumeImpl() checks wait state, detaches callback + // → returns f.get() → 42 + // + // 6. val = 42 + + co_return val; +} +``` + +--- + +## Cancellation + +### How cancellation is triggered + +Cancellation flows through the SAV refcount system: + +1. The caller drops the last `Future` reference to an actor +2. `Future::~Future()` calls `sav->delFutureRef()` +3. `delFutureRef()` sees futures hit 0 and promises > 0 → calls `cancel()` +4. `cancel()` is virtual, dispatched to the actor subclass + +Or explicitly: the caller calls `f.cancel()` which calls `sav->cancel()`. + +### Normal cancellation (CoroActor) + +```cpp +void cancel() override { + auto prev_wait_state = actor_wait_state; + actor_wait_state = ACTOR_WAIT_STATE_CANCELLED; + if (actorWaitStateIsWaiting(prev_wait_state)) { + handle.resume(); // resume the coroutine + } +} +``` + +If the coroutine is suspended at a `co_await`, it is resumed. The awaiter's +`resumeImpl()` sees `CANCELLED` and throws `actor_cancelled()`. The exception +unwinds through the coroutine body — running destructors, entering catch +blocks — until it reaches `unhandled_exception()`, which stores the error on +the SAV. + +If the coroutine is not waiting (between co_await points, which shouldn't +happen in cooperative scheduling), the flag is just set. The next +`await_ready()` will detect it. + +### `Uncancellable` coroutines + +`CoroActor` — the `cancel()` body is compiled away +(`if constexpr (IsCancellable)` is false). The coroutine runs to completion +regardless of whether futures are dropped. + +### `NoThrowOnCancel` coroutines + +Uses `NoThrowOnCancelCoroActor` instead of `CoroActor`. Key differences: + +1. **The actor is heap-allocated separately** from the coroutine frame + (via `new ActorType()` in the promise). This is necessary because cancel + destroys the frame while the SAV must survive for Future observers. + +2. **`cancel()` destroys the frame directly** instead of resuming: + +```cpp +void cancel() override { + if (!canBeSet()) return; + + if (cancelHandler) { + // Detach the awaiter's callback before destroying the frame + cancelHandler->cancelWait(); + } + + destroyFrame(); // handle.destroy() — RAII destructors run + + if (futures) { + sendErrorAndDelPromiseRef(actor_cancelled()); + } else { + promises = 0; + delete this; + } +} +``` + +3. **`final_suspend` returns `false`** from `await_suspend`. After completing + the SAV and clearing the handle, the frame auto-destructs (the coroutine was + never suspended at the final suspend point). This is well-defined per the + C++ standard: returning `false` means the coroutine resumes, control flows + off the end, and the frame is destroyed. + +4. **`AwaitCancelHandler`**: A virtual interface that awaiters implement. Each + awaiter registers itself via `pt->setCancelHandler(this)` in + `await_suspend` and clears itself in `resumeImpl`. When `cancel()` fires, + it calls `cancelHandler->cancelWait()` to detach the awaiter's callback + from whatever Future it's waiting on, *before* destroying the frame. Without + this, the destroyed awaiter would be a dangling node in the Future's + callback list. + +**Behavioral difference from normal cancellation:** + +- RAII destructors run (frame destruction invokes ~locals) +- `catch` blocks inside the coroutine are **not** entered for cancellation +- The coroutine never "sees" the cancellation — it simply ceases to exist + +--- + +## `AsyncResult` + +``` +flow/include/flow/Coroutines.h, CoroutinesImpl.h +``` + +An alternative to `Future` that transfers ownership of the result value +through `co_await`. Unlike `Future` where `co_await` returns `T const&`, +`co_await AsyncResult` returns `T` by value (moved out). This avoids copies +for expensive payloads. + +`AsyncResult` is backed by `AsyncResultState` (separate from SAV) with +its own reference counting. The coroutine promise is `AsyncResultPromise`, +which manages `producerHandle` and `producerWaitState` directly rather than +inheriting from Actor. + +Cancellation for AsyncResult coroutines goes through +`AsyncResultState::cancelProducer()`, which mirrors the Actor cancel logic: +either resumes the producer to throw `actor_cancelled()` or (with +`noThrowOnCancel`) destroys the producer frame directly. + +--- + +## `choose` / `when` for Coroutines + +``` +flow/include/flow/CoroUtils.h +``` + +The coroutine equivalent of the actor compiler's `choose { when(...) { ... } }` +pattern. Two forms: + +### `Choose().When(future, handler).When(...).run()` + +Callback-based. Creates a `ChooseImplActor` that registers callbacks on all +futures simultaneously. When the first one fires, it removes all other +callbacks and runs the handler. Returns `Future`. + +**Limitations:** handlers are synchronous functions — you cannot `co_await` +inside a `When` handler. + +### `co_await race(future1, future2, ...)` + +Template-based alternative that supports heterogeneous future types. Returns +`Future>` where each `Ti` is the value type of the +corresponding future. When the first future completes, `race` cancels the +others and constructs the variant with `std::in_place_index` for the +winning future's position. + +Use `result.index()` to determine which future won, and `std::get(result)` +or `std::visit(...)` to extract the value: + +```cpp +auto result = co_await race(futureInt, futureString); +if (result.index() == 0) { + int val = std::get<0>(result); +} else { + std::string val = std::get<1>(result); +} +``` + +--- + +## Marker Parameters + +FDB uses defaulted function parameters as compile-time markers: + +```cpp +Future foo(int x, NoThrowOnCancel = {}) { + co_await bar(); + co_return; +} +``` + +The compiler sees the parameter types `(int, NoThrowOnCancel)` and passes them +through `coroutine_traits, int, NoThrowOnCancel>`, which selects +the appropriate promise type. + +| Marker | Effect | +|---|---| +| (none) | Normal cancellable coroutine | +| `Uncancellable` | `cancel()` is a no-op; coroutine runs to completion | +| `NoThrowOnCancel` | `cancel()` destroys the frame directly; no `actor_cancelled` thrown | +| `ExplicitVoid` | `co_await Future` produces `Void` instead of `void` | + +`Uncancellable` and `NoThrowOnCancel` are mutually exclusive (enforced by +`static_assert`). Neither is compatible with `AsyncGenerator`. + +--- + +## Lifecycle Summary + +### Normal completion + +1. Coroutine is called → frame allocated, `get_return_object()` returns + `Future` to caller +2. `initial_suspend()` returns `suspend_never` → body runs immediately +3. Body executes, suspending/resuming at `co_await` points +4. `co_return value` → `promise.return_value(value)` stores result in SAV +5. `final_suspend()` → `await_suspend` fires callbacks via + `finishSendAndDelPromiseRef()`, decrements promise count +6. Frame is either left suspended (normal) or auto-destroyed + (NoThrowOnCancel) +7. When both refcounts hit 0, `destroy()` frees the frame (if still alive) + and the SAV + +### Cancellation (normal) + +1. Last `Future` dropped → `delFutureRef()` → `cancel()` on the actor +2. `cancel()` sets `actor_wait_state = CANCELLED`, resumes the coroutine +3. Awaiter's `resumeImpl()` throws `actor_cancelled()` +4. Exception unwinds through the body (running destructors and catch blocks) +5. `unhandled_exception()` stores the error on the SAV (sets `error_state`) +6. `final_suspend()` → `await_suspend` fires waiting callbacks via + `finishSendErrorAndDelPromiseRef()` and decrements the promise count +7. Frame cleanup proceeds as in normal completion + +### Cancellation (NoThrowOnCancel) + +1. Last `Future` dropped → `delFutureRef()` → `cancel()` on the actor +2. `cancel()` calls `cancelHandler->cancelWait()` to detach the awaiter +3. `cancel()` calls `handle.destroy()` — frame is destroyed, RAII runs +4. `cancel()` sends `actor_cancelled()` to any remaining Futures via SAV +5. Actor (heap-allocated) survives until all Future references are dropped + +### Refcount flow + +``` +Actor constructor: promises=1 futures=1 +get_return_object(): Future takes the futures=1 ref (no increment) +Caller copies Future: futures=2 +co_await registers cb: futures stays same (cb replaces future ref) +Future fires cb: coroutine resumes +final_suspend: promises-- (via finishSendAndDelPromiseRef) +Caller drops Future: futures-- → if both 0, destroy() +``` diff --git a/design/file-level-encryption-backup.md b/design/file-level-encryption-backup.md new file mode 100644 index 00000000000..265af6c00ea --- /dev/null +++ b/design/file-level-encryption-backup.md @@ -0,0 +1,370 @@ +# File-Level Encryption in FoundationDB Backups + +This document describes the design and usage of file-level encryption for +FoundationDB backups and restores. + +--- + +## 1. Overview + +File-level encryption protects backup data files written to local +directories or object stores (e.g., S3) by encrypting each file with a +user-supplied key before it is uploaded. Restore requires the same key to +be available to all participating processes. + +**What is encrypted:** + +- All **snapshot**, **range**, and **log** files. + +**What is *not* encrypted:** + +- The metadata files in the `properties/` folder of the backup container, + so tools like `fdbbackup describe` and `fdbbackup expire` can work + without holding the key. +- Anything inside the live cluster — mutations, keys, and values are + unchanged on the wire and in storage. + +**Cipher:** AES-256-GCM. The same key must be supplied to backup, restore, +modify (when re-encrypting), and decode operations. + +--- + +## 2. Encryption Key + +The encryption key is a **32-byte (256-bit)** value in raw binary format — +the exact key length AES-256 requires. + +The key value itself is **never** passed on the command line. The user +provides the **path** to a file holding the 32 raw bytes via +`--encryption-key-file `, and the CLI reads the file to load the key. +The file must contain exactly 32 bytes of raw binary data — no headers, no +encoding, no trailing newline. + +For example, a key file can be generated using OpenSSL: + +``` +openssl rand 32 > key_file +``` + +--- + +## 3. CLI Arguments + +Two command-line arguments drive the encrypted-backup workflow: + +### `--encryption-key-file ` + +Path to the file holding the AES-256 key. Encryption is opt-in: a backup +is encrypted only when this argument is supplied to `fdbbackup start`. The +same argument must be supplied to `fdbrestore` and `fdbdecode` so they can +read the ciphertext. + +### `--encryption-block-size ` + +Block size used to encrypt each data file. Optional on `fdbbackup start`, +defaults to `1048576` (1 MB). The chosen value is recorded in the backup's +metadata so that restore and decode pick it up automatically — those tools +do not accept the flag themselves. + +Validation rules on `fdbbackup start`: + +- Must be a positive integer; non-numeric or non-positive values are + rejected with `ERROR: Invalid encryption block size`. +- Must be combined with `--encryption-key-file`. Supplying it without a + key file is rejected with + `ERROR: --encryption-block-size option requires --encryption-key-file to be set`. + +### Per-command summary + +| Command | `--encryption-key-file` | `--encryption-block-size` | Notes | +| ------------------ | :---------------------: | :-----------------------: | -------------------------------------------------------------------------------------- | +| `fdbbackup start` | required for encryption | optional, default 1 MB | Starts a new encrypted backup. | +| `fdbbackup modify` | conditional | n/a | Required only when the destination URL changes and future files must be re-encrypted. | +| `fdbrestore start` | required if encrypted | n/a | Block size is read from the backup's metadata file, not the CLI. | +| `fdbdecode` | required if encrypted | n/a | Decodes encrypted log files with the supplied key. | + +--- + +## 4. Usage + +The same workflow applies whether the destination is S3 or a local file +directory. + +### 4.1 Start a backup + +`--encryption-block-size` is optional; it defaults to 1 MB (`1048576`) +when omitted. The example below sets it explicitly to half a megabyte +(`524288`): + +``` +fdbbackup start \ + -C $CLUSTER_FILE \ + -t test_backup \ + -w \ + -d $BACKUP_URL \ + -k '"" \xff' \ + --log --logdir=$LOG_DIR \ + --blob-credentials $BLOB_CREDENTIALS_FILE \ + --knob_blobstore_encryption_type=aws:kms \ + --encryption-key-file /root/backup_testing/test_encryption_key_file \ + --encryption-block-size 524288 +``` + +### 4.2 Stop a backup + +``` +fdbbackup discontinue -C $CLUSTER_FILE -t test_backup +``` + +### 4.3 Restore a backup + +Restore must use the **same key** that was used at backup time: + +``` +fdbrestore start \ + --dest-cluster-file $CLUSTER_FILE \ + -t test_backup \ + -w \ + -r $BACKUP_URL \ + --log --logdir=$LOG_DIR \ + --blob-credentials $BLOB_CREDENTIALS_FILE \ + --knob_blobstore_encryption_type=aws:kms \ + --encryption-key-file /root/backup_testing/test_encryption_key_file +``` + +`fdbrestore` does **not** take `--encryption-block-size`; the block size +is read from the backup's `encryption_metadata` file. + +### 4.4 Modify a backup destination + +``` +fdbbackup modify \ + -C $CLUSTER_FILE \ + -t mybackup \ + -d $BACKUP_URL \ + --encryption-key-file /root/local_testing/key_file \ + --log --logdir=$LOG_DIR +``` + +Caveats: + +- If the **same URL** is provided with a **different** key, `modify` + rejects the change to avoid mixing differently-encrypted files at the + same URL. +- If a **different URL** is provided, the encryption key may be different, + the same, or omitted entirely. + +### 4.5 Decode encrypted log files + +``` +fdbdecode \ + -r $BACKUP_URL \ + --encryption-key-file /root/local_testing/key_file +``` + +`--encryption-key-file` is required only when the backup at `BACKUP_URL` +is encrypted. + +--- + +## 5. Verifying Encryption Status + +### 5.1 `fdbbackup describe` + +`fdbbackup describe` reports whether a backup is encrypted and the block +size used. **No encryption key is required.** + +``` +fdbbackup describe \ + -C $CLUSTER_FILE \ + -d $BACKUP_URL \ + --blob-credentials $BLOB_CREDENTIALS_FILE +``` + +Text output: + +``` +URL: ... +Restorable: true +Partitioned logs: false +File-level encryption: true +Encryption block size: 1048576 +Snapshot: ... +``` + +JSON output (`--json`) adds `FileLevelEncryption` and +`EncryptionBlockSize`: + +```json +{ + "SchemaVersion": "1.0.0", + "URL": "file:///root/local_testing/backup_before/backup-2026-01-06-04-03-08.320838/", + "Restorable": true, + "Partitioned": false, + "FileLevelEncryption": true, + "EncryptionBlockSize": 1048576, + "Snapshots": [ ... ] +} +``` + +### 5.2 `fdbcli` status JSON + +Each backup agent reports an array of encryption key file paths along +with read success status, so distribution failures are visible at a +glance: + +```json +"encryption_keys": [ + { "path": "/root/local_testing/key1", "success": false }, + { "path": "/root/local_testing/key_file", "success": true } +] +``` + +Each backup tag reports its key file path and encryption status: + +```json +"encryption_key_file": "/root/local_testing/key1", +"file_level_encryption": true +``` + +A complete fragment: + +```json +"instances": { + "4747c54caca01d682a2780d27b7dc9f6": { + "blob_stats": { ... }, + "configured_workers": 10, + "encryption_keys": [ + { "path": "/root/local_testing/key1", "success": false }, + { "path": "/root/local_testing/key_file", "success": true } + ], + "id": "4747c54caca01d682a2780d27b7dc9f6" + } +}, +"tags": { + "test_backup5": { + "current_container": "file:///root/backup_testing/backupfolder/backup-2026-01-21-22-47-47.952405", + "current_status": "has been completed", + "encryption_key_file": "/root/local_testing/key1", + "file_level_encryption": true, + ... + }, + "test_backup8": { + "current_container": "file:///root/backup_testing/backupfolder/backup-2026-01-21-22-57-08.832311", + "current_status": "has been completed", + "file_level_encryption": false, + ... + } +} +``` + +--- + +## 6. Errors and Conflicting Scenarios + +| Scenario | Error | +| --------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | +| Restoring with a key when backup is **not** encrypted | `ERROR: Backup is not encrypted, please remove the encryption key file path.` | +| Restoring **without** a key when backup **is** encrypted | `ERROR: Backup is encrypted, please provide the encryption key file path.` | +| Restoring with the **wrong** key | `ERROR: Failed to read data. Verify that backup and restore encryption keys match (if provided) or the data is corrupted.` | +| `modify` with same URL and different key | `Destination URL matches the existing backup URL for tag '...', but the encryption key file does not match. Fatal Error: Backup error` | +| `--encryption-block-size` without `--encryption-key-file` | `ERROR: --encryption-block-size option requires --encryption-key-file to be set` | +| Invalid `--encryption-block-size` value | `ERROR: Invalid encryption block size ''` | + +--- + +## 7. Key Distribution Requirements + +The encryption key file **path** is added to the backup container and +persisted in `BackupConfig`. That same path is then propagated to every +process that touches the encrypted files: + +- Every **`backup_agent`** host participating in the backup. +- Every **`backup_agent`** host participating in a restore (restore + workers read the encrypted files and must decrypt them). +- The **`fdbrestore`** client process that drives the restore — it opens + the backup container directly to read metadata and validate the + configuration. + +Each of these hosts **must** have the **same** encryption key file +stored locally **at the exact same path**. Otherwise individual agents or +the `fdbrestore` process will fail to read or write encrypted files and +the backup/restore will be unable to make progress. + +`fdbcli status json` (Section 5.2) is the operational signal for whether +the key file is present and readable on every agent. + +--- + +## 8. Design Internals + +### 8.1 What "file-level" means + +"File-level" refers to the **scope** of encryption: each whole backup +file (snapshot, range, or log) is the unit that gets encrypted, as +opposed to encrypting individual mutations, keys/values, or live cluster +storage. Mutations and the live cluster are unchanged; only the on-disk +backup artifacts are ciphertext. Filenames, directory layout, and files +under `properties/` stay plaintext so `fdbbackup describe` and +`fdbbackup expire` can work without the key. + +### 8.2 Block layout inside a file + +Inside each file, the data is split into fixed-size blocks (the +**encryption block size**) and each block is encrypted independently with +AES-256-GCM using the 32-byte key from `--encryption-key-file`. The +blocks are an implementation detail of the AES-GCM stream cipher — not a +change in scope — and they exist so restore can read a small byte range +without decrypting the entire file. + +Per-block IVs are derived deterministically: the first 12 bytes come from +a hash of the filename, and the last 4 bytes are the block index. This +gives every block a unique IV while keeping reads seekable. On restore, +the `encryption_metadata` file is read first to recover the +enable flag and the block size, so the reader splits each file into the +exact same blocks that were produced at backup time. + +### 8.3 Metadata files + +Metadata lives in the `properties/` folder of the backup container and +stays unencrypted. + +| File | Contents | Purpose | +| -------------------------- | ----------------------------------- | -------------------------------------------------------------------------------------- | +| `encryption_metadata` | JSON object (see below) | Indicates whether the backup is encrypted and the encryption block size used. | +| `mutation_log_type` | `"1"` or `"0"` (1 byte) | Whether the backup uses partitioned mutation logs. | +| `log_begin_version` | Version number as text (10 bytes) | Earliest mutation-log version present in the backup. | +| `log_end_version` | Version number as text (10 bytes) | Latest mutation-log version present in the backup. | + +`encryption_metadata` is a JSON object containing both the on/off flag +and the block size used to encrypt the data files: + +```json +{ + "is_encryption_enabled": true, + "encryption_block_size": 1048576 +} +``` + +- `is_encryption_enabled` — `true` if all snapshot/range/log files are + encrypted, `false` otherwise. +- `encryption_block_size` — block size in bytes used to encrypt the data + files, or `0` if the backup is not encrypted. The block size must match + what restore/decode tools use for the same backup, so it is persisted + with the backup itself. + +The `encryption_metadata` metadata file is created immediately when a +backup starts. The other metadata files are generated only when +`fdbrestore` or `fdbbackup describe` runs. Those operations scan +filenames only, not file contents, when reconstructing the on-disk view. + +Example container layout: + +``` +ls -ltrh properties/ +total 16K +-rw-r--r-- 1 root root 74 May 5 21:04 encryption_metadata +-rw-r--r-- 1 root root 1 May 5 21:06 mutation_log_type +-rw-r--r-- 1 root root 10 May 5 21:41 log_end_version +-rw-r--r-- 1 root root 10 May 5 21:41 log_begin_version +``` diff --git a/design/global-tag-throttling.md b/design/global-tag-throttling.md deleted file mode 100644 index 1ea0db9a354..00000000000 --- a/design/global-tag-throttling.md +++ /dev/null @@ -1,131 +0,0 @@ -## Global Tag Throttling - -When the `GLOBAL_TAG_THROTTLING` knob is enabled, the ratekeeper will use the [transaction tagging feature](https://apple.github.io/foundationdb/transaction-tagging.html) to throttle tags according to the global tag throttling algorithm. This page describes the implementation of this algorithm. - -### Tag Quotas -The global tag throttler bases throttling decisions on "quotas" provided by clients through the tag quota API. Each tag quota has two components: - -* Reserved quota -* Total quota - -The global tag throttler cannot throttle tags to a throughput below the reserved quota, and it cannot allow throughput to exceed the total quota. - -### Cost -Internally, the units for these quotas are bytes. The cost of an operation is rounded up to the nearest page size. The cost of a read operation is computed as: - -``` -readCost = ceiling(bytesRead / CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE) * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE; -``` - -The cost of a write operation is computed as: - -``` -writeCost = CLIENT_KNOBS->GLOBAL_TAG_THROTTLING_RW_FUNGIBILITY_RATIO * ceiling(bytesWritten / CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE) * CLIENT_KNOBS->TAG_THROTTLING_PAGE_SIZE; -``` - -Here `bytesWritten` includes cleared bytes. The size of range clears is estimated at commit time. - -### Tuple Layer -Tag quotas are stored inside of the system keyspace (with prefix `\xff/tagQuota/`). They are stored using the tuple layer, in a tuple of form: `(reservedQuota, totalQuota)`. There is currently no custom code in the bindings for manipulating these system keys. However, in any language for which bindings are available, it is possible to use the tuple layer to manipulate tag quotas. - -### fdbcli -The easiest way for an external client to interact with tag quotas is through `fdbcli`. To get the quota (in bytes/second) of a particular tag, run the following command: - -``` -fdbcli> quota get [reserved_throughput|total_throughput] -``` - -To set the quota through `fdbcli`, run: - -``` -fdbcli> quota set [reserved_throughput|total_throughput] -``` - -To clear a both reserved and total throughput quotas for a tag, run: - -``` -fdbcli> quota clear -``` - -### Limit Calculation -The transaction budget that ratekeeper calculates and distributes to clients (via GRV proxies) for each tag is calculated based on several intermediate rate calculations, outlined in this section. - -* Reserved Rate: Based on reserved quota and the average transaction cost, a reserved TPS rate is computed for each tag. - -* Desired Rate: Based on total quota and the average transaction cost, a desired TPS rate is computed for each tag. - -* Limiting Rate: When a storage server is near saturation, tags contributing notably to the workload on this storage server will receive a limiting TPS rate, computed to relieve the workload on the storage server. - -* Target Rate: The target rate is the cluster-wide rate enforced by the global tag throttler. This rate is computed as: - -``` -targetTps = max(reservedTps, min(desiredTps, limitingTps)); -``` - -* Per-Client Rate: While the target rate represents the cluster-wide desired throughput according to the global tag throttler, this budget must be shared across potentially many clients. Therefore, based on observed throughput from various clients, each client will receive an equal budget based on a per-client, per-tag rate computed by the global tag throttler. This rate is in the end what will be sent to clients to enforce throttling. - -## Implementation - -### Stat collection -The transaction rates and costs of all transactions must be visible to the global tag throttler. Whenever a client tags a transaction, sampling is performed to determine whether to attach the tag to messages sent to storage servers and commit proxies. - -For read operations that are sampled (with probability `CLIENT_KNOBS->READ_TAG_SAMPLE_RATE`), read costs are aggregated on storage servers using the `TransactionTagCounter` class. This class tracks the busyness of the top-k tags affecting the storage server with read load (here `k` is determined by `SERVER_KNOBS->SS_THROTTLE_TAGS_TRACKED`). Storage servers periodically send per-tag read cost statistics to the ratekeeper through `StorageQueuingMetricsReply` messages. - -For write operations that are sampled (with probability `COMMIT_SAMPLE_COST`), write costs are aggregated on commit proxies in the `ProxyCommitData::ssTrTagCommitCost` object. Per-storage, per-tag write cost statistics are periodically sent from commit proxies to the ratekeeper through `ReportCommitCostEstimationRequest` messages. - -The ratekeeper tracks per-storage, per-tag cost statistics in the `GlobalTagThrottlerImpl::throughput` object. - -The ratekeeper must also track the rate of transactions performed with each tag. Each GRV proxy agreggates a per-tag counter of transactions started (without sampling). These are sent to the ratekeeper through `GetRateInfoRequest` messages. The global tag throttler then tracks per tag transaction rates in the `GlobalTagThrottlerImpl::tagStatistics` object. - -### Average Cost Calculation -Quotas are expressed in terms of cost, but because throttling is enforced at the beginning of transactions, budgets need to be calculated in terms of transactions per second. To make this conversion, it is necessary to track the average cost of transactions (per-tag, and per-tag on a particular storage server). - -Both cost and transaction counters are exponentially smoothed over time, with knob-configurable smoothing intervals. - -### Reserved Rate Calculation -The global tag throttler periodically reads reserved quotas from the system keyspace. Using these reserved quotas and the average cost of transactions with the given tag, a reserved TPS rate is computed. Read and write rates are aggregated as follows: - -``` -reservedTps = max(reservedReadTps, reservedWriteTps); -``` - -### Desired Rate Calculation -Similar to reserved rate calculation, the total quota is read from the system key space. Then, using the average cost of transactions with the given tag, a desired TPS rate is computed. Read and write rates are aggregated as follows: - -``` -desiredTps = min(desiredReadTps, desiredWriteTps); -``` - -### Limiting Rate Calculation -In addition to tag busyness statistics, the `StorageQueuingMetricsReply` messages sent from storage servers to the ratekeeper also contain metrics on the health of storage servers. The ratekeeper uses these metrics as part of its calculation of a global transaction rate (independent of tag throttling). - -The global tag throttler also makes use of these metrics to compute a "throttling ratio" for each storage server. This throttling ratio is computed in `StorageQueueInfo::getThrottlingRatio`. The global tag throttler uses the throttling ratio for each tracked storage server to compute a "limiting transaction rate" for each combination of storage server and tag. - -In the "healthy" case where no metrics are near saturation, the throttling ratio will be an empty `Optional`, indicating that the storage server is not near saturation. If, on the other hand, the metrics indicate approaching saturation, the throttling ratio will be a number between 0 and 2 indicating the ratio of current throughput the storage server can serve. In this case, the global tag throttler looks at the current cost being served by the storage server, multiplies it by the throttling ratio, and computes a limiting cost for the storage server. Among all tags using significant resources on this storage server, this limiting cost is divided up according to the relative total quotas allocated to these tags. Next, a transaction limit is determined for each tag, based on how much the average transaction for the given tag affects the given storage server. - -These per-tag, per-storage limiting transaction rates are aggregated to compute per-tag limiting transaction rates: - -``` -limitingTps(tag) = min{limitingTps(tag, storage) : all storage servers} -``` - -If the throttling ratio is empty for all storage servers affected by a tag, then the per-tag, per-storage limiting TPS rate is also empty. In this case the target rate for this tag is simply the desired rate. - -If an individual zone is unhealthy, it may cause the throttling ratio for storage servers in that zone to shoot up. This should not be misinterpreted as a workload issue that requires active throttling. Therefore, the zone with the worst throttling ratios is ignored when computing the limiting transaction rate for a tag (similar to the calculation of the global transaction limit in `Ratekeeper::updateRate`). - -### Client Rate Calculation -The smoothed per-client rate for each tag is tracked within `GlobalTagThrottlerImpl::PerTagStatistics`. Once a target rate has been computed, this is passed to `GlobalTagThrottlerImpl::PerTagStatistics::updateAndGetPerClientRate` which adjusts the per-client rate. The per-client rate is meant to limit the busiest clients, so that at equilibrium, the per-client rate will remain constant and the sum of throughput from all clients will match the target rate. - -## Simulation Testing -The `ThroughputQuota.toml` test provides a simple end-to-end test using the global tag throttler. Quotas are set using the internal tag quota API in the `ThroughputQuota` workload. This is run with the `Cycle` workload, which randomly tags transactions. - -In addition to this end-to-end test, there is a suite of unit tests with the `/GlobalTagThrottler/` prefix. These tests run in a mock environment, with mock storage servers providing simulated storage queue statistics and tag busyness reports. Mock clients simulate workload on these mock storage servers, and get throttling feedback directly from a global tag throttler which is monitoring the mock storage servers. - -In each unit test, the `GlobalTagThrottlerTesting::monitor` function is used to periodically check whether or not a desired equilibrium state has been reached. If the desired state is reached and maintained for a sufficient period of time, the test passes. If the unit test is unable to reach this desired equilibrium state before a timeout, the test will fail. Commonly, the desired state is for the global tag throttler to report a client rate sufficiently close to the desired rate specified as an input to the `GlobalTagThrottlerTesting::rateIsNear` function. - -## Visibility - -### Tracing -On the ratekeeper, every `SERVER_KNOBS->TAG_THROTTLE_PUSH_INTERVAL` seconds, the ratekeeper will call `GlobalTagThrottler::getClientRates`. At the end of the rate calculation for each tag, a trace event of type `GlobalTagThrottler_GotClientRate` is produced. This trace event reports the relevant inputs that went in to the rate calculation, and can be used for debugging. - -On storage servers, every `SERVER_KNOBS->TAG_MEASUREMENT_INTERVAL` seconds, there are `BusyReadTag` events for every tag that has sufficient read cost to be reported to the ratekeeper. Both cost and fractional busyness are reported. diff --git a/design/mocks3server_chaos_design.md b/design/mocks3server_chaos_design.md index 25c03ce0f7c..0e41be8cbab 100644 --- a/design/mocks3server_chaos_design.md +++ b/design/mocks3server_chaos_design.md @@ -53,7 +53,7 @@ if (BUGGIFY) { * **HTTP errors**: 429 (throttling), 503 (service unavailable), 500/502 (server errors) * **Auth errors**: 401 (unauthorized), 406 (not acceptable) -* **S3-specific errors**: InvalidToken, ExpiredToken (matching [`S3BlobStore.actor.cpp`](https://github.com/apple/foundationdb/tree/main/fdbclient/S3BlobStore.actor.cpp#L1241) patterns) +* **S3-specific errors**: InvalidToken, ExpiredToken (matching [`S3BlobStore.cpp`](https://github.com/apple/foundationdb/tree/main/fdbclient/S3BlobStore.cpp#L1241) patterns) * **Connection issues**: Connection drops, timeouts * **Data corruption**: Malformed responses, bit flips @@ -119,4 +119,3 @@ wait(startMockS3ServerChaos(listenAddress)); ``` Chaos behavior is controlled by **S3FaultInjector rates** (0.0-1.0), with **BUGGIFY providing occasional extra chaos** - no master boolean switch. - diff --git a/design/recovery-internals.md b/design/recovery-internals.md index 4b4ac34ebb6..9f77bc60e15 100644 --- a/design/recovery-internals.md +++ b/design/recovery-internals.md @@ -71,7 +71,7 @@ This phase locks the coordinated state (cstate) to make sure there is only one C Recall that `ServerDBInfo` has CC's interface and is propagated by CC to every process in a cluster. The current running tLogs can use the CC interface in its `ServerDBInfo` to send its interface to CC. CC simply waits on receiving the `TLogRejoinRequest` streams: for each tLog’s interface received, the CC compares the interface ID with the tLog ID read from cstate. Once the CC collects enough old tLog interfaces, it will use the interfaces to lock those tLogs. The logic of collecting tLogs’ interfaces is implemented in `trackRejoins()` function. -The logic of locking the tLogs is implemented in `epochEnd()` function in [TagPartitionedLogSystems.actor.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/TagPartitionedLogSystem.actor.cpp). +The logic of locking the tLogs is implemented in the `epochEnd()` function in [LogSystem.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp). Once we lock the cstate, we bump the `recoveryCount` by 1 and write the `recoveryCount` to the cstate. Each tLog in a recovery attempt records the `recoveryCount` and monitors the change of the variable. If the `recoveryCount` increases, becoming larger than the recorded value, the tLog will terminate itself. This mechanism makes sure that when multiple recovery attempts happen concurrently, only tLogs in the most recent recovery will be running. tLogs in other recovery attempts can release their memory earlier, reducing the memory pressure during recovery. This is an important memory optimization before shared tLogs, which allows tLogs in different generations to share the same memory, is introduced. @@ -100,7 +100,7 @@ Once the CC gets enough tLogs, it calculates the known committed version (i.e., The CC's finds the maximum `knownCommittedVersion` of all the tLogs. This defines the lower bound of what version range of mutations needs to be copied to the new generation. That is, any versions larger than the CC's `knownCommittedVersion` are not guaranteed to persist on all replicas. The CC chooses a *recovery version*, which is the minimum of durable versions on all tLogs of the old generation, and recruits a new set of tLogs that copy all data between `knownCommittedVersion + 1` and `recoveryVersion` from old tLogs. This copy makes sure data within the range has enough replicas to satisfy the replication policy. -Later, the CC will use the recruited tLogs and `recoveryVersion` to create a new `TagPartitionedLogSystem` for the new generation. +Later, the CC will use the recruited tLogs and `recoveryVersion` to create a new `LogSystem` for the new generation. Note that tLogs can continue to rejoin the cluster after the `recoveryVersion` has been calculated but before the log system has been created. **An example of `knownCommittedVersion` and `recoveryVersion`:** @@ -117,7 +117,7 @@ Consider an old generation with three TLogs: `A, B, C`. Their durable versions a * Situation 1: Too many tLogs in the previous generation permanently died, say due to hardware failure. If force recovery is allowed by system administrator, the CC can choose to force recovery, which can cause data loss; otherwise, to unblock the recovery, system administrator has to bring up those died tLogs, for example by copying their files onto new hardware. -* Situation 2: A tLog may die after it reports alive to the CC in the RECRUITING phase. This may cause the `RecoveryVersion` calculated by the CC in this phase to no longer be valid in the next phases (see [getDurableVersion()](https://github.com/apple/foundationdb/blob/cf7c8f41b22612c4a8a632ea6f173d891b00380d/fdbserver/TagPartitionedLogSystem.actor.cpp#L1940)). When this happens, the CC will detect it (that the previous computed recovery version is no longer viable, because the tail of versions are no longer retrievable from current set of live TLogs), terminate the current recovery, and start a new recovery. See https://github.com/apple/foundationdb/blob/cf7c8f41b22612c4a8a632ea6f173d891b00380d/fdbserver/TagPartitionedLogSystem.actor.cpp#L2468-L2525, where changes to TLogs can cause changes to old log system (`outLogSystem->set(logSystem);`), which in turn causes the recovery loop https://github.com/apple/foundationdb/blob/cf7c8f41b22612c4a8a632ea6f173d891b00380d/fdbserver/ClusterRecovery.actor.cpp#L1582-L1610 to abandon progress and restart. +* Situation 2: A tLog may die after it reports alive to the CC in the RECRUITING phase. This may cause the `RecoveryVersion` calculated by the CC in this phase to no longer be valid in the next phases (see `getDurableVersion()` in [LogSystem.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp)). When this happens, the CC will detect it (that the previous computed recovery version is no longer viable, because the tail of versions are no longer retrievable from current set of live TLogs), terminate the current recovery, and start a new recovery. Changes to TLogs can cause changes to old log system (`outLogSystem->set(logSystem);`), which in turn causes the recovery loop in [ClusterRecovery.actor.cpp](https://github.com/apple/foundationdb/blob/main/fdbserver/clustercontroller/ClusterRecovery.actor.cpp) to abandon progress and restart. Once we have a `knownCommittedVersion`, the CC will reconstruct the [transaction state store](https://github.com/apple/foundationdb/blob/main/design/transaction-state-store.md) by peeking the txnStateTag in oldLogSystem. Recall that the txnStateStore includes the transaction system’s configuration, such as the assignment of shards to SS and to tLogs and that the txnStateStore was durable on disk in the oldLogSystem. diff --git a/design/restore_v1.md b/design/restore_v1.md index 95145597042..0a64b60bf05 100644 --- a/design/restore_v1.md +++ b/design/restore_v1.md @@ -106,7 +106,7 @@ In the restore mechanism, we define five core tasks: 2. `RESTORE_DISPATCH_ADDTASK_SIZE` specifies how many works are dispatched by a single transaction, where we do not want this value too large to make the transaction too large to commit successfully. We do not want this value too small to have too much overhead of transactions to dispatch tasks. - - When `usePartitionedLog` is set, the `StartFullRestoreTaskFunc` spawns `RestoreDispatchPartitionedTaskFunc` instead, as the core of Restore V2. We omit it here since we focus on the Restore V1. + - When `mutation-log-type partitioned-log-experimental` is set, the `StartFullRestoreTaskFunc` spawns `RestoreDispatchPartitionedTaskFunc` instead, as the core of Restore V2. We omit it here since we focus on the Restore V1. 3. `RestoreRangeTaskFunc` (aka. snapshot restore task) diff --git a/design/transaction-state-store.md b/design/transaction-state-store.md index 5cb126bea4b..13a5bed33fc 100644 --- a/design/transaction-state-store.md +++ b/design/transaction-state-store.md @@ -114,4 +114,4 @@ during recovery, `txnStateStore` is recovered by reading all data from this vers * Commit proxies only write metadata mutations in its own transaction batch to TLogs: https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L772-L774 adds mutations to `storeCommits`. Later in `postResolution()`, https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L1162-L1176, only the last one in `storeCommits` are send to TLogs. -* Commit proxies clear the buffered data, and advance the transaction state store's pop version `poppedUpTo`, in `LogSystemDiskQueueAdapter` after TLog push: https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L1283-L1287 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/KeyValueStoreMemory.actor.cpp#L1033-L1034 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/LogSystemDiskQueueAdapter.actor.cpp#L213. +* Commit proxies clear the buffered data, and advance the transaction state store's pop version `poppedUpTo`, in `LogSystemDiskQueueAdapter` after TLog push: https://github.com/apple/foundationdb/blob/6281e647784e74dccb3a6cb88efb9d8b9cccd376/fdbserver/CommitProxyServer.actor.cpp#L1283-L1287 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/KeyValueStoreMemory.cpp#L1033-L1034 https://github.com/apple/foundationdb/blob/4ce8a7c8b414936662622c83d99a67fe93783a81/fdbserver/LogSystemDiskQueueAdapter.actor.cpp#L213. diff --git a/design/validating_restored_data_using_one_cluster.md b/design/validating_restored_data_using_one_cluster.md index 34cac712288..ebb16a379dd 100644 --- a/design/validating_restored_data_using_one_cluster.md +++ b/design/validating_restored_data_using_one_cluster.md @@ -83,5 +83,3 @@ Alternative Design Considerations * Add new AuditType **ValidateRestore.** * New audit actor auditRestoreQ() in StorageServer similar to auditStorageShardReplicaQ() * Change the code to read the range appended with the prefix restored_data_prefix. Can compare, update metadata and error mechanism in the same way. [Range](https://github.com/apple/foundationdb/blob/release-7.4/fdbserver/storageserver.actor.cpp#L5671) - - diff --git a/documentation/coro_tutorial/tutorial.cpp b/documentation/coro_tutorial/tutorial.cpp index 91686bec934..fe52bb1d90d 100644 --- a/documentation/coro_tutorial/tutorial.cpp +++ b/documentation/coro_tutorial/tutorial.cpp @@ -24,7 +24,7 @@ #include "flow/DeterministicRandom.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" #include "fdbrpc/Net2FileSystem.h" #include #include @@ -194,8 +194,7 @@ Future echoServer() { [&requests](StreamRequest const& req) { requests.add([](StreamRequest req) -> Future { req.reply.setByteLimit(1024); - int i = 0; - for (; i < 100; ++i) { + for (int i = 0; i < 100; ++i) { co_await req.reply.onReady(); std::cout << "Send " << i << std::endl; req.reply.send(StreamReply{ i }); @@ -439,7 +438,7 @@ Future fdbClientStream() { GetRangeLimits()); loop { Standalone range = co_await results.getFuture(); - if (range.size()) { + if (!range.empty()) { bytes += range.expectedSize(); next = keyAfter(range.back().key); } @@ -555,17 +554,6 @@ Future fdbClient() { } } -Future fdbStatusStresser() { - Database db = Database::createDatabase(clusterFile, 300); - Key statusJson(std::string("\xff\xff/status/json")); - loop { - co_await runRYWTransaction(db, [&statusJson](ReadYourWritesTransaction* tr) -> Future { - co_await tr->get(statusJson); - co_return; - }); - } -} - AsyncGenerator> readBlocks(Reference file, int64_t blockSize) { auto sz = co_await file->size(); decltype(sz) offset = 0; @@ -646,9 +634,8 @@ std::unordered_map()>> actors = { { "fdbClientStream", &fdbClientStream }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientStream { "fdbClientGetRange", &fdbClientGetRange }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientGetRange { "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient - { "fdbStatusStresser", &fdbStatusStresser }, { "testReadLines", &testReadLines } -}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser +}; int main(int argc, char* argv[]) { bool isServer = false; diff --git a/documentation/sphinx/source/api-c.rst b/documentation/sphinx/source/api-c.rst index a97fa5900cc..4a807d6f7e8 100644 --- a/documentation/sphinx/source/api-c.rst +++ b/documentation/sphinx/source/api-c.rst @@ -870,11 +870,11 @@ Applications must provide error handling and an appropriate retry loop around th .. function:: FDBFuture* fdb_transaction_get_tag_throttled_duration(FDBTransaction* transaction) - |future-return0| the time (in seconds) that the transaction was throttled by the tag throttler in the returned future. |future-return1| call :func:`fdb_future_get_double()` to extract the duration, |future-return2| + |future-return0| ``0``. This compatibility API used to report proxy-side tag throttling time, but proxy-side tag throttling has been removed. |future-return1| call :func:`fdb_future_get_double()` to extract the duration, |future-return2| .. function:: FDBFuture* fdb_transaction_get_total_cost(FDBTransaction* transaction) - |future-return0| the cost of the transaction so far (in bytes) in the returned future, as computed by the tag throttler, and used for tag throttling if throughput quotas are specified. |future-return1| call :func:`fdb_future_get_int64()` to extract the cost, |future-return2| + |future-return0| the cost of the transaction so far (in bytes) in the returned future, as computed by the tag throttler. |future-return1| call :func:`fdb_future_get_int64()` to extract the cost, |future-return2| .. function:: FDBFuture* fdb_transaction_get_approximate_size(FDBTransaction* transaction) diff --git a/documentation/sphinx/source/backups.rst b/documentation/sphinx/source/backups.rst index 77323530ee4..459dee628c1 100644 --- a/documentation/sphinx/source/backups.rst +++ b/documentation/sphinx/source/backups.rst @@ -284,7 +284,7 @@ The ``start`` subcommand is used to start a backup. If there is already a backu :: - user@host$ fdbbackup start [-t ] -d [-z] [-s ] [--partitioned-log-experimental] [-w] [-k '[ ]']... + user@host$ fdbbackup start [-t ] -d [-z] [-s ] [--mutation-log-type partitioned-log-experimental] [-w] [-k '[ ]']... ``-z`` Perform the backup continuously rather than terminating once a restorable backup is achieved. Database mutations within the backup's target key ranges will be continuously written to the backup as well as repeated inconsistent snapshots at the configured snapshot rate. @@ -295,7 +295,7 @@ The ``start`` subcommand is used to start a backup. If there is already a backu ``--initial-snapshot-interval `` Specifies the duration, in seconds, of the first inconsistent snapshot written to the backup. The default is 0, which means as fast as possible. -``--partitioned-log-experimental`` +``--mutation-log-type partitioned-log-experimental`` Specifies the backup uses the partitioned mutation logs generated by backup workers. Since FDB version 6.3, this option is experimental and requires using fast restore for restoring the database from the generated files. The default is to use non-partitioned mutation logs generated by backup agents. ``-w`` diff --git a/documentation/sphinx/source/clang-format.rst b/documentation/sphinx/source/clang-format.rst new file mode 100644 index 00000000000..52ecbf41ce4 --- /dev/null +++ b/documentation/sphinx/source/clang-format.rst @@ -0,0 +1,68 @@ +############ +Clang-Format +############ + +``clang-format`` enforces consistent code style across the FoundationDB codebase. It runs as part of CI on every pull request. If your changes introduce formatting violations, the PR build will fail. + +Configuration is in the ``.clang-format`` file at the repository root, based on the Mozilla style with FDB-specific adjustments (120 column limit, tab width 4, etc.). + +Running clang-format locally +============================ + +Format only your changed lines +------------------------------- + +Use ``git clang-format`` to format only the lines you changed, avoiding reformatting untouched code: + +.. code-block:: shell + + # Format staged changes (before committing) + git clang-format + + # Format changes between your branch and main + git clang-format origin/main + + # Preview what would change (dry run) + git clang-format --diff origin/main + +Format specific files +--------------------- + +If ``git clang-format`` complains about unstaged changes, or you want to format an entire file (not just your changed lines), use ``clang-format`` directly: + +.. code-block:: shell + + clang-format -i path/to/file.cpp + + # Format multiple files + clang-format -i fdbserver/DataDistribution.actor.cpp fdbclient/SystemData.cpp + +.. note:: + + This reformats the entire file, not just your changes. This may produce a large diff if the file was not previously formatted. + +IDE integration +=============== + +**VS Code** (with clangd extension): + +1. Open Settings (Cmd+,) +2. Set **Editor: Default Formatter** to ``llvm-vs-code-extensions.vscode-clangd`` +3. Check **Editor: Format On Save** +4. Set **Editor: Format On Save Mode** to ``file`` + +Files will be automatically formatted on save using the ``.clang-format`` configuration. + +Key style rules +=============== + +The ``.clang-format`` configuration enforces: + +* **Column limit**: 120 characters +* **Indentation**: Tabs with width 4 +* **Braces**: Attach style (opening brace on same line) +* **Arguments**: One per line when they don't fit +* **Pointer alignment**: Left (``int* p``, not ``int *p``) +* **Short functions**: Inline functions may be on a single line + +See the ``.clang-format`` file in the repository root for the full configuration. diff --git a/documentation/sphinx/source/clang-tidy.rst b/documentation/sphinx/source/clang-tidy.rst new file mode 100644 index 00000000000..145a7209bb2 --- /dev/null +++ b/documentation/sphinx/source/clang-tidy.rst @@ -0,0 +1,186 @@ +########## +Clang-Tidy +########## + +``clang-tidy`` is a static analysis tool that detects common programming errors, enforces coding standards, and suggests modern C++ improvements. +It runs as part of CI on every pull request targeting a branch that has a ``.clang-tidy`` file (currently only ``main``). Findings are reported in the build log but do not currently fail the build. To enforce failures, add ``WarningsAsErrors: '*'`` to ``.clang-tidy``. + +This guide explains how to run ``clang-tidy`` locally so you can fix issues before pushing. + +What clang-tidy checks +====================== + +FoundationDB enables 13 checks configured in the ``.clang-tidy`` file at the repository root. The +intent is to enable more as we go forward. Here are some example rules: + +* **3 Bugprone rules** -- catch potential runtime errors (e.g., ``bugprone-use-after-move``) +* **4 Modernize rules** -- encourage modern C++ practices (e.g., ``modernize-use-auto``, ``modernize-use-override``) +* **1 Performance rule** -- avoid unnecessary copies (``performance-for-range-copy``) +* **5 Readability rules** -- improve code clarity (e.g., ``readability-container-contains``, ``readability-container-size-empty``) + +Basic examples of ``clang-tidy`` style and performance improvement changes: + +.. code-block:: cpp + + // Use empty() instead of size() == 0 + -if (str.size() == 0) { + +if (str.empty()) { + + // Use contains() instead of count() > 0 + -if (classPath.count(path) > 0) { + +if (classPath.contains(path)) { + + // Avoid unnecessary copies in range-for loops + -for (auto state : states) + +for (const auto& state : states) + + // Remove unnecessary const on return types + -const Key keyServersKey(const KeyRef& k) { + +Key keyServersKey(const KeyRef& k) { + +clang-tidy vs :doc:`clang-format` +================================= + +Both run in CI but serve different purposes: + +.. list-table:: + :header-rows: 1 + :widths: 15 40 45 + + * - Feature + - ``clang-format`` (Code Style) + - ``clang-tidy`` (Code Quality) + * - **Focus** + - Whitespace, indentation, braces + - Bugs, performance, modern practices + * - **Speed** + - Very fast (syntax-only) + - Slower (requires compilation database) + * - **Action** + - Modifies files in-place (``-i``) + - Reports violations (read-only) + * - **Example** + - ``if(x){...}`` to ``if (x) { ... }`` + - ``vec.size() == 0`` to ``vec.empty()`` + +Running clang-tidy locally +========================== + +Step 1: Generate the compilation database +------------------------------------------ + +``clang-tidy`` needs a ``compile_commands.json`` file to understand include paths and macros. Add ``-DCMAKE_EXPORT_COMPILE_COMMANDS=ON`` when you configure your build: + +.. code-block:: shell + + cmake -B ~/build_output -S . -G Ninja -DCMAKE_EXPORT_COMPILE_COMMANDS=ON + +Then symlink it to your source root: + +.. code-block:: shell + + ln -sf ~/build_output/compile_commands.json . + +.. note:: + + A full build is recommended so that generated files (e.g., ``.actor.g.cpp`` headers) exist and include paths resolve correctly. Without a build, ``clang-tidy`` may report false errors on files that depend on generated code. If your ``compile_commands.json`` was generated on a different machine (e.g., Okteto), fix the paths with: + + .. code-block:: shell + + sed -i '' 's|/root/src/foundationdb/|/Users/YOU/checkouts/foundationdb/|g' compile_commands.json + +Step 2: Locate the helper script +-------------------------------- + +``clang-tidy-diff.py`` runs ``clang-tidy`` only on the lines changed in your diff, avoiding noise from existing code you did not touch. + +On macOS (with Homebrew LLVM): + +.. code-block:: shell + + export TIDY_DIFF=$(find $(brew --prefix llvm)/share/clang -name "clang-tidy-diff.py") + +On Linux: + +.. code-block:: shell + + export TIDY_DIFF=$(find /usr/lib/llvm-*/share/clang -name "clang-tidy-diff.py" | head -n 1) + +To make this permanent, add to your ``~/.bashrc`` or ``~/.zshrc``: + +.. code-block:: shell + + alias fdb-tidy='python3 $(find $(brew --prefix llvm 2>/dev/null || echo "/usr/lib/llvm-*") -name "clang-tidy-diff.py" | head -n 1) -p 1 -path .' + +Step 3: Run against your changes +--------------------------------- + +Check all changes between your branch and ``main``: + +.. code-block:: shell + + git diff -U0 origin/main...HEAD | grep -v -E '\.actor\.cpp' | python3 "$TIDY_DIFF" -p 1 -path . + + # Or with the alias: + git diff -U0 origin/main...HEAD | grep -v -E '\.actor\.cpp' | fdb-tidy + +Check a specific commit: + +.. code-block:: shell + + git show -U0 | python3 "$TIDY_DIFF" -p 1 -path . + +.. important:: + + Use ``-U0`` (no context lines). Without it, ``clang-tidy`` may lint surrounding lines that you did not change, reporting pre-existing issues. + +Understanding the flags +----------------------- + +The ``clang-tidy-diff.py`` script has different flag conventions from ``clang-tidy`` itself: + +* ``-p 1`` -- strip one directory prefix from git diff paths (``a/src/main.cpp`` becomes ``src/main.cpp``) +* ``-path .`` -- directory containing ``compile_commands.json`` +* ``-quiet`` -- suppress per-file warning count summaries +* ``-j N`` -- run N clang-tidy instances in parallel + +.. warning:: + + In the standard ``clang-tidy`` command, ``-p`` points to the build directory. In ``clang-tidy-diff.py``, ``-path`` points to the build directory and ``-p`` is for path stripping. + +Running clang-tidy during builds (CMake integration) +===================================================== + +FoundationDB's CMake build supports running ``clang-tidy`` automatically during C/C++ compilation: + +.. code-block:: shell + + cmake -S . -B build -G Ninja -DUSE_CLANG_TIDY=ON + ninja -C build fdbserver + +This runs ``clang-tidy`` on every file as it compiles. It is slower than using ``clang-tidy-diff.py`` on your diff but catches issues in all compiled code. + +Optional CMake variables: + +* ``CLANG_TIDY`` -- path to the ``clang-tidy`` executable (auto-detected by default) +* ``CLANG_TIDY_EXTRA_ARGS`` -- additional space-separated arguments passed to ``clang-tidy`` + +Known limitations +----------------- + +**``.actor.cpp`` files cannot be analyzed.** These files use FoundationDB's custom actor compiler syntax (``ACTOR``, ``wait()``, ``state``) that ``clang-tidy`` cannot parse. Exclude them from your diff when running locally: + +Quick reference +=============== + +.. code-block:: shell + + git diff -U0 origin/main...HEAD | grep -v -E '\.actor\.cpp' | python3 "$TIDY_DIFF" -p 1 -path . + +**GCC-built compile_commands.json with clang-tidy.** If your ``compile_commands.json`` was generated with GCC, it may contain GCC-specific flags that ``clang-tidy`` (which uses the clang frontend) does not recognize. Add ``-extra-arg=-Wno-unknown-warning-option`` to suppress these errors: + +.. code-block:: shell + + git diff -U0 origin/main...HEAD | python3 "$TIDY_DIFF" -p 1 -path . -extra-arg=-Wno-unknown-warning-option + +Quick reference diff --git a/documentation/sphinx/source/command-line-interface.rst b/documentation/sphinx/source/command-line-interface.rst index 779a1975c55..390c271e33a 100644 --- a/documentation/sphinx/source/command-line-interface.rst +++ b/documentation/sphinx/source/command-line-interface.rst @@ -64,7 +64,7 @@ The ``commit`` command commits the current transaction. Any sets or clears execu configure --------- -The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=] [perpetual_storage_wiggle_locality=<:|0>] [storage_migration_type={disabled|aggressive|gradual}]``. +The ``configure`` command changes the database configuration. Its syntax is ``configure [new|tss] [single|double|triple|three_data_hall|three_datacenter] [ssd|memory] [grv_proxies=] [commit_proxies=] [resolvers=] [logs=] [count=] [perpetual_storage_wiggle=] [perpetual_storage_wiggle_locality=<:|0>] [perpetual_storage_wiggle_engine=] [storage_migration_type={disabled|aggressive|gradual}]``. The ``new`` option, if present, initializes a new database with the given configuration rather than changing the configuration of an existing one. When ``new`` is used, both a redundancy mode and a storage engine must be specified. diff --git a/documentation/sphinx/source/developer-tools.rst b/documentation/sphinx/source/developer-tools.rst new file mode 100644 index 00000000000..15a100c6382 --- /dev/null +++ b/documentation/sphinx/source/developer-tools.rst @@ -0,0 +1,13 @@ +############### +Developer Tools +############### + +Tools and guides for FoundationDB contributors. + +.. toctree:: + :maxdepth: 1 + + local-dev + clang-format + clang-tidy + internal-dev-tools diff --git a/documentation/sphinx/source/global-configuration.rst b/documentation/sphinx/source/global-configuration.rst index 663ad26eb4e..56316e7c52a 100644 --- a/documentation/sphinx/source/global-configuration.rst +++ b/documentation/sphinx/source/global-configuration.rst @@ -79,9 +79,9 @@ Values must always be encoded according to the :ref:`api-python-tuple-layer`. .. code-block:: cpp - // In GlobalConfig.actor.h + // In GlobalConfig.h extern const KeyRef myGlobalConfigKey; - // In GlobalConfig.actor.cpp + // In GlobalConfig.cpp const KeyRef myGlobalConfigKey = "config/key"_sr; // When you want to set the value.. diff --git a/documentation/sphinx/source/ha-write-path.rst b/documentation/sphinx/source/ha-write-path.rst index deea5d0b931..92d86b363e9 100644 --- a/documentation/sphinx/source/ha-write-path.rst +++ b/documentation/sphinx/source/ha-write-path.rst @@ -118,9 +118,9 @@ tLogs also maintain two properties: At primary SS ------------- -**Primary tLog of a SS.** Since a SS’s tag is identically mapped to one tLog. The tLog has all mutations for the SS and is the primary tLog for the SS. When the SS peeks data from tLogs, it will prefer to peek data from its primary tLog. If the primary tLog crashes, it will contact the rest of tLogs, ask for mutations with the SS’s tag, and merge them together. This complex merge operation is abstracted in the TagPartitionedLogSystem interface. +**Primary tLog of a SS.** Since a SS’s tag is identically mapped to one tLog. The tLog has all mutations for the SS and is the primary tLog for the SS. When the SS peeks data from tLogs, it will prefer to peek data from its primary tLog. If the primary tLog crashes, it will contact the rest of tLogs, ask for mutations with the SS’s tag, and merge them together. This complex merge operation is abstracted in the LogSystem interface. -**Pulling data from tLogs.** Each SS in the primary DC keeps pulling mutations, whose tag is the SS’s tag, from tLogs. Once mutations before a version V1 are made durable on a SS, the SS pops the tag upto the version V1 from *all* tLogs. The pop operation is an RPC to tLogs through the TagPartitionedLogSystem interface. +**Pulling data from tLogs.** Each SS in the primary DC keeps pulling mutations, whose tag is the SS’s tag, from tLogs. Once mutations before a version V1 are made durable on a SS, the SS pops the tag upto the version V1 from *all* tLogs. The pop operation is an RPC to tLogs through the LogSystem interface. Since the mutation m1 has three tags for primary SSes, the mutation will be made durable on three primary SSes. This marks the end of the mutation’s journey in the primary DC. @@ -164,7 +164,7 @@ https://github.com/apple/foundationdb/blob/7eabdf784a21bca102f84e7eaf14bafc54605 Mutation Serialization (WiP) ============================ -This section will go into detail on how mutations are serialized as preparation for ingestion into the TagPartitionedLogSystem. This has also been covered at: +This section will go into detail on how mutations are serialized as preparation for ingestion into the LogSystem. This has also been covered at: https://drive.google.com/file/d/1OaP5bqH2kst1VxD6RWj8h2cdr9rhhBHy/view diff --git a/documentation/sphinx/source/index.rst b/documentation/sphinx/source/index.rst index 167bebda43f..55f3d733a95 100644 --- a/documentation/sphinx/source/index.rst +++ b/documentation/sphinx/source/index.rst @@ -44,13 +44,14 @@ The latest changes are detailed in :ref:`release-notes`. The documentation has t * :doc:`visibility` contains documentation related to Visibility into FoundationDB. +* :doc:`developer-tools` contains documentation for FoundationDB contributors, including local development setup, static analysis, and internal testing tools. + .. toctree:: :maxdepth: 1 :titlesonly: :hidden: - local-dev - internal-dev-tools + developer-tools why-foundationdb technical-overview client-design diff --git a/documentation/sphinx/source/mr-status-json-schemas.rst.inc b/documentation/sphinx/source/mr-status-json-schemas.rst.inc index 060ff759ec0..a6df96e873e 100644 --- a/documentation/sphinx/source/mr-status-json-schemas.rst.inc +++ b/documentation/sphinx/source/mr-status-json-schemas.rst.inc @@ -348,6 +348,7 @@ "trace_log_file_write_error", "trace_log_could_not_create_file", "trace_log_writer_thread_unresponsive", + "exclude_from_tlog_recruitment_low_disk", "process_error", "io_error", "io_timeout", @@ -625,7 +626,8 @@ "incorrect_cluster_file_contents", "trace_log_file_write_error", "trace_log_could_not_create_file", - "trace_log_writer_thread_unresponsive" + "trace_log_writer_thread_unresponsive", + "exclude_from_tlog_recruitment_low_disk" ] }, "description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally." diff --git a/documentation/sphinx/source/read-write-path.rst b/documentation/sphinx/source/read-write-path.rst index 8257f8905f9..277950b832f 100644 --- a/documentation/sphinx/source/read-write-path.rst +++ b/documentation/sphinx/source/read-write-path.rst @@ -179,7 +179,7 @@ Implementation of FDB read path * Proxy confirm queuing system is alive: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/MasterProxyServer.actor.cpp#L1199 * How is confirmEpochLive(..) implemented for the above item: - https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbserver/TagPartitionedLogSystem.actor.cpp#L1216-L1225 + https://github.com/apple/foundationdb/blob/main/fdbserver/logsystem/LogSystem.cpp * **Step 4 (Locality request)**: https://github.com/apple/foundationdb/blob/4b0fba6ea89b51b82df7868ca24b81f6997db4e4/fdbclient/NativeAPI.actor.cpp#L1312-L1313 diff --git a/documentation/sphinx/source/release-notes/release-notes-700.rst b/documentation/sphinx/source/release-notes/release-notes-700.rst index de56447519c..e350341fb0a 100644 --- a/documentation/sphinx/source/release-notes/release-notes-700.rst +++ b/documentation/sphinx/source/release-notes/release-notes-700.rst @@ -32,7 +32,7 @@ Features Performance ----------- * Improved Deque copy performance. `(PR #3197) `_ -* Increased performance of dr_agent when copying the mutation log. The ``COPY_LOG_BLOCK_SIZE``, ``COPY_LOG_BLOCKS_PER_TASK``, ``COPY_LOG_PREFETCH_BLOCKS``, ``COPY_LOG_READ_AHEAD_BYTES`` and ``COPY_LOG_TASK_DURATION_NANOS`` knobs can be set. `(PR #3436) `_ +* Increased performance of dr_agent when copying the mutation log. The ``COPY_LOG_BLOCK_SIZE``, ``COPY_LOG_BLOCKS_PER_TASK``, ``COPY_LOG_PREFETCH_BLOCKS``, ``COPY_LOG_READ_AHEAD_BYTES`` and ``COPY_LOG_TASK_DURATION_SECONDS`` knobs can be set. `(PR #3436) `_ `(PR #12733) `_ * Added multiple new microbenchmarks for PromiseStream, Reference, IRandom, and timer, as well as support for benchmarking actors. `(PR #3590) `_ * Use xxhash3 for SQLite page checksums. `(PR #4075) `_ * fdbserver now uses jemalloc on Linux instead of the system malloc. `(PR #4222) `_ diff --git a/documentation/sphinx/source/release-notes/release-notes-730.rst b/documentation/sphinx/source/release-notes/release-notes-730.rst index f00ea980918..72935f9148c 100644 --- a/documentation/sphinx/source/release-notes/release-notes-730.rst +++ b/documentation/sphinx/source/release-notes/release-notes-730.rst @@ -2,6 +2,16 @@ Release Notes ############# +7.3.77 +====== +* Same as 7.3.76 release with AVX enabled. + +7.3.76 +====== +* Fixed peer disconnect detection in waitValueOrSignal so dead connections are detected immediately instead of hanging indefinitely. `(PR #12935) `_ +* Made getTeamByServers O(1) in time which was sometimes causing Data Distributor (DD) to get stuck during initialization when shard_encode_location_metadata was enabled. `(PR #12938) `_ +* Added miscellaneous observability improvements for CPU and memory usage, DD startup, and S3 backup subsystems. `(PR #12937) `_, `(PR #12913) `_, and `(PR #12997) `_ + 7.3.75 ====== * Same as 7.3.74 release with AVX enabled. diff --git a/documentation/tutorial/dining_philosophers.actor.cpp b/documentation/tutorial/dining_philosophers.actor.cpp index b2825fd57cd..8f6bc7065f2 100644 --- a/documentation/tutorial/dining_philosophers.actor.cpp +++ b/documentation/tutorial/dining_philosophers.actor.cpp @@ -24,7 +24,7 @@ #include "flow/DeterministicRandom.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" #include #include #include @@ -102,8 +102,8 @@ struct GetForkRequest { ForkState forkState; ReplyPromise reply; - GetForkRequest(ForkState fork_state) : forkState(fork_state) {} - GetForkRequest() {} + explicit GetForkRequest(ForkState fork_state) : forkState(fork_state) {} + GetForkRequest() = default; template void serialize(Ar& ar) { @@ -116,8 +116,8 @@ struct ReleaseForkRequest { ForkState forkState; ReplyPromise reply; - ReleaseForkRequest(ForkState fork_state) : forkState(fork_state) {} - ReleaseForkRequest() {} + explicit ReleaseForkRequest(ForkState fork_state) : forkState(fork_state) {} + ReleaseForkRequest() = default; template void serialize(Ar& ar) { diff --git a/documentation/tutorial/make_h2o.actor.cpp b/documentation/tutorial/make_h2o.actor.cpp index d3f0977e96a..d3739750450 100644 --- a/documentation/tutorial/make_h2o.actor.cpp +++ b/documentation/tutorial/make_h2o.actor.cpp @@ -24,7 +24,7 @@ #include "flow/DeterministicRandom.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" #include "flow/actorcompiler.h" #include diff --git a/documentation/tutorial/play.actor.cpp b/documentation/tutorial/play.actor.cpp index 12eef72324a..a8ee1db16b8 100644 --- a/documentation/tutorial/play.actor.cpp +++ b/documentation/tutorial/play.actor.cpp @@ -22,7 +22,7 @@ #include "flow/Arena.h" #include "flow/flow.h" #include "flow/Platform.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" #include "flow/actorcompiler.h" #include diff --git a/documentation/tutorial/play_network.actor.cpp b/documentation/tutorial/play_network.actor.cpp index 707ccde612c..3eafaf64f31 100644 --- a/documentation/tutorial/play_network.actor.cpp +++ b/documentation/tutorial/play_network.actor.cpp @@ -22,7 +22,7 @@ #include "flow/Arena.h" #include "flow/flow.h" #include "flow/Platform.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" #include "flow/actorcompiler.h" #include diff --git a/documentation/tutorial/print_in_order.actor.cpp b/documentation/tutorial/print_in_order.actor.cpp index 97adf841ebc..174f202bfe4 100644 --- a/documentation/tutorial/print_in_order.actor.cpp +++ b/documentation/tutorial/print_in_order.actor.cpp @@ -24,7 +24,7 @@ #include "flow/DeterministicRandom.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" #include "flow/actorcompiler.h" #include diff --git a/documentation/tutorial/tutorial.actor.cpp b/documentation/tutorial/tutorial.actor.cpp index 29888d0b69e..5a706a6f837 100644 --- a/documentation/tutorial/tutorial.actor.cpp +++ b/documentation/tutorial/tutorial.actor.cpp @@ -25,7 +25,7 @@ #include "flow/DeterministicRandom.h" #include "fdbclient/NativeAPI.actor.h" #include "fdbclient/ReadYourWrites.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" #include #include #include @@ -518,20 +518,6 @@ ACTOR Future fdbClient() { } } -ACTOR Future fdbStatusStresser() { - state Database db = Database::createDatabase(clusterFile, 300); - state ReadYourWritesTransaction tx(db); - state Key statusJson(std::string("\xff\xff/status/json")); - loop { - try { - tx.reset(); - Optional _ = wait(tx.get(statusJson)); - } catch (Error& e) { - wait(tx.onError(e)); - } - } -} - std::unordered_map()>> actors = { { "timer", &simpleTimer }, // ./tutorial timer { "promiseDemo", &promiseDemo }, // ./tutorial promiseDemo @@ -543,9 +529,8 @@ std::unordered_map()>> actors = { { "multipleClients", &multipleClients }, // ./tutorial -s 127.0.0.1:6666 multipleClients { "fdbClientStream", &fdbClientStream }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientStream { "fdbClientGetRange", &fdbClientGetRange }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClientGetRange - { "fdbClient", &fdbClient }, // ./tutorial -C $CLUSTER_FILE_PATH fdbClient - { "fdbStatusStresser", &fdbStatusStresser } -}; // ./tutorial -C $CLUSTER_FILE_PATH fdbStatusStresser + { "fdbClient", &fdbClient } // ./tutorial -C $CLUSTER_FILE_PATH fdbClient +}; int main(int argc, char* argv[]) { bool isServer = false; diff --git a/fdbbackup/CMakeLists.txt b/fdbbackup/CMakeLists.txt index ff0b0c2d559..63e5b394fba 100644 --- a/fdbbackup/CMakeLists.txt +++ b/fdbbackup/CMakeLists.txt @@ -1,22 +1,22 @@ set(FDBBACKUP_SRCS Decode.cpp - backup.actor.cpp) + backup.cpp) add_flow_target(EXECUTABLE NAME fdbbackup SRCS ${FDBBACKUP_SRCS}) -target_include_directories(fdbbackup PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include") +target_include_directories(fdbbackup PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") target_link_libraries(fdbbackup PRIVATE fdbclient) set(FDBCONVERT_SRCS - FileConverter.actor.cpp) + FileConverter.cpp) add_flow_target(EXECUTABLE NAME fdbconvert SRCS ${FDBCONVERT_SRCS}) -target_include_directories(fdbconvert PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include") +target_include_directories(fdbconvert PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") target_link_libraries(fdbconvert PRIVATE fdbclient) set(FDBDECODE_SRCS Decode.cpp - FileDecoder.actor.cpp) + FileDecoder.cpp) add_flow_target(EXECUTABLE NAME fdbdecode SRCS ${FDBDECODE_SRCS}) -target_include_directories(fdbdecode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include") +target_include_directories(fdbdecode PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") target_link_libraries(fdbdecode PRIVATE fdbclient) if(NOT OPEN_FOR_IDE) @@ -39,19 +39,25 @@ if(NOT OPEN_FOR_IDE) symlink_files( LOCATION packages/bin SOURCE fdbbackup - TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_tool) + TARGETS fdbdr dr_agent backup_agent fdbrestore) symlink_files( LOCATION bin SOURCE fdbbackup - TARGETS fdbdr dr_agent backup_agent fdbrestore fastrestore_tool) + TARGETS fdbdr dr_agent backup_agent fdbrestore) - # Test version of backup.actor.cpp without main() function - add_flow_target(EXECUTABLE NAME backup_tests SRCS backup.actor.cpp) - target_include_directories(backup_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include") + # Test version of backup.cpp without main() function + add_flow_target(EXECUTABLE NAME backup_tests SRCS backup.cpp) + target_include_directories(backup_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") target_link_libraries(backup_tests PRIVATE fdbclient) target_compile_definitions(backup_tests PRIVATE EXCLUDE_MAIN_FUNCTION=1) add_test(NAME BackupTests COMMAND backup_tests) + # Test version of FileDecoder.cpp without main() function + add_flow_target(EXECUTABLE NAME fdbdecode_tests SRCS Decode.cpp FileDecoder.cpp) + target_include_directories(fdbdecode_tests PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") + target_link_libraries(fdbdecode_tests PRIVATE fdbclient) + target_compile_definitions(fdbdecode_tests PRIVATE EXCLUDE_MAIN_FUNCTION=1) + add_test(NAME FileDecoderTests COMMAND fdbdecode_tests) endif() if (GPERFTOOLS_FOUND) @@ -62,7 +68,7 @@ endif() if (NOT WIN32 AND NOT OPEN_FOR_IDE) enable_testing() add_test(NAME dir_backup_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/dir_backup_test.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) - add_test(NAME s3_backup_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/s3_backup_test.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR} --encrypt-at-random) + add_test(NAME blob_backup_restore_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/blob_backup_restore_test.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) # Note: No --encrypt flag - BulkLoad doesn't support encryption yet add_test(NAME s3_backup_bulkdump_bulkload_tests COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/tests/s3_backup_bulkdump_bulkload.sh ${CMAKE_SOURCE_DIR} ${CMAKE_BINARY_DIR}) endif() diff --git a/fdbbackup/FileConverter.actor.cpp b/fdbbackup/FileConverter.actor.cpp deleted file mode 100644 index 7ba2ae04e7c..00000000000 --- a/fdbbackup/FileConverter.actor.cpp +++ /dev/null @@ -1,626 +0,0 @@ -/* - * FileConverter.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbbackup/FileConverter.h" - -#include -#include -#include -#include -#include - -#include "fdbclient/BackupAgent.actor.h" -#include "fdbclient/BackupContainer.h" -#include "fdbclient/MutationList.h" -#include "flow/flow.h" -#include "flow/serialize.h" -#include "fdbclient/BuildFlags.h" -#include "flow/actorcompiler.h" // has to be last include - -namespace file_converter { - -void printConvertUsage() { - std::cout << "\n" - << " -r, --container Container URL.\n" - << " -b, --begin BEGIN\n" - << " Begin version.\n" - << " -e, --end END End version.\n" - << " --log Enables trace file logging for the CLI session.\n" - << " --logdir PATH Specifies the output directory for trace files. If\n" - << " unspecified, defaults to the current directory. Has\n" - << " no effect unless --log is specified.\n" - << " --loggroup LOG_GROUP\n" - << " Sets the LogGroup field with the specified value for all\n" - << " events in the trace output (defaults to `default').\n" - << " --trace-format FORMAT\n" - << " Select the format of the trace files. xml (the default) and json are supported.\n" - << " Has no effect unless --log is specified.\n" - << " --build-flags Print build information and exit.\n" - << " -h, --help Display this help and exit.\n" - << "\n"; - - return; -} - -void printBuildInformation() { - printf("%s", jsonBuildInformation().c_str()); -} - -void printLogFiles(std::string msg, const std::vector& files) { - std::cout << msg << " " << files.size() << " log files\n"; - for (const auto& file : files) { - std::cout << file.toString() << "\n"; - } - std::cout << std::endl; -} - -std::vector getRelevantLogFiles(const std::vector& files, Version begin, Version end) { - std::vector filtered; - for (const auto& file : files) { - if (file.beginVersion <= end && file.endVersion >= begin && file.tagId >= 0 && file.fileSize > 0) { - filtered.push_back(file); - } - } - std::sort(filtered.begin(), filtered.end()); - - // Remove duplicates. This is because backup workers may store the log for - // old epochs successfully, but do not update the progress before another - // recovery happened. As a result, next epoch will retry and creates - // duplicated log files. - std::vector sorted; - int i = 0; - for (int j = 1; j < filtered.size(); j++) { - if (!filtered[i].isSubset(filtered[j])) { - sorted.push_back(filtered[i]); - } - i = j; - } - if (i < filtered.size()) { - sorted.push_back(filtered[i]); - } - - return sorted; -} - -struct ConvertParams { - std::string container_url; - Optional proxy; - Version begin = invalidVersion; - Version end = invalidVersion; - bool log_enabled = false; - std::string log_dir, trace_format, trace_log_group; - - bool isValid() const { return begin != invalidVersion && end != invalidVersion && !container_url.empty(); } - - std::string toString() const { - std::string s; - s.append("ContainerURL:"); - s.append(container_url); - if (proxy.present()) { - s.append(" Proxy:"); - s.append(proxy.get()); - } - s.append(" Begin:"); - s.append(format("%" PRId64, begin)); - s.append(" End:"); - s.append(format("%" PRId64, end)); - if (log_enabled) { - if (!log_dir.empty()) { - s.append(" LogDir:").append(log_dir); - } - if (!trace_format.empty()) { - s.append(" Format:").append(trace_format); - } - if (!trace_log_group.empty()) { - s.append(" LogGroup:").append(trace_log_group); - } - } - return s; - } -}; - -struct VersionedData { - LogMessageVersion version; - StringRef message; // Serialized mutation. - Arena arena; // The arena that contains mutation. - - VersionedData() : version(invalidVersion, -1) {} - VersionedData(LogMessageVersion v, StringRef m, Arena a) : version(v), message(m), arena(a) {} -}; - -struct MutationFilesReadProgress : public ReferenceCounted { - MutationFilesReadProgress(std::vector& logs, Version begin, Version end) - : files(logs), beginVersion(begin), endVersion(end) {} - - struct FileProgress : public ReferenceCounted { - FileProgress(Reference f, int index) : fd(f), idx(index), offset(0), eof(false) {} - - bool operator<(const FileProgress& rhs) const { - if (rhs.mutations.empty()) - return true; - if (mutations.empty()) - return false; - return mutations[0].version < rhs.mutations[0].version; - } - bool operator<=(const FileProgress& rhs) const { - if (rhs.mutations.empty()) - return true; - if (mutations.empty()) - return false; - return mutations[0].version <= rhs.mutations[0].version; - } - bool empty() { return eof && mutations.empty(); } - - // Decodes the block into mutations and save them if >= minVersion and < maxVersion. - // Returns true if new mutations has been saved. - bool decodeBlock(const Standalone& buf, int len, Version minVersion, Version maxVersion) { - StringRef block(buf.begin(), len); - StringRefReader reader(block, restore_corrupted_data()); - int count = 0, inserted = 0; - Version msgVersion = invalidVersion; - - try { - // Read block header - if (reader.consume() != PARTITIONED_MLOG_VERSION) - throw restore_unsupported_file_version(); - - while (1) { - // If eof reached or first key len bytes is 0xFF then end of block was reached. - if (reader.eof() || *reader.rptr == 0xFF) - break; - - // Deserialize messages written in saveMutationsToFile(). - msgVersion = bigEndian64(reader.consume()); - uint32_t sub = bigEndian32(reader.consume()); - int msgSize = bigEndian32(reader.consume()); - const uint8_t* message = reader.consume(msgSize); - - ArenaReader rd( - buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion())); - MutationRef m; - rd >> m; - count++; - if (msgVersion >= maxVersion) { - TraceEvent("FileDecodeEnd") - .detail("MaxV", maxVersion) - .detail("Version", msgVersion) - .detail("File", fd->getFilename()); - eof = true; - break; // skip - } - if (msgVersion >= minVersion) { - mutations.emplace_back( - LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena()); - inserted++; - } - } - offset += len; - - TraceEvent("Decoded") - .detail("Name", fd->getFilename()) - .detail("Count", count) - .detail("Insert", inserted) - .detail("BlockOffset", reader.rptr - buf.begin()) - .detail("Total", mutations.size()) - .detail("EOF", eof) - .detail("Version", msgVersion) - .detail("NewOffset", offset); - return inserted > 0; - } catch (Error& e) { - TraceEvent(SevWarn, "CorruptLogFileBlock") - .error(e) - .detail("Filename", fd->getFilename()) - .detail("BlockOffset", offset) - .detail("BlockLen", len) - .detail("ErrorRelativeOffset", reader.rptr - buf.begin()) - .detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset); - throw; - } - } - - Reference fd; - int idx; // index in the MutationFilesReadProgress::files vector - int64_t offset; // offset of the file to be read - bool eof; // If EOF is seen so far or endVersion is encountered. If true, the file can't be read further. - std::vector mutations; // Buffered mutations read so far - }; - - bool hasMutations() { - for (const auto& fp : fileProgress) { - if (!fp->empty()) - return true; - } - return false; - } - - void dumpProgress(std::string msg) { - std::cout << msg << "\n "; - for (const auto& fp : fileProgress) { - std::cout << fp->fd->getFilename() << " " << fp->mutations.size() << " mutations"; - if (fp->mutations.size() > 0) { - std::cout << ", range " << fp->mutations[0].version.toString() << " " - << fp->mutations.back().version.toString() << "\n"; - } else { - std::cout << "\n\n"; - } - } - } - - // Sorts files according to their first mutation version and removes files without mutations. - void sortAndRemoveEmpty() { - std::sort(fileProgress.begin(), - fileProgress.end(), - [](const Reference& a, const Reference& b) { return (*a) < (*b); }); - while (!fileProgress.empty() && fileProgress.back()->empty()) { - fileProgress.pop_back(); - } - } - - // Requires hasMutations() return true before calling this function. - // The caller must hold on the the arena associated with the mutation. - Future getNextMutation() { return getMutationImpl(this); } - - ACTOR static Future getMutationImpl(MutationFilesReadProgress* self) { - ASSERT(!self->fileProgress.empty() && !self->fileProgress[0]->mutations.empty()); - - state Reference fp = self->fileProgress[0]; - state VersionedData data = fp->mutations[0]; - fp->mutations.erase(fp->mutations.begin()); - if (fp->mutations.empty()) { - // decode one more block - wait(decodeToVersion(fp, /*version=*/0, self->endVersion, self->getLogFile(fp->idx))); - } - - if (fp->empty()) { - self->fileProgress.erase(self->fileProgress.begin()); - } else { - // Keep fileProgress sorted - for (int i = 1; i < self->fileProgress.size(); i++) { - if (*self->fileProgress[i - 1] <= *self->fileProgress[i]) { - break; - } - std::swap(self->fileProgress[i - 1], self->fileProgress[i]); - } - } - return data; - } - - LogFile& getLogFile(int index) { return files[index]; } - - Future openLogFiles(Reference container) { return openLogFilesImpl(this, container); } - - // Opens log files in the progress and starts decoding until the beginVersion is seen. - ACTOR static Future openLogFilesImpl(MutationFilesReadProgress* progress, - Reference container) { - state std::vector>> asyncFiles; - for (const auto& file : progress->files) { - asyncFiles.push_back(container->readFile(file.fileName)); - } - wait(waitForAll(asyncFiles)); // open all files - - // Attempt decode the first few blocks of log files until beginVersion is consumed - std::vector> fileDecodes; - for (int i = 0; i < asyncFiles.size(); i++) { - auto fp = makeReference(asyncFiles[i].get(), i); - progress->fileProgress.push_back(fp); - fileDecodes.push_back( - decodeToVersion(fp, progress->beginVersion, progress->endVersion, progress->getLogFile(i))); - } - - wait(waitForAll(fileDecodes)); - - progress->sortAndRemoveEmpty(); - - return Void(); - } - - // Decodes the file until EOF or an mutation >= minVersion and saves these mutations. - // Skip mutations >= maxVersion. - ACTOR static Future decodeToVersion(Reference fp, - Version minVersion, - Version maxVersion, - LogFile file) { - if (fp->empty()) - return Void(); - - if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) - return Void(); - - state int64_t len; - try { - // Read block by block until we see the minVersion - loop { - len = std::min(file.blockSize, file.fileSize - fp->offset); - if (len == 0) { - fp->eof = true; - return Void(); - } - - state Standalone buf = makeString(len); - int rLen = wait(fp->fd->read(mutateString(buf), len, fp->offset)); - if (len != rLen) - throw restore_bad_read(); - - TraceEvent("ReadFile") - .detail("Name", fp->fd->getFilename()) - .detail("Length", rLen) - .detail("Offset", fp->offset); - if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) - break; - } - return Void(); - } catch (Error& e) { - TraceEvent(SevWarn, "CorruptedLogFileBlock") - .error(e) - .detail("Filename", fp->fd->getFilename()) - .detail("BlockOffset", fp->offset) - .detail("BlockLen", len); - throw; - } - } - - std::vector files; - const Version beginVersion, endVersion; - std::vector> fileProgress; -}; - -// Writes a log file in the old backup format, described in backup-dataFormat.md. -// This is similar to the LogFileWriter in FileBackupAgent.actor.cpp. -struct LogFileWriter { - LogFileWriter() : blockSize(-1) {} - LogFileWriter(Reference f, int bsize) : file(f), blockSize(bsize) {} - - // Returns the block key, i.e., `Param1`, in the back file. The format is - // `hash_value|commitVersion|part`. - static Standalone getBlockKey(Version commitVersion, int part) { - const int32_t version = commitVersion / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; - - BinaryWriter wr(Unversioned()); - wr << (uint8_t)hashlittle(&version, sizeof(version), 0); - wr << bigEndian64(commitVersion); - wr << bigEndian32(part); - return wr.toValue(); - } - - // Start a new block if needed, then write the key and value - ACTOR static Future writeKV_impl(LogFileWriter* self, Key k, Value v) { - // If key and value do not fit in this block, end it and start a new one - int toWrite = sizeof(int32_t) + k.size() + sizeof(int32_t) + v.size(); - if (self->file->size() + toWrite > self->blockEnd) { - // Write padding if needed - int bytesLeft = self->blockEnd - self->file->size(); - if (bytesLeft > 0) { - state Value paddingFFs = fileBackup::makePadding(bytesLeft); - wait(self->file->append(paddingFFs.begin(), bytesLeft)); - } - - // Set new blockEnd - self->blockEnd += self->blockSize; - - // write Header - wait(self->file->append((uint8_t*)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION))); - } - - wait(self->file->appendStringRefWithLen(k)); - wait(self->file->appendStringRefWithLen(v)); - - // At this point we should be in whatever the current block is or the block size is too small - if (self->file->size() > self->blockEnd) - throw backup_bad_block_size(); - - return Void(); - } - - Future writeKV(Key k, Value v) { return writeKV_impl(this, k, v); } - - // Adds a new mutation to an internal buffer and writes out when encountering - // a new commitVersion or exceeding the block size. - ACTOR static Future addMutation(LogFileWriter* self, Version commitVersion, MutationListRef mutations) { - state Standalone value = BinaryWriter::toValue(mutations, IncludeVersion()); - - state int part = 0; - for (; part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE < value.size(); part++) { - StringRef partBuf = value.substr( - part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE, - std::min(value.size() - part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE, CLIENT_KNOBS->MUTATION_BLOCK_SIZE)); - Standalone key = getBlockKey(commitVersion, part); - wait(writeKV_impl(self, key, partBuf)); - } - return Void(); - } - -private: - Reference file; - int blockSize; - int64_t blockEnd = 0; -}; - -ACTOR Future convert(ConvertParams params) { - state Reference container = - IBackupContainer::openContainer(params.container_url, params.proxy, {}); - state BackupFileList listing = wait(container->dumpFileList()); - std::sort(listing.logs.begin(), listing.logs.end()); - TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size()); - state BackupDescription desc = wait(container->describeBackup()); - std::cout << "\n" << desc.toString() << "\n"; - - // std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec << - // "\n"; - - std::vector logs = getRelevantLogFiles(listing.logs, params.begin, params.end); - printLogFiles("Range has", logs); - - state Reference progress(new MutationFilesReadProgress(logs, params.begin, params.end)); - - wait(progress->openLogFiles(container)); - - state int blockSize = CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE; - state Reference outFile = wait(container->writeLogFile(params.begin, params.end, blockSize)); - state LogFileWriter logFile(outFile, blockSize); - std::cout << "Output file: " << outFile->getFileName() << "\n"; - - state MutationList list; - state Arena arena; - state Version version = invalidVersion; - while (progress->hasMutations()) { - state VersionedData data = wait(progress->getNextMutation()); - - // emit a mutation batch to file when encounter a new version - if (list.totalSize() > 0 && version != data.version.version) { - wait(LogFileWriter::addMutation(&logFile, version, list)); - list = MutationList(); - arena = Arena(); - } - - // keep getting data until a new version is encounter, then flush all data buffered and start to buffer for a - // new version. - ArenaReader rd(data.arena, data.message, AssumeVersion(g_network->protocolVersion())); - MutationRef m; - rd >> m; - std::cout << data.version.toString() << " m = " << m.toString() << "\n"; - list.push_back_deep(arena, m); - version = data.version.version; - } - if (list.totalSize() > 0) { - wait(LogFileWriter::addMutation(&logFile, version, list)); - } - - wait(outFile->finish()); - - return Void(); -} - -int parseCommandLine(ConvertParams* param, CSimpleOpt* args) { - while (args->Next()) { - auto lastError = args->LastError(); - switch (lastError) { - case SO_SUCCESS: - break; - - default: - std::cerr << "ERROR: argument given for option: " << args->OptionText() << "\n"; - return FDB_EXIT_ERROR; - break; - } - - int optId = args->OptionId(); - const char* arg = args->OptionArg(); - switch (optId) { - case OPT_HELP: - printConvertUsage(); - return FDB_EXIT_ERROR; - - case OPT_BEGIN_VERSION: - if (!sscanf(arg, "%" SCNd64, ¶m->begin)) { - std::cerr << "ERROR: could not parse begin version " << arg << "\n"; - printConvertUsage(); - return FDB_EXIT_ERROR; - } - break; - - case OPT_END_VERSION: - if (!sscanf(arg, "%" SCNd64, ¶m->end)) { - std::cerr << "ERROR: could not parse end version " << arg << "\n"; - printConvertUsage(); - return FDB_EXIT_ERROR; - } - break; - - case OPT_CONTAINER: - param->container_url = args->OptionArg(); - break; - - case OPT_TRACE: - param->log_enabled = true; - break; - - case OPT_TRACE_DIR: - param->log_dir = args->OptionArg(); - break; - - case OPT_TRACE_FORMAT: - if (!validateTraceFormat(args->OptionArg())) { - std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n"; - return FDB_EXIT_ERROR; - } - param->trace_format = args->OptionArg(); - break; - - case OPT_TRACE_LOG_GROUP: - param->trace_log_group = args->OptionArg(); - break; - case OPT_BUILD_FLAGS: - printBuildInformation(); - return FDB_EXIT_ERROR; - break; - } - } - return FDB_EXIT_SUCCESS; -} - -} // namespace file_converter - -int main(int argc, char** argv) { - try { - CSimpleOpt* args = - new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - file_converter::ConvertParams param; - int status = file_converter::parseCommandLine(¶m, args); - std::cout << "Params: " << param.toString() << "\n"; - if (status != FDB_EXIT_SUCCESS || !param.isValid()) { - file_converter::printConvertUsage(); - return status; - } - - if (param.log_enabled) { - if (param.log_dir.empty()) { - setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); - } else { - setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param.log_dir)); - } - if (!param.trace_format.empty()) { - setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format)); - } - if (!param.trace_log_group.empty()) { - setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group)); - } - } - - platformInit(); - Error::init(); - - StringRef url(param.container_url); - setupNetwork(0, UseMetrics::True); - - TraceEvent::setNetworkThread(); - openTraceFile({}, 10 << 20, 10 << 20, param.log_dir, "convert", param.trace_log_group); - - auto f = stopAfter(convert(param)); - - runNetwork(); - return status; - } catch (Error& e) { - fprintf(stderr, "ERROR: %s\n", e.what()); - return FDB_EXIT_ERROR; - } catch (std::exception& e) { - TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); - return FDB_EXIT_MAIN_EXCEPTION; - } -} \ No newline at end of file diff --git a/fdbbackup/FileConverter.cpp b/fdbbackup/FileConverter.cpp new file mode 100644 index 00000000000..27cf69243c1 --- /dev/null +++ b/fdbbackup/FileConverter.cpp @@ -0,0 +1,617 @@ +/* + * FileConverter.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbbackup/FileConverter.h" + +#include +#include +#include +#include +#include + +#include "fdbclient/BackupAgent.h" +#include "fdbclient/BackupContainer.h" +#include "fdbclient/MutationList.h" +#include "flow/flow.h" +#include "flow/serialize.h" +#include "fdbclient/BuildFlags.h" + +namespace file_converter { + +void printConvertUsage() { + std::cout << "\n" + << " -r, --container Container URL.\n" + << " -b, --begin BEGIN\n" + << " Begin version.\n" + << " -e, --end END End version.\n" + << " --log Enables trace file logging for the CLI session.\n" + << " --logdir PATH Specifies the output directory for trace files. If\n" + << " unspecified, defaults to the current directory. Has\n" + << " no effect unless --log is specified.\n" + << " --loggroup LOG_GROUP\n" + << " Sets the LogGroup field with the specified value for all\n" + << " events in the trace output (defaults to `default').\n" + << " --trace-format FORMAT\n" + << " Select the format of the trace files. xml (the default) and json are supported.\n" + << " Has no effect unless --log is specified.\n" + << " --build-flags Print build information and exit.\n" + << " -h, --help Display this help and exit.\n" + << "\n"; + + return; +} + +void printBuildInformation() { + printf("%s", jsonBuildInformation().c_str()); +} + +void printLogFiles(std::string msg, const std::vector& files) { + std::cout << msg << " " << files.size() << " log files\n"; + for (const auto& file : files) { + std::cout << file.toString() << "\n"; + } + std::cout << std::endl; +} + +std::vector getRelevantLogFiles(const std::vector& files, Version begin, Version end) { + std::vector filtered; + for (const auto& file : files) { + if (file.beginVersion <= end && file.endVersion >= begin && file.tagId >= 0 && file.fileSize > 0) { + filtered.push_back(file); + } + } + std::sort(filtered.begin(), filtered.end()); + + // Remove duplicates. This is because backup workers may store the log for + // old epochs successfully, but do not update the progress before another + // recovery happened. As a result, next epoch will retry and creates + // duplicated log files. + std::vector sorted; + int i = 0; + for (int j = 1; j < filtered.size(); j++) { + if (!filtered[i].isSubset(filtered[j])) { + sorted.push_back(filtered[i]); + } + i = j; + } + if (i < filtered.size()) { + sorted.push_back(filtered[i]); + } + + return sorted; +} + +struct ConvertParams { + std::string container_url; + Optional proxy; + Version begin = invalidVersion; + Version end = invalidVersion; + bool log_enabled = false; + std::string log_dir, trace_format, trace_log_group; + + bool isValid() const { return begin != invalidVersion && end != invalidVersion && !container_url.empty(); } + + std::string toString() const { + std::string s; + s.append("ContainerURL:"); + s.append(container_url); + if (proxy.present()) { + s.append(" Proxy:"); + s.append(proxy.get()); + } + s.append(" Begin:"); + s.append(format("%" PRId64, begin)); + s.append(" End:"); + s.append(format("%" PRId64, end)); + if (log_enabled) { + if (!log_dir.empty()) { + s.append(" LogDir:").append(log_dir); + } + if (!trace_format.empty()) { + s.append(" Format:").append(trace_format); + } + if (!trace_log_group.empty()) { + s.append(" LogGroup:").append(trace_log_group); + } + } + return s; + } +}; + +struct VersionedData { + LogMessageVersion version; + StringRef message; // Serialized mutation. + Arena arena; // The arena that contains mutation. + + VersionedData() : version(invalidVersion, -1) {} + VersionedData(LogMessageVersion v, StringRef m, Arena a) : version(v), message(m), arena(a) {} +}; + +struct MutationFilesReadProgress : public ReferenceCounted { + MutationFilesReadProgress(std::vector& logs, Version begin, Version end) + : files(logs), beginVersion(begin), endVersion(end) {} + + struct FileProgress : public ReferenceCounted { + FileProgress(Reference f, int index) : fd(f), idx(index), offset(0), eof(false) {} + + bool operator<(const FileProgress& rhs) const { + if (rhs.mutations.empty()) + return true; + if (mutations.empty()) + return false; + return mutations[0].version < rhs.mutations[0].version; + } + bool operator<=(const FileProgress& rhs) const { + if (rhs.mutations.empty()) + return true; + if (mutations.empty()) + return false; + return mutations[0].version <= rhs.mutations[0].version; + } + bool empty() { return eof && mutations.empty(); } + + // Decodes the block into mutations and save them if >= minVersion and < maxVersion. + // Returns true if new mutations has been saved. + bool decodeBlock(const Standalone& buf, int len, Version minVersion, Version maxVersion) { + StringRef block(buf.begin(), len); + StringRefReader reader(block, restore_corrupted_data()); + int count = 0, inserted = 0; + Version msgVersion = invalidVersion; + + try { + // Read block header + if (reader.consume() != PARTITIONED_MLOG_VERSION) + throw restore_unsupported_file_version(); + + while (1) { + // If eof reached or first key len bytes is 0xFF then end of block was reached. + if (reader.eof() || *reader.rptr == 0xFF) + break; + + // Deserialize messages written in saveMutationsToFile(). + msgVersion = bigEndian64(reader.consume()); + uint32_t sub = bigEndian32(reader.consume()); + int msgSize = bigEndian32(reader.consume()); + const uint8_t* message = reader.consume(msgSize); + + ArenaReader rd( + buf.arena(), StringRef(message, msgSize), AssumeVersion(g_network->protocolVersion())); + MutationRef m; + rd >> m; + count++; + if (msgVersion >= maxVersion) { + TraceEvent("FileDecodeEnd") + .detail("MaxV", maxVersion) + .detail("Version", msgVersion) + .detail("File", fd->getFilename()); + eof = true; + break; // skip + } + if (msgVersion >= minVersion) { + mutations.emplace_back( + LogMessageVersion(msgVersion, sub), StringRef(message, msgSize), buf.arena()); + inserted++; + } + } + offset += len; + + TraceEvent("Decoded") + .detail("Name", fd->getFilename()) + .detail("Count", count) + .detail("Insert", inserted) + .detail("BlockOffset", reader.rptr - buf.begin()) + .detail("Total", mutations.size()) + .detail("EOF", eof) + .detail("Version", msgVersion) + .detail("NewOffset", offset); + return inserted > 0; + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptLogFileBlock") + .error(e) + .detail("Filename", fd->getFilename()) + .detail("BlockOffset", offset) + .detail("BlockLen", len) + .detail("ErrorRelativeOffset", reader.rptr - buf.begin()) + .detail("ErrorAbsoluteOffset", reader.rptr - buf.begin() + offset); + throw; + } + } + + Reference fd; + int idx; // index in the MutationFilesReadProgress::files vector + int64_t offset; // offset of the file to be read + bool eof; // If EOF is seen so far or endVersion is encountered. If true, the file can't be read further. + std::vector mutations; // Buffered mutations read so far + }; + + bool hasMutations() { + for (const auto& fp : fileProgress) { + if (!fp->empty()) + return true; + } + return false; + } + + void dumpProgress(std::string msg) { + std::cout << msg << "\n "; + for (const auto& fp : fileProgress) { + std::cout << fp->fd->getFilename() << " " << fp->mutations.size() << " mutations"; + if (!fp->mutations.empty()) { + std::cout << ", range " << fp->mutations[0].version.toString() << " " + << fp->mutations.back().version.toString() << "\n"; + } else { + std::cout << "\n\n"; + } + } + } + + // Sorts files according to their first mutation version and removes files without mutations. + void sortAndRemoveEmpty() { + std::sort(fileProgress.begin(), + fileProgress.end(), + [](const Reference& a, const Reference& b) { return (*a) < (*b); }); + while (!fileProgress.empty() && fileProgress.back()->empty()) { + fileProgress.pop_back(); + } + } + + // Requires hasMutations() return true before calling this function. + // The caller must hold on the the arena associated with the mutation. + Future getNextMutation() { return getMutationImpl(this); } + + static Future getMutationImpl(MutationFilesReadProgress* self) { + ASSERT(!self->fileProgress.empty() && !self->fileProgress[0]->mutations.empty()); + + Reference fp = self->fileProgress[0]; + VersionedData data = fp->mutations[0]; + fp->mutations.erase(fp->mutations.begin()); + if (fp->mutations.empty()) { + // decode one more block + co_await decodeToVersion(fp, /*version=*/0, self->endVersion, self->getLogFile(fp->idx)); + } + + if (fp->empty()) { + self->fileProgress.erase(self->fileProgress.begin()); + } else { + // Keep fileProgress sorted + for (int i = 1; i < self->fileProgress.size(); i++) { + if (*self->fileProgress[i - 1] <= *self->fileProgress[i]) { + break; + } + std::swap(self->fileProgress[i - 1], self->fileProgress[i]); + } + } + co_return data; + } + + LogFile& getLogFile(int index) { return files[index]; } + + Future openLogFiles(Reference container) { return openLogFilesImpl(this, container); } + + // Opens log files in the progress and starts decoding until the beginVersion is seen. + static Future openLogFilesImpl(MutationFilesReadProgress* progress, Reference container) { + std::vector>> asyncFiles; + for (const auto& file : progress->files) { + asyncFiles.push_back(container->readFile(file.fileName)); + } + co_await waitForAll(asyncFiles); // open all files + + // Attempt decode the first few blocks of log files until beginVersion is consumed + std::vector> fileDecodes; + for (int i = 0; i < asyncFiles.size(); i++) { + auto fp = makeReference(asyncFiles[i].get(), i); + progress->fileProgress.push_back(fp); + fileDecodes.push_back( + decodeToVersion(fp, progress->beginVersion, progress->endVersion, progress->getLogFile(i))); + } + + co_await waitForAll(fileDecodes); + + progress->sortAndRemoveEmpty(); + } + + // Decodes the file until EOF or an mutation >= minVersion and saves these mutations. + // Skip mutations >= maxVersion. + static Future decodeToVersion(Reference fp, + Version minVersion, + Version maxVersion, + LogFile file) { + if (fp->empty()) { + co_return; + } + + if (!fp->mutations.empty() && fp->mutations.back().version.version >= minVersion) { + co_return; + } + + int64_t len = 0; + try { + // Read block by block until we see the minVersion + while (true) { + len = std::min(file.blockSize, file.fileSize - fp->offset); + if (len == 0) { + fp->eof = true; + co_return; + } + + Standalone buf = makeString(len); + int rLen = co_await fp->fd->read(mutateString(buf), len, fp->offset); + if (len != rLen) + throw restore_bad_read(); + + TraceEvent("ReadFile") + .detail("Name", fp->fd->getFilename()) + .detail("Length", rLen) + .detail("Offset", fp->offset); + if (fp->decodeBlock(buf, rLen, minVersion, maxVersion)) + break; + } + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptedLogFileBlock") + .error(e) + .detail("Filename", fp->fd->getFilename()) + .detail("BlockOffset", fp->offset) + .detail("BlockLen", len); + throw; + } + } + + std::vector files; + const Version beginVersion, endVersion; + std::vector> fileProgress; +}; + +// Writes a log file in the old backup format, described in backup-dataFormat.md. +// This is similar to the LogFileWriter in FileBackupAgent.cpp. +struct LogFileWriter { + LogFileWriter() : blockSize(-1) {} + LogFileWriter(Reference f, int bsize) : file(f), blockSize(bsize) {} + + // Returns the block key, i.e., `Param1`, in the back file. The format is + // `hash_value|commitVersion|part`. + static Standalone getBlockKey(Version commitVersion, int part) { + const int32_t version = commitVersion / CLIENT_KNOBS->LOG_RANGE_BLOCK_SIZE; + + BinaryWriter wr(Unversioned()); + wr << (uint8_t)hashlittle(&version, sizeof(version), 0); + wr << bigEndian64(commitVersion); + wr << bigEndian32(part); + return wr.toValue(); + } + + // Start a new block if needed, then write the key and value + static Future writeKV_impl(LogFileWriter* self, Key k, Value v) { + // If key and value do not fit in this block, end it and start a new one + int toWrite = sizeof(int32_t) + k.size() + sizeof(int32_t) + v.size(); + if (self->file->size() + toWrite > self->blockEnd) { + // Write padding if needed + int bytesLeft = self->blockEnd - self->file->size(); + if (bytesLeft > 0) { + Value paddingFFs = fileBackup::makePadding(bytesLeft); + co_await self->file->append(paddingFFs.begin(), bytesLeft); + } + + // Set new blockEnd + self->blockEnd += self->blockSize; + + // write Header + co_await self->file->append((uint8_t*)&BACKUP_AGENT_MLOG_VERSION, sizeof(BACKUP_AGENT_MLOG_VERSION)); + } + + co_await self->file->appendStringRefWithLen(k); + co_await self->file->appendStringRefWithLen(v); + + // At this point we should be in whatever the current block is or the block size is too small + if (self->file->size() > self->blockEnd) + throw backup_bad_block_size(); + } + + Future writeKV(Key k, Value v) { return writeKV_impl(this, k, v); } + + // Adds a new mutation to an internal buffer and writes out when encountering + // a new commitVersion or exceeding the block size. + static Future addMutation(LogFileWriter* self, Version commitVersion, MutationListRef mutations) { + Standalone value = BinaryWriter::toValue(mutations, IncludeVersion()); + + int part = 0; + for (; part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE < value.size(); part++) { + StringRef partBuf = value.substr( + part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE, + std::min(value.size() - part * CLIENT_KNOBS->MUTATION_BLOCK_SIZE, CLIENT_KNOBS->MUTATION_BLOCK_SIZE)); + Standalone key = getBlockKey(commitVersion, part); + co_await writeKV_impl(self, key, partBuf); + } + } + +private: + Reference file; + int blockSize; + int64_t blockEnd = 0; +}; + +Future convert(ConvertParams params) { + Reference container = IBackupContainer::openContainer(params.container_url, params.proxy, {}, 0); + BackupFileList listing = co_await container->dumpFileList(); + std::sort(listing.logs.begin(), listing.logs.end()); + TraceEvent("Container").detail("URL", params.container_url).detail("Logs", listing.logs.size()); + BackupDescription desc = co_await container->describeBackup(); + std::cout << "\n" << desc.toString() << "\n"; + + // std::cout << "Using Protocol Version: 0x" << std::hex << g_network->protocolVersion().version() << std::dec << + // "\n"; + + std::vector logs = getRelevantLogFiles(listing.logs, params.begin, params.end); + printLogFiles("Range has", logs); + + Reference progress(new MutationFilesReadProgress(logs, params.begin, params.end)); + + co_await progress->openLogFiles(container); + + int blockSize = CLIENT_KNOBS->BACKUP_LOGFILE_BLOCK_SIZE; + Reference outFile = co_await container->writeLogFile(params.begin, params.end, blockSize); + LogFileWriter logFile(outFile, blockSize); + std::cout << "Output file: " << outFile->getFileName() << "\n"; + + MutationList list; + Arena arena; + Version version = invalidVersion; + while (progress->hasMutations()) { + VersionedData data = co_await progress->getNextMutation(); + + // emit a mutation batch to file when encounter a new version + if (list.totalSize() > 0 && version != data.version.version) { + co_await LogFileWriter::addMutation(&logFile, version, list); + list = MutationList(); + arena = Arena(); + } + + // keep getting data until a new version is encounter, then flush all data buffered and start to buffer for a + // new version. + ArenaReader rd(data.arena, data.message, AssumeVersion(g_network->protocolVersion())); + MutationRef m; + rd >> m; + std::cout << data.version.toString() << " m = " << m.toString() << "\n"; + list.push_back_deep(arena, m); + version = data.version.version; + } + if (list.totalSize() > 0) { + co_await LogFileWriter::addMutation(&logFile, version, list); + } + + co_await outFile->finish(); +} + +int parseCommandLine(ConvertParams* param, CSimpleOpt* args) { + while (args->Next()) { + auto lastError = args->LastError(); + switch (lastError) { + case SO_SUCCESS: + break; + + default: + std::cerr << "ERROR: argument given for option: " << args->OptionText() << "\n"; + return FDB_EXIT_ERROR; + break; + } + + int optId = args->OptionId(); + const char* arg = args->OptionArg(); + switch (optId) { + case OPT_HELP: + printConvertUsage(); + return FDB_EXIT_ERROR; + + case OPT_BEGIN_VERSION: + if (!sscanf(arg, "%" SCNd64, ¶m->begin)) { + std::cerr << "ERROR: could not parse begin version " << arg << "\n"; + printConvertUsage(); + return FDB_EXIT_ERROR; + } + break; + + case OPT_END_VERSION: + if (!sscanf(arg, "%" SCNd64, ¶m->end)) { + std::cerr << "ERROR: could not parse end version " << arg << "\n"; + printConvertUsage(); + return FDB_EXIT_ERROR; + } + break; + + case OPT_CONTAINER: + param->container_url = args->OptionArg(); + break; + + case OPT_TRACE: + param->log_enabled = true; + break; + + case OPT_TRACE_DIR: + param->log_dir = args->OptionArg(); + break; + + case OPT_TRACE_FORMAT: + if (!validateTraceFormat(args->OptionArg())) { + std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n"; + return FDB_EXIT_ERROR; + } + param->trace_format = args->OptionArg(); + break; + + case OPT_TRACE_LOG_GROUP: + param->trace_log_group = args->OptionArg(); + break; + case OPT_BUILD_FLAGS: + printBuildInformation(); + return FDB_EXIT_ERROR; + break; + } + } + return FDB_EXIT_SUCCESS; +} + +} // namespace file_converter + +int main(int argc, char** argv) { + try { + CSimpleOpt* args = + new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + file_converter::ConvertParams param; + int status = file_converter::parseCommandLine(¶m, args); + std::cout << "Params: " << param.toString() << "\n"; + if (status != FDB_EXIT_SUCCESS || !param.isValid()) { + file_converter::printConvertUsage(); + return status; + } + + if (param.log_enabled) { + if (param.log_dir.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); + } else { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param.log_dir)); + } + if (!param.trace_format.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param.trace_format)); + } + if (!param.trace_log_group.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param.trace_log_group)); + } + } + + platformInit(); + Error::init(); + + StringRef url(param.container_url); + setupNetwork(0, UseMetrics::True); + + TraceEvent::setNetworkThread(); + openTraceFile({}, 10 << 20, 10 << 20, param.log_dir, "convert", param.trace_log_group); + + auto f = stopAfter(convert(param)); + + runNetwork(); + return status; + } catch (Error& e) { + fprintf(stderr, "ERROR: %s\n", e.what()); + return FDB_EXIT_ERROR; + } catch (std::exception& e) { + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + return FDB_EXIT_MAIN_EXCEPTION; + } +} diff --git a/fdbbackup/FileDecoder.actor.cpp b/fdbbackup/FileDecoder.actor.cpp deleted file mode 100644 index 71872010481..00000000000 --- a/fdbbackup/FileDecoder.actor.cpp +++ /dev/null @@ -1,957 +0,0 @@ -/* - * FileDecoder.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef _WIN32 -#include -#endif - -#include "fdbclient/BackupTLSConfig.h" -#include "fdbclient/BuildFlags.h" -#include "fdbbackup/FileConverter.h" -#include "fdbbackup/Decode.h" -#include "fdbclient/BackupAgent.actor.h" -#include "fdbclient/BackupContainer.h" -#include "fdbclient/BackupContainerFileSystem.h" -#include "fdbclient/BuildFlags.h" -#include "fdbclient/CommitTransaction.h" -#include "fdbclient/FDBTypes.h" -#include "fdbclient/IKnobCollection.h" -#include "fdbclient/KeyRangeMap.h" -#include "fdbclient/Knobs.h" -#include "fdbclient/MutationList.h" -#include "fdbclient/SystemData.h" -#include "fdbclient/versions.h" -#include "flow/ArgParseUtil.h" -#include "flow/FastRef.h" -#include "flow/IRandom.h" -#include "flow/Platform.h" -#include "flow/Trace.h" -#include "flow/flow.h" -#include "flow/serialize.h" - -#include "flow/actorcompiler.h" // has to be last include - -#define SevDecodeInfo SevVerbose - -extern bool g_crashOnError; -extern const char* getSourceVersion(); - -namespace file_converter { - -void printDecodeUsage() { - std::cout << "Decoder for FoundationDB backup mutation logs.\n" - "Usage: fdbdecode [OPTIONS]\n" - " -r, --container URL\n" - " Backup container URL, e.g., file:///some/path/.\n" - " -i, --input FILE\n" - " Log file filter, only matched files are decoded.\n" - " --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifies the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n" - " --loggroup LOG_GROUP\n" - " Sets the LogGroup field with the specified value for all\n" - " events in the trace output (defaults to `default').\n" - " --trace-format FORMAT\n" - " Select the format of the trace files, xml (the default) or json.\n" - " Has no effect unless --log is specified.\n" - " --crash Crash on serious error.\n" - " --blob-credentials FILE\n" - " File containing blob credentials in JSON format.\n" - " The same credential format/file fdbbackup uses.\n" TLS_HELP - " -t, --file-type [log|range|both]\n" - " Specifies the backup file type to decode.\n" - " --build-flags Print build information and exit.\n" - " --list-only Print file list and exit.\n" - " --validate-filters Validate the default RangeMap filtering logic with a slower one.\n" - " -k KEY_PREFIX Use a single prefix for filtering mutations.\n" - " --filters PREFIX_FILTER_FILE\n" - " A file containing a list of prefix filters in HEX format separated by \";\",\n" - " e.g., \"\\x05\\x01;\\x15\\x2b\"\n" - " --hex-prefix HEX_PREFIX\n" - " The prefix specified in HEX format, e.g., --hex-prefix \"\\\\x05\\\\x01\".\n" - " --begin-version-filter BEGIN_VERSION\n" - " The version range's begin version (inclusive) for filtering.\n" - " --end-version-filter END_VERSION\n" - " The version range's end version (exclusive) for filtering.\n" - " --knob-KNOBNAME KNOBVALUE\n" - " Changes a knob value. KNOBNAME should be lowercase.\n" - " -s, --save Save a copy of downloaded files (default: not saving).\n" - "\n"; - return; -} - -void printBuildInformation() { - std::cout << jsonBuildInformation() << "\n"; -} - -struct DecodeParams : public ReferenceCounted { - std::string container_url; - Optional proxy; - std::string fileFilter; // only files match the filter will be decoded - bool log_enabled = true; - std::string log_dir, trace_format, trace_log_group; - BackupTLSConfig tlsConfig; - bool list_only = false; - bool decode_logs = true; - bool decode_range = true; - bool save_file_locally = false; - bool validate_filters = false; - std::vector prefixes; // Key prefixes for filtering - // more efficient data structure for intersection queries than "prefixes" - fileBackup::RangeMapFilters filters; - Version beginVersionFilter = 0; - Version endVersionFilter = std::numeric_limits::max(); - - std::vector> knobs; - - // Returns if [begin, end) overlap with the filter range - bool overlap(Version begin, Version end) const { - // Filter [100, 200), [50,75) [200, 300) - return !(begin >= endVersionFilter || end <= beginVersionFilter); - } - - bool overlap(Version version) const { return version >= beginVersionFilter && version < endVersionFilter; } - - void updateRangeMap() { filters.updateFilters(prefixes); } - - bool matchFilters(const MutationRef& m) const { - bool match = filters.match(m); - if (!validate_filters) { - return match; - } - - // If we choose to validate the filters, go through filters one by one - for (const auto& prefix : prefixes) { - if (isSingleKeyMutation((MutationRef::Type)m.type)) { - if (m.param1.startsWith(StringRef(prefix))) { - ASSERT(match); - return true; - } - } else if (m.type == MutationRef::ClearRange) { - KeyRange range(KeyRangeRef(m.param1, m.param2)); - KeyRange range2 = prefixRange(StringRef(prefix)); - if (range.intersects(range2)) { - ASSERT(match); - return true; - } - } else { - ASSERT(false); - } - } - ASSERT(!match); - return false; - } - - bool matchFilters(const KeyRange& range) const { - bool match = filters.match(range); - if (!validate_filters) { - return match; - } - - for (const auto& prefix : prefixes) { - if (range.intersects(prefixRange(StringRef(prefix)))) { - ASSERT(match); - return true; - } - } - return false; - } - - bool matchFilters(KeyValueRef kv) const { - bool match = filters.match(kv); - - if (!validate_filters) { - return match; - } - - for (const auto& prefix : prefixes) { - if (kv.key.startsWith(StringRef(prefix))) { - ASSERT(match); - return true; - } - } - - return match; - } - - std::string toString() { - std::string s; - s.append("ContainerURL: "); - s.append(container_url); - if (proxy.present()) { - s.append(", Proxy: "); - s.append(proxy.get()); - } - s.append(", FileFilter: "); - s.append(fileFilter); - if (log_enabled) { - if (!log_dir.empty()) { - s.append(" LogDir:").append(log_dir); - } - if (!trace_format.empty()) { - s.append(" Format:").append(trace_format); - } - if (!trace_log_group.empty()) { - s.append(" LogGroup:").append(trace_log_group); - } - } - s.append(", list_only: ").append(list_only ? "true" : "false"); - s.append(", validate_filters: ").append(validate_filters ? "true" : "false"); - if (beginVersionFilter != 0) { - s.append(", beginVersionFilter: ").append(std::to_string(beginVersionFilter)); - } - if (endVersionFilter < std::numeric_limits::max()) { - s.append(", endVersionFilter: ").append(std::to_string(endVersionFilter)); - } - if (!prefixes.empty()) { - s.append(", KeyPrefixes: ").append(printable(describe(prefixes))); - } - for (const auto& [knob, value] : knobs) { - s.append(", KNOB-").append(knob).append(" = ").append(value); - } - s.append(", SaveFile: ").append(save_file_locally ? "true" : "false"); - return s; - } - - void updateKnobs() { - IKnobCollection::setupKnobs(knobs); - - // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs - IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False); - } -}; - -// Parses and returns a ";" separated HEX encoded strings. So the ";" in -// the string should be escaped as "\;". -// Sets "err" to true if there is any parsing error. -std::vector parsePrefixesLine(const std::string& line, bool& err) { - std::vector results; - err = false; - - int p = 0; - while (p < line.size()) { - int end = line.find_first_of(';', p); - if (end == line.npos) { - end = line.size(); - } - auto prefix = decode_hex_string(line.substr(p, end - p), err); - if (err) { - return results; - } - results.push_back(prefix); - p = end + 1; - } - return results; -} - -std::vector parsePrefixFile(const std::string& filename, bool& err) { - std::string line = readFileBytes(filename, 64 * 1024 * 1024); - return parsePrefixesLine(line, err); -} - -int parseDecodeCommandLine(Reference param, CSimpleOpt* args) { - bool err = false; - - while (args->Next()) { - auto lastError = args->LastError(); - switch (lastError) { - case SO_SUCCESS: - break; - - default: - std::cerr << "ERROR: argument given for option: " << args->OptionText() << "\n"; - return FDB_EXIT_ERROR; - break; - } - int optId = args->OptionId(); - switch (optId) { - case OPT_HELP: - return FDB_EXIT_ERROR; - - case OPT_CONTAINER: - param->container_url = args->OptionArg(); - break; - - case OPT_FILE_TYPE: { - auto ftype = std::string(args->OptionArg()); - if (ftype == "log") { - param->decode_range = false; - } else if (ftype == "range") { - param->decode_logs = false; - } else if (ftype != "both" && ftype != "") { - err = true; - std::cerr << "ERROR: Unrecognized backup file type option: " << args->OptionArg() << "\n"; - return FDB_EXIT_ERROR; - } - break; - } - - case OPT_LIST_ONLY: - param->list_only = true; - break; - - case OPT_VALIDATE_FILTERS: - param->validate_filters = true; - break; - - case OPT_KEY_PREFIX: - param->prefixes.push_back(args->OptionArg()); - break; - - case OPT_FILTERS: - param->prefixes = parsePrefixFile(args->OptionArg(), err); - if (err) { - throw std::runtime_error("ERROR:" + std::string(args->OptionArg()) + "contains invalid prefix(es)"); - } - break; - - case OPT_HEX_KEY_PREFIX: - param->prefixes.push_back(decode_hex_string(args->OptionArg(), err)); - break; - - case OPT_PROXY: - param->proxy = args->OptionArg(); - break; - - case OPT_BEGIN_VERSION_FILTER: - param->beginVersionFilter = std::atoll(args->OptionArg()); - break; - - case OPT_END_VERSION_FILTER: - param->endVersionFilter = std::atoll(args->OptionArg()); - break; - - case OPT_CRASHONERROR: - g_crashOnError = true; - break; - - case OPT_INPUT_FILE: - param->fileFilter = args->OptionArg(); - break; - - case OPT_TRACE: - param->log_enabled = true; - break; - - case OPT_TRACE_DIR: - param->log_dir = args->OptionArg(); - break; - - case OPT_TRACE_FORMAT: - if (!selectTraceFormatter(args->OptionArg())) { - std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n"; - return FDB_EXIT_ERROR; - } - param->trace_format = args->OptionArg(); - break; - - case OPT_TRACE_LOG_GROUP: - param->trace_log_group = args->OptionArg(); - break; - - case OPT_BLOB_CREDENTIALS: - param->tlsConfig.blobCredentials.push_back(args->OptionArg()); - break; - - case OPT_KNOB: { - Optional knobName = extractPrefixedArgument("--knob", args->OptionSyntax()); - if (!knobName.present()) { - std::cerr << "ERROR: unable to parse knob option '" << args->OptionSyntax() << "'\n"; - return FDB_EXIT_ERROR; - } - param->knobs.emplace_back(knobName.get(), args->OptionArg()); - break; - } - - case OPT_SAVE_FILE: - param->save_file_locally = true; - break; - - case TLSConfig::OPT_TLS_PLUGIN: - args->OptionArg(); - break; - - case TLSConfig::OPT_TLS_CERTIFICATES: - param->tlsConfig.tlsCertPath = args->OptionArg(); - break; - - case TLSConfig::OPT_TLS_PASSWORD: - param->tlsConfig.tlsPassword = args->OptionArg(); - break; - - case TLSConfig::OPT_TLS_CA_FILE: - param->tlsConfig.tlsCAPath = args->OptionArg(); - break; - - case TLSConfig::OPT_TLS_KEY: - param->tlsConfig.tlsKeyPath = args->OptionArg(); - break; - - case TLSConfig::OPT_TLS_VERIFY_PEERS: - param->tlsConfig.tlsVerifyPeers = args->OptionArg(); - break; - - case OPT_BUILD_FLAGS: - printBuildInformation(); - return FDB_EXIT_ERROR; - break; - } - } - return FDB_EXIT_SUCCESS; -} - -template -void printLogFiles(std::string msg, const std::vector& files) { - std::cout << msg << " " << files.size() << " total\n"; - for (const auto& file : files) { - std::cout << file.toString() << "\n"; - } - std::cout << std::endl; -} - -std::vector getRelevantLogFiles(const std::vector& files, const Reference params) { - std::vector filtered; - for (const auto& file : files) { - if (file.fileName.find(params->fileFilter) != std::string::npos && - params->overlap(file.beginVersion, file.endVersion + 1)) { - filtered.push_back(file); - } - } - return filtered; -} - -std::vector getRelevantRangeFiles(const std::vector& files, - const Reference params) { - std::vector filtered; - for (const auto& file : files) { - if (file.fileName.find(params->fileFilter) != std::string::npos && params->overlap(file.version)) { - filtered.push_back(file); - } - } - return filtered; -} - -struct VersionedMutations { - Version version; - std::vector mutations; - std::string serializedMutations; // buffer that contains mutations -}; - -/* - * Model a decoding progress for a mutation file. Usage is: - * - * DecodeProgress progress(logfile); - * wait(progress->openFile(container)); - * while (1) { - * Optional batch = wait(progress->getNextBatch()); - * if (!batch.present()) break; - * ... // process the batch mutations - * } - * - * Internally, the decoding process is done block by block -- each block is - * decoded into a list of key/value pairs, which are then decoded into batches - * of mutations. Because a version's mutations can be split into many key/value - * pairs, the decoding of mutation needs to look ahead to find all batches that - * belong to the same version. - */ -class DecodeProgress { - std::vector>> blocks; - std::unordered_map mutationBlocksByVersion; - -public: - DecodeProgress() = default; - DecodeProgress(const LogFile& file, bool save) : file(file), save(save) {} - - ~DecodeProgress() { - if (lfd != -1) { - close(lfd); - } - } - - // Open and loads file into memory - Future openFile(Reference container) { return openFileImpl(this, container); } - - // The following are private APIs: - - // Returns the next batch of mutations along with the arena backing it. - // Note the returned batch can be empty when the file has unfinished - // version batch data that are in the next file. - Optional getNextBatch() { - for (auto& [version, m] : mutationBlocksByVersion) { - if (m.isComplete()) { - VersionedMutations vms; - vms.version = version; - vms.serializedMutations = m.serializedMutations; - vms.mutations = fileBackup::decodeMutationLogValue(vms.serializedMutations); - TraceEvent("Decode").detail("Version", vms.version).detail("N", vms.mutations.size()); - mutationBlocksByVersion.erase(version); - return vms; - } - } - - // No complete versions - if (!mutationBlocksByVersion.empty()) { - TraceEvent(SevWarn, "UnfishedBlocks").detail("NumberOfVersions", mutationBlocksByVersion.size()); - } - return Optional(); - } - - ACTOR static Future openFileImpl(DecodeProgress* self, Reference container) { - Reference fd = wait(container->readFile(self->file.fileName)); - self->fd = fd; - state Standalone buf = makeString(self->file.fileSize); - int rLen = wait(self->fd->read(mutateString(buf), self->file.fileSize, 0)); - if (rLen != self->file.fileSize) { - throw restore_bad_read(); - } - - if (self->save) { - std::string dir = self->file.fileName; - std::size_t found = self->file.fileName.find_last_of('/'); - if (found != std::string::npos) { - std::string path = self->file.fileName.substr(0, found); - if (!directoryExists(path)) { - platform::createDirectory(path); - } - } - self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (self->lfd == -1) { - TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName); - throw platform_error(); - } - int wlen = write(self->lfd, buf.begin(), self->file.fileSize); - if (wlen != self->file.fileSize) { - TraceEvent(SevError, "WriteLocalFileFailed") - .detail("File", self->file.fileName) - .detail("Len", self->file.fileSize); - throw platform_error(); - } - TraceEvent("WriteLocalFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize); - } - - self->decodeFile(buf); - return Void(); - } - - // Add chunks to mutationBlocksByVersion - void addBlockKVPairs(VectorRef chunks) { - for (auto& kv : chunks) { - auto versionAndChunkNumber = fileBackup::decodeMutationLogKey(kv.key); - mutationBlocksByVersion[versionAndChunkNumber.first].addChunk(versionAndChunkNumber.second, kv); - } - } - - // Reads a file a file content in the buffer, decodes it into key/value pairs, and stores these pairs. - void decodeFile(const Standalone& buf) { - try { - loop { - int64_t len = std::min(file.blockSize, file.fileSize - offset); - if (len == 0) { - return; - } - - // Decode a file block into log_key and log_value chunks - Standalone> chunks = - fileBackup::decodeMutationLogFileBlock(buf.substr(offset, len)); - blocks.push_back(chunks); - addBlockKVPairs(chunks); - offset += len; - } - } catch (Error& e) { - TraceEvent(SevWarn, "CorruptLogFileBlock") - .error(e) - .detail("Filename", file.fileName) - .detail("BlockOffset", offset) - .detail("BlockLen", file.blockSize); - throw; - } - } - - LogFile file; - Reference fd; - int64_t offset = 0; - bool eof = false; - bool save = false; - int lfd = -1; // local file descriptor -}; - -class DecodeRangeProgress { -public: - std::vector>> blocks; - - DecodeRangeProgress() = default; - DecodeRangeProgress(const RangeFile& file, bool save) : file(file), save(save) {} - ~DecodeRangeProgress() { - if (lfd != -1) { - close(lfd); - } - } - - // Open and loads file into memory - Future openFile(Reference container) { return openFileImpl(this, container); } - - ACTOR static Future openFileImpl(DecodeRangeProgress* self, Reference container) { - TraceEvent("ReadFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize); - - Reference fd = wait(container->readFile(self->file.fileName)); - self->fd = fd; - state Standalone buf = makeString(self->file.fileSize); - int rLen = wait(self->fd->read(mutateString(buf), self->file.fileSize, 0)); - if (rLen != self->file.fileSize) { - throw restore_bad_read(); - } - - if (self->save) { - std::string dir = self->file.fileName; - std::size_t found = self->file.fileName.find_last_of('/'); - if (found != std::string::npos) { - std::string path = self->file.fileName.substr(0, found); - if (!directoryExists(path)) { - platform::createDirectory(path); - } - } - - self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); - if (self->lfd == -1) { - TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName); - throw platform_error(); - } - int wlen = write(self->lfd, buf.begin(), self->file.fileSize); - if (wlen != self->file.fileSize) { - TraceEvent(SevError, "WriteLocalFileFailed") - .detail("File", self->file.fileName) - .detail("Len", self->file.fileSize); - throw platform_error(); - } - TraceEvent("WriteLocalFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize); - } - - self->decodeFile(buf); - return Void(); - } - - // Reads a file content in the buffer, decodes it into key/value pairs, and stores these pairs. - void decodeFile(const Standalone& buf) { - try { - loop { - // process one block at a time - int64_t len = std::min(file.blockSize, file.fileSize - offset); - if (len == 0) { - return; - } - - Standalone> chunks = fileBackup::decodeRangeFileBlock(buf.substr(offset, len)); - blocks.push_back(chunks); - offset += len; - } - } catch (Error& e) { - TraceEvent(SevWarn, "CorruptRangeFileBlock") - .error(e) - .detail("Filename", file.fileName) - .detail("BlockOffset", offset) - .detail("BlockLen", file.blockSize); - throw; - } - } - - RangeFile file; - Reference fd; - int64_t offset = 0; - bool save = false; - int lfd = -1; // local file descriptor -}; - -// convert a StringRef to Hex string -std::string hexStringRef(const StringRef& s) { - std::string result; - result.reserve(s.size() * 2); - for (int i = 0; i < s.size(); i++) { - result.append(format("%02x", s[i])); - } - return result; -} - -ACTOR Future process_range_file(Reference container, - RangeFile file, - UID uid, - Reference params) { - - if (file.fileSize == 0) { - TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName); - return Void(); - } - - state DecodeRangeProgress progress(file, params->save_file_locally); - wait(progress.openFile(container)); - - for (auto& block : progress.blocks) { - for (const auto& kv : block) { - bool print = params->prefixes.empty(); // no filtering - - if (!print) { - print = params->matchFilters(kv); - } - - if (print) { - TraceEvent(format("KVPair_%llu", file.version).c_str(), uid) - .detail("Version", file.version) - .setMaxFieldLength(1000) - .detail("KV", kv); - std::cout << file.version << " key: " << hexStringRef(kv.key) << " value: " << hexStringRef(kv.value) - << std::endl; - } - } - } - TraceEvent("ProcessRangeFileDone", uid).detail("File", file.fileName); - - return Void(); -} - -ACTOR Future process_file(Reference container, - LogFile file, - UID uid, - Reference params) { - if (file.fileSize == 0) { - TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName); - return Void(); - } - - state DecodeProgress progress(file, params->save_file_locally); - wait(progress.openFile(container)); - while (true) { - auto batch = progress.getNextBatch(); - if (!batch.present()) - break; - - const VersionedMutations& vms = batch.get(); - if (vms.version < params->beginVersionFilter || vms.version >= params->endVersionFilter) { - TraceEvent("SkipVersion").detail("Version", vms.version); - continue; - } - - int sub = 0; - for (const auto& m : vms.mutations) { - sub++; // sub sequence number starts at 1 - bool print = params->prefixes.empty(); // no filtering - - if (!print) { - print = params->matchFilters(m); - } - if (print) { - TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid) - .detail("Version", vms.version) - .setMaxFieldLength(1000) - .detail("M", m.toString()); - std::cout << vms.version << "." << sub << " " << typeString[(int)m.type] - << " param1: " << hexStringRef(m.param1) << " param2: " << hexStringRef(m.param2) << "\n"; - } - } - } - TraceEvent("ProcessFileDone", uid).detail("File", file.fileName); - return Void(); -} - -// Use the snapshot metadata to quickly identify relevant range files and -// then filter by versions. -ACTOR Future> getRangeFiles(Reference bc, Reference params) { - state std::vector snapshots = - wait((dynamic_cast(bc.getPtr()))->listKeyspaceSnapshots()); - state std::vector files; - - state int i = 0; - for (; i < snapshots.size(); i++) { - try { - std::pair, std::map> results = - wait((dynamic_cast(bc.getPtr()))->readKeyspaceSnapshot(snapshots[i])); - for (const auto& rangeFile : results.first) { - const auto& keyRange = results.second.at(rangeFile.fileName); - if (params->matchFilters(keyRange)) { - files.push_back(rangeFile); - } - } - } catch (Error& e) { - TraceEvent("ReadKeyspaceSnapshotError").error(e).detail("I", i); - if (e.code() != error_code_restore_missing_data) { - throw; - } - } - } - return getRelevantRangeFiles(files, params); -} - -ACTOR Future decode_logs(Reference params) { - state Reference container = - IBackupContainer::openContainer(params->container_url, params->proxy, {}); - state UID uid = deterministicRandom()->randomUniqueID(); - state BackupFileList listing = wait(container->dumpFileList()); - // remove partitioned logs - listing.logs.erase(std::remove_if(listing.logs.begin(), - listing.logs.end(), - [](const LogFile& file) { - std::string prefix("plogs/"); - return file.fileName.substr(0, prefix.size()) == prefix; - }), - listing.logs.end()); - std::sort(listing.logs.begin(), listing.logs.end()); - TraceEvent("Container", uid).detail("URL", params->container_url).detail("Logs", listing.logs.size()); - TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params->toString()); - - BackupDescription desc = wait(container->describeBackup()); - std::cout << "\n" << desc.toString() << "\n"; - - state std::vector logFiles; - state std::vector rangeFiles; - - if (params->decode_logs) { - logFiles = getRelevantLogFiles(listing.logs, params); - printLogFiles("Relevant log files are: ", logFiles); - } - - if (params->decode_range) { - // rangeFiles = getRelevantRangeFiles(filteredRangeFiles, params); - std::vector files = wait(getRangeFiles(container, params)); - rangeFiles = files; - printLogFiles("Relevant range files are: ", rangeFiles); - } - - TraceEvent("TotalFiles", uid).detail("LogFiles", logFiles.size()).detail("RangeFiles", rangeFiles.size()); - - if (params->list_only) - return Void(); - - // Decode log files. - state int idx = 0; - if (params->decode_logs) { - while (idx < logFiles.size()) { - TraceEvent("ProcessFile").detail("Name", logFiles[idx].fileName).detail("I", idx); - wait(process_file(container, logFiles[idx], uid, params)); - idx++; - } - TraceEvent("DecodeLogsDone", uid).log(); - } - - // Decode range files. - if (params->decode_range) { - idx = 0; - while (idx < rangeFiles.size()) { - TraceEvent("ProcessFile").detail("Name", rangeFiles[idx].fileName).detail("I", idx); - wait(process_range_file(container, rangeFiles[idx], uid, params)); - idx++; - } - TraceEvent("DecodeRangeFileDone", uid).log(); - } - - return Void(); -} - -} // namespace file_converter - -int main(int argc, char** argv) { - std::string commandLine; - for (int a = 0; a < argc; a++) { - if (a) - commandLine += ' '; - commandLine += argv[a]; - } - - try { - std::unique_ptr args( - new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE)); - auto param = makeReference(); - int status = file_converter::parseDecodeCommandLine(param, args.get()); - std::cout << "Params: " << param->toString() << "\n"; - param->updateRangeMap(); - if (status != FDB_EXIT_SUCCESS) { - file_converter::printDecodeUsage(); - return status; - } - - if (param->log_enabled) { - if (param->log_dir.empty()) { - setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); - } else { - setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param->log_dir)); - } - if (!param->trace_format.empty()) { - setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param->trace_format)); - } else { - setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, "json"_sr); - } - if (!param->trace_log_group.empty()) { - setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param->trace_log_group)); - } - } - - if (!param->tlsConfig.setupTLS()) { - TraceEvent(SevError, "TLSError").log(); - throw tls_error(); - } - - platformInit(); - Error::init(); - - StringRef url(param->container_url); - setupNetwork(0, UseMetrics::True); - - // Must be called after setupNetwork() to be effective - param->updateKnobs(); - - TraceEvent("ProgramStart") - .setMaxEventLength(12000) - .detail("SourceVersion", getSourceVersion()) - .detail("Version", FDB_VT_VERSION) - .detail("PackageName", FDB_VT_PACKAGE_NAME) - .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) - .setMaxFieldLength(10000) - .detail("CommandLine", commandLine) - .setMaxFieldLength(0) - .trackLatest("ProgramStart"); - - TraceEvent::setNetworkThread(); - openTraceFile({}, 10 << 20, 500 << 20, param->log_dir, "decode", param->trace_log_group); - param->tlsConfig.setupBlobCredentials(); - - auto f = stopAfter(decode_logs(param)); - - runNetwork(); - - flushTraceFileVoid(); - fflush(stdout); - closeTraceFile(); - - return status; - } catch (Error& e) { - std::cerr << "ERROR: " << e.what() << "\n"; - return FDB_EXIT_ERROR; - } catch (std::exception& e) { - TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); - return FDB_EXIT_MAIN_EXCEPTION; - } -} diff --git a/fdbbackup/FileDecoder.cpp b/fdbbackup/FileDecoder.cpp new file mode 100644 index 00000000000..bc192a80b64 --- /dev/null +++ b/fdbbackup/FileDecoder.cpp @@ -0,0 +1,1003 @@ +/* + * FileDecoder.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#endif + +#include "fdbclient/BackupTLSConfig.h" +#include "fdbclient/BuildFlags.h" +#include "fdbbackup/FileConverter.h" +#include "fdbbackup/Decode.h" +#include "fdbclient/BackupAgent.h" +#include "fdbclient/BackupContainer.h" +#include "fdbclient/BackupContainerFileSystem.h" +#include "fdbclient/CommitTransaction.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/KeyRangeMap.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/MutationList.h" +#include "fdbclient/SystemData.h" +#include "fdbclient/versions.h" +#include "flow/ArgParseUtil.h" +#include "flow/FastRef.h" +#include "flow/IRandom.h" +#include "flow/Platform.h" +#include "flow/Trace.h" +#include "flow/flow.h" +#include "flow/serialize.h" + +#define SevDecodeInfo SevVerbose + +extern bool g_crashOnError; +extern const char* getSourceVersion(); + +namespace file_converter { + +void printDecodeUsage() { + std::cout << "Decoder for FoundationDB backup mutation logs.\n" + "Usage: fdbdecode [OPTIONS]\n" + " -r, --container URL\n" + " Backup container URL, e.g., file:///some/path/.\n" + " -i, --input FILE\n" + " Log file filter, only matched files are decoded.\n" + " --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifies the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n" + " --loggroup LOG_GROUP\n" + " Sets the LogGroup field with the specified value for all\n" + " events in the trace output (defaults to `default').\n" + " --trace-format FORMAT\n" + " Select the format of the trace files, xml (the default) or json.\n" + " Has no effect unless --log is specified.\n" + " --crash Crash on serious error.\n" + " --blob-credentials FILE\n" + " File containing blob credentials in JSON format.\n" + " The same credential format/file fdbbackup uses.\n" TLS_HELP + " -t, --file-type [log|range|both]\n" + " Specifies the backup file type to decode.\n" + " --build-flags Print build information and exit.\n" + " --list-only Print file list and exit.\n" + " --validate-filters Validate the default RangeMap filtering logic with a slower one.\n" + " -k KEY_PREFIX Use a single prefix for filtering mutations.\n" + " --filters PREFIX_FILTER_FILE\n" + " A file containing a list of prefix filters in HEX format separated by \";\",\n" + " e.g., \"\\x05\\x01;\\x15\\x2b\"\n" + " --hex-prefix HEX_PREFIX\n" + " The prefix specified in HEX format, e.g., --hex-prefix \"\\\\x05\\\\x01\".\n" + " --begin-version-filter BEGIN_VERSION\n" + " The version range's begin version (inclusive) for filtering.\n" + " --end-version-filter END_VERSION\n" + " The version range's end version (exclusive) for filtering.\n" + " --knob-KNOBNAME KNOBVALUE\n" + " Changes a knob value. KNOBNAME should be lowercase.\n" + " -s, --save Save a copy of downloaded files (default: not saving).\n" + " --encryption-key-file FILE\n" + " AES-128-GCM encryption key file for encrypted backups.\n" + "\n"; + return; +} + +void printBuildInformation() { + std::cout << jsonBuildInformation() << "\n"; +} + +struct DecodeParams : public ReferenceCounted { + std::string container_url; + Optional proxy; + std::string fileFilter; // only files match the filter will be decoded + bool log_enabled = true; + std::string log_dir, trace_format, trace_log_group; + BackupTLSConfig tlsConfig; + bool list_only = false; + bool decode_logs = true; + bool decode_range = true; + bool save_file_locally = false; + bool validate_filters = false; + std::vector prefixes; // Key prefixes for filtering + // more efficient data structure for intersection queries than "prefixes" + fileBackup::RangeMapFilters filters; + Version beginVersionFilter = 0; + Version endVersionFilter = std::numeric_limits::max(); + + std::vector> knobs; + Optional encryptionKeyFileName; + + // Returns if [begin, end) overlap with the filter range + bool overlap(Version begin, Version end) const { + // Filter [100, 200), [50,75) [200, 300) + return !(begin >= endVersionFilter || end <= beginVersionFilter); + } + + bool overlap(Version version) const { return version >= beginVersionFilter && version < endVersionFilter; } + + bool validVersionFilters() { return beginVersionFilter < endVersionFilter; } + + void updateRangeMap() { filters.updateFilters(prefixes); } + + bool matchFilters(const MutationRef& m) const { + bool match = filters.match(m); + if (!validate_filters) { + return match; + } + + // If we choose to validate the filters, go through filters one by one + for (const auto& prefix : prefixes) { + if (isSingleKeyMutation((MutationRef::Type)m.type)) { + if (m.param1.startsWith(StringRef(prefix))) { + ASSERT(match); + return true; + } + } else if (m.type == MutationRef::ClearRange) { + KeyRange range(KeyRangeRef(m.param1, m.param2)); + KeyRange range2 = prefixRange(StringRef(prefix)); + if (range.intersects(range2)) { + ASSERT(match); + return true; + } + } else { + ASSERT(false); + } + } + ASSERT(!match); + return false; + } + + bool matchFilters(const KeyRange& range) const { + bool match = filters.match(range); + if (!validate_filters) { + return match; + } + + for (const auto& prefix : prefixes) { + if (range.intersects(prefixRange(StringRef(prefix)))) { + ASSERT(match); + return true; + } + } + return false; + } + + bool matchFilters(KeyValueRef kv) const { + bool match = filters.match(kv); + + if (!validate_filters) { + return match; + } + + for (const auto& prefix : prefixes) { + if (kv.key.startsWith(StringRef(prefix))) { + ASSERT(match); + return true; + } + } + + return match; + } + + std::string toString() { + std::string s; + s.append("ContainerURL: "); + s.append(container_url); + if (proxy.present()) { + s.append(", Proxy: "); + s.append(proxy.get()); + } + s.append(", FileFilter: "); + s.append(fileFilter); + if (log_enabled) { + if (!log_dir.empty()) { + s.append(" LogDir:").append(log_dir); + } + if (!trace_format.empty()) { + s.append(" Format:").append(trace_format); + } + if (!trace_log_group.empty()) { + s.append(" LogGroup:").append(trace_log_group); + } + } + s.append(", list_only: ").append(list_only ? "true" : "false"); + s.append(", validate_filters: ").append(validate_filters ? "true" : "false"); + if (beginVersionFilter != 0) { + s.append(", beginVersionFilter: ").append(std::to_string(beginVersionFilter)); + } + if (endVersionFilter < std::numeric_limits::max()) { + s.append(", endVersionFilter: ").append(std::to_string(endVersionFilter)); + } + if (!prefixes.empty()) { + s.append(", KeyPrefixes: ").append(printable(describe(prefixes))); + } + for (const auto& [knob, value] : knobs) { + s.append(", KNOB-").append(knob).append(" = ").append(value); + } + s.append(", SaveFile: ").append(save_file_locally ? "true" : "false"); + if (encryptionKeyFileName.present()) { + s.append(", EncryptionKeyFile: ").append(encryptionKeyFileName.get()); + } + return s; + } + + void updateKnobs() { + setupClientKnobs(knobs); + + // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs + initializeClientKnobs(Randomize::False, IsSimulated::False); + } +}; + +// Parses and returns a ";" separated HEX encoded strings. So the ";" in +// the string should be escaped as "\;". +// Sets "err" to true if there is any parsing error. +std::vector parsePrefixesLine(const std::string& line, bool& err) { + std::vector results; + err = false; + + int p = 0; + while (p < line.size()) { + int end = line.find_first_of(';', p); + if (end == line.npos) { + end = line.size(); + } + auto prefix = decode_hex_string(line.substr(p, end - p), err); + if (err) { + return results; + } + results.push_back(prefix); + p = end + 1; + } + return results; +} + +std::vector parsePrefixFile(const std::string& filename, bool& err) { + std::string line = readFileBytes(filename, 64 * 1024 * 1024); + return parsePrefixesLine(line, err); +} + +int parseDecodeCommandLine(Reference param, CSimpleOpt* args) { + bool err = false; + + while (args->Next()) { + auto lastError = args->LastError(); + switch (lastError) { + case SO_SUCCESS: + break; + + default: + std::cerr << "ERROR: argument given for option: " << args->OptionText() << "\n"; + return FDB_EXIT_ERROR; + break; + } + int optId = args->OptionId(); + switch (optId) { + case OPT_HELP: + return FDB_EXIT_ERROR; + + case OPT_CONTAINER: + param->container_url = args->OptionArg(); + break; + + case OPT_FILE_TYPE: { + auto ftype = std::string(args->OptionArg()); + if (ftype == "log") { + param->decode_range = false; + } else if (ftype == "range") { + param->decode_logs = false; + } else if (ftype != "both" && !ftype.empty()) { + err = true; + std::cerr << "ERROR: Unrecognized backup file type option: " << args->OptionArg() << "\n"; + return FDB_EXIT_ERROR; + } + break; + } + + case OPT_LIST_ONLY: + param->list_only = true; + break; + + case OPT_VALIDATE_FILTERS: + param->validate_filters = true; + break; + + case OPT_KEY_PREFIX: + param->prefixes.push_back(args->OptionArg()); + break; + + case OPT_FILTERS: + param->prefixes = parsePrefixFile(args->OptionArg(), err); + if (err) { + throw std::runtime_error("ERROR:" + std::string(args->OptionArg()) + "contains invalid prefix(es)"); + } + break; + + case OPT_HEX_KEY_PREFIX: + param->prefixes.push_back(decode_hex_string(args->OptionArg(), err)); + break; + + case OPT_PROXY: + param->proxy = args->OptionArg(); + break; + + case OPT_BEGIN_VERSION_FILTER: + param->beginVersionFilter = std::atoll(args->OptionArg()); + break; + + case OPT_END_VERSION_FILTER: + param->endVersionFilter = std::atoll(args->OptionArg()); + break; + + case OPT_CRASHONERROR: + g_crashOnError = true; + break; + + case OPT_INPUT_FILE: + param->fileFilter = args->OptionArg(); + break; + + case OPT_TRACE: + param->log_enabled = true; + break; + + case OPT_TRACE_DIR: + param->log_dir = args->OptionArg(); + break; + + case OPT_TRACE_FORMAT: + if (!selectTraceFormatter(args->OptionArg())) { + std::cerr << "ERROR: Unrecognized trace format " << args->OptionArg() << "\n"; + return FDB_EXIT_ERROR; + } + param->trace_format = args->OptionArg(); + break; + + case OPT_TRACE_LOG_GROUP: + param->trace_log_group = args->OptionArg(); + break; + + case OPT_BLOB_CREDENTIALS: + param->tlsConfig.blobCredentials.push_back(args->OptionArg()); + break; + + case OPT_KNOB: { + Optional knobName = extractPrefixedArgument("--knob", args->OptionSyntax()); + if (!knobName.present()) { + std::cerr << "ERROR: unable to parse knob option '" << args->OptionSyntax() << "'\n"; + return FDB_EXIT_ERROR; + } + param->knobs.emplace_back(knobName.get(), args->OptionArg()); + break; + } + + case OPT_SAVE_FILE: + param->save_file_locally = true; + break; + + case OPT_ENCRYPTION_KEY_FILE: + param->encryptionKeyFileName = args->OptionArg(); + break; + + case TLSConfig::OPT_TLS_PLUGIN: + args->OptionArg(); + break; + + case TLSConfig::OPT_TLS_CERTIFICATES: + param->tlsConfig.tlsCertPath = args->OptionArg(); + break; + + case TLSConfig::OPT_TLS_PASSWORD: + param->tlsConfig.tlsPassword = args->OptionArg(); + break; + + case TLSConfig::OPT_TLS_CA_FILE: + param->tlsConfig.tlsCAPath = args->OptionArg(); + break; + + case TLSConfig::OPT_TLS_KEY: + param->tlsConfig.tlsKeyPath = args->OptionArg(); + break; + + case TLSConfig::OPT_TLS_VERIFY_PEERS: + param->tlsConfig.tlsVerifyPeers = args->OptionArg(); + break; + + case OPT_BUILD_FLAGS: + printBuildInformation(); + return FDB_EXIT_ERROR; + break; + } + } + return FDB_EXIT_SUCCESS; +} + +template +void printLogFiles(std::string msg, const std::vector& files) { + std::cout << msg << " " << files.size() << " total\n"; + for (const auto& file : files) { + std::cout << file.toString() << "\n"; + } + std::cout << std::endl; +} + +std::vector getRelevantLogFiles(const std::vector& files, const Reference params) { + std::vector filtered; + for (const auto& file : files) { + if (file.fileName.find(params->fileFilter) != std::string::npos && + params->overlap(file.beginVersion, file.endVersion + 1)) { + filtered.push_back(file); + } + } + return filtered; +} + +std::vector getRelevantRangeFiles(const std::vector& files, + const Reference params) { + std::vector filtered; + for (const auto& file : files) { + if (file.fileName.find(params->fileFilter) != std::string::npos && params->overlap(file.version)) { + filtered.push_back(file); + } + } + return filtered; +} + +struct VersionedMutations { + Version version; + std::vector mutations; + std::string serializedMutations; // buffer that contains mutations +}; + +/* + * Model a decoding progress for a mutation file. Usage is: + * + * DecodeProgress progress(logfile); + * co_await progress.openFile(container); + * while (1) { + * Optional batch = progress.getNextBatch(); + * if (!batch.present()) break; + * ... // process the batch mutations + * } + * + * Internally, the decoding process is done block by block -- each block is + * decoded into a list of key/value pairs, which are then decoded into batches + * of mutations. Because a version's mutations can be split into many key/value + * pairs, the decoding of mutation needs to look ahead to find all batches that + * belong to the same version. + */ +class DecodeProgress { + std::vector>> blocks; + std::unordered_map mutationBlocksByVersion; + +public: + DecodeProgress() = default; + DecodeProgress(const LogFile& file, bool save) : file(file), save(save) {} + + ~DecodeProgress() { + if (lfd != -1) { + close(lfd); + } + } + + // Open and loads file into memory + Future openFile(Reference container) { return openFileImpl(this, container); } + + // The following are private APIs: + + // Returns the next batch of mutations along with the arena backing it. + // Note the returned batch can be empty when the file has unfinished + // version batch data that are in the next file. + Optional getNextBatch() { + for (auto& [version, m] : mutationBlocksByVersion) { + if (m.isComplete()) { + VersionedMutations vms; + vms.version = version; + vms.serializedMutations = m.serializedMutations; + vms.mutations = fileBackup::decodeMutationLogValue(vms.serializedMutations); + TraceEvent("Decode").detail("Version", vms.version).detail("N", vms.mutations.size()); + mutationBlocksByVersion.erase(version); + return vms; + } + } + + // No complete versions + if (!mutationBlocksByVersion.empty()) { + TraceEvent(SevWarn, "UnfishedBlocks").detail("NumberOfVersions", mutationBlocksByVersion.size()); + } + return Optional(); + } + + static Future openFileImpl(DecodeProgress* self, Reference container) { + self->fd = co_await container->readFile(self->file.fileName); + Standalone buf = makeString(self->file.fileSize); + int rLen = co_await self->fd->read(mutateString(buf), self->file.fileSize, 0); + if (rLen != self->file.fileSize) { + throw restore_bad_read(); + } + + if (self->save) { + std::string dir = self->file.fileName; + std::size_t found = self->file.fileName.find_last_of('/'); + if (found != std::string::npos) { + std::string path = self->file.fileName.substr(0, found); + if (!directoryExists(path)) { + platform::createDirectory(path); + } + } + self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (self->lfd == -1) { + TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName); + throw platform_error(); + } + int wlen = write(self->lfd, buf.begin(), self->file.fileSize); + if (wlen != self->file.fileSize) { + TraceEvent(SevError, "WriteLocalFileFailed") + .detail("File", self->file.fileName) + .detail("Len", self->file.fileSize); + throw platform_error(); + } + TraceEvent("WriteLocalFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize); + } + + self->decodeFile(buf); + } + + // Add chunks to mutationBlocksByVersion + void addBlockKVPairs(VectorRef chunks) { + for (auto& kv : chunks) { + auto versionAndChunkNumber = fileBackup::decodeMutationLogKey(kv.key); + mutationBlocksByVersion[versionAndChunkNumber.first].addChunk(versionAndChunkNumber.second, kv); + } + } + + // Reads a file a file content in the buffer, decodes it into key/value pairs, and stores these pairs. + void decodeFile(const Standalone& buf) { + try { + while (true) { + int64_t len = std::min(file.blockSize, file.fileSize - offset); + if (len == 0) { + return; + } + + // Decode a file block into log_key and log_value chunks + Standalone> chunks = + fileBackup::decodeMutationLogFileBlock(buf.substr(offset, len)); + blocks.push_back(chunks); + addBlockKVPairs(chunks); + offset += len; + } + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptLogFileBlock") + .error(e) + .detail("Filename", file.fileName) + .detail("BlockOffset", offset) + .detail("BlockLen", file.blockSize); + throw; + } + } + + LogFile file; + Reference fd; + int64_t offset = 0; + bool eof = false; + bool save = false; + int lfd = -1; // local file descriptor +}; + +class DecodeRangeProgress { +public: + std::vector>> blocks; + + DecodeRangeProgress() = default; + DecodeRangeProgress(const RangeFile& file, bool save) : file(file), save(save) {} + ~DecodeRangeProgress() { + if (lfd != -1) { + close(lfd); + } + } + + // Open and loads file into memory + Future openFile(Reference container) { return openFileImpl(this, container); } + + static Future openFileImpl(DecodeRangeProgress* self, Reference container) { + TraceEvent("ReadFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize); + + self->fd = co_await container->readFile(self->file.fileName); + Standalone buf = makeString(self->file.fileSize); + int rLen = co_await self->fd->read(mutateString(buf), self->file.fileSize, 0); + if (rLen != self->file.fileSize) { + throw restore_bad_read(); + } + + if (self->save) { + std::string dir = self->file.fileName; + std::size_t found = self->file.fileName.find_last_of('/'); + if (found != std::string::npos) { + std::string path = self->file.fileName.substr(0, found); + if (!directoryExists(path)) { + platform::createDirectory(path); + } + } + + self->lfd = open(self->file.fileName.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0600); + if (self->lfd == -1) { + TraceEvent(SevError, "OpenLocalFileFailed").detail("File", self->file.fileName); + throw platform_error(); + } + int wlen = write(self->lfd, buf.begin(), self->file.fileSize); + if (wlen != self->file.fileSize) { + TraceEvent(SevError, "WriteLocalFileFailed") + .detail("File", self->file.fileName) + .detail("Len", self->file.fileSize); + throw platform_error(); + } + TraceEvent("WriteLocalFile").detail("Name", self->file.fileName).detail("Len", self->file.fileSize); + } + + self->decodeFile(buf); + } + + // Reads a file content in the buffer, decodes it into key/value pairs, and stores these pairs. + void decodeFile(const Standalone& buf) { + try { + while (true) { + // process one block at a time + int64_t len = std::min(file.blockSize, file.fileSize - offset); + if (len == 0) { + return; + } + + Standalone> chunks = fileBackup::decodeRangeFileBlock(buf.substr(offset, len)); + blocks.push_back(chunks); + offset += len; + } + } catch (Error& e) { + TraceEvent(SevWarn, "CorruptRangeFileBlock") + .error(e) + .detail("Filename", file.fileName) + .detail("BlockOffset", offset) + .detail("BlockLen", file.blockSize); + throw; + } + } + + RangeFile file; + Reference fd; + int64_t offset = 0; + bool save = false; + int lfd = -1; // local file descriptor +}; + +// convert a StringRef to Hex string +std::string hexStringRef(const StringRef& s) { + std::string result; + result.reserve(s.size() * 2); + for (int i = 0; i < s.size(); i++) { + result.append(format("%02x", s[i])); + } + return result; +} + +Future process_range_file(Reference container, + RangeFile file, + UID uid, + Reference params) { + + if (file.fileSize == 0) { + TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName); + co_return; + } + + DecodeRangeProgress progress(file, params->save_file_locally); + co_await progress.openFile(container); + + for (auto& block : progress.blocks) { + for (const auto& kv : block) { + bool print = params->prefixes.empty(); // no filtering + + if (!print) { + print = params->matchFilters(kv); + } + + if (print) { + TraceEvent(format("KVPair_%llu", file.version).c_str(), uid) + .detail("Version", file.version) + .setMaxFieldLength(1000) + .detail("KV", kv); + std::cout << file.version << " key: " << hexStringRef(kv.key) << " value: " << hexStringRef(kv.value) + << std::endl; + } + } + } + TraceEvent("ProcessRangeFileDone", uid).detail("File", file.fileName); +} + +Future process_file(Reference container, + LogFile file, + UID uid, + Reference params) { + if (file.fileSize == 0) { + TraceEvent("SkipEmptyFile", uid).detail("Name", file.fileName); + co_return; + } + + DecodeProgress progress(file, params->save_file_locally); + co_await progress.openFile(container); + while (true) { + auto batch = progress.getNextBatch(); + if (!batch.present()) + break; + + const VersionedMutations& vms = batch.get(); + if (vms.version < params->beginVersionFilter || vms.version >= params->endVersionFilter) { + TraceEvent("SkipVersion").detail("Version", vms.version); + continue; + } + + int sub = 0; + for (const auto& m : vms.mutations) { + sub++; // sub sequence number starts at 1 + bool print = params->prefixes.empty(); // no filtering + + if (!print) { + print = params->matchFilters(m); + } + if (print) { + TraceEvent(format("Mutation_%llu_%d", vms.version, sub).c_str(), uid) + .detail("Version", vms.version) + .setMaxFieldLength(1000) + .detail("M", m.toString()); + std::cout << vms.version << "." << sub << " " << typeString[(int)m.type] + << " param1: " << hexStringRef(m.param1) << " param2: " << hexStringRef(m.param2) << "\n"; + } + } + } + TraceEvent("ProcessFileDone", uid).detail("File", file.fileName); +} + +// Use the snapshot metadata to quickly identify relevant range files and +// then filter by versions. +Future> getRangeFiles(Reference bc, Reference params) { + std::vector snapshots = + co_await (dynamic_cast(bc.getPtr()))->listKeyspaceSnapshots(); + std::vector files; + + for (int i = 0; i < snapshots.size(); i++) { + try { + std::pair, std::map> results = + co_await (dynamic_cast(bc.getPtr()))->readKeyspaceSnapshot(snapshots[i]); + for (const auto& rangeFile : results.first) { + const auto& keyRange = results.second.at(rangeFile.fileName); + if (params->matchFilters(keyRange)) { + files.push_back(rangeFile); + } + } + } catch (Error& e) { + TraceEvent("ReadKeyspaceSnapshotError").error(e).detail("I", i); + if (e.code() != error_code_restore_missing_data) { + throw; + } + } + } + co_return getRelevantRangeFiles(files, params); +} + +Future decode_logs(Reference params) { + Reference container = + IBackupContainer::openContainer(params->container_url, params->proxy, params->encryptionKeyFileName, 0); + UID uid = deterministicRandom()->randomUniqueID(); + + BackupFileList listing = co_await container->dumpFileList(); + + // remove partitioned logs + listing.logs.erase(std::remove_if(listing.logs.begin(), + listing.logs.end(), + [](const LogFile& file) { + std::string prefix("plogs/"); + return file.fileName.substr(0, prefix.size()) == prefix; + }), + listing.logs.end()); + std::sort(listing.logs.begin(), listing.logs.end()); + TraceEvent("Container", uid).detail("URL", params->container_url).detail("Logs", listing.logs.size()); + TraceEvent("DecodeParam", uid).setMaxFieldLength(100000).detail("Value", params->toString()); + + BackupDescription desc = co_await container->describeBackup(); + + container->setEncryptionBlockSize(desc.encryptionBlockSize); + + std::cout << "\n" << desc.toString() << "\n"; + + std::vector logFiles; + std::vector rangeFiles; + + if (params->decode_logs) { + logFiles = getRelevantLogFiles(listing.logs, params); + printLogFiles("Relevant log files are: ", logFiles); + } + + if (params->decode_range) { + // rangeFiles = getRelevantRangeFiles(filteredRangeFiles, params); + std::vector files = co_await getRangeFiles(container, params); + rangeFiles = files; + printLogFiles("Relevant range files are: ", rangeFiles); + } + + TraceEvent("TotalFiles", uid).detail("LogFiles", logFiles.size()).detail("RangeFiles", rangeFiles.size()); + + if (params->list_only) + co_return; + + // Decode log files. + int idx = 0; + if (params->decode_logs) { + while (idx < logFiles.size()) { + TraceEvent("ProcessFile").detail("Name", logFiles[idx].fileName).detail("I", idx); + co_await process_file(container, logFiles[idx], uid, params); + idx++; + } + TraceEvent("DecodeLogsDone", uid).log(); + } + + // Decode range files. + if (params->decode_range) { + idx = 0; + while (idx < rangeFiles.size()) { + TraceEvent("ProcessFile").detail("Name", rangeFiles[idx].fileName).detail("I", idx); + co_await process_range_file(container, rangeFiles[idx], uid, params); + idx++; + } + TraceEvent("DecodeRangeFileDone", uid).log(); + } +} + +} // namespace file_converter + +#ifndef EXCLUDE_MAIN_FUNCTION +int main(int argc, char** argv) { + std::string commandLine; + for (int a = 0; a < argc; a++) { + if (a) + commandLine += ' '; + commandLine += argv[a]; + } + + try { + std::unique_ptr args( + new CSimpleOpt(argc, argv, file_converter::gConverterOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE)); + auto param = makeReference(); + int status = file_converter::parseDecodeCommandLine(param, args.get()); + std::cout << "Params: " << param->toString() << "\n"; + param->updateRangeMap(); + if (status != FDB_EXIT_SUCCESS) { + file_converter::printDecodeUsage(); + return status; + } + + // Check if the beginVersionFilter is greater than the endVersionFilter, otherwise the filtering will be + // invalid. + if (!param->validVersionFilters()) { + std::cerr << "--begin-version-filter " << param->beginVersionFilter + << " cannot be equal or greater than --end-version-filter " << param->endVersionFilter << "\n"; + file_converter::printDecodeUsage(); + return FDB_EXIT_ERROR; + } + + if (param->log_enabled) { + if (param->log_dir.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); + } else { + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(param->log_dir)); + } + if (!param->trace_format.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(param->trace_format)); + } else { + setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, "json"_sr); + } + if (!param->trace_log_group.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(param->trace_log_group)); + } + } + + if (!param->tlsConfig.setupTLS()) { + TraceEvent(SevError, "TLSError").log(); + throw tls_error(); + } + + platformInit(); + Error::init(); + + StringRef url(param->container_url); + setupNetwork(0, UseMetrics::True); + + // Must be called after setupNetwork() to be effective + param->updateKnobs(); + + TraceEvent("ProgramStart") + .setMaxEventLength(12000) + .detail("SourceVersion", getSourceVersion()) + .detail("Version", FDB_VT_VERSION) + .detail("PackageName", FDB_VT_PACKAGE_NAME) + .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr)) + .setMaxFieldLength(10000) + .detail("CommandLine", commandLine) + .setMaxFieldLength(0) + .trackLatest("ProgramStart"); + + TraceEvent::setNetworkThread(); + openTraceFile({}, 10 << 20, 500 << 20, param->log_dir, "decode", param->trace_log_group); + param->tlsConfig.setupBlobCredentials(); + + auto f = stopAfter(decode_logs(param)); + + runNetwork(); + + flushTraceFileVoid(); + fflush(stdout); + closeTraceFile(); + + return status; + } catch (Error& e) { + std::cerr << "ERROR: " << e.what() << "\n"; + return FDB_EXIT_ERROR; + } catch (std::exception& e) { + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + return FDB_EXIT_MAIN_EXCEPTION; + } +} +#else // EXCLUDE_MAIN_FUNCTION + +int main() { + auto assertValid = [](file_converter::DecodeParams& p, bool expected, const char* label) { + bool result = p.validVersionFilters(); + if (result != expected) { + fprintf(stderr, "FAIL [%s]: expected %s\n", label, expected ? "valid" : "invalid"); + return false; + } + printf("PASS [%s]\n", label); + return true; + }; + + bool ok = true; + file_converter::DecodeParams p; + + ok &= assertValid(p, true, "defaults"); + + p.beginVersionFilter = 100; + p.endVersionFilter = 200; + ok &= assertValid(p, true, "begin < end"); + + p.beginVersionFilter = 200; + p.endVersionFilter = 200; + ok &= assertValid(p, false, "begin == end"); + + p.beginVersionFilter = 300; + p.endVersionFilter = 200; + ok &= assertValid(p, false, "begin > end"); + + return ok ? 0 : 1; +} +#endif // EXCLUDE_MAIN_FUNCTION diff --git a/fdbbackup/backup.actor.cpp b/fdbbackup/backup.actor.cpp deleted file mode 100644 index 168bd2ccb62..00000000000 --- a/fdbbackup/backup.actor.cpp +++ /dev/null @@ -1,5074 +0,0 @@ -/* - * backup.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "flow/ApiVersion.h" -#include "fmt/format.h" -#include "fdbclient/BackupTLSConfig.h" -#include "fdbbackup/Decode.h" -#include "fdbclient/JsonBuilder.h" -#include "flow/Arena.h" -#include "flow/ArgParseUtil.h" -#include "flow/Error.h" -#include "flow/SystemMonitor.h" -#include "flow/Trace.h" -#define BOOST_DATE_TIME_NO_LIB -#include - -#include "flow/flow.h" -#include "flow/FastAlloc.h" -#include "flow/serialize.h" -#include "flow/IRandom.h" -#include "flow/genericactors.actor.h" -#include "flow/TLSConfig.actor.h" - -#include "fdbclient/DatabaseContext.h" -#include "fdbclient/FDBTypes.h" -#include "fdbclient/BackupAgent.actor.h" -#include "fdbclient/Status.h" -#include "fdbclient/BackupContainer.h" -#include "fdbclient/ClusterConnectionFile.h" -#include "fdbclient/KeyBackedTypes.actor.h" -#include "fdbclient/IKnobCollection.h" -#include "fdbclient/RunRYWTransaction.actor.h" -#include "fdbclient/S3BlobStore.h" -#include "fdbclient/SystemData.h" -#include "fdbclient/json_spirit/json_spirit_writer_template.h" -#include "fdbclient/BulkLoading.h" -#include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/BackupContainer.h" - -#include "flow/Platform.h" - -#include -#include -#include -#include // std::transform -#include -#include -#include - -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#define NOMINMAX -#include -#endif -#include - -#ifdef __linux__ -#include -#ifdef ALLOC_INSTRUMENTATION -#include -#endif -#endif - -#include "fdbclient/versions.h" -#include "fdbclient/BuildFlags.h" - -#include "SimpleOpt/SimpleOpt.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -// Type of program being executed -enum class ProgramExe { AGENT, BACKUP, RESTORE, FASTRESTORE_TOOL, DR_AGENT, DB_BACKUP, UNDEFINED }; - -enum class BackupType { - UNDEFINED = 0, - START, - MODIFY, - STATUS, - ABORT, - WAIT, - DISCONTINUE, - PAUSE, - RESUME, - EXPIRE, - DELETE_BACKUP, - DESCRIBE, - LIST, - QUERY, - DUMP, - CLEANUP, - TAGS, -}; - -enum class DBType { UNDEFINED = 0, START, STATUS, SWITCH, ABORT, PAUSE, RESUME }; - -// New fast restore reuses the type from legacy slow restore -enum class RestoreType { UNKNOWN, START, STATUS, ABORT, WAIT }; - -// -enum { - // Backup constants - OPT_DESTCONTAINER, - OPT_SNAPSHOTINTERVAL, - OPT_INITIAL_SNAPSHOT_INTERVAL, - OPT_ERRORLIMIT, - OPT_NOSTOPWHENDONE, - OPT_EXPIRE_BEFORE_VERSION, - OPT_EXPIRE_BEFORE_DATETIME, - OPT_EXPIRE_DELETE_BEFORE_DAYS, - OPT_EXPIRE_RESTORABLE_AFTER_VERSION, - OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, - OPT_EXPIRE_MIN_RESTORABLE_DAYS, - OPT_BASEURL, - OPT_BLOB_CREDENTIALS, - OPT_DESCRIBE_DEEP, - OPT_DESCRIBE_TIMESTAMPS, - OPT_DUMP_BEGIN, - OPT_DUMP_END, - OPT_JSON, - OPT_DELETE_DATA, - OPT_MIN_CLEANUP_SECONDS, - OPT_USE_PARTITIONED_LOG, - OPT_MODE, - - // Backup and Restore constants - OPT_PROXY, - OPT_TAGNAME, - OPT_BACKUPKEYS, - OPT_BACKUPKEYS_FILE, - OPT_WAITFORDONE, - OPT_BACKUPKEYS_FILTER, - OPT_INCREMENTALONLY, - OPT_ENCRYPTION_KEY_FILE, - - // Backup Modify - OPT_MOD_ACTIVE_INTERVAL, - OPT_MOD_VERIFY_UID, - - // Restore constants - OPT_RESTORECONTAINER, - OPT_RESTORE_VERSION, - OPT_RESTORE_SNAPSHOT_VERSION, - OPT_RESTORE_TIMESTAMP, - OPT_PREFIX_ADD, - OPT_PREFIX_REMOVE, - OPT_RESTORE_CLUSTERFILE_DEST, - OPT_RESTORE_CLUSTERFILE_ORIG, - OPT_RESTORE_BEGIN_VERSION, - OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, - // The two restore options below allow callers of fdbrestore to divide a normal restore into one which restores just - // the system keyspace and another that restores just the user key space. This is unlike the backup command where - // all keys (both system and user) will be backed up together - OPT_RESTORE_USER_DATA, - OPT_RESTORE_SYSTEM_DATA, - - // Shared constants - OPT_CLUSTERFILE, - OPT_QUIET, - OPT_DRYRUN, - OPT_FORCE, - OPT_HELP, - OPT_DEVHELP, - OPT_VERSION, - OPT_BUILD_FLAGS, - OPT_PARENTPID, - OPT_CRASHONERROR, - OPT_NOBUFSTDOUT, - OPT_BUFSTDOUTERR, - OPT_TRACE, - OPT_TRACE_DIR, - OPT_KNOB, - OPT_TRACE_LOG_GROUP, - OPT_MEMLIMIT, - OPT_VMEMLIMIT, - OPT_LOCALITY, - - // DB constants - OPT_SOURCE_CLUSTER, - OPT_DEST_CLUSTER, - OPT_CLEANUP, - OPT_DSTONLY, - - OPT_TRACE_FORMAT, -}; - -// Top level binary commands. -CSimpleOpt::SOption g_rgOptions[] = { { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_BUILD_FLAGS, "--build-flags", SO_NONE }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - - SO_END_OF_OPTIONS }; - -CSimpleOpt::SOption g_rgAgentOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_BUILD_FLAGS, "--build-flags", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_LOCALITY, "--locality-", SO_REQ_SEP }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupStartOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, - { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - // Enable "-p" option after GA - // { OPT_USE_PARTITIONED_LOG, "-p", SO_NONE }, - { OPT_USE_PARTITIONED_LOG, "--partitioned-log-experimental", SO_NONE }, - { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "--snapshot-interval", SO_REQ_SEP }, - { OPT_INITIAL_SNAPSHOT_INTERVAL, "--initial-snapshot-interval", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_DRYRUN, "-n", SO_NONE }, - { OPT_DRYRUN, "--dryrun", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, - { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP }, - { OPT_MODE, "--mode", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupModifyOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_MOD_VERIFY_UID, "--verify-uid", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, - { OPT_SNAPSHOTINTERVAL, "--snapshot-interval", SO_REQ_SEP }, - { OPT_MOD_ACTIVE_INTERVAL, "--active-snapshot-interval", SO_REQ_SEP }, - { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupStatusOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_JSON, "--json", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupAbortOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupCleanupOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_DELETE_DATA, "--delete-data", SO_NONE }, - { OPT_MIN_CLEANUP_SECONDS, "--min-cleanup-seconds", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupWaitOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, - { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupPauseOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupExpireOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_FORCE, "--force", SO_NONE }, - { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, "--restorable-after-version", SO_REQ_SEP }, - { OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, "--restorable-after-timestamp", SO_REQ_SEP }, - { OPT_EXPIRE_BEFORE_VERSION, "--expire-before-version", SO_REQ_SEP }, - { OPT_EXPIRE_BEFORE_DATETIME, "--expire-before-timestamp", SO_REQ_SEP }, - { OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min-restorable-days", SO_REQ_SEP }, - { OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete-before-days", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupDeleteOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupDescribeOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_DESCRIBE_DEEP, "--deep", SO_NONE }, - { OPT_DESCRIBE_TIMESTAMPS, "--version-timestamps", SO_NONE }, - { OPT_JSON, "--json", SO_NONE }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupDumpOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP }, - { OPT_DUMP_END, "--end", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupTagsOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, - { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupListOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_BASEURL, "-b", SO_REQ_SEP }, - { OPT_BASEURL, "--base-url", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgBackupQueryOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_RESTORE_TIMESTAMP, "--query-restore-timestamp", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, - { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "--query-restore-version", SO_REQ_SEP }, - { OPT_RESTORE_SNAPSHOT_VERSION, "--query-restore-snapshot-version", SO_REQ_SEP }, - { OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -// g_rgRestoreOptions is used by fdbrestore and fastrestore_tool -CSimpleOpt::SOption g_rgRestoreOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_RESTORE_CLUSTERFILE_DEST, "--dest-cluster-file", SO_REQ_SEP }, - { OPT_RESTORE_CLUSTERFILE_ORIG, "--orig-cluster-file", SO_REQ_SEP }, - { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_RESTORECONTAINER, "-r", SO_REQ_SEP }, - { OPT_PROXY, "--proxy", SO_REQ_SEP }, - { OPT_PREFIX_ADD, "--add-prefix", SO_REQ_SEP }, - { OPT_PREFIX_REMOVE, "--remove-prefix", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_WAITFORDONE, "-w", SO_NONE }, - { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, - { OPT_RESTORE_USER_DATA, "--user-data", SO_NONE }, - { OPT_RESTORE_SYSTEM_DATA, "--system-metadata", SO_NONE }, - { OPT_MODE, "--mode", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "--version", SO_REQ_SEP }, - { OPT_RESTORE_VERSION, "-v", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_DRYRUN, "-n", SO_NONE }, - { OPT_DRYRUN, "--dryrun", SO_NONE }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, - { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, - { OPT_RESTORE_BEGIN_VERSION, "--begin-version", SO_REQ_SEP }, - { OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, "--inconsistent-snapshot-only", SO_NONE }, - { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgDBAgentOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_BUILD_FLAGS, "--build-flags", SO_NONE }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_LOCALITY, "--locality-", SO_REQ_SEP }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgDBStartOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, - { OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP }, - { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgDBStatusOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, - { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgDBSwitchOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_FORCE, "-f", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgDBAbortOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_CLEANUP, "--cleanup", SO_NONE }, - { OPT_DSTONLY, "--dstonly", SO_NONE }, - { OPT_TAGNAME, "-t", SO_REQ_SEP }, - { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -CSimpleOpt::SOption g_rgDBPauseOptions[] = { -#ifdef _WIN32 - { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, -#endif - { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, - { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, - { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, - { OPT_QUIET, "-q", SO_NONE }, - { OPT_QUIET, "--quiet", SO_NONE }, - { OPT_CRASHONERROR, "--crash", SO_NONE }, - { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, - { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, - { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_DEVHELP, "--dev-help", SO_NONE }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS -}; - -const KeyRef exeAgent = "backup_agent"_sr; -const KeyRef exeBackup = "fdbbackup"_sr; -const KeyRef exeRestore = "fdbrestore"_sr; -const KeyRef exeFastRestoreTool = "fastrestore_tool"_sr; // must be lower case -const KeyRef exeDatabaseAgent = "dr_agent"_sr; -const KeyRef exeDatabaseBackup = "fdbdr"_sr; - -extern const char* getSourceVersion(); - -#ifdef _WIN32 -void parentWatcher(void* parentHandle) { - HANDLE parent = (HANDLE)parentHandle; - int signal = WaitForSingleObject(parent, INFINITE); - CloseHandle(parentHandle); - if (signal == WAIT_OBJECT_0) - criticalError(FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited"); - TraceEvent(SevError, "ParentProcessWaitFailed").detail("RetCode", signal).GetLastError(); -} - -#endif - -static void printVersion() { - printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("source version %s\n", getSourceVersion()); - printf("protocol %llx\n", (long long)currentProtocolVersion().version()); -} - -static void printBuildInformation() { - printf("%s", jsonBuildInformation().c_str()); -} - -const char* BlobCredentialInfo = - " BLOB CREDENTIALS\n" - " Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n" - " loaded, if possible, from 1 or more blob credentials definition files.\n\n" - " These files can be specified with the --blob-credentials argument described above or via the environment " - "variable\n" - " FDB_BLOB_CREDENTIALS, whose value is a colon-separated list of files. The command line takes priority over\n" - " over the environment but all files from both sources are used.\n\n" - " At connect time, the specified files are read in order and the first matching account specification " - "(user@host)\n" - " will be used to obtain the secret key.\n\n" - " The JSON schema is:\n" - " { \"accounts\" : { \"user@host\" : { \"secret\" : \"SECRETKEY\" }, \"user2@host2\" : { \"secret\" : " - "\"SECRET\" } } }\n"; - -static void printHelpTeaser(const char* name) { - fprintf(stderr, "Try `%s --help' for more information.\n", name); -} - -static void printAgentUsage(bool devhelp) { - printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s [OPTIONS]\n\n", exeAgent.toString().c_str()); - printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", - platform::getDefaultClusterFilePath().c_str()); - printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifies the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); - printf(" --loggroup LOG_GROUP\n" - " Sets the LogGroup field with the specified value for all\n" - " events in the trace output (defaults to `default').\n"); - printf(" --trace-format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); - printf(" -m SIZE, --memory SIZE\n" - " Memory limit. The default value is 8GiB. When specified\n" - " without a unit, MiB is assumed.\n"); - printf(TLS_HELP); - printf(" --build-flags Print build information and exit.\n"); - printf(" -v, --version Print version information and exit.\n"); - printf(" -h, --help Display this help and exit.\n"); - - if (devhelp) { -#ifdef _WIN32 - printf(" -n Create a new console.\n"); - printf(" -q Disable error dialog on crash.\n"); - printf(" --parentpid PID\n"); - printf(" Specify a process after whose termination to exit.\n"); -#endif - } - - printf("\n"); - puts(BlobCredentialInfo); - - return; -} - -void printBackupContainerInfo() { - printf(" Backup URL forms:\n\n"); - std::vector formats = IBackupContainer::getURLFormats(); - for (const auto& f : formats) - printf(" %s\n", f.c_str()); - printf("\n"); -} - -static void printBackupUsage(bool devhelp) { - printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | abort | wait | discontinue | pause | resume | expire | " - "delete | describe | list | query | cleanup | tags) [ACTION_OPTIONS]\n\n", - exeBackup.toString().c_str()); - printf(" TOP LEVEL OPTIONS:\n"); - printf(" --build-flags Print build information and exit.\n"); - printf(" -v, --version Print version information and exit.\n"); - printf(" -h, --help Display this help and exit.\n"); - printf("\n"); - - printf(" ACTION OPTIONS:\n"); - printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", - platform::getDefaultClusterFilePath().c_str()); - printf(" -d, --destcontainer URL\n" - " The Backup container URL for start, modify, describe, query, expire, and delete " - "operations.\n"); - printBackupContainerInfo(); - printf(" -b, --base-url BASEURL\n" - " Base backup URL for list operations. This looks like a Backup URL but without a backup " - "name.\n"); - printf(" --blob-credentials FILE\n" - " File containing blob credentials in JSON format. Can be specified multiple times for " - "multiple files. See below for more details.\n"); - printf(" --expire-before-timestamp DATETIME\n" - " Datetime cutoff for expire operations. Requires a cluster file and will use " - "version/timestamp metadata\n" - " in the database to obtain a cutoff version very close to the timestamp given in %s.\n", - BackupAgentBase::timeFormat().c_str()); - printf(" --expire-before-version VERSION\n" - " Version cutoff for expire operations. Deletes data files containing no data at or after " - "VERSION.\n"); - printf(" --delete-before-days NUM_DAYS\n" - " Another way to specify version cutoff for expire operations. Deletes data files " - "containing no data at or after a\n" - " version approximately NUM_DAYS days worth of versions prior to the latest log version in " - "the backup.\n"); - printf(" --query-restore-snapshot-version VERSION\n" - " For query operations, set the snapshot version, inclusive, used to restore a backup.\n" - " Set -1 to use the latest valid snapshot,\n" - " Set -3 to use the oldest valid snapshot.\n"); - printf(" -qrv --query-restore-version VERSION\n" - " For query operations, set target version for restoring a backup. Set -1 for maximum\n" - " restorable version (default) and -3 for minimum restorable version.\n"); - printf( - " --query-restore-timestamp DATETIME\n" - " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", - BackupAgentBase::timeFormat().c_str()); - printf( - " and it will be converted to a version from that time using metadata in the cluster file.\n"); - printf(" --restorable-after-timestamp DATETIME\n" - " For expire operations, set minimum acceptable restorability to the version equivalent of " - "DATETIME and later.\n"); - printf(" --restorable-after-version VERSION\n" - " For expire operations, set minimum acceptable restorability to the VERSION and later.\n"); - printf(" --min-restorable-days NUM-DAYS\n" - " For expire operations, set minimum acceptable restorability to approximately NUM_DAYS " - "days worth of versions\n" - " prior to the latest log version in the backup.\n"); - printf(" --version-timestamps\n"); - printf(" For describe operations, lookup versions in the database to obtain timestamps. A cluster " - "file is required.\n"); - printf( - " -f, --force For expire operations, force expiration even if minimum restorability would be violated.\n"); - printf(" -s, --snapshot-interval DURATION\n" - " For start or modify operations, specifies the backup's default target snapshot interval " - "as DURATION seconds. Defaults to %d for start operations.\n", - CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC); - printf(" --mode MODE Snapshot mechanism to use: bulkdump, rangefile (default, legacy), or both.\n" - " bulkdump: Uses BulkDump SST files for faster restore performance\n" - " rangefile: Traditional range files for backward compatibility\n" - " both: Generate both formats for validation (increases backup size)\n"); - printf(" --active-snapshot-interval DURATION\n" - " For modify operations, sets the desired interval for the backup's currently active " - "snapshot, relative to the start of the snapshot.\n"); - printf(" --verify-uid UID\n" - " Specifies a UID to verify against the BackupUID of the running backup. If provided, the " - "UID is verified in the same transaction\n" - " which sets the new backup parameters (if the UID matches).\n"); - printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); - printf(" -k KEYS List of key ranges to backup or to filter the backup in query operations.\n" - " If not specified, the entire database will be backed up or no filter will be applied.\n"); - printf(" --keys-file FILE\n" - " Same as -k option, except keys are specified in the input file.\n"); - printf(" --partitioned-log-experimental Starts with new type of backup system using partitioned logs.\n"); - printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); - printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifies the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); - printf(" --loggroup LOG_GROUP\n" - " Sets the LogGroup field with the specified value for all\n" - " events in the trace output (defaults to `default').\n"); - printf(" --trace-format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); - printf(" --max-cleanup-seconds SECONDS\n" - " Specifies the amount of time a backup or DR needs to be stale before cleanup will\n" - " remove mutations for it. By default this is set to one hour.\n"); - printf(" --delete-data\n" - " This flag will cause cleanup to remove mutations for the most stale backup or DR.\n"); - printf(" --incremental\n" - " Performs incremental backup without the base backup.\n" - " This option indicates to the backup agent that it will only need to record the log files, " - "and ignore the range files.\n"); - printf(" --encryption-key-file" - " The AES-256-GCM key in the provided file is used for encrypting backup files.\n" - " For modify operations, need to pass encryption key file only if Backup container URL is " - "changed to " - "re-encrypt all future backup files. \n"); - - printf(TLS_HELP); - printf(" -w, --wait Wait for the backup to complete (allowed with `start' and `discontinue').\n"); - printf(" -z, --no-stop-when-done\n" - " Do not stop backup when restorable.\n"); - printf(" -h, --help Display this help and exit.\n"); - - if (devhelp) { -#ifdef _WIN32 - printf(" -n Create a new console.\n"); - printf(" -q Disable error dialog on crash.\n"); - printf(" --parentpid PID\n"); - printf(" Specify a process after whose termination to exit.\n"); -#endif - printf(" --deep For describe operations, do not use cached metadata. Warning: Very slow\n"); - } - printf("\n" - " KEYS FORMAT: \" \" [...]\n"); - printf("\n"); - puts(BlobCredentialInfo); - - return; -} - -static void printRestoreUsage(bool devhelp) { - printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | abort | wait) [OPTIONS]\n\n", - exeRestore.toString().c_str()); - - printf(" TOP LEVEL OPTIONS:\n"); - printf(" --build-flags Print build information and exit.\n"); - printf(" -v, --version Print version information and exit.\n"); - printf(" -h, --help Display this help and exit.\n"); - printf("\n"); - - printf(" ACTION OPTIONS:\n"); - // printf(" FOLDERS Paths to folders containing the backup files.\n"); - printf(" Options for all commands:\n\n"); - printf(" --dest-cluster-file CONNFILE\n"); - printf(" The cluster file to restore data into.\n"); - printf(" -t, --tagname TAGNAME\n"); - printf(" The restore tag to act on. Default is 'default'\n"); - printf("\n"); - printf(" Options for start:\n\n"); - printf(" -r URL The Backup URL for the restore to read from.\n"); - printBackupContainerInfo(); - printf(" -w, --waitfordone\n"); - printf(" Wait for the restore to complete before exiting. Prints progress updates.\n"); - printf(" -k KEYS List of key ranges from the backup to restore.\n"); - printf(" --keys-file FILE\n" - " Same as -k option, except keys are specified in the input file.\n"); - printf(" --remove-prefix PREFIX\n"); - printf(" Prefix to remove from the restored keys.\n"); - printf(" --add-prefix PREFIX\n"); - printf(" Prefix to add to the restored keys\n"); - printf(" -n, --dryrun Perform a trial run with no changes made.\n"); - printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifies the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); - printf(" --loggroup LOG_GROUP\n" - " Sets the LogGroup field with the specified value for all\n" - " events in the trace output (defaults to `default').\n"); - printf(" --trace-format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); - printf(" --incremental\n" - " Performs incremental restore without the base backup.\n" - " This tells the backup agent to only replay the log files from the backup source.\n" - " This also allows a restore to be performed into a non-empty destination database.\n"); - printf(" --begin-version\n" - " To be used in conjunction with incremental restore.\n" - " Indicates to the backup agent to only begin replaying log files from a certain version, " - "instead of the entire set.\n"); - printf( - " --mode MODE Restore mechanism to use: rangefile (default), bulkload.\n" - " rangefile: Traditional range file restore from kvranges/\n" - " bulkload: Use BulkLoad for faster range data restoration if BulkDump dataset is available\n" - " If incomplete dataset: restore returns error with clear message directing user to retry with " - "--mode rangefile.\n"); - printf(" --encryption-key-file" - " The AES-256-GCM key in the provided file is used for decrypting backup files.\n"); - printf(TLS_HELP); - printf(" -v DBVERSION The version at which the database will be restored.\n"); - printf(" --timestamp Instead of a numeric version, use this to specify a timestamp in %s\n", - BackupAgentBase::timeFormat().c_str()); - printf( - " and it will be converted to a version from that time using metadata in orig_cluster_file.\n"); - printf(" --orig-cluster-file CONNFILE\n"); - printf(" The cluster file for the original database from which the backup was created. The " - "original database\n"); - printf(" is only needed to convert a --timestamp argument to a database version.\n"); - printf(" --user-data\n" - " Restore only the user keyspace. This option should NOT be used alongside " - "--system-metadata (below) and CANNOT be used alongside other specified key ranges.\n"); - printf( - " --system-metadata\n" - " Restore only the relevant system keyspace. This option " - "should NOT be used alongside --user-data (above) and CANNOT be used alongside other specified key ranges.\n"); - - if (devhelp) { -#ifdef _WIN32 - printf(" -q Disable error dialog on crash.\n"); - printf(" --parentpid PID\n"); - printf(" Specify a process after whose termination to exit.\n"); -#endif - } - - printf("\n" - " KEYS FORMAT: \" \" [...]\n"); - printf("\n"); - puts(BlobCredentialInfo); - - return; -} - -static void printFastRestoreUsage(bool devhelp) { - printf(" NOTE: Fast restore aims to support the same fdbrestore option list.\n"); - printf(" But fast restore is still under development. The options may not be fully supported.\n"); - printf(" Supported options are: --dest-cluster-file, -r, --waitfordone, --logdir\n"); - printRestoreUsage(devhelp); - return; -} - -static void printDBAgentUsage(bool devhelp) { - printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s [OPTIONS]\n\n", exeDatabaseAgent.toString().c_str()); - printf(" -d, --destination CONNFILE\n" - " The path of a file containing the connection string for the\n" - " destination FoundationDB cluster.\n"); - printf(" -s, --source CONNFILE\n" - " The path of a file containing the connection string for the\n" - " source FoundationDB cluster.\n"); - printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifies the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); - printf(" --loggroup LOG_GROUP\n" - " Sets the LogGroup field with the specified value for all\n" - " events in the trace output (defaults to `default').\n"); - printf(" --trace-format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); - printf(" -m, --memory SIZE\n" - " Memory limit. The default value is 8GiB. When specified\n" - " without a unit, MiB is assumed.\n"); - printf(TLS_HELP); - printf(" --build-flags Print build information and exit.\n"); - printf(" -v, --version Print version information and exit.\n"); - printf(" -h, --help Display this help and exit.\n"); - if (devhelp) { -#ifdef _WIN32 - printf(" -n Create a new console.\n"); - printf(" -q Disable error dialog on crash.\n"); - printf(" --parentpid PID\n"); - printf(" Specify a process after whose termination to exit.\n"); -#endif - } - - return; -} - -static void printDBBackupUsage(bool devhelp) { - printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", - exeDatabaseBackup.toString().c_str()); - - printf(" TOP LEVEL OPTIONS:\n"); - printf(" --build-flags Print build information and exit.\n"); - printf(" -v, --version Print version information and exit.\n"); - printf(" -h, --help Display this help and exit.\n"); - printf("\n"); - - printf(" ACTION OPTIONS:\n"); - printf(" -d, --destination CONNFILE\n" - " The path of a file containing the connection string for the\n"); - printf(" destination FoundationDB cluster.\n"); - printf(" -s, --source CONNFILE\n" - " The path of a file containing the connection string for the\n" - " source FoundationDB cluster.\n"); - printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); - printf(" -k KEYS List of key ranges to backup.\n" - " If not specified, the entire database will be backed up.\n"); - printf(" --keys-file FILE\n" - " Same as -k option, except keys are specified in the input file.\n"); - printf(" --cleanup Abort will attempt to stop mutation logging on the source cluster.\n"); - printf(" --dstonly Abort will not make any changes on the source cluster.\n"); - printf(TLS_HELP); - printf(" --log Enables trace file logging for the CLI session.\n" - " --logdir PATH Specifies the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n"); - printf(" --loggroup LOG_GROUP\n" - " Sets the LogGroup field with the specified value for all\n" - " events in the trace output (defaults to `default').\n"); - printf(" --trace-format FORMAT\n" - " Select the format of the trace files. xml (the default) and json are supported.\n" - " Has no effect unless --log is specified.\n"); - printf(" -h, --help Display this help and exit.\n"); - printf("\n" - " KEYS FORMAT: \" \" [...]\n"); - - if (devhelp) { -#ifdef _WIN32 - printf(" -n Create a new console.\n"); - printf(" -q Disable error dialog on crash.\n"); - printf(" --parentpid PID\n"); - printf(" Specify a process after whose termination to exit.\n"); -#endif - } - - return; -} - -static void printUsage(ProgramExe programExe, bool devhelp) { - - switch (programExe) { - case ProgramExe::AGENT: - printAgentUsage(devhelp); - break; - case ProgramExe::BACKUP: - printBackupUsage(devhelp); - break; - case ProgramExe::RESTORE: - printRestoreUsage(devhelp); - break; - case ProgramExe::FASTRESTORE_TOOL: - printFastRestoreUsage(devhelp); - break; - case ProgramExe::DR_AGENT: - printDBAgentUsage(devhelp); - break; - case ProgramExe::DB_BACKUP: - printDBBackupUsage(devhelp); - break; - case ProgramExe::UNDEFINED: - default: - break; - } - - return; -} - -extern bool g_crashOnError; - -// Return the type of program executable based on the name of executable file -ProgramExe getProgramType(std::string programExe) { - ProgramExe enProgramExe = ProgramExe::UNDEFINED; - - std::transform(programExe.begin(), programExe.end(), programExe.begin(), ::tolower); - - // Remove the extension, if Windows -#ifdef _WIN32 - size_t lastDot = programExe.find_last_of("."); - if (lastDot != std::string::npos) { - size_t lastSlash = programExe.find_last_of("\\"); - - // Ensure last dot is after last slash, if present - if ((lastSlash == std::string::npos) || (lastSlash < lastDot)) { - programExe = programExe.substr(0, lastDot); - } - } -#endif - // For debugging convenience, remove .debug suffix if present. - if (StringRef(programExe).endsWith(".debug"_sr)) - programExe = programExe.substr(0, programExe.size() - 6); - - // Check if backup agent - if ((programExe.length() >= exeAgent.size()) && - (programExe.compare(programExe.length() - exeAgent.size(), exeAgent.size(), (const char*)exeAgent.begin()) == - 0)) { - enProgramExe = ProgramExe::AGENT; - } - - // Check if backup - else if ((programExe.length() >= exeBackup.size()) && - (programExe.compare( - programExe.length() - exeBackup.size(), exeBackup.size(), (const char*)exeBackup.begin()) == 0)) { - enProgramExe = ProgramExe::BACKUP; - } - - // Check if restore - else if ((programExe.length() >= exeRestore.size()) && - (programExe.compare( - programExe.length() - exeRestore.size(), exeRestore.size(), (const char*)exeRestore.begin()) == 0)) { - enProgramExe = ProgramExe::RESTORE; - } - - // Check if restore - else if ((programExe.length() >= exeFastRestoreTool.size()) && - (programExe.compare(programExe.length() - exeFastRestoreTool.size(), - exeFastRestoreTool.size(), - (const char*)exeFastRestoreTool.begin()) == 0)) { - enProgramExe = ProgramExe::FASTRESTORE_TOOL; - } - - // Check if db agent - else if ((programExe.length() >= exeDatabaseAgent.size()) && - (programExe.compare(programExe.length() - exeDatabaseAgent.size(), - exeDatabaseAgent.size(), - (const char*)exeDatabaseAgent.begin()) == 0)) { - enProgramExe = ProgramExe::DR_AGENT; - } - - // Check if db backup - else if ((programExe.length() >= exeDatabaseBackup.size()) && - (programExe.compare(programExe.length() - exeDatabaseBackup.size(), - exeDatabaseBackup.size(), - (const char*)exeDatabaseBackup.begin()) == 0)) { - enProgramExe = ProgramExe::DB_BACKUP; - } - - return enProgramExe; -} - -BackupType getBackupType(std::string backupType) { - BackupType enBackupType = BackupType::UNDEFINED; - - std::transform(backupType.begin(), backupType.end(), backupType.begin(), ::tolower); - - static std::map values; - if (values.empty()) { - values["start"] = BackupType::START; - values["status"] = BackupType::STATUS; - values["abort"] = BackupType::ABORT; - values["cleanup"] = BackupType::CLEANUP; - values["wait"] = BackupType::WAIT; - values["discontinue"] = BackupType::DISCONTINUE; - values["pause"] = BackupType::PAUSE; - values["resume"] = BackupType::RESUME; - values["expire"] = BackupType::EXPIRE; - values["delete"] = BackupType::DELETE_BACKUP; - values["describe"] = BackupType::DESCRIBE; - values["list"] = BackupType::LIST; - values["query"] = BackupType::QUERY; - values["dump"] = BackupType::DUMP; - values["modify"] = BackupType::MODIFY; - values["tags"] = BackupType::TAGS; - } - - auto i = values.find(backupType); - if (i != values.end()) - enBackupType = i->second; - - return enBackupType; -} - -Optional getSnapshotMode(std::string mode) { - std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); - - if (mode == "rangefile") - return SnapshotMode::RANGEFILE; - if (mode == "bulkdump") - return SnapshotMode::BULKDUMP; - if (mode == "both") - return SnapshotMode::BOTH; - return Optional(); -} - -Optional getRestoreMode(std::string mode) { - std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); - - if (mode == "rangefile") - return RestoreMode::RANGEFILE; - if (mode == "bulkload") - return RestoreMode::BULKLOAD; - return Optional(); -} - -RestoreType getRestoreType(std::string name) { - if (name == "start") - return RestoreType::START; - if (name == "abort") - return RestoreType::ABORT; - if (name == "status") - return RestoreType::STATUS; - if (name == "wait") - return RestoreType::WAIT; - return RestoreType::UNKNOWN; -} - -DBType getDBType(std::string dbType) { - DBType enBackupType = DBType::UNDEFINED; - - std::transform(dbType.begin(), dbType.end(), dbType.begin(), ::tolower); - - static std::map values; - if (values.empty()) { - values["start"] = DBType::START; - values["status"] = DBType::STATUS; - values["switch"] = DBType::SWITCH; - values["abort"] = DBType::ABORT; - values["pause"] = DBType::PAUSE; - values["resume"] = DBType::RESUME; - } - - auto i = values.find(dbType); - if (i != values.end()) - enBackupType = i->second; - - return enBackupType; -} - -ACTOR Future getLayerStatus(Reference tr, - IPAddress localIP, - std::string name, - std::string id, - ProgramExe exe, - Database dest, - Snapshot snapshot = Snapshot::False) { - // This process will write a document that looks like this: - // { backup : { $expires : {}, version: } - // so that the value under 'backup' will eventually expire to null and thus be ignored by - // readers of status. This is because if all agents die then they can no longer clean up old - // status docs from other dead agents. - - state Version readVer = wait(tr->getReadVersion()); - - state json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root - JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root - JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation - // Create the $expires key which is where the rest of the status output will go - - state JSONDoc layerRoot = op.subDoc("$expires"); - // Set the version argument in the $expires operator object. - op.create("version") = readVer + 120 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; - - layerRoot.create("instances_running.$sum") = 1; - layerRoot.create("last_updated.$max") = now(); - - state JSONDoc o = layerRoot.subDoc("instances." + id); - - o.create("version") = FDB_VT_VERSION; - o.create("id") = id; - o.create("last_updated") = now(); - o.create("memory_usage") = (int64_t)getMemoryUsage(); - o.create("resident_size") = (int64_t)getResidentMemoryUsage(); - o.create("main_thread_cpu_seconds") = getProcessorTimeThread(); - o.create("process_cpu_seconds") = getProcessorTimeProcess(); - o.create("configured_workers") = CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; - o.create("processID") = ::getpid(); - o.create("locality") = tr->getDatabase()->clientLocality.toJSON(); - o.create("networkAddress") = localIP.toString(); - - if (exe == ProgramExe::AGENT) { - static S3BlobStoreEndpoint::Stats last_stats; - static double last_ts = 0; - S3BlobStoreEndpoint::Stats current_stats = S3BlobStoreEndpoint::s_stats; - JSONDoc blobstats = o.create("blob_stats"); - blobstats.create("total") = current_stats.getJSON(); - S3BlobStoreEndpoint::Stats diff = current_stats - last_stats; - json_spirit::mObject diffObj = diff.getJSON(); - if (last_ts > 0) - diffObj["bytes_per_second"] = double(current_stats.bytes_sent - last_stats.bytes_sent) / (now() - last_ts); - blobstats.create("recent") = diffObj; - last_stats = current_stats; - last_ts = now(); - - JSONDoc totalBlobStats = layerRoot.subDoc("blob_recent_io"); - for (auto& p : diffObj) - totalBlobStats.create(p.first + ".$sum") = p.second; - - state FileBackupAgent fba; - state std::vector backupTags = wait(getAllBackupTags(tr, snapshot)); - state std::vector>> tagLastRestorableVersions; - state std::vector> tagStates; - state std::vector>> tagContainers; - state std::vector> tagRangeBytes; - state std::vector> tagLogBytes; - state Future> fBackupPaused = tr->get(fba.taskBucket->getPauseKey(), snapshot); - - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state std::vector::iterator tag; - state std::vector backupTagUids; - for (tag = backupTags.begin(); tag != backupTags.end(); tag++) { - UidAndAbortedFlagT uidAndAbortedFlag = wait(tag->getOrThrow(tr, snapshot)); - BackupConfig config(uidAndAbortedFlag.first); - backupTagUids.push_back(config.getUid()); - - tagStates.push_back(config.stateEnum().getOrThrow(tr, snapshot)); - tagRangeBytes.push_back(config.rangeBytesWritten().getD(tr, snapshot, 0)); - tagLogBytes.push_back(config.logBytesWritten().getD(tr, snapshot, 0)); - tagContainers.push_back(config.backupContainer().getOrThrow(tr, snapshot)); - tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName), snapshot)); - } - - wait(waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && - waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); - - state std::vector> encryptionSetupResults; - state std::vector encryptionContainerIndices; - - for (int i = 0; i < tagContainers.size(); i++) { - if (tagContainers[i].get()->getEncryptionKeyFileName().present()) { - encryptionSetupResults.push_back(tagContainers[i].get()->encryptionSetupComplete()); - encryptionContainerIndices.push_back(i); - } - } - wait(waitForAllReady(encryptionSetupResults)); - json_spirit::mArray keysArr; - std::unordered_set seenKeyPaths; - for (int j = 0; j < encryptionContainerIndices.size() && j < 1e6; j++) { - int i = encryptionContainerIndices[j]; - std::string keyPath = tagContainers[i].get()->getEncryptionKeyFileName().get(); - - if (seenKeyPaths.find(keyPath) == seenKeyPaths.end()) { - seenKeyPaths.insert(keyPath); - json_spirit::mObject keyObj; - keyObj["path"] = tagContainers[i].get()->getEncryptionKeyFileName().get(); - keyObj["success"] = !encryptionSetupResults[j].isError(); - keysArr.push_back(keyObj); - } - } - o.create("encryption_keys") = keysArr; - - JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest"); - layerRoot.create("tags.timestamp") = now(); - layerRoot.create("total_workers.$sum") = - fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; - layerRoot.create("paused.$latest") = fBackupPaused.get().present(); - - int j = 0; - for (KeyBackedTag eachTag : backupTags) { - EBackupState status = tagStates[j].get(); - const char* statusText = fba.getStateText(status); - - // The object for this backup tag inside this instance's subdocument - JSONDoc tagRoot = tagsRoot.subDoc(eachTag.tagName); - tagRoot.create("current_container") = tagContainers[j].get()->getURL(); - tagRoot.create("current_status") = statusText; - if (tagLastRestorableVersions[j].get().present()) { - Version last_restorable_version = tagLastRestorableVersions[j].get().get(); - double last_restorable_seconds_behind = - ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; - tagRoot.create("last_restorable_version") = last_restorable_version; - tagRoot.create("last_restorable_seconds_behind") = last_restorable_seconds_behind; - } - tagRoot.create("running_backup") = - (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING); - tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL); - tagRoot.create("range_bytes_written") = tagRangeBytes[j].get(); - tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get(); - tagRoot.create("mutation_stream_id") = backupTagUids[j].toString(); - tagRoot.create("file_level_encryption") = - tagContainers[j].get()->getEncryptionKeyFileName().present() ? true : false; - if (tagContainers[j].get()->getEncryptionKeyFileName().present()) { - tagRoot.create("encryption_key_file") = tagContainers[j].get()->getEncryptionKeyFileName().get(); - } - j++; - } - } else if (exe == ProgramExe::DR_AGENT) { - state DatabaseBackupAgent dba; - state Reference tr2(new ReadYourWritesTransaction(dest)); - tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr2->setOption(FDBTransactionOptions::LOCK_AWARE); - state RangeResult tagNames = wait(tr2->getRange(dba.tagNames.range(), 10000, snapshot)); - state std::vector>> backupVersion; - state std::vector> backupStatus; - state std::vector> tagRangeBytesDR; - state std::vector> tagLogBytesDR; - state Future> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot); - - state std::vector drTagUids; - for (int i = 0; i < tagNames.size(); i++) { - backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin), snapshot)); - UID tagUID = BinaryReader::fromStringRef(tagNames[i].value, Unversioned()); - drTagUids.push_back(tagUID); - backupStatus.push_back(dba.getStateValue(tr2, tagUID, snapshot)); - tagRangeBytesDR.push_back(dba.getRangeBytesWritten(tr2, tagUID, snapshot)); - tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID, snapshot)); - } - - wait(waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && - waitForAll(tagLogBytesDR) && success(fDRPaused)); - - JSONDoc tagsRoot = layerRoot.subDoc("tags.$latest"); - layerRoot.create("tags.timestamp") = now(); - layerRoot.create("total_workers.$sum") = fDRPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; - layerRoot.create("paused.$latest") = fDRPaused.get().present(); - - for (int i = 0; i < tagNames.size(); i++) { - std::string tagName = dba.sourceTagNames.unpack(tagNames[i].key).getString(0).toString(); - - auto status = backupStatus[i].get(); - - JSONDoc tagRoot = tagsRoot.create(tagName); - tagRoot.create("running_backup") = - (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING); - tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL); - tagRoot.create("range_bytes_written") = tagRangeBytesDR[i].get(); - tagRoot.create("mutation_log_bytes_written") = tagLogBytesDR[i].get(); - tagRoot.create("mutation_stream_id") = drTagUids[i].toString(); - - if (backupVersion[i].get().present()) { - double seconds_behind = ((double)readVer - BinaryReader::fromStringRef( - backupVersion[i].get().get(), Unversioned())) / - CLIENT_KNOBS->CORE_VERSIONSPERSECOND; - tagRoot.create("seconds_behind") = seconds_behind; - //TraceEvent("BackupMetrics").detail("SecondsBehind", seconds_behind); - } - - tagRoot.create("backup_state") = BackupAgentBase::getStateText(status); - } - } - - std::string json = json_spirit::write_string(layersRootValue); - return json; -} - -// Check for unparsable or expired statuses and delete them. -// First checks the first doc in the key range, and if it is valid, alive and not "me" then -// returns. Otherwise, checks the rest of the range as well. -ACTOR Future cleanupStatus(Reference tr, - std::string rootKey, - std::string name, - std::string id, - int limit = 1) { - state RangeResult docs = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True)); - state bool readMore = false; - state int i; - for (i = 0; i < docs.size(); ++i) { - json_spirit::mValue docValue; - try { - json_spirit::read_string(docs[i].value.toString(), docValue); - JSONDoc doc(docValue); - // Update the reference version for $expires - JSONDoc::expires_reference_version = tr->getReadVersion().get(); - // Evaluate the operators in the document, which will reduce to nothing if it is expired. - doc.cleanOps(); - if (!doc.has(name + ".last_updated")) - throw Error(); - - // Alive and valid. - // If limit == 1 and id is present then read more - if (limit == 1 && doc.has(name + ".instances." + id)) - readMore = true; - } catch (Error& e) { - // If doc can't be parsed or isn't alive, delete it. - TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").errorUnsuppressed(e).detail("Key", docs[i].key); - tr->clear(docs[i].key); - // If limit is 1 then read more. - if (limit == 1) - readMore = true; - } - if (readMore) { - limit = 10000; - RangeResult docs2 = wait(tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True)); - docs = std::move(docs2); - readMore = false; - } - } - - return Void(); -} - -// Get layer status document for just this layer -ACTOR Future getLayerStatus(Database src, std::string rootKey) { - state Transaction tr(src); - - loop { - try { - tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - state RangeResult kvPairs = - wait(tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), GetRangeLimits::ROW_LIMIT_UNLIMITED)); - state json_spirit::mObject statusDoc; - state JSONDoc modifier(statusDoc); - for (auto& kv : kvPairs) { - state json_spirit::mValue docValue; - json_spirit::read_string(kv.value.toString(), docValue); - wait(yield()); - modifier.absorb(docValue); - wait(yield()); - } - JSONDoc::expires_reference_version = (uint64_t)tr.getReadVersion().get(); - modifier.cleanOps(); - return statusDoc; - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - -// Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay -// based on that and BACKUP_AGGREGATE_POLL_RATE -ACTOR Future updateAgentPollRate(Database src, - std::string rootKey, - std::string name, - std::shared_ptr pollDelay) { - loop { - try { - json_spirit::mObject status = wait(getLayerStatus(src, rootKey)); - int64_t processes = 0; - // If instances count is present and greater than 0 then update pollDelay - if (JSONDoc(status).tryGet(name + ".instances_running", processes) && processes > 0) { - // The aggregate poll rate is the target poll rate for all agent processes in the cluster - // The poll rate (polls/sec) for a single processes is aggregate poll rate / processes, and pollDelay is - // the inverse of that - *pollDelay = (double)processes / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; - } - } catch (Error& e) { - TraceEvent(SevWarn, "BackupAgentPollRateUpdateError").error(e); - } - wait(delay(CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL)); - } -} - -ACTOR Future statusUpdateActor(Database statusUpdateDest, - std::string name, - ProgramExe exe, - std::shared_ptr pollDelay, - Database taskDest = Database(), - std::string id = nondeterministicRandom()->randomUniqueID().toString()) { - state std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name; - state std::string rootKey = backupStatusPrefixRange.begin.toString() + name + "/json"; - state std::string instanceKey = rootKey + "/" + "agent-" + id; - state Reference tr(new ReadYourWritesTransaction(statusUpdateDest)); - state Future pollRateUpdater; - - // In order to report a useful networkAddress to the cluster's layer status JSON object, determine which local - // network interface IP will be used to talk to the cluster. This is a blocking call, so it is only done once, - // and in a retry loop because if we can't connect to the cluster we can't do any work anyway. - state IPAddress localIP; - - loop { - try { - localIP = statusUpdateDest->getConnectionRecord()->getConnectionString().determineLocalSourceIP(); - break; - } catch (Error& e) { - TraceEvent(SevWarn, "AgentCouldNotDetermineLocalIP").error(e); - wait(delay(1.0)); - } - } - - // Register the existence of this layer in the meta key space - loop { - tr->reset(); - try { - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - tr->set(metaKey, rootKey); - wait(tr->commit()); - break; - } catch (Error& e) { - TraceEvent(SevWarnAlways, "LayerStatusMetaKeyUpdateError").errorUnsuppressed(e); - wait(tr->onError(e)); // Non-retryable txns throws back the error. - } - } - break; - } catch (Error& e) { - // For non-retryable txns, do delay, reset txn and retry - TraceEvent(SevWarnAlways, "UnableToWriteLayerStatusMetaKey").errorUnsuppressed(e); - wait(delay(5.0)); - } - } - - // Write status periodically - loop { - tr->reset(); - try { - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state Future futureStatusDoc = - getLayerStatus(tr, localIP, name, id, exe, taskDest, Snapshot::True); - wait(cleanupStatus(tr, rootKey, name, id)); - std::string statusdoc = wait(futureStatusDoc); - tr->set(instanceKey, statusdoc); - wait(tr->commit()); - break; - } catch (Error& e) { - TraceEvent(SevWarnAlways, "LayerBackupStatusUpdateError").errorUnsuppressed(e); - wait(tr->onError(e)); - } - } - - wait(delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY * - ((1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER) + - 2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER))); - - // Now that status was written at least once by this process (and hopefully others), start the poll rate - // control updater if it wasn't started yet - if (!pollRateUpdater.isValid()) - pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay); - } catch (Error& e) { - TraceEvent(SevWarnAlways, "UnableToWriteBackupStatus").error(e); - wait(delay(10.0)); - } - } -} - -ACTOR Future runDBAgent(Database src, Database dest) { - state std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); - std::string id = nondeterministicRandom()->randomUniqueID().toString(); - state Future status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, pollDelay, dest, id); - state Future status_other = - statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, pollDelay, dest, id); - - state DatabaseBackupAgent backupAgent(src); - - loop { - try { - wait(backupAgent.run(dest, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); - break; - } catch (Error& e) { - if (e.code() == error_code_operation_cancelled) - throw; - - TraceEvent(SevError, "DA_runAgent").error(e); - fprintf(stderr, "ERROR: DR agent encountered fatal error `%s'\n", e.what()); - - wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); - } - } - - return Void(); -} - -ACTOR Future runAgent(Database db) { - state std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); - state Future status = statusUpdateActor(db, "backup", ProgramExe::AGENT, pollDelay); - - state FileBackupAgent backupAgent; - - loop { - try { - wait(backupAgent.run(db, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT)); - break; - } catch (Error& e) { - if (e.code() == error_code_operation_cancelled) - throw; - - TraceEvent(SevError, "BA_runAgent").error(e); - fprintf(stderr, "ERROR: backup agent encountered fatal error `%s'\n", e.what()); - - wait(delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY)); - } - } - - return Void(); -} - -ACTOR Future submitDBBackup(Database src, - Database dest, - Standalone> backupRanges, - std::string tagName) { - try { - state DatabaseBackupAgent backupAgent(src); - ASSERT(!backupRanges.empty()); - - wait(backupAgent.submitBackup( - dest, KeyRef(tagName), backupRanges, StopWhenDone::False, StringRef(), StringRef(), LockDB::True)); - - // Check if a backup agent is running - bool agentRunning = wait(backupAgent.checkActive(dest)); - - if (!agentRunning) { - printf("The DR on tag `%s' was successfully submitted but no DR agents are responding.\n", - printable(StringRef(tagName)).c_str()); - - // Throw an error that will not display any additional information - throw actor_cancelled(); - } else { - printf("The DR on tag `%s' was successfully submitted.\n", printable(StringRef(tagName)).c_str()); - } - } - - catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - switch (e.code()) { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); - break; - case error_code_backup_duplicate: - fprintf(stderr, "ERROR: A DR is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); - break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); - break; - } - - throw backup_error(); - } - - return Void(); -} - -ACTOR Future submitBackup(Database db, - std::string url, - Optional proxy, - int initialSnapshotIntervalSeconds, - int snapshotIntervalSeconds, - Standalone> backupRanges, - std::string tagName, - bool dryRun, - WaitForComplete waitForCompletion, - StopWhenDone stopWhenDone, - UsePartitionedLog usePartitionedLog, - IncrementalBackupOnly incrementalBackupOnly, - Optional encryptionKeyFile, - SnapshotMode snapshotMode = SnapshotMode::RANGEFILE) { - try { - state FileBackupAgent backupAgent; - ASSERT(!backupRanges.empty()); - - if (dryRun) { - state KeyBackedTag tag = makeBackupTag(tagName); - Optional uidFlag = wait(tag.get(db.getReference())); - - if (uidFlag.present()) { - BackupConfig config(uidFlag.get().first); - EBackupState backupStatus = wait(config.stateEnum().getOrThrow(db.getReference())); - - // Throw error if a backup is currently running until we support parallel backups - if (BackupAgentBase::isRunnable(backupStatus)) { - throw backup_duplicate(); - } - } - - if (waitForCompletion) { - printf("Submitted and now waiting for the backup on tag `%s' to complete. (DRY RUN)\n", - printable(StringRef(tagName)).c_str()); - } - - else { - // Check if a backup agent is running - bool agentRunning = wait(backupAgent.checkActive(db)); - - if (!agentRunning) { - printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. " - "(DRY RUN)\n", - printable(StringRef(tagName)).c_str()); - - // Throw an error that will not display any additional information - throw actor_cancelled(); - } else { - printf("The backup on tag `%s' was successfully submitted. (DRY RUN)\n", - printable(StringRef(tagName)).c_str()); - } - } - } - - else { - wait(backupAgent.submitBackup(db, - KeyRef(url), - proxy, - initialSnapshotIntervalSeconds, - snapshotIntervalSeconds, - tagName, - backupRanges, - stopWhenDone, - usePartitionedLog, - incrementalBackupOnly, - encryptionKeyFile, - static_cast(snapshotMode))); - - // Wait for the backup to complete, if requested - if (waitForCompletion) { - printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", - printable(StringRef(tagName)).c_str()); - wait(success(backupAgent.waitBackup(db, tagName))); - } else { - // Check if a backup agent is running - bool agentRunning = wait(backupAgent.checkActive(db)); - - if (!agentRunning) { - printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n", - printable(StringRef(tagName)).c_str()); - - // Throw an error that will not display any additional information - throw actor_cancelled(); - } else { - printf("The backup on tag `%s' was successfully submitted.\n", - printable(StringRef(tagName)).c_str()); - } - } - } - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - switch (e.code()) { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); - break; - case error_code_backup_duplicate: - fprintf(stderr, "ERROR: A backup is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); - break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); - break; - } - - throw backup_error(); - } - - return Void(); -} - -ACTOR Future switchDBBackup(Database src, - Database dest, - Standalone> backupRanges, - std::string tagName, - ForceAction forceAction) { - try { - state DatabaseBackupAgent backupAgent(src); - ASSERT(!backupRanges.empty()); - - wait(backupAgent.atomicSwitchover(dest, KeyRef(tagName), backupRanges, StringRef(), StringRef(), forceAction)); - printf("The DR on tag `%s' was successfully switched.\n", printable(StringRef(tagName)).c_str()); - } - - catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - switch (e.code()) { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); - break; - case error_code_backup_duplicate: - fprintf(stderr, "ERROR: A DR is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); - break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); - break; - } - - throw backup_error(); - } - - return Void(); -} - -ACTOR Future statusDBBackup(Database src, Database dest, std::string tagName, int errorLimit) { - try { - state DatabaseBackupAgent backupAgent(src); - - std::string statusText = wait(backupAgent.getStatus(dest, errorLimit, StringRef(tagName))); - printf("%s\n", statusText.c_str()); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future statusBackup(Database db, std::string tagName, ShowErrors showErrors, bool json) { - try { - state FileBackupAgent backupAgent; - - std::string statusText = - wait(json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName)); - printf("%s\n", statusText.c_str()); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future abortDBBackup(Database src, - Database dest, - std::string tagName, - PartialBackup partial, - DstOnly dstOnly) { - try { - state DatabaseBackupAgent backupAgent(src); - - wait(backupAgent.abortBackup(dest, Key(tagName), partial, AbortOldBackup::False, dstOnly)); - wait(backupAgent.unlockBackup(dest, Key(tagName))); - - printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - switch (e.code()) { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); - break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A DR was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); - break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); - break; - } - throw; - } - - return Void(); -} - -ACTOR Future abortBackup(Database db, std::string tagName) { - try { - state FileBackupAgent backupAgent; - - wait(backupAgent.abortBackup(db, tagName)); - - printf("The backup on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - switch (e.code()) { - case error_code_backup_error: - fprintf(stderr, "ERROR: An error was encountered during submission\n"); - break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A backup was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); - break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); - break; - } - throw; - } - - return Void(); -} - -ACTOR Future cleanupMutations(Database db, DeleteData deleteData) { - try { - wait(cleanupBackup(db, deleteData)); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future waitBackup(Database db, std::string tagName, StopWhenDone stopWhenDone) { - try { - state FileBackupAgent backupAgent; - - EBackupState status = wait(backupAgent.waitBackup(db, tagName, stopWhenDone)); - - printf("The backup on tag `%s' %s.\n", - printable(StringRef(tagName)).c_str(), - BackupAgentBase::getStateText(status)); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future discontinueBackup(Database db, std::string tagName, WaitForComplete waitForCompletion) { - try { - state FileBackupAgent backupAgent; - - wait(backupAgent.discontinueBackup(db, StringRef(tagName))); - - // Wait for the backup to complete, if requested - if (waitForCompletion) { - printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", - printable(StringRef(tagName)).c_str()); - wait(success(backupAgent.waitBackup(db, tagName))); - } else { - printf("The backup on tag `%s' was successfully discontinued.\n", printable(StringRef(tagName)).c_str()); - } - - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - switch (e.code()) { - case error_code_backup_error: - fprintf(stderr, "ERROR: An encounter was error during submission\n"); - break; - case error_code_backup_unneeded: - fprintf(stderr, "ERROR: A backup in not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); - break; - case error_code_backup_duplicate: - fprintf(stderr, - "ERROR: The backup on tag `%s' is already discontinued\n", - printable(StringRef(tagName)).c_str()); - break; - default: - fprintf(stderr, "ERROR: %s\n", e.what()); - break; - } - throw; - } - - return Void(); -} - -ACTOR Future changeBackupResumed(Database db, bool pause) { - try { - FileBackupAgent backupAgent; - wait(backupAgent.changePause(db, pause)); - printf("All backup agents have been %s.\n", pause ? "paused" : "resumed"); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future changeDBBackupResumed(Database src, Database dest, bool pause) { - try { - state DatabaseBackupAgent backupAgent(src); - wait(backupAgent.taskBucket->changePause(dest, pause)); - printf("All DR agents have been %s.\n", pause ? "paused" : "resumed"); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -Reference openBackupContainer(const char* name, - const std::string& destinationContainer, - const Optional& proxy, - const Optional& encryptionKeyFile) { - // Error, if no dest container was specified - if (destinationContainer.empty()) { - fprintf(stderr, "ERROR: No backup destination was specified.\n"); - printHelpTeaser(name); - throw backup_error(); - } - - if (destinationContainer.find("../") != std::string::npos) { - fprintf( - stderr, "ERROR: Backup Container URL '%s' contains directory traversals\n", destinationContainer.c_str()); - printHelpTeaser(name); - throw backup_invalid_url(); - } - - Reference c; - try { - c = IBackupContainer::openContainer(destinationContainer, proxy, encryptionKeyFile); - } catch (Error& e) { - std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); - if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { - msg += format(": %s", IBackupContainer::lastOpenError.c_str()); - } - fprintf(stderr, "%s\n", msg.c_str()); - printHelpTeaser(name); - throw; - } - - return c; -} - -// Submit the restore request to the database if "performRestore" is true. Otherwise, -// check if the restore can be performed. -ACTOR Future runRestore(Database db, - std::string originalClusterFile, - std::string tagName, - std::string container, - Optional proxy, - Standalone> ranges, - Version beginVersion, - Version targetVersion, - std::string targetTimestamp, - bool performRestore, - Verbose verbose, - WaitForComplete waitForDone, - std::string addPrefix, - std::string removePrefix, - OnlyApplyMutationLogs onlyApplyMutationLogs, - InconsistentSnapshotOnly inconsistentSnapshotOnly, - Optional encryptionKeyFile, - RestoreMode restoreMode = RestoreMode::RANGEFILE) { - ASSERT(!ranges.empty()); - - if (targetVersion != invalidVersion && !targetTimestamp.empty()) { - fprintf(stderr, "Restore target version and target timestamp cannot both be specified\n"); - throw restore_error(); - } - - state Optional origDb; - - // Resolve targetTimestamp if given - if (!targetTimestamp.empty()) { - if (originalClusterFile.empty()) { - fprintf(stderr, - "An original cluster file must be given in order to resolve restore target timestamp '%s'\n", - targetTimestamp.c_str()); - throw restore_error(); - } - - if (!fileExists(originalClusterFile)) { - fprintf( - stderr, "Original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); - throw restore_error(); - } - - origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION); - Version v = wait(timeKeeperVersionFromDatetime(targetTimestamp, origDb.get())); - fmt::print("Timestamp '{0}' resolves to version {1}\n", targetTimestamp, v); - targetVersion = v; - } - - try { - state FileBackupAgent backupAgent; - - state Reference bc = - openBackupContainer(exeRestore.toString().c_str(), container, proxy, encryptionKeyFile); - - // If targetVersion is unset then use the maximum restorable version from the backup description - if (targetVersion == invalidVersion) { - if (verbose) - printf( - "No restore target version given, will use maximum restorable version from backup description.\n"); - - // For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties - BackupDescription desc = wait(bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0)); - - if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) { - targetVersion = desc.contiguousLogEnd.get() - 1; - } else if (desc.maxRestorableVersion.present()) { - targetVersion = desc.maxRestorableVersion.get(); - } else { - fprintf(stderr, "The specified backup is not restorable to any version.\n"); - throw restore_error(); - } - - if (verbose) { - fmt::print("Using target restore version {}\n", targetVersion); - } - } - - if (performRestore) { - Version restoredVersion = wait(backupAgent.restore(db, - origDb, - KeyRef(tagName), - KeyRef(container), - proxy, - ranges, - waitForDone, - targetVersion, - verbose, - KeyRef(addPrefix), - KeyRef(removePrefix), - LockDB::True, - UnlockDB::True, - onlyApplyMutationLogs, - inconsistentSnapshotOnly, - beginVersion, - encryptionKeyFile, - {}, - restoreMode == RestoreMode::RANGEFILE)); - - if (waitForDone && verbose) { - // If restore is now complete then report version restored - fmt::print("Restored to version {}\n", restoredVersion); - } - } else { - state Optional rset = wait(bc->getRestoreSet(targetVersion, ranges)); - - if (!rset.present()) { - fmt::print(stderr, - "Insufficient data to restore to version {}. Describe backup for more information.\n", - targetVersion); - throw restore_invalid_version(); - } - - fmt::print("Backup can be used to restore to version {}\n", targetVersion); - } - - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -// Fast restore agent that kicks off the restore: send restore requests to restore workers. -ACTOR Future runFastRestoreTool(Database db, - std::string tagName, - std::string container, - Optional proxy, - Standalone> ranges, - Version dbVersion, - bool performRestore, - Verbose verbose, - WaitForComplete waitForDone) { - try { - state FileBackupAgent backupAgent; - state Version restoreVersion = invalidVersion; - - if (ranges.size() > 1) { - fprintf(stdout, "[WARNING] Currently only a single restore range is tested!\n"); - } - - if (ranges.size() == 0) { - ranges.push_back(ranges.arena(), normalKeys); - } - - printf("[INFO] runFastRestoreTool: restore_ranges:%d first range:%s\n", - ranges.size(), - ranges.front().toString().c_str()); - TraceEvent ev("FastRestoreTool"); - ev.detail("RestoreRanges", ranges.size()); - for (int i = 0; i < ranges.size(); ++i) { - ev.detail(format("Range%d", i), ranges[i]); - } - - if (performRestore) { - if (dbVersion == invalidVersion) { - TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", "Largest restorable version"); - // For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties - BackupDescription desc = - wait(IBackupContainer::openContainer(container, proxy, {}) - ->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0)); - if (!desc.maxRestorableVersion.present()) { - fprintf(stderr, "The specified backup is not restorable to any version.\n"); - throw restore_error(); - } - - dbVersion = desc.maxRestorableVersion.get(); - TraceEvent("FastRestoreTool").detail("TargetRestoreVersion", dbVersion); - } - state UID randomUID = deterministicRandom()->randomUniqueID(); - TraceEvent("FastRestoreTool") - .detail("SubmitRestoreRequests", ranges.size()) - .detail("RestoreUID", randomUID); - wait(backupAgent.submitParallelRestore(db, - KeyRef(tagName), - ranges, - KeyRef(container), - proxy, - dbVersion, - LockDB::True, - randomUID, - ""_sr, - ""_sr)); - // TODO: Support addPrefix and removePrefix - if (waitForDone) { - // Wait for parallel restore to finish and unlock DB after that - TraceEvent("FastRestoreTool").detail("BackupAndParallelRestore", "WaitForRestoreToFinish"); - wait(backupAgent.parallelRestoreFinish(db, randomUID)); - TraceEvent("FastRestoreTool").detail("BackupAndParallelRestore", "RestoreFinished"); - } else { - TraceEvent("FastRestoreTool") - .detail("RestoreUID", randomUID) - .detail("OperationGuide", "Manually unlock DB when restore finishes"); - printf("WARNING: DB will be in locked state after restore. Need UID:%s to unlock DB\n", - randomUID.toString().c_str()); - } - - restoreVersion = dbVersion; - } else { - state Reference bc = IBackupContainer::openContainer(container, proxy, {}); - // For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties - state BackupDescription description = - wait(bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0)); - - if (dbVersion <= 0) { - wait(description.resolveVersionTimes(db)); - if (description.maxRestorableVersion.present()) - restoreVersion = description.maxRestorableVersion.get(); - else { - fprintf(stderr, "Backup is not restorable\n"); - throw restore_invalid_version(); - } - } else { - restoreVersion = dbVersion; - } - - state Optional rset = wait(bc->getRestoreSet(restoreVersion)); - if (!rset.present()) { - fmt::print(stderr, "Insufficient data to restore to version {}\n", restoreVersion); - throw restore_invalid_version(); - } - - // Display the restore information, if requested - if (verbose) { - fmt::print("[DRY RUN] Restoring backup to version: {}\n", restoreVersion); - fmt::print("{}\n", description.toString()); - } - } - - if (waitForDone && verbose) { - // If restore completed then report version restored - fmt::print("Restored to version {0}{1}\n", restoreVersion, (performRestore) ? "" : " (DRY RUN)"); - } - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future dumpBackupData(const char* name, - std::string destinationContainer, - Optional proxy, - Version beginVersion, - Version endVersion) { - state Reference c = openBackupContainer(name, destinationContainer, proxy, {}); - - if (beginVersion < 0 || endVersion < 0) { - BackupDescription desc = wait(c->describeBackup()); - - if (!desc.maxLogEnd.present()) { - fprintf(stderr, "ERROR: Backup must have log data in order to use relative begin/end versions.\n"); - throw backup_invalid_info(); - } - - if (beginVersion < 0) { - beginVersion += desc.maxLogEnd.get(); - } - - if (endVersion < 0) { - endVersion += desc.maxLogEnd.get(); - } - } - - fmt::print("Scanning version range {0} to {1}\n", beginVersion, endVersion); - BackupFileList files = wait(c->dumpFileList(beginVersion, endVersion)); - files.toStream(stdout); - - return Void(); -} - -ACTOR Future expireBackupData(const char* name, - std::string destinationContainer, - Optional proxy, - Version endVersion, - std::string endDatetime, - Database db, - bool force, - Version restorableAfterVersion, - std::string restorableAfterDatetime, - Optional encryptionKeyFile) { - if (!endDatetime.empty()) { - Version v = wait(timeKeeperVersionFromDatetime(endDatetime, db)); - endVersion = v; - } - - if (!restorableAfterDatetime.empty()) { - Version v = wait(timeKeeperVersionFromDatetime(restorableAfterDatetime, db)); - restorableAfterVersion = v; - } - - if (endVersion == invalidVersion) { - fprintf(stderr, "ERROR: No version or date/time is specified.\n"); - printHelpTeaser(name); - throw backup_error(); - ; - } - - try { - Reference c = openBackupContainer(name, destinationContainer, proxy, encryptionKeyFile); - - state IBackupContainer::ExpireProgress progress; - state std::string lastProgress; - state Future expire = c->expireData(endVersion, force, &progress, restorableAfterVersion); - - loop { - choose { - when(wait(delay(5))) { - std::string p = progress.toString(); - if (p != lastProgress) { - int spaces = lastProgress.size() - p.size(); - printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); - lastProgress = p; - } - } - when(wait(expire)) { - break; - } - } - } - - std::string p = progress.toString(); - int spaces = lastProgress.size() - p.size(); - printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); - - if (endVersion < 0) - fmt::print("All data before {0} versions ({1}" - " days) prior to latest backup log has been deleted.\n", - -endVersion, - -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); - else - fmt::print("All data before version {} has been deleted.\n", endVersion); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - if (e.code() == error_code_backup_cannot_expire) - fprintf(stderr, - "ERROR: Requested expiration would be unsafe. Backup would not meet minimum restorability. Use " - "--force to delete data anyway.\n"); - else - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future deleteBackupContainer(const char* name, - std::string destinationContainer, - Optional proxy) { - try { - state Reference c = openBackupContainer(name, destinationContainer, proxy, {}); - state int numDeleted = 0; - state Future done = c->deleteContainer(&numDeleted); - - state int lastUpdate = -1; - printf("Deleting %s...\n", destinationContainer.c_str()); - - loop { - choose { - when(wait(done)) { - break; - } - when(wait(delay(5))) { - if (numDeleted != lastUpdate) { - printf("\r%d...", numDeleted); - lastUpdate = numDeleted; - } - } - } - } - printf("\r%d objects deleted\n", numDeleted); - printf("The entire container has been deleted.\n"); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -ACTOR Future describeBackup(const char* name, - std::string destinationContainer, - Optional proxy, - bool deep, - Optional cx, - bool json, - Optional encryptionKeyFile) { - try { - Reference c = openBackupContainer(name, destinationContainer, proxy, encryptionKeyFile); - state BackupDescription desc = wait(c->describeBackup(deep)); - if (cx.present()) - wait(desc.resolveVersionTimes(cx.get())); - printf("%s\n", (json ? desc.toJSON() : desc.toString()).c_str()); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, "ERROR: %s\n", e.what()); - throw; - } - - return Void(); -} - -static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) { - result["error"] = errorMessage; - printf("%s\n", result.getJson().c_str()); - TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage); -} - -std::pair getMaxMinRestorableVersions(const BackupDescription& desc, bool mayOnlyApplyMutationLog) { - Version maxRestorableVersion = invalidVersion; - Version minRestorableVersion = invalidVersion; - if (desc.maxRestorableVersion.present()) { - maxRestorableVersion = desc.maxRestorableVersion.get(); - } else if (mayOnlyApplyMutationLog && desc.contiguousLogEnd.present()) { - maxRestorableVersion = desc.contiguousLogEnd.get(); - } - - if (desc.minRestorableVersion.present()) { - minRestorableVersion = desc.minRestorableVersion.get(); - } else if (mayOnlyApplyMutationLog && desc.minLogBegin.present()) { - minRestorableVersion = desc.minLogBegin.get(); - } - - return std::make_pair(maxRestorableVersion, minRestorableVersion); -} - -// If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for -// selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overridden to the version -// resolved to that timestamp. -ACTOR Future queryBackup(const char* name, - std::string destinationContainer, - Optional proxy, - Standalone> keyRangesFilter, - Version restoreVersion, - Version snapshotVersion, - std::string originalClusterFile, - std::string restoreTimestamp, - Verbose verbose, - Optional cx) { - state UID operationId = deterministicRandom()->randomUniqueID(); - state JsonBuilderObject result; - state std::string errorMessage; - result["key_ranges_filter"] = printable(keyRangesFilter); - result["destination_container"] = destinationContainer; - - TraceEvent("BackupQueryStart") - .detail("OperationId", operationId) - .detail("DestinationContainer", destinationContainer) - .detail("KeyRangesFilter", printable(keyRangesFilter)) - .detail("SpecifiedRestoreVersion", restoreVersion) - .detail("SpecifiedSnapshotVersion", snapshotVersion) - .detail("RestoreTimestamp", restoreTimestamp) - .detail("BackupClusterFile", originalClusterFile); - - // Resolve restoreTimestamp if given - if (!restoreTimestamp.empty()) { - if (originalClusterFile.empty()) { - reportBackupQueryError( - operationId, - result, - format("an original cluster file must be given in order to resolve restore target timestamp '%s'", - restoreTimestamp.c_str())); - return Void(); - } - - if (!fileExists(originalClusterFile)) { - reportBackupQueryError(operationId, - result, - format("The specified original source database cluster file '%s' does not exist\n", - originalClusterFile.c_str())); - return Void(); - } - - Database origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION); - Version v = wait(timeKeeperVersionFromDatetime(restoreTimestamp, origDb)); - result["restore_timestamp"] = restoreTimestamp; - result["restore_timestamp_resolved_version"] = v; - restoreVersion = v; - } - - state int64_t totalRangeFilesSize = 0; - state int64_t totalLogFilesSize = 0; - state JsonBuilderArray rangeFilesJson; - state JsonBuilderArray logFilesJson; - try { - state Reference bc = openBackupContainer(name, destinationContainer, proxy, {}); - BackupDescription desc = wait(bc->describeBackup()); - // Use continuous log end version for the maximum restorable version for the key ranges when a restorable - // version doesn't exist. - auto [maxRestorableVersion, minRestorableVersion] = getMaxMinRestorableVersions(desc, !keyRangesFilter.empty()); - if (restoreVersion == invalidVersion) { - restoreVersion = maxRestorableVersion; - } - if (restoreVersion == earliestVersion) { - restoreVersion = minRestorableVersion; - } - if (snapshotVersion == earliestVersion) { - snapshotVersion = minRestorableVersion; - } - TraceEvent("BackupQueryResolveRestoreVersion") - .detail("RestoreVersion", restoreVersion) - .detail("SnapshotVersion", snapshotVersion); - if (restoreVersion < 0) { - reportBackupQueryError(operationId, - result, - errorMessage = - format("the specified restorable version %lld is not valid", restoreVersion)); - return Void(); - } - - state Optional fileSet; - if (snapshotVersion != invalidVersion) { - // When a snapshot version is specified, we will first get a restore set using the latest snapshot file to - // restore to the snapshot version. After snapshot version, we will only use mutation logs to restore. - wait(store(fileSet, bc->getRestoreSet(snapshotVersion, keyRangesFilter))); - if (fileSet.present()) { - result["snapshot_version"] = fileSet.get().targetVersion; - for (const auto& rangeFile : fileSet.get().ranges) { - JsonBuilderObject object; - object["file_name"] = rangeFile.fileName; - object["file_size"] = rangeFile.fileSize; - object["version"] = rangeFile.version; - object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0 - ? "none" - : fileSet.get().keyRanges.at(rangeFile.fileName).toString(); - rangeFilesJson.push_back(object); - totalRangeFilesSize += rangeFile.fileSize; - } - for (const auto& log : fileSet.get().logs) { - JsonBuilderObject object; - object["file_name"] = log.fileName; - object["file_size"] = log.fileSize; - object["begin_version"] = log.beginVersion; - object["end_version"] = log.endVersion; - logFilesJson.push_back(object); - totalLogFilesSize += log.fileSize; - } - - snapshotVersion = fileSet.get().targetVersion; - - TraceEvent("BackupQueryReceivedRestorableFilesSetFromSnapshot") - .detail("SnapshotVersion", snapshotVersion) - .detail("DestinationContainer", destinationContainer) - .detail("KeyRangesFilter", printable(keyRangesFilter)) - .detail("ActualRestoreVersion", fileSet.get().targetVersion) - .detail("NumRangeFiles", fileSet.get().ranges.size()) - .detail("NumLogFiles", fileSet.get().logs.size()) - .detail("RangeFilesBytes", totalRangeFilesSize) - .detail("LogFilesBytes", totalLogFilesSize); - } else { - reportBackupQueryError( - operationId, - result, - format("no restorable files set found for specified key ranges from snapshotVersion %lld", - snapshotVersion)); - return Void(); - } - - // We only need to know all the mutation logs from `snapshotVersion` to `restoreVersion`. - wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter, /*logOnly=*/true, snapshotVersion))); - } else { - // When a snapshot version is not specified, we use the latest snapshot to restore to the `restoreVersion`. - wait(store(fileSet, bc->getRestoreSet(restoreVersion, keyRangesFilter))); - } - - if (fileSet.present()) { - result["restore_version"] = fileSet.get().targetVersion; - for (const auto& rangeFile : fileSet.get().ranges) { - JsonBuilderObject object; - object["file_name"] = rangeFile.fileName; - object["file_size"] = rangeFile.fileSize; - object["version"] = rangeFile.version; - object["key_range"] = fileSet.get().keyRanges.count(rangeFile.fileName) == 0 - ? "none" - : fileSet.get().keyRanges.at(rangeFile.fileName).toString(); - rangeFilesJson.push_back(object); - totalRangeFilesSize += rangeFile.fileSize; - } - for (const auto& log : fileSet.get().logs) { - JsonBuilderObject object; - object["file_name"] = log.fileName; - object["file_size"] = log.fileSize; - object["begin_version"] = log.beginVersion; - object["end_version"] = log.endVersion; - logFilesJson.push_back(object); - totalLogFilesSize += log.fileSize; - } - - TraceEvent("BackupQueryReceivedRestorableFilesSet") - .detail("DestinationContainer", destinationContainer) - .detail("KeyRangesFilter", printable(keyRangesFilter)) - .detail("ActualRestoreVersion", fileSet.get().targetVersion) - .detail("NumRangeFiles", fileSet.get().ranges.size()) - .detail("NumLogFiles", fileSet.get().logs.size()) - .detail("RangeFilesBytes", totalRangeFilesSize) - .detail("LogFilesBytes", totalLogFilesSize); - } else if (snapshotVersion == invalidVersion) { - reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges"); - return Void(); - } - - } catch (Error& e) { - reportBackupQueryError(operationId, result, e.what()); - return Void(); - } - - result["total_range_files_size"] = totalRangeFilesSize; - result["total_log_files_size"] = totalLogFilesSize; - - if (verbose) { - result["ranges"] = rangeFilesJson; - result["logs"] = logFilesJson; - } - - printf("%s\n", result.getJson().c_str()); - return Void(); -} - -ACTOR Future listBackup(std::string baseUrl, Optional proxy) { - try { - std::vector containers = wait(IBackupContainer::listContainers(baseUrl, proxy)); - for (std::string container : containers) { - printf("%s\n", container.c_str()); - } - } catch (Error& e) { - std::string msg = format("ERROR: %s", e.what()); - if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { - msg += format(": %s", IBackupContainer::lastOpenError.c_str()); - } - fprintf(stderr, "%s\n", msg.c_str()); - throw; - } - - return Void(); -} - -ACTOR Future listBackupTags(Database cx) { - state Reference tr = makeReference(cx); - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - std::vector tags = wait(getAllBackupTags(tr)); - for (const auto& tag : tags) { - printf("%s\n", tag.tagName.c_str()); - } - return Void(); - } catch (Error& e) { - wait(tr->onError(e)); - } - } -} - -struct BackupModifyOptions { - Optional verifyUID; - Optional destURL; - Optional proxy; - Optional snapshotIntervalSeconds; - Optional activeSnapshotIntervalSeconds; - Optional encryptionKeyFile; - bool hasChanges() const { - return destURL.present() || snapshotIntervalSeconds.present() || activeSnapshotIntervalSeconds.present(); - } -}; - -ACTOR Future modifyBackup(Database db, std::string tagName, BackupModifyOptions options) { - if (!options.hasChanges()) { - fprintf(stderr, "No changes were specified, nothing to do!\n"); - throw backup_error(); - } - - state KeyBackedTag tag = makeBackupTag(tagName); - - state Reference tr(new ReadYourWritesTransaction(db)); - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - - state Optional uidFlag = wait(tag.get(db.getReference())); - - if (!uidFlag.present()) { - fprintf(stderr, "No backup exists on tag '%s'\n", tagName.c_str()); - throw backup_error(); - } - - if (uidFlag.get().second) { - fprintf(stderr, "Cannot modify aborted backup on tag '%s'\n", tagName.c_str()); - throw backup_error(); - } - - state BackupConfig config(uidFlag.get().first); - EBackupState s = wait(config.stateEnum().getOrThrow(tr, Snapshot::False, backup_invalid_info())); - if (!FileBackupAgent::isRunnable(s)) { - fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str()); - throw backup_error(); - } - - if (options.verifyUID.present() && options.verifyUID.get() != uidFlag.get().first.toString()) { - fprintf(stderr, - "UID verification failed, backup on tag '%s' is '%s' but '%s' was specified.\n", - tagName.c_str(), - uidFlag.get().first.toString().c_str(), - options.verifyUID.get().c_str()); - throw backup_error(); - } - - if (options.destURL.present()) { - state Reference prevContainer = - wait(config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info())); - std::string prevURL = prevContainer->getURL(); - std::string newURL = options.destURL.get(); - if (!prevURL.empty() && prevURL.back() == '/') { - prevURL.pop_back(); - } - if (!newURL.empty() && newURL.back() == '/') { - newURL.pop_back(); - } - - if (prevURL == newURL) { - if ((options.encryptionKeyFile.present() && !prevContainer->getEncryptionKeyFileName().present()) || - (!options.encryptionKeyFile.present() && prevContainer->getEncryptionKeyFileName().present()) || - (options.encryptionKeyFile.present() && prevContainer->getEncryptionKeyFileName().present() && - options.encryptionKeyFile.get() != prevContainer->getEncryptionKeyFileName().get())) { - fprintf(stderr, - "Destination URL matches the existing backup URL for tag '%s', " - "but the encryption key file does not match.\n", - tagName.c_str()); - throw backup_error(); - } - } - - state Reference bc; - TraceEvent("ModifyBackupSetNewContainer") - .detail("TagName", tagName) - .detail("DestURL", options.destURL.get()) - .detail("EncryptionKeyFile", - options.encryptionKeyFile.present() ? options.encryptionKeyFile.get() : "None"); - bc = openBackupContainer( - exeBackup.toString().c_str(), options.destURL.get(), options.proxy, options.encryptionKeyFile); - try { - wait(timeoutError(bc->create(), 30)); - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - fprintf(stderr, - "ERROR: Could not create backup container at '%s': %s\n", - options.destURL.get().c_str(), - e.what()); - throw backup_error(); - } - - config.backupContainer().set(tr, bc); - wait(bc->writeEncryptionMetadata()); - } else if (options.encryptionKeyFile.present()) { - fprintf(stdout, - " Encryption key file specified without a new destination URL." - " The encryption key will not be used.\n"); - } - - if (options.snapshotIntervalSeconds.present()) { - config.snapshotIntervalSeconds().set(tr, options.snapshotIntervalSeconds.get()); - } - - if (options.activeSnapshotIntervalSeconds.present()) { - Version begin = wait(config.snapshotBeginVersion().getOrThrow(tr, Snapshot::False, backup_error())); - config.snapshotTargetEndVersion().set(tr, - begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * - CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); - } - - wait(tr->commit()); - break; - } catch (Error& e) { - wait(tr->onError(e)); - } - } - - return Void(); -} - -static std::vector> parseLine(std::string& line, bool& err, bool& partial) { - err = false; - partial = false; - - bool quoted = false; - std::vector buf; - std::vector> ret; - - size_t i = line.find_first_not_of(' '); - size_t offset = i; - - bool forcetoken = false; - - while (i <= line.length()) { - switch (line[i]) { - case ';': - if (!quoted) { - if (i > offset) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - ret.push_back(std::move(buf)); - offset = i = line.find_first_not_of(' ', i + 1); - } else - i++; - break; - case '"': - quoted = !quoted; - line.erase(i, 1); - if (quoted) - forcetoken = true; - break; - case ' ': - if (!quoted) { - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - offset = i = line.find_first_not_of(' ', i); - forcetoken = false; - } else - i++; - break; - case '\\': - if (i + 2 > line.length()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - switch (line[i + 1]) { - char ent, save; - case '"': - case '\\': - case ' ': - case ';': - line.erase(i, 1); - break; - case 'x': - if (i + 4 > line.length()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - char* pEnd; - save = line[i + 4]; - line[i + 4] = 0; - ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); - if (*pEnd) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - line[i + 4] = save; - line.replace(i, 4, 1, ent); - break; - default: - err = true; - ret.push_back(std::move(buf)); - return ret; - } - default: - i++; - } - } - - i -= 1; - if (i > offset || forcetoken) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - - ret.push_back(std::move(buf)); - - if (quoted) - partial = true; - - return ret; -} - -static void addKeyRange(std::string optionValue, Standalone>& keyRanges) { - bool err = false, partial = false; - [[maybe_unused]] int tokenArray = 0; - - auto parsed = parseLine(optionValue, err, partial); - - for (auto tokens : parsed) { - tokenArray++; - - /* - int tokenIndex = 0; - for (auto token : tokens) - { - tokenIndex++; - - printf("%4d token #%2d: %s\n", tokenArray, tokenIndex, printable(token).c_str()); - } - */ - - // Process the keys - // [end] - switch (tokens.size()) { - // empty - case 0: - break; - - // single key range - case 1: - keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), strinc(tokens.at(0)))); - break; - - // full key range - case 2: - try { - keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), tokens.at(1))); - } catch (Error& e) { - fprintf(stderr, - "ERROR: Invalid key range `%s %s' reported error %s\n", - tokens.at(0).toString().c_str(), - tokens.at(1).toString().c_str(), - e.what()); - throw invalid_option_value(); - } - break; - - // Too many keys - default: - fmt::print(stderr, "ERROR: Invalid key range identified with {} keys", tokens.size()); - throw invalid_option_value(); - break; - } - } - - return; -} - -Version parseVersion(const char* str) { - StringRef s((const uint8_t*)str, strlen(str)); - - if (s.endsWith("days"_sr) || s.endsWith("d"_sr)) { - float days; - if (sscanf(str, "%f", &days) != 1) { - fprintf(stderr, "Could not parse version: %s\n", str); - flushAndExit(FDB_EXIT_ERROR); - } - return (double)CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 3600 * -days; - } - - Version ver; - if (sscanf(str, "%" SCNd64, &ver) != 1) { - fprintf(stderr, "Could not parse version: %s\n", str); - flushAndExit(FDB_EXIT_ERROR); - } - return ver; -} - -// Creates a connection to a cluster. Optionally prints an error if the connection fails. -Optional connectToCluster(std::string const& clusterFile, - LocalityData const& localities, - bool quiet = false) { - auto resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(clusterFile); - Reference ccf; - - Optional db; - - try { - ccf = makeReference(resolvedClusterFile.first); - } catch (Error& e) { - if (!quiet) - fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str()); - return db; - } - - try { - db = Database::createDatabase(ccf, ApiVersion::LATEST_VERSION, IsInternal::True, localities); - } catch (Error& e) { - if (!quiet) { - fprintf(stderr, "ERROR: %s\n", e.what()); - fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getLocation().c_str()); - } - return db; - } - - return db; -}; - -static constexpr CSimpleOpt::SOption* const allOptionArrays[] = { g_rgOptions, - g_rgAgentOptions, - g_rgBackupStartOptions, - g_rgBackupModifyOptions, - g_rgBackupStatusOptions, - g_rgBackupAbortOptions, - g_rgBackupCleanupOptions, - g_rgBackupDiscontinueOptions, - g_rgBackupWaitOptions, - g_rgBackupPauseOptions, - g_rgBackupExpireOptions, - g_rgBackupDeleteOptions, - g_rgBackupDescribeOptions, - g_rgBackupDumpOptions, - g_rgBackupTagsOptions, - g_rgBackupListOptions, - g_rgBackupQueryOptions, - g_rgRestoreOptions, - g_rgDBAgentOptions, - g_rgDBStartOptions, - g_rgDBStatusOptions, - g_rgDBSwitchOptions, - g_rgDBAbortOptions, - g_rgDBPauseOptions }; - -// The last element in SOption arrays is always END_MARKER = SO_END_OF_OPTIONS. -constexpr CSimpleOpt::SOption END_MARKER = SO_END_OF_OPTIONS; - -/** - * Validates and processes a command-line option. - * - * This function checks if the current argument (argv[i]) matches any known option. - * If the option requires a parameter, it consumes the next argument as the parameter. - * The processed option (and its parameter, if any) are added to the 'options' vector. - * - * Parameters: - * argc - Total number of command-line arguments. - * argv - Array of command-line argument strings. - * i - Current index in argv; incremented if a parameter is consumed. - * options - Vector to which valid options (and their parameters, if any) are appended. - * - * Returns: - * true if the option is recognized and valid (and its parameter, if required, is present); - * false otherwise. - * - * This function is used to reorder and validate command-line arguments. - */ -static bool processOption(int argc, char* argv[], int& i, std::vector& options) { - std::string_view option = argv[i]; - - options.emplace_back(argv[i]); - size_t equalPos = option.find('='); - - if (equalPos != std::string_view::npos) { - option = option.substr(0, equalPos); - } - - for (auto* opt : allOptionArrays) { - for (int j = 0; opt[j].nId != END_MARKER.nId; ++j) { - const char* knownOpt = opt[j].pszArg; - size_t knownOptLen = strlen(knownOpt); - bool isPrefixOpt = knownOptLen > 1 && knownOpt[knownOptLen - 1] == '-'; - - // Create normalized versions for hyphen-underscore equivalence - std::string optNorm(option); - std::replace(optNorm.begin(), optNorm.end(), '-', '_'); - std::string knownNorm(knownOpt, isPrefixOpt ? knownOptLen - 1 : knownOptLen); - std::replace(knownNorm.begin(), knownNorm.end(), '-', '_'); - - if (optNorm == knownNorm || (isPrefixOpt && optNorm.size() >= knownNorm.size() && - optNorm.compare(0, knownNorm.size(), knownNorm) == 0)) { - if (opt[j].nArgType == SO_REQ_SEP && equalPos == std::string_view::npos) { - ++i; - if (i >= argc) { - fmt::print(stderr, "ERROR: Option {} requires a parameter\n", option); - return false; - } - options.emplace_back(argv[i]); - } - return true; - } - } - } - fmt::print(stderr, "ERROR: unknown option '{}'\n", option); - return false; -} - -/** - * Reorders command-line arguments so that all non-option parameters (subcommands) are placed before options. - * Validates all options using processOption. Returns the reordered arguments via the output parameters - * newArgC (argument count) and newArgV (argument vector). - * - * Note: The returned newArgV pointer points to static storage (argvStorage) - */ -static bool reorderArguments(int argc, char* argv[], int& newArgC, char**& newArgV) { - static std::vector argvStorage; - std::vector parameters; - std::vector options; - - parameters.push_back(argv[0]); // program name - auto isOptions = [](const char* arg) -> bool { return arg && *arg == '-'; }; - - for (int i = 1; i < argc; ++i) { - char* arg = argv[i]; - - if (isOptions(arg)) { - if (!processOption(argc, argv, i, options)) - return false; - } else { - parameters.emplace_back(arg); - } - } - - argvStorage.reserve(parameters.size() + options.size() + 1); // +1 for null terminator - argvStorage = parameters; - argvStorage.insert(argvStorage.end(), options.begin(), options.end()); - - newArgC = static_cast(argvStorage.size()); - - argvStorage.push_back(nullptr); // Null-terminate the argv array - newArgV = argvStorage.data(); - - return true; -} - -#ifndef EXCLUDE_MAIN_FUNCTION - -int main(int argc, char* argv[]) { - platformInit(); - - int status = FDB_EXIT_SUCCESS; - - std::string commandLine; - for (int a = 0; a < argc; a++) { - if (a) - commandLine += ' '; - commandLine += argv[a]; - } - - try { - registerCrashHandler(); - - // Set default of line buffering standard out and error - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - - ProgramExe programExe = getProgramType(argv[0]); - BackupType backupType = BackupType::UNDEFINED; - RestoreType restoreType = RestoreType::UNKNOWN; - DBType dbType = DBType::UNDEFINED; - - std::unique_ptr args; - - char** newArgV{}; - int newArgC{}; - - if (!reorderArguments(argc, argv, newArgC, newArgV)) { - return FDB_EXIT_ERROR; - } - - switch (programExe) { - case ProgramExe::AGENT: - args = std::make_unique( - newArgC, newArgV, g_rgAgentOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case ProgramExe::DR_AGENT: - args = std::make_unique( - newArgC, newArgV, g_rgDBAgentOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case ProgramExe::BACKUP: - // Display backup help, if no arguments - if (newArgC < 2) { - printBackupUsage(false); - return FDB_EXIT_ERROR; - } else { - // Get the backup type - backupType = getBackupType(newArgV[1]); - - // Create the appropriate simple opt - switch (backupType) { - case BackupType::START: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupStartOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::STATUS: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupStatusOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::ABORT: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupAbortOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::CLEANUP: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupCleanupOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::WAIT: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupWaitOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::DISCONTINUE: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupDiscontinueOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::PAUSE: - case BackupType::RESUME: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::EXPIRE: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupExpireOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::DELETE_BACKUP: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupDeleteOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::DESCRIBE: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupDescribeOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::DUMP: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupDumpOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::LIST: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupListOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::QUERY: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupQueryOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::MODIFY: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupModifyOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::TAGS: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgBackupTagsOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case BackupType::UNDEFINED: - default: - args = std::make_unique( - newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - } - } - break; - case ProgramExe::DB_BACKUP: - // Display backup help, if no arguments - if (newArgC < 2) { - printDBBackupUsage(false); - return FDB_EXIT_ERROR; - } else { - // Get the backup type - dbType = getDBType(newArgV[1]); - - // Create the appropriate simple opt - switch (dbType) { - case DBType::START: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgDBStartOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case DBType::STATUS: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgDBStatusOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case DBType::SWITCH: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgDBSwitchOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case DBType::ABORT: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgDBAbortOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case DBType::PAUSE: - case DBType::RESUME: - args = std::make_unique( - newArgC - 1, &newArgV[1], g_rgDBPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - case DBType::UNDEFINED: - default: - args = std::make_unique( - newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - break; - } - } - break; - case ProgramExe::RESTORE: - if (newArgC < 2) { - printRestoreUsage(false); - return FDB_EXIT_ERROR; - } - // Get the restore operation type - restoreType = getRestoreType(newArgV[1]); - if (restoreType == RestoreType::UNKNOWN) { - args = - std::make_unique(newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - } else { - args = std::make_unique( - newArgC - 1, newArgV + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - } - break; - case ProgramExe::FASTRESTORE_TOOL: - if (newArgC < 2) { - printFastRestoreUsage(false); - return FDB_EXIT_ERROR; - } - // Get the restore operation type - restoreType = getRestoreType(newArgV[1]); - if (restoreType == RestoreType::UNKNOWN) { - args = - std::make_unique(newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - } else { - args = std::make_unique( - newArgC - 1, newArgV + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - } - break; - case ProgramExe::UNDEFINED: - default: - fprintf(stderr, "FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); - fprintf(stderr, "ERROR: Unable to determine program type based on executable `%s'\n", newArgV[0]); - return FDB_EXIT_ERROR; - break; - } - - Optional proxy; - std::string p; - if (platform::getEnvironmentVar("HTTP_PROXY", p) || platform::getEnvironmentVar("HTTPS_PROXY", p)) { - proxy = p; - } - std::string destinationContainer; - bool describeDeep = false; - bool describeTimestamps = false; - int initialSnapshotIntervalSeconds = - 0; // The initial snapshot has a desired duration of 0, meaning go as fast as possible. - int snapshotIntervalSeconds = CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC; - std::string clusterFile; - std::string sourceClusterFile; - std::string baseUrl; - std::string expireDatetime; - Version expireVersion = invalidVersion; - std::string expireRestorableAfterDatetime; - Version expireRestorableAfterVersion = std::numeric_limits::max(); - std::vector> knobs; - std::string tagName = BackupAgentBase::getDefaultTag().toString(); - bool tagProvided = false; - std::string restoreContainer; - std::string addPrefix; - std::string removePrefix; - Standalone> backupKeys; - Standalone> backupKeysFilter; - int maxErrors = 20; - Version beginVersion = invalidVersion; - Version restoreVersion = invalidVersion; - Version snapshotVersion = invalidVersion; - std::string restoreTimestamp; - WaitForComplete waitForDone{ false }; - StopWhenDone stopWhenDone{ true }; - UsePartitionedLog usePartitionedLog{ false }; // Set to true to use new backup system - IncrementalBackupOnly incrementalBackupOnly{ false }; - OnlyApplyMutationLogs onlyApplyMutationLogs{ false }; - InconsistentSnapshotOnly inconsistentSnapshotOnly{ false }; - ForceAction forceAction{ false }; - bool trace = false; - bool quietDisplay = false; - bool dryRun = false; - bool restoreSystemKeys = false; - bool restoreUserKeys = false; - RestoreMode restoreMode = RestoreMode::RANGEFILE; // Default to traditional range file restore - std::string traceDir = ""; - std::string traceFormat = ""; - std::string traceLogGroup; - uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE; - uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE; - ESOError lastError; - PartialBackup partial{ true }; - DstOnly dstOnly{ false }; - LocalityData localities; - uint64_t memLimit = 8LL << 30; - uint64_t virtualMemLimit = 0; // unlimited - Optional ti; - BackupTLSConfig tlsConfig; - Version dumpBegin = 0; - Version dumpEnd = std::numeric_limits::max(); - std::string restoreClusterFileDest; - std::string restoreClusterFileOrig; - bool jsonOutput = false; - DeleteData deleteData{ false }; - Optional encryptionKeyFile; - Optional blobManifestUrl; - SnapshotMode snapshotMode = SnapshotMode::RANGEFILE; // Default to legacy rangefile mode - - BackupModifyOptions modifyOptions; - - if (newArgC == 1) { - printUsage(programExe, false); - return FDB_EXIT_ERROR; - } - -#ifdef _WIN32 - // Windows needs a gentle nudge to format floats correctly - //_set_output_format(_TWO_DIGIT_EXPONENT); -#endif - - while (args->Next()) { - lastError = args->LastError(); - - switch (lastError) { - case SO_SUCCESS: - break; - - case SO_ARG_INVALID_DATA: - fprintf(stderr, "ERROR: invalid argument to option `%s'\n", args->OptionText()); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - - case SO_ARG_INVALID: - fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText()); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - - case SO_ARG_MISSING: - fprintf(stderr, "ERROR: missing argument for option `%s'\n", args->OptionText()); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - - case SO_OPT_INVALID: - fprintf(stderr, "ERROR: unknown option `%s'\n", args->OptionText()); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - - default: - fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText()); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - } - - int optId = args->OptionId(); - switch (optId) { - case OPT_HELP: - printUsage(programExe, false); - return FDB_EXIT_SUCCESS; - break; - case OPT_DEVHELP: - printUsage(programExe, true); - return FDB_EXIT_SUCCESS; - break; - case OPT_VERSION: - printVersion(); - return FDB_EXIT_SUCCESS; - break; - case OPT_BUILD_FLAGS: - printBuildInformation(); - return FDB_EXIT_SUCCESS; - break; - case OPT_NOBUFSTDOUT: - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - break; - case OPT_BUFSTDOUTERR: - setvbuf(stdout, NULL, _IOFBF, BUFSIZ); - setvbuf(stderr, NULL, _IOFBF, BUFSIZ); - break; - case OPT_QUIET: - quietDisplay = true; - break; - case OPT_DRYRUN: - dryRun = true; - break; - case OPT_DELETE_DATA: - deleteData.set(true); - break; - case OPT_MIN_CLEANUP_SECONDS: - knobs.emplace_back("min_cleanup_seconds", args->OptionArg()); - break; - case OPT_FORCE: - forceAction.set(true); - break; - case OPT_TRACE: - trace = true; - break; - case OPT_TRACE_DIR: - trace = true; - traceDir = args->OptionArg(); - break; - case OPT_TRACE_FORMAT: - if (!validateTraceFormat(args->OptionArg())) { - fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg()); - } - traceFormat = args->OptionArg(); - break; - case OPT_TRACE_LOG_GROUP: - traceLogGroup = args->OptionArg(); - break; - case OPT_LOCALITY: { - Optional localityKey = extractPrefixedArgument("--locality", args->OptionSyntax()); - if (!localityKey.present()) { - fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", args->OptionSyntax()); - return FDB_EXIT_ERROR; - } - Standalone key = StringRef(localityKey.get()); - std::transform(key.begin(), key.end(), mutateString(key), ::tolower); - localities.set(key, Standalone(std::string(args->OptionArg()))); - break; - } - case OPT_EXPIRE_BEFORE_DATETIME: - expireDatetime = args->OptionArg(); - break; - case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME: - expireRestorableAfterDatetime = args->OptionArg(); - break; - case OPT_EXPIRE_BEFORE_VERSION: - case OPT_EXPIRE_RESTORABLE_AFTER_VERSION: - case OPT_EXPIRE_MIN_RESTORABLE_DAYS: - case OPT_EXPIRE_DELETE_BEFORE_DAYS: { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - - // Interpret the value as days worth of versions relative to now (negative) - if (optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) { - ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; - } - - if (optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) - expireVersion = ver; - else - expireRestorableAfterVersion = ver; - break; - } - case OPT_RESTORE_TIMESTAMP: - restoreTimestamp = args->OptionArg(); - break; - case OPT_BASEURL: - baseUrl = args->OptionArg(); - break; - case OPT_RESTORE_CLUSTERFILE_DEST: - restoreClusterFileDest = args->OptionArg(); - break; - case OPT_RESTORE_CLUSTERFILE_ORIG: - restoreClusterFileOrig = args->OptionArg(); - break; - case OPT_CLUSTERFILE: - case OPT_DEST_CLUSTER: - clusterFile = args->OptionArg(); - break; - case OPT_SOURCE_CLUSTER: - sourceClusterFile = args->OptionArg(); - break; - case OPT_CLEANUP: - partial.set(false); - break; - case OPT_DSTONLY: - dstOnly.set(true); - break; - case OPT_KNOB: { - Optional knobName = extractPrefixedArgument("--knob", args->OptionSyntax()); - if (!knobName.present()) { - fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", args->OptionSyntax()); - return FDB_EXIT_ERROR; - } - knobs.emplace_back(knobName.get(), args->OptionArg()); - break; - } - case OPT_BACKUPKEYS: - try { - addKeyRange(args->OptionArg(), backupKeys); - } catch (Error&) { - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - break; - case OPT_BACKUPKEYS_FILE: - try { - std::string line = readFileBytes(args->OptionArg(), 64 * 1024 * 1024); - addKeyRange(line, backupKeys); - } catch (Error&) { - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - break; - case OPT_BACKUPKEYS_FILTER: - try { - addKeyRange(args->OptionArg(), backupKeysFilter); - } catch (Error&) { - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - break; - case OPT_PROXY: - proxy = args->OptionArg(); - if (!Hostname::isHostname(proxy.get()) && !NetworkAddress::parseOptional(proxy.get()).present()) { - fprintf(stderr, "ERROR: Proxy format should be either IP:port or host:port\n"); - return FDB_EXIT_ERROR; - } - modifyOptions.proxy = proxy; - break; - case OPT_DESTCONTAINER: - destinationContainer = args->OptionArg(); - // If the url starts with '/' then prepend "file://" for backwards compatibility - if (StringRef(destinationContainer).startsWith("/"_sr)) - destinationContainer = std::string("file://") + destinationContainer; - modifyOptions.destURL = destinationContainer; - break; - case OPT_SNAPSHOTINTERVAL: - case OPT_INITIAL_SNAPSHOT_INTERVAL: - case OPT_MOD_ACTIVE_INTERVAL: { - const char* a = args->OptionArg(); - int seconds; - if (!sscanf(a, "%d", &seconds)) { - fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - if (optId == OPT_SNAPSHOTINTERVAL) { - snapshotIntervalSeconds = seconds; - modifyOptions.snapshotIntervalSeconds = seconds; - } else if (optId == OPT_INITIAL_SNAPSHOT_INTERVAL) { - initialSnapshotIntervalSeconds = seconds; - } else if (optId == OPT_MOD_ACTIVE_INTERVAL) { - modifyOptions.activeSnapshotIntervalSeconds = seconds; - } - break; - } - case OPT_MOD_VERIFY_UID: - modifyOptions.verifyUID = args->OptionArg(); - break; - case OPT_WAITFORDONE: - waitForDone.set(true); - break; - case OPT_NOSTOPWHENDONE: - stopWhenDone.set(false); - break; - case OPT_USE_PARTITIONED_LOG: - usePartitionedLog.set(true); - break; - case OPT_INCREMENTALONLY: - incrementalBackupOnly.set(true); - onlyApplyMutationLogs.set(true); - break; - case OPT_ENCRYPTION_KEY_FILE: - encryptionKeyFile = args->OptionArg(); - modifyOptions.encryptionKeyFile = encryptionKeyFile; - break; - case OPT_RESTORECONTAINER: - restoreContainer = args->OptionArg(); - // If the url starts with '/' then prepend "file://" for backwards compatibility - if (StringRef(restoreContainer).startsWith("/"_sr)) - restoreContainer = std::string("file://") + restoreContainer; - break; - case OPT_DESCRIBE_DEEP: - describeDeep = true; - break; - case OPT_DESCRIBE_TIMESTAMPS: - describeTimestamps = true; - break; - case OPT_PREFIX_ADD: { - bool err = false; - addPrefix = decode_hex_string(args->OptionArg(), err); - if (err) { - fprintf(stderr, "ERROR: Could not parse add prefix\n"); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - break; - } - case OPT_PREFIX_REMOVE: { - bool err = false; - removePrefix = decode_hex_string(args->OptionArg(), err); - if (err) { - fprintf(stderr, "ERROR: Could not parse remove prefix\n"); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - break; - } - case OPT_ERRORLIMIT: { - const char* a = args->OptionArg(); - if (!sscanf(a, "%d", &maxErrors)) { - fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - break; - } - case OPT_RESTORE_BEGIN_VERSION: { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse database beginVersion `%s'\n", a); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - beginVersion = ver; - break; - } - case OPT_RESTORE_VERSION: { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - restoreVersion = ver; - break; - } - case OPT_RESTORE_SNAPSHOT_VERSION: { - const char* a = args->OptionArg(); - long long ver = 0; - if (!sscanf(a, "%lld", &ver)) { - fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - snapshotVersion = ver; - break; - } - case OPT_RESTORE_USER_DATA: { - restoreUserKeys = true; - break; - } - case OPT_RESTORE_SYSTEM_DATA: { - restoreSystemKeys = true; - break; - } - case OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY: { - inconsistentSnapshotOnly.set(true); - break; - } -#ifdef _WIN32 - case OPT_PARENTPID: { - auto pid_str = args->OptionArg(); - int parent_pid = atoi(pid_str); - auto pHandle = OpenProcess(SYNCHRONIZE, FALSE, parent_pid); - if (!pHandle) { - TraceEvent("ParentProcessOpenError").GetLastError(); - fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError()); - throw platform_error(); - } - startThread(&parentWatcher, pHandle); - break; - } -#endif - case OPT_TAGNAME: - tagName = args->OptionArg(); - tagProvided = true; - break; - case OPT_CRASHONERROR: - g_crashOnError = true; - break; - case OPT_MEMLIMIT: - ti = parse_with_suffix(args->OptionArg(), "MiB"); - if (!ti.present()) { - fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg()); - printHelpTeaser(newArgV[0]); - flushAndExit(FDB_EXIT_ERROR); - } - memLimit = ti.get(); - break; - case OPT_VMEMLIMIT: - ti = parse_with_suffix(args->OptionArg(), "MiB"); - if (!ti.present()) { - fprintf(stderr, "ERROR: Could not parse virtual memory limit from `%s'\n", args->OptionArg()); - printHelpTeaser(newArgV[0]); - flushAndExit(FDB_EXIT_ERROR); - } - virtualMemLimit = ti.get(); - break; - case OPT_BLOB_CREDENTIALS: - tlsConfig.blobCredentials.push_back(args->OptionArg()); - break; - case TLSConfig::OPT_TLS_PLUGIN: - args->OptionArg(); - break; - case TLSConfig::OPT_TLS_CERTIFICATES: - tlsConfig.tlsCertPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_PASSWORD: - tlsConfig.tlsPassword = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_CA_FILE: - tlsConfig.tlsCAPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_KEY: - tlsConfig.tlsKeyPath = args->OptionArg(); - break; - case TLSConfig::OPT_TLS_VERIFY_PEERS: - tlsConfig.tlsVerifyPeers = args->OptionArg(); - break; - case OPT_DUMP_BEGIN: - dumpBegin = parseVersion(args->OptionArg()); - break; - case OPT_DUMP_END: - dumpEnd = parseVersion(args->OptionArg()); - break; - case OPT_JSON: - jsonOutput = true; - break; - case OPT_MODE: - // Handle mode parameter for both backup and restore - if (programExe == ProgramExe::BACKUP) { - // Validate and store mode parameter for snapshot generation - auto parsedMode = getSnapshotMode(args->OptionArg()); - if (!parsedMode.present()) { - fprintf(stderr, - "ERROR: Unknown snapshot mode '%s'. Valid modes are: rangefile, bulkdump, both\n", - args->OptionArg()); - return FDB_EXIT_ERROR; - } - snapshotMode = parsedMode.get(); - } else if (programExe == ProgramExe::RESTORE || programExe == ProgramExe::FASTRESTORE_TOOL) { - // Validate and store mode parameter for restore mechanism - auto parsedMode = getRestoreMode(args->OptionArg()); - if (!parsedMode.present()) { - fprintf(stderr, - "ERROR: Unknown restore mode '%s'. Valid modes are: rangefile, bulkload\n", - args->OptionArg()); - return FDB_EXIT_ERROR; - } - restoreMode = parsedMode.get(); - } - break; - } - } - - // Process the extra arguments - for (int argLoop = 0; argLoop < args->FileCount(); argLoop++) { - switch (programExe) { - case ProgramExe::AGENT: - fprintf(stderr, "ERROR: Backup Agent does not support argument value `%s'\n", args->File(argLoop)); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - - // Add the backup key range - case ProgramExe::BACKUP: - // Error, if the keys option was not specified - if (backupKeys.size() == 0) { - fprintf(stderr, "ERROR: Unknown backup option value `%s'\n", args->File(argLoop)); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - // Otherwise, assume the item is a key range - else { - try { - addKeyRange(args->File(argLoop), backupKeys); - } catch (Error&) { - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - } - break; - - case ProgramExe::RESTORE: - fprintf(stderr, "ERROR: FDB Restore does not support argument value `%s'\n", args->File(argLoop)); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - - case ProgramExe::FASTRESTORE_TOOL: - fprintf( - stderr, "ERROR: FDB Fast Restore Tool does not support argument value `%s'\n", args->File(argLoop)); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - - case ProgramExe::DR_AGENT: - fprintf(stderr, "ERROR: DR Agent does not support argument value `%s'\n", args->File(argLoop)); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - - case ProgramExe::DB_BACKUP: - // Error, if the keys option was not specified - if (backupKeys.size() == 0) { - fprintf(stderr, "ERROR: Unknown DR option value `%s'\n", args->File(argLoop)); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - // Otherwise, assume the item is a key range - else { - try { - addKeyRange(args->File(argLoop), backupKeys); - } catch (Error&) { - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - } - } - break; - - case ProgramExe::UNDEFINED: - default: - return FDB_EXIT_ERROR; - } - } - - if (restoreSystemKeys && restoreUserKeys) { - fprintf(stderr, "ERROR: Please only specify one of --user-data or --system-metadata, not both\n"); - return FDB_EXIT_ERROR; - } - - if (trace) { - if (!traceLogGroup.empty()) - setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(traceLogGroup)); - - if (traceDir.empty()) - setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); - else - setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(traceDir)); - if (!traceFormat.empty()) { - setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(traceFormat)); - } - - setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING); - } - setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING); - - // deferred TLS options - if (!tlsConfig.setupTLS()) { - return 1; - } - - Error::init(); - std::set_new_handler(&platform::outOfMemory); - setMemoryQuota(virtualMemLimit); - - Database db; - Database sourceDb; - FileBackupAgent ba; - Key tag; - Future> f; - Future> fstatus; - Reference c; - - try { - setupNetwork(0, UseMetrics::True); - } catch (Error& e) { - fprintf(stderr, "ERROR: %s\n", e.what()); - return FDB_EXIT_ERROR; - } - - Future memoryUsageMonitor = startMemoryUsageMonitor(memLimit); - - IKnobCollection::setupKnobs(knobs); - // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs - IKnobCollection::getMutableGlobalKnobCollection().initialize(Randomize::False, IsSimulated::False); - - TraceEvent("ProgramStart") - .setMaxEventLength(12000) - .detail("SourceVersion", getSourceVersion()) - .detail("Version", FDB_VT_VERSION) - .detail("PackageName", FDB_VT_PACKAGE_NAME) - .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(NULL)) - .setMaxFieldLength(10000) - .detail("CommandLine", commandLine) - .setMaxFieldLength(0) - .detail("MemoryLimit", memLimit) - .detail("Proxy", proxy.orDefault("")) - .trackLatest("ProgramStart"); - - // Ordinarily, this is done when the network is run. However, network thread should be set before - // TraceEvents are logged. This thread will eventually run the network, so call it now. - TraceEvent::setNetworkThread(); - - // Sets up blob credentials, including one from the environment FDB_BLOB_CREDENTIALS. - tlsConfig.setupBlobCredentials(); - - // Opens a trace file if trace is set (and if a trace file isn't already open) - // For most modes, initCluster() will open a trace file, but some fdbbackup operations do not require - // a cluster so they should use this instead. - auto initTraceFile = [&]() { - if (trace) - openTraceFile({}, traceRollSize, traceMaxLogsSize, traceDir, "trace", traceLogGroup); - }; - - auto initCluster = [&](bool quiet = false) { - Optional result = connectToCluster(clusterFile, localities, quiet); - if (result.present()) { - db = result.get(); - // Make sure we are setting a transaction timeout and retry limit to prevent cases - // where the fdbbackup command hangs infinitely. 60 seconds should be more than - // enough for all cases to finish and 5 retries should also be good enough for - // most cases. - int64_t timeout = 60000; - db->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, - Optional(StringRef((const uint8_t*)&timeout, sizeof(timeout)))); - int64_t retryLimit = 5; - db->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, - Optional(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit)))); - } - - return result.present(); - }; - - auto initSourceCluster = [&](bool required, bool quiet = false) { - if (!sourceClusterFile.size() && required) { - if (!quiet) { - fprintf(stderr, "ERROR: source cluster file is required\n"); - } - return false; - } - - Optional result = connectToCluster(sourceClusterFile, localities, quiet); - if (result.present()) { - sourceDb = result.get(); - // Make sure we are setting a transaction timeout and retry limit to prevent cases - // where the fdbbackup command hangs infinitely. 60 seconds should be more than - // enough for all cases to finish and 5 retries should also be good enough for - // most cases. - int64_t timeout = 60000; - sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, - Optional(StringRef((const uint8_t*)&timeout, sizeof(timeout)))); - int64_t retryLimit = 5; - sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, - Optional(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit)))); - } - - return result.present(); - }; - - // The fastrestore tool does not yet support multiple ranges and is incompatible with - // features that back up data in the system keys. - if (!restoreSystemKeys && !restoreUserKeys && backupKeys.empty() && - programExe != ProgramExe::FASTRESTORE_TOOL) { - addDefaultBackupRanges(backupKeys); - } - - if ((restoreSystemKeys || restoreUserKeys) && programExe == ProgramExe::FASTRESTORE_TOOL) { - fprintf(stderr, "ERROR: Options: --user-data and --system-metadata are not supported with fastrestore\n"); - return FDB_EXIT_ERROR; - } - - if ((restoreUserKeys || restoreSystemKeys) && !backupKeys.empty()) { - fprintf(stderr, - "ERROR: Cannot specify additional ranges when using --user-data or --system-metadata " - "options\n"); - return FDB_EXIT_ERROR; - } - if (restoreUserKeys) { - backupKeys.push_back_deep(backupKeys.arena(), normalKeys); - } else if (restoreSystemKeys) { - for (const auto& r : getSystemBackupRanges()) { - backupKeys.push_back_deep(backupKeys.arena(), r); - } - } - - switch (programExe) { - case ProgramExe::AGENT: - if (!initCluster()) - return FDB_EXIT_ERROR; - fileBackupAgentProxy = proxy; - f = stopAfter(runAgent(db)); - break; - case ProgramExe::BACKUP: - switch (backupType) { - case BackupType::START: { - if (!initCluster()) - return FDB_EXIT_ERROR; - // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually - // writeable. - openBackupContainer(newArgV[0], destinationContainer, proxy, encryptionKeyFile); - f = stopAfter(submitBackup(db, - destinationContainer, - proxy, - initialSnapshotIntervalSeconds, - snapshotIntervalSeconds, - backupKeys, - tagName, - dryRun, - waitForDone, - stopWhenDone, - usePartitionedLog, - incrementalBackupOnly, - encryptionKeyFile, - snapshotMode)); - break; - } - - case BackupType::MODIFY: { - if (!initCluster()) - return FDB_EXIT_ERROR; - - f = stopAfter(modifyBackup(db, tagName, modifyOptions)); - break; - } - - case BackupType::STATUS: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(statusBackup(db, tagName, ShowErrors::True, jsonOutput)); - break; - - case BackupType::ABORT: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(abortBackup(db, tagName)); - break; - - case BackupType::CLEANUP: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(cleanupMutations(db, deleteData)); - break; - - case BackupType::WAIT: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(waitBackup(db, tagName, stopWhenDone)); - break; - - case BackupType::DISCONTINUE: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(discontinueBackup(db, tagName, waitForDone)); - break; - - case BackupType::PAUSE: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(changeBackupResumed(db, true)); - break; - - case BackupType::RESUME: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(changeBackupResumed(db, false)); - break; - - case BackupType::EXPIRE: - initTraceFile(); - // Must have a usable cluster if either expire DateTime options were used - if (!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) { - if (!initCluster()) - return FDB_EXIT_ERROR; - } - f = stopAfter(expireBackupData(newArgV[0], - destinationContainer, - proxy, - expireVersion, - expireDatetime, - db, - forceAction, - expireRestorableAfterVersion, - expireRestorableAfterDatetime, - encryptionKeyFile)); - break; - - case BackupType::DELETE_BACKUP: - initTraceFile(); - f = stopAfter(deleteBackupContainer(newArgV[0], destinationContainer, proxy)); - break; - - case BackupType::DESCRIBE: - initTraceFile(); - // If timestamp lookups are desired, require a cluster file - if (describeTimestamps && !initCluster()) - return FDB_EXIT_ERROR; - - // Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was - // given, but quietly skip them if not. - f = stopAfter(describeBackup(newArgV[0], - destinationContainer, - proxy, - describeDeep, - describeTimestamps ? Optional(db) : Optional(), - jsonOutput, - encryptionKeyFile)); - break; - - case BackupType::LIST: - initTraceFile(); - f = stopAfter(listBackup(baseUrl, proxy)); - break; - - case BackupType::TAGS: - if (!initCluster()) - return FDB_EXIT_ERROR; - f = stopAfter(listBackupTags(db)); - break; - - case BackupType::QUERY: - initTraceFile(); - f = stopAfter(queryBackup(newArgV[0], - destinationContainer, - proxy, - backupKeysFilter, - restoreVersion, - snapshotVersion, - restoreClusterFileOrig, - restoreTimestamp, - Verbose{ !quietDisplay }, - db)); - break; - - case BackupType::DUMP: - initTraceFile(); - f = stopAfter(dumpBackupData(newArgV[0], destinationContainer, proxy, dumpBegin, dumpEnd)); - break; - - case BackupType::UNDEFINED: - default: - fprintf(stderr, "ERROR: Unsupported backup action %s\n", newArgV[1]); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - } - - break; - case ProgramExe::RESTORE: - if (dryRun) { - if (restoreType != RestoreType::START) { - fprintf(stderr, "Restore dry run only works for 'start' command\n"); - return FDB_EXIT_ERROR; - } - - // Must explicitly call trace file options handling if not calling Database::createDatabase() - initTraceFile(); - } else { - if (restoreClusterFileDest.empty()) { - fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n"); - return FDB_EXIT_ERROR; - } - - if (!fileExists(restoreClusterFileDest)) { - fprintf(stderr, - "Restore destination cluster file '%s' does not exist.\n", - restoreClusterFileDest.c_str()); - return FDB_EXIT_ERROR; - } - - try { - db = Database::createDatabase(restoreClusterFileDest, ApiVersion::LATEST_VERSION); - } catch (Error& e) { - fprintf(stderr, - "Restore destination cluster file '%s' invalid: %s\n", - restoreClusterFileDest.c_str(), - e.what()); - return FDB_EXIT_ERROR; - } - } - - switch (restoreType) { - case RestoreType::START: - f = stopAfter(runRestore(db, - restoreClusterFileOrig, - tagName, - restoreContainer, - proxy, - backupKeys, - beginVersion, - restoreVersion, - restoreTimestamp, - !dryRun, - Verbose{ !quietDisplay }, - waitForDone, - addPrefix, - removePrefix, - onlyApplyMutationLogs, - inconsistentSnapshotOnly, - encryptionKeyFile, - restoreMode)); // Pass RestoreMode directly - break; - case RestoreType::WAIT: - f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), Verbose::True))); - break; - case RestoreType::ABORT: - f = stopAfter( - map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void { - printf("RESTORE_ABORT Tag: %s State: %s\n", - tagName.c_str(), - FileBackupAgent::restoreStateText(s).toString().c_str()); - return Void(); - })); - break; - case RestoreType::STATUS: - // If no tag is specifically provided then print all tag status, don't just use "default" - if (tagProvided) - tag = tagName; - f = stopAfter(map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void { - printf("%s\n", s.c_str()); - return Void(); - })); - break; - default: - throw restore_error(); - } - break; - case ProgramExe::FASTRESTORE_TOOL: - // Support --dest-cluster-file option as fdbrestore does - if (dryRun) { - if (restoreType != RestoreType::START) { - fprintf(stderr, "Restore dry run only works for 'start' command\n"); - return FDB_EXIT_ERROR; - } - - // Must explicitly call trace file options handling if not calling Database::createDatabase() - initTraceFile(); - } else { - if (restoreClusterFileDest.empty()) { - fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n"); - return FDB_EXIT_ERROR; - } - - if (!fileExists(restoreClusterFileDest)) { - fprintf(stderr, - "Restore destination cluster file '%s' does not exist.\n", - restoreClusterFileDest.c_str()); - return FDB_EXIT_ERROR; - } - - try { - db = Database::createDatabase(restoreClusterFileDest, ApiVersion::LATEST_VERSION); - } catch (Error& e) { - fprintf(stderr, - "Restore destination cluster file '%s' invalid: %s\n", - restoreClusterFileDest.c_str(), - e.what()); - return FDB_EXIT_ERROR; - } - } - // TODO: We have not implemented the code commented out in this case - switch (restoreType) { - case RestoreType::START: - f = stopAfter(runFastRestoreTool(db, - tagName, - restoreContainer, - proxy, - backupKeys, - restoreVersion, - !dryRun, - Verbose{ !quietDisplay }, - waitForDone)); - break; - case RestoreType::WAIT: - printf("[TODO][ERROR] FastRestore does not support RESTORE_WAIT yet!\n"); - throw restore_error(); - // f = stopAfter( success(ba.waitRestore(db, KeyRef(tagName), true)) ); - break; - case RestoreType::ABORT: - printf("[TODO][ERROR] FastRestore does not support RESTORE_ABORT yet!\n"); - throw restore_error(); - // f = stopAfter( map(ba.abortRestore(db, KeyRef(tagName)), - //[tagName](FileBackupAgent::ERestoreState s) -> Void { printf("Tag: %s - // State: %s\n", tagName.c_str(), - // FileBackupAgent::restoreStateText(s).toString().c_str()); return Void(); - // }) ); - break; - case RestoreType::STATUS: - printf("[TODO][ERROR] FastRestore does not support RESTORE_STATUS yet!\n"); - throw restore_error(); - // If no tag is specifically provided then print all tag status, don't just use "default" - if (tagProvided) - tag = tagName; - // f = stopAfter( map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void - //{ printf("%s\n", s.c_str()); return Void(); - // }) ); - break; - default: - throw restore_error(); - } - break; - case ProgramExe::DR_AGENT: - if (!initCluster() || !initSourceCluster(true)) { - return FDB_EXIT_ERROR; - } - f = stopAfter(runDBAgent(sourceDb, db)); - break; - case ProgramExe::DB_BACKUP: - if (!initCluster() || !initSourceCluster(dbType != DBType::ABORT || !dstOnly)) { - return FDB_EXIT_ERROR; - } - switch (dbType) { - case DBType::START: - f = stopAfter(submitDBBackup(sourceDb, db, backupKeys, tagName)); - break; - case DBType::STATUS: - f = stopAfter(statusDBBackup(sourceDb, db, tagName, maxErrors)); - break; - case DBType::SWITCH: - f = stopAfter(switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction)); - break; - case DBType::ABORT: - f = stopAfter(abortDBBackup(sourceDb, db, tagName, partial, dstOnly)); - break; - case DBType::PAUSE: - f = stopAfter(changeDBBackupResumed(sourceDb, db, true)); - break; - case DBType::RESUME: - f = stopAfter(changeDBBackupResumed(sourceDb, db, false)); - break; - case DBType::UNDEFINED: - default: - fprintf(stderr, "ERROR: Unsupported DR action %s\n", newArgV[1]); - printHelpTeaser(newArgV[0]); - return FDB_EXIT_ERROR; - break; - } - break; - case ProgramExe::UNDEFINED: - default: - return FDB_EXIT_ERROR; - } - - runNetwork(); - - if (f.isValid() && f.isReady() && !f.isError() && !f.get().present()) { - status = FDB_EXIT_ERROR; - } - - if (fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) { - status = fstatus.get().get(); - } - -#ifdef ALLOC_INSTRUMENTATION - { - std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " " - << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " " - << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " " - << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " " - << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " " - << FastAllocator<16384>::pageCount << std::endl; - - std::vector> typeNames; - for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) { - std::string s; - -#ifdef __linux__ - char* demangled = abi::__cxa_demangle(i->first, NULL, NULL, NULL); - if (demangled) { - s = demangled; - if (StringRef(s).startsWith("(anonymous namespace)::"_sr)) - s = s.substr("(anonymous namespace)::"_sr.size()); - free(demangled); - } else - s = i->first; -#else - s = i->first; - if (StringRef(s).startsWith("class `anonymous namespace'::"_sr)) - s = s.substr("class `anonymous namespace'::"_sr.size()); - else if (StringRef(s).startsWith("class "_sr)) - s = s.substr("class "_sr.size()); - else if (StringRef(s).startsWith("struct "_sr)) - s = s.substr("struct "_sr.size()); -#endif - - typeNames.emplace_back(s, i->first); - } - std::sort(typeNames.begin(), typeNames.end()); - for (int i = 0; i < typeNames.size(); i++) { - const char* n = typeNames[i].second; - auto& f = allocInstr[n]; - printf("%+d\t%+d\t%d\t%d\t%s\n", - f.allocCount, - -f.deallocCount, - f.allocCount - f.deallocCount, - f.maxAllocated, - typeNames[i].first.c_str()); - } - - // We're about to exit and clean up data structures, this will wreak havoc on allocation recording - memSample_entered = true; - } -#endif - } catch (Error& e) { - TraceEvent(SevError, "MainError").error(e); - status = FDB_EXIT_MAIN_ERROR; - } catch (boost::system::system_error& e) { - if (g_network) { - TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); - } else { - fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code().value()); - } - status = FDB_EXIT_MAIN_EXCEPTION; - } catch (std::exception& e) { - TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); - status = FDB_EXIT_MAIN_EXCEPTION; - } - - flushAndExit(status); -} - -#else // EXCLUDE_MAIN_FUNCTION - -int main() { - - printf("=== Running ParsedArgs Tests ===\n"); - - auto testOptionParsing = [](std::initializer_list args, - const std::vector& expectedOptions = {}, - bool shouldSucceed = true, - const char* testName = "", - bool expectCSimpleOptions = false) -> bool { - printf("\n--- Test: %s ---\n", testName); - static std::vector persistentArgs; - persistentArgs.clear(); - persistentArgs.reserve(args.size()); - for (const char* arg : args) { - persistentArgs.emplace_back(arg); - } - int argc = static_cast(persistentArgs.size()); - - std::vector argv; - for (auto& arg : persistentArgs) { - argv.push_back(arg.data()); - } - argv.push_back(nullptr); - - int argcNew{}; - char** argvNew{}; - - printf("DEBUG: argc: %d\n", argc); - for (int i = 0; i < argv.size(); ++i) { - printf("DEBUG: argv[%d]: %s\n", i, argv[i]); - } - - bool success = reorderArguments(argc, argv.data(), argcNew, argvNew); - - printf("DEBUG: argcNew: %d\n", argcNew); - for (int i = 0; i < argcNew; ++i) { - printf("DEBUG: argvNew[%d]: %s\n", i, argvNew[i]); - } - - if (success != shouldSucceed) { - printf("%s: FAIL - Expected %s but got %s\n", - testName, - shouldSucceed ? "success" : "failure", - success ? "success" : "failure"); - return false; - } - if (!shouldSucceed) { - printf("\n\t--- Test PASSED: %s (expected failure)---\n", testName); - return true; - } - - // Test CSimpleOpt conversion - std::vector actualOptions; - for (int i = 1; i < argcNew; i++) { - actualOptions.push_back(argvNew[i]); - } - - if (actualOptions != expectedOptions) { - printf("%s: FAIL - Options mismatch\n", testName); - printf(" Expected options (%zu): ", expectedOptions.size()); - for (const auto& opt : expectedOptions) - printf("'%s' ", opt.c_str()); - printf("\n Actual options (%zu): ", actualOptions.size()); - for (const auto& opt : actualOptions) - printf("'%s' ", opt.c_str()); - printf("\n"); - return false; - } - - // Test with actual CSimpleOpt if expected - if (expectCSimpleOptions && !expectedOptions.empty()) { - try { - std::unique_ptr simpleOpt = std::make_unique( - argcNew, const_cast(argvNew), g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); - - ESOError lastError = SO_SUCCESS; - bool foundExpectedOptions = true; - - while (simpleOpt->Next()) { - lastError = simpleOpt->LastError(); - if (lastError != SO_SUCCESS) { - printf("CSimpleOpt parsing error: %d\n", lastError); - foundExpectedOptions = false; - break; - } - - int optId = simpleOpt->OptionId(); - printf("CSimpleOpt found option: id=%d, text='%s', arg='%s'\n", - optId, - simpleOpt->OptionText(), - simpleOpt->OptionArg() ? simpleOpt->OptionArg() : "null"); - } - - if (!foundExpectedOptions) { - printf("%s: FAIL - CSimpleOpt parsing failed\n", testName); - return false; - } - } catch (const std::exception& e) { - printf("%s: FAIL - CSimpleOpt exception: %s\n", testName, e.what()); - return false; - } - } - - printf("\n\t--- Test PASSED: %s ---\n", testName); - return true; - }; - - printf("\n1) Basic Command Tests:\n"); - bool allPassed = true; - allPassed &= testOptionParsing({ "fdbbackup", "status" }, { "status" }, true, "1.1 Single command"); - allPassed &= testOptionParsing({ "fdbbackup" }, {}, true, "1.2 No commands"); - allPassed &= testOptionParsing({ "fdbbackup", "unknown" }, { "unknown" }, true, "1.3 Unknown command"); - allPassed &= testOptionParsing( - { "fdbbackup", "unknown1", "unknown2" }, { "unknown1", "unknown2" }, true, "1.4 Several unknown commands"); - - printf("\n2) Command Positioning Tests:\n"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file", "/cluster" }, - { "start", "--cluster-file", "/cluster" }, - true, - "2.1 Command before options"); - allPassed &= testOptionParsing({ "fdbbackup", "--cluster-file", "/cluster", "start" }, - { "start", "--cluster-file", "/cluster" }, - true, - "2.2 Command after options"); - allPassed &= testOptionParsing({ "fdbbackup", "--cluster-file", "/cluster", "list", "--json" }, - { "list", "--cluster-file", "/cluster", "--json" }, - true, - "2.3 Options before and after command"); - - printf("\n3) Option Parameter Tests:\n"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "-C", "/cluster" }, - { "start", "-C", "/cluster" }, - true, - "3.1 Short option with parameter"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--snapshot-interval", "30" }, - { "start", "--snapshot-interval", "30" }, - true, - "3.2 Option with parameter"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--logdir", "/logs", "--trace-format", "json" }, - { "start", "--logdir", "/logs", "--trace-format", "json" }, - true, - "3.3 Multiple options with parameters"); - - printf("\n4) Equal Sign Parameter Tests:\n"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file=/cluster" }, - { "start", "--cluster-file=/cluster" }, - true, - "4.1 Option with equals"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--snapshot-interval", "30", "--cluster-file=/cluster" }, - { "start", "--snapshot-interval", "30", "--cluster-file=/cluster" }, - true, - "4.2 Multiple options using both equals and space separators"); - - printf("\n5) Prefix Option Tests:\n"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--knob-max_workers", "10" }, - { "start", "--knob-max_workers", "10" }, - true, - "5.1 Knob option with parameter"); - - printf("\n6) Global flag options and CSimpleOpt Tests:\n"); - allPassed &= - testOptionParsing({ "fdbbackup", "--version", "-h" }, { "--version", "-h" }, true, "6.1 Version flag", true); - - printf("\n7) Error Tests:\n"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--unknown-option" }, {}, false, "7.1 Unknown option"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file" }, {}, false, "7.2 Missing parameter"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file", "--help" }, - { "start", "--cluster-file", "--help" }, - true, - "7.3 Option as parameter"); - allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file=/cluster", "-C=" }, - { "start", "--cluster-file=/cluster", "-C=" }, - true, - "7.4 Option with empty parameter value using equals"); - allPassed &= testOptionParsing( - { "fdbbackup", "start", "-C=" }, { "start", "-C=" }, true, "7.5 Empty parameter value with equals"); - - printf("\n=== %s ===\n", allPassed ? "All tests PASSED!" : "Some tests FAILED!"); - return allPassed ? 0 : 1; -} - -#endif // EXCLUDE_MAIN_FUNCTION diff --git a/fdbbackup/backup.cpp b/fdbbackup/backup.cpp new file mode 100644 index 00000000000..0c24028f33a --- /dev/null +++ b/fdbbackup/backup.cpp @@ -0,0 +1,4847 @@ +/* + * backup.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flow/ApiVersion.h" +#include "fmt/format.h" +#include "fdbclient/BackupTLSConfig.h" +#include "fdbbackup/Decode.h" +#include "fdbclient/JsonBuilder.h" +#include "flow/Arena.h" +#include "flow/ArgParseUtil.h" +#include "flow/Error.h" +#include "flow/SystemMonitor.h" +#include "flow/Trace.h" +#include "flow/CoroUtils.h" +#define BOOST_DATE_TIME_NO_LIB +#include + +#include "flow/flow.h" +#include "flow/FastAlloc.h" +#include "flow/serialize.h" +#include "flow/IRandom.h" +#include "flow/genericactors.actor.h" +#include "flow/TLSConfig.h" + +#include "fdbclient/DatabaseContext.h" +#include "fdbclient/FDBTypes.h" +#include "fdbclient/BackupAgent.h" +#include "fdbclient/Status.h" +#include "fdbclient/BackupContainer.h" +#include "fdbclient/ClusterConnectionFile.h" +#include "fdbclient/KeyBackedTypes.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/RunRYWTransaction.h" +#include "fdbclient/IBlobStore.h" +#include "fdbclient/SystemData.h" +#include "fdbclient/json_spirit/json_spirit_writer_template.h" +#include "fdbclient/BulkLoading.h" +#include "fdbclient/ManagementAPI.h" + +#include "flow/Platform.h" + +#include +#include +#include +#include // std::transform +#include +#include +#include + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#endif +#include + +#ifdef __linux__ +#include +#ifdef ALLOC_INSTRUMENTATION +#include +#endif +#endif + +#include "fdbclient/versions.h" +#include "fdbclient/BuildFlags.h" + +#include "SimpleOpt/SimpleOpt.h" + +// Type of program being executed +enum class ProgramExe { AGENT, BACKUP, RESTORE, DR_AGENT, DB_BACKUP, UNDEFINED }; + +enum class BackupType { + UNDEFINED = 0, + START, + MODIFY, + STATUS, + ABORT, + WAIT, + DISCONTINUE, + PAUSE, + RESUME, + EXPIRE, + DELETE_BACKUP, + DESCRIBE, + LIST, + QUERY, + DUMP, + CLEANUP, + TAGS, +}; + +enum class DBType { UNDEFINED = 0, START, STATUS, SWITCH, ABORT, PAUSE, RESUME }; + +// New fast restore reuses the type from legacy slow restore +enum class RestoreType { UNKNOWN, START, STATUS, ABORT, WAIT }; + +// +enum { + // Backup constants + OPT_DESTCONTAINER, + OPT_SNAPSHOTINTERVAL, + OPT_INITIAL_SNAPSHOT_INTERVAL, + OPT_ERRORLIMIT, + OPT_NOSTOPWHENDONE, + OPT_EXPIRE_BEFORE_VERSION, + OPT_EXPIRE_BEFORE_DATETIME, + OPT_EXPIRE_DELETE_BEFORE_DAYS, + OPT_EXPIRE_RESTORABLE_AFTER_VERSION, + OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, + OPT_EXPIRE_MIN_RESTORABLE_DAYS, + OPT_BASEURL, + OPT_BLOB_CREDENTIALS, + OPT_DESCRIBE_DEEP, + OPT_DESCRIBE_TIMESTAMPS, + OPT_DUMP_BEGIN, + OPT_DUMP_END, + OPT_JSON, + OPT_DELETE_DATA, + OPT_MIN_CLEANUP_SECONDS, + OPT_MUTATION_LOG_TYPE, + OPT_MODE, + + // Backup and Restore constants + OPT_PROXY, + OPT_TAGNAME, + OPT_BACKUPKEYS, + OPT_BACKUPKEYS_FILE, + OPT_WAITFORDONE, + OPT_BACKUPKEYS_FILTER, + OPT_INCREMENTALONLY, + OPT_ENCRYPTION_KEY_FILE, + OPT_ENCRYPTION_BLOCK_SIZE, + + // Backup Modify + OPT_MOD_ACTIVE_INTERVAL, + OPT_MOD_VERIFY_UID, + + // Restore constants + OPT_RESTORECONTAINER, + OPT_RESTORE_VERSION, + OPT_RESTORE_SNAPSHOT_VERSION, + OPT_RESTORE_TIMESTAMP, + OPT_PREFIX_ADD, + OPT_PREFIX_REMOVE, + OPT_RESTORE_CLUSTERFILE_DEST, + OPT_RESTORE_CLUSTERFILE_ORIG, + OPT_RESTORE_BEGIN_VERSION, + OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, + // The two restore options below allow callers of fdbrestore to divide a normal restore into one which restores just + // the system keyspace and another that restores just the user key space. This is unlike the backup command where + // all keys (both system and user) will be backed up together + OPT_RESTORE_USER_DATA, + OPT_RESTORE_SYSTEM_DATA, + + // Shared constants + OPT_CLUSTERFILE, + OPT_QUIET, + OPT_DRYRUN, + OPT_FORCE, + OPT_HELP, + OPT_DEVHELP, + OPT_VERSION, + OPT_BUILD_FLAGS, + OPT_PARENTPID, + OPT_CRASHONERROR, + OPT_NOBUFSTDOUT, + OPT_BUFSTDOUTERR, + OPT_TRACE, + OPT_TRACE_DIR, + OPT_KNOB, + OPT_TRACE_LOG_GROUP, + OPT_MEMLIMIT, + OPT_VMEMLIMIT, + OPT_LOCALITY, + + // DB constants + OPT_SOURCE_CLUSTER, + OPT_DEST_CLUSTER, + OPT_CLEANUP, + OPT_DSTONLY, + + OPT_TRACE_FORMAT, +}; + +// Top level binary commands. +CSimpleOpt::SOption g_rgOptions[] = { { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_BUILD_FLAGS, "--build-flags", SO_NONE }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + + SO_END_OF_OPTIONS }; + +CSimpleOpt::SOption g_rgAgentOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_BUILD_FLAGS, "--build-flags", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_LOCALITY, "--locality-", SO_REQ_SEP }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupStartOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, + { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_MUTATION_LOG_TYPE, "--mutation-log-type", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "--snapshot-interval", SO_REQ_SEP }, + { OPT_INITIAL_SNAPSHOT_INTERVAL, "--initial-snapshot-interval", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_DRYRUN, "-n", SO_NONE }, + { OPT_DRYRUN, "--dryrun", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP }, + { OPT_ENCRYPTION_BLOCK_SIZE, "--encryption-block-size", SO_REQ_SEP }, + { OPT_MODE, "--mode", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupModifyOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_MOD_VERIFY_UID, "--verify-uid", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "-s", SO_REQ_SEP }, + { OPT_SNAPSHOTINTERVAL, "--snapshot-interval", SO_REQ_SEP }, + { OPT_MOD_ACTIVE_INTERVAL, "--active-snapshot-interval", SO_REQ_SEP }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupStatusOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_JSON, "--json", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupAbortOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupCleanupOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_DELETE_DATA, "--delete-data", SO_NONE }, + { OPT_MIN_CLEANUP_SECONDS, "--min-cleanup-seconds", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupDiscontinueOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupWaitOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_NOSTOPWHENDONE, "-z", SO_NONE }, + { OPT_NOSTOPWHENDONE, "--no-stop-when-done", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupPauseOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupExpireOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_FORCE, "--force", SO_NONE }, + { OPT_EXPIRE_RESTORABLE_AFTER_VERSION, "--restorable-after-version", SO_REQ_SEP }, + { OPT_EXPIRE_RESTORABLE_AFTER_DATETIME, "--restorable-after-timestamp", SO_REQ_SEP }, + { OPT_EXPIRE_BEFORE_VERSION, "--expire-before-version", SO_REQ_SEP }, + { OPT_EXPIRE_BEFORE_DATETIME, "--expire-before-timestamp", SO_REQ_SEP }, + { OPT_EXPIRE_MIN_RESTORABLE_DAYS, "--min-restorable-days", SO_REQ_SEP }, + { OPT_EXPIRE_DELETE_BEFORE_DAYS, "--delete-before-days", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupDeleteOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupDescribeOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_DESCRIBE_DEEP, "--deep", SO_NONE }, + { OPT_DESCRIBE_TIMESTAMPS, "--version-timestamps", SO_NONE }, + { OPT_JSON, "--json", SO_NONE }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupDumpOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_DUMP_BEGIN, "--begin", SO_REQ_SEP }, + { OPT_DUMP_END, "--end", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupTagsOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_CLUSTERFILE, "-C", SO_REQ_SEP }, + { OPT_CLUSTERFILE, "--cluster-file", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupListOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_BASEURL, "-b", SO_REQ_SEP }, + { OPT_BASEURL, "--base-url", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgBackupQueryOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_RESTORE_TIMESTAMP, "--query-restore-timestamp", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "-d", SO_REQ_SEP }, + { OPT_DESTCONTAINER, "--destcontainer", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-qrv", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "--query-restore-version", SO_REQ_SEP }, + { OPT_RESTORE_SNAPSHOT_VERSION, "--query-restore-snapshot-version", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILTER, "--keys", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +// g_rgRestoreOptions is used by fdbrestore +CSimpleOpt::SOption g_rgRestoreOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_RESTORE_CLUSTERFILE_DEST, "--dest-cluster-file", SO_REQ_SEP }, + { OPT_RESTORE_CLUSTERFILE_ORIG, "--orig-cluster-file", SO_REQ_SEP }, + { OPT_RESTORE_TIMESTAMP, "--timestamp", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_RESTORECONTAINER, "-r", SO_REQ_SEP }, + { OPT_PROXY, "--proxy", SO_REQ_SEP }, + { OPT_PREFIX_ADD, "--add-prefix", SO_REQ_SEP }, + { OPT_PREFIX_REMOVE, "--remove-prefix", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_WAITFORDONE, "-w", SO_NONE }, + { OPT_WAITFORDONE, "--waitfordone", SO_NONE }, + { OPT_RESTORE_USER_DATA, "--user-data", SO_NONE }, + { OPT_RESTORE_SYSTEM_DATA, "--system-metadata", SO_NONE }, + { OPT_MODE, "--mode", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "--version", SO_REQ_SEP }, + { OPT_RESTORE_VERSION, "-v", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_DRYRUN, "-n", SO_NONE }, + { OPT_DRYRUN, "--dryrun", SO_NONE }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_BLOB_CREDENTIALS, "--blob-credentials", SO_REQ_SEP }, + { OPT_INCREMENTALONLY, "--incremental", SO_NONE }, + { OPT_RESTORE_BEGIN_VERSION, "--begin-version", SO_REQ_SEP }, + { OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY, "--inconsistent-snapshot-only", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgDBAgentOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + { OPT_VERSION, "--version", SO_NONE }, + { OPT_VERSION, "-v", SO_NONE }, + { OPT_BUILD_FLAGS, "--build-flags", SO_NONE }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_LOCALITY, "--locality-", SO_REQ_SEP }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgDBStartOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "-k", SO_REQ_SEP }, + { OPT_BACKUPKEYS_FILE, "--keys-file", SO_REQ_SEP }, + { OPT_BACKUPKEYS, "--keys", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgDBStatusOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "-e", SO_REQ_SEP }, + { OPT_ERRORLIMIT, "--errorlimit", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgDBSwitchOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_FORCE, "-f", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgDBAbortOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_CLEANUP, "--cleanup", SO_NONE }, + { OPT_DSTONLY, "--dstonly", SO_NONE }, + { OPT_TAGNAME, "-t", SO_REQ_SEP }, + { OPT_TAGNAME, "--tagname", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +CSimpleOpt::SOption g_rgDBPauseOptions[] = { +#ifdef _WIN32 + { OPT_PARENTPID, "--parentpid", SO_REQ_SEP }, +#endif + { OPT_SOURCE_CLUSTER, "-s", SO_REQ_SEP }, + { OPT_SOURCE_CLUSTER, "--source", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "-d", SO_REQ_SEP }, + { OPT_DEST_CLUSTER, "--destination", SO_REQ_SEP }, + { OPT_TRACE, "--log", SO_NONE }, + { OPT_TRACE_DIR, "--logdir", SO_REQ_SEP }, + { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, + { OPT_TRACE_LOG_GROUP, "--loggroup", SO_REQ_SEP }, + { OPT_QUIET, "-q", SO_NONE }, + { OPT_QUIET, "--quiet", SO_NONE }, + { OPT_CRASHONERROR, "--crash", SO_NONE }, + { OPT_MEMLIMIT, "-m", SO_REQ_SEP }, + { OPT_MEMLIMIT, "--memory", SO_REQ_SEP }, + { OPT_VMEMLIMIT, "--memory-vsize", SO_REQ_SEP }, + { OPT_HELP, "-?", SO_NONE }, + { OPT_HELP, "-h", SO_NONE }, + { OPT_HELP, "--help", SO_NONE }, + { OPT_DEVHELP, "--dev-help", SO_NONE }, + { OPT_KNOB, "--knob-", SO_REQ_SEP }, + TLS_OPTION_FLAGS, + SO_END_OF_OPTIONS +}; + +const KeyRef exeAgent = "backup_agent"_sr; +const KeyRef exeBackup = "fdbbackup"_sr; +const KeyRef exeRestore = "fdbrestore"_sr; +const KeyRef exeDatabaseAgent = "dr_agent"_sr; +const KeyRef exeDatabaseBackup = "fdbdr"_sr; + +extern const char* getSourceVersion(); + +#ifdef _WIN32 +void parentWatcher(void* parentHandle) { + HANDLE parent = (HANDLE)parentHandle; + int signal = WaitForSingleObject(parent, INFINITE); + CloseHandle(parentHandle); + if (signal == WAIT_OBJECT_0) + criticalError(FDB_EXIT_SUCCESS, "ParentProcessExited", "Parent process exited"); + TraceEvent(SevError, "ParentProcessWaitFailed").detail("RetCode", signal).GetLastError(); +} + +#endif + +static void printVersion() { + printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); + printf("source version %s\n", getSourceVersion()); + printf("protocol %llx\n", (long long)currentProtocolVersion().version()); +} + +static void printBuildInformation() { + printf("%s", jsonBuildInformation().c_str()); +} + +const char* BlobCredentialInfo = + " BLOB CREDENTIALS\n" + " Blob account secret keys can optionally be omitted from blobstore:// URLs, in which case they will be\n" + " loaded, if possible, from 1 or more blob credentials definition files.\n\n" + " These files can be specified with the --blob-credentials argument described above or via the environment " + "variable\n" + " FDB_BLOB_CREDENTIALS, whose value is a colon-separated list of files. The command line takes priority over\n" + " over the environment but all files from both sources are used.\n\n" + " At connect time, the specified files are read in order and the first matching account specification " + "(user@host)\n" + " will be used to obtain the secret key.\n\n" + " The JSON schema is:\n" + " { \"accounts\" : { \"user@host\" : { \"secret\" : \"SECRETKEY\" }, \"user2@host2\" : { \"secret\" : " + "\"SECRET\" } } }\n"; + +static void printHelpTeaser(const char* name) { + fprintf(stderr, "Try `%s --help' for more information.\n", name); +} + +static void printAgentUsage(bool devhelp) { + printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); + printf("Usage: %s [OPTIONS]\n\n", exeAgent.toString().c_str()); + printf(" -C CONNFILE The path of a file containing the connection string for the\n" + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifies the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --loggroup LOG_GROUP\n" + " Sets the LogGroup field with the specified value for all\n" + " events in the trace output (defaults to `default').\n"); + printf(" --trace-format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" -m SIZE, --memory SIZE\n" + " Memory limit. The default value is 8GiB. When specified\n" + " without a unit, MiB is assumed.\n"); + printf(TLS_HELP); + printf(" --build-flags Print build information and exit.\n"); + printf(" -v, --version Print version information and exit.\n"); + printf(" -h, --help Display this help and exit.\n"); + + if (devhelp) { +#ifdef _WIN32 + printf(" -n Create a new console.\n"); + printf(" -q Disable error dialog on crash.\n"); + printf(" --parentpid PID\n"); + printf(" Specify a process after whose termination to exit.\n"); +#endif + } + + printf("\n"); + puts(BlobCredentialInfo); + + return; +} + +void printBackupContainerInfo() { + printf(" Backup URL forms:\n\n"); + std::vector formats = IBackupContainer::getURLFormats(); + for (const auto& f : formats) + printf(" %s\n", f.c_str()); + printf("\n"); +} + +static void printBackupUsage(bool devhelp) { + printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); + printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | abort | wait | discontinue | pause | resume | expire | " + "delete | describe | list | query | cleanup | tags) [ACTION_OPTIONS]\n\n", + exeBackup.toString().c_str()); + printf(" TOP LEVEL OPTIONS:\n"); + printf(" --build-flags Print build information and exit.\n"); + printf(" -v, --version Print version information and exit.\n"); + printf(" -h, --help Display this help and exit.\n"); + printf("\n"); + + printf(" ACTION OPTIONS:\n"); + printf(" -C CONNFILE The path of a file containing the connection string for the\n" + " FoundationDB cluster. The default is first the value of the\n" + " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" + " then `%s'.\n", + platform::getDefaultClusterFilePath().c_str()); + printf(" -d, --destcontainer URL\n" + " The Backup container URL for start, modify, describe, query, expire, and delete " + "operations.\n"); + printBackupContainerInfo(); + printf(" -b, --base-url BASEURL\n" + " Base backup URL for list operations. This looks like a Backup URL but without a backup " + "name.\n"); + printf(" --blob-credentials FILE\n" + " File containing blob credentials in JSON format. Can be specified multiple times for " + "multiple files. See below for more details.\n"); + printf(" --expire-before-timestamp DATETIME\n" + " Datetime cutoff for expire operations. Requires a cluster file and will use " + "version/timestamp metadata\n" + " in the database to obtain a cutoff version very close to the timestamp given in %s.\n", + BackupAgentBase::timeFormat().c_str()); + printf(" --expire-before-version VERSION\n" + " Version cutoff for expire operations. Deletes data files containing no data at or after " + "VERSION.\n"); + printf(" --delete-before-days NUM_DAYS\n" + " Another way to specify version cutoff for expire operations. Deletes data files " + "containing no data at or after a\n" + " version approximately NUM_DAYS days worth of versions prior to the latest log version in " + "the backup.\n"); + printf(" --query-restore-snapshot-version VERSION\n" + " For query operations, set the snapshot version, inclusive, used to restore a backup.\n" + " Set -1 to use the latest valid snapshot,\n" + " Set -3 to use the oldest valid snapshot.\n"); + printf(" -qrv --query-restore-version VERSION\n" + " For query operations, set target version for restoring a backup. Set -1 for maximum\n" + " restorable version (default) and -3 for minimum restorable version.\n"); + printf( + " --query-restore-timestamp DATETIME\n" + " For query operations, instead of a numeric version, use this to specify a timestamp in %s\n", + BackupAgentBase::timeFormat().c_str()); + printf( + " and it will be converted to a version from that time using metadata in the cluster file.\n"); + printf(" --restorable-after-timestamp DATETIME\n" + " For expire operations, set minimum acceptable restorability to the version equivalent of " + "DATETIME and later.\n"); + printf(" --restorable-after-version VERSION\n" + " For expire operations, set minimum acceptable restorability to the VERSION and later.\n"); + printf(" --min-restorable-days NUM-DAYS\n" + " For expire operations, set minimum acceptable restorability to approximately NUM_DAYS " + "days worth of versions\n" + " prior to the latest log version in the backup.\n"); + printf(" --version-timestamps\n"); + printf(" For describe operations, lookup versions in the database to obtain timestamps. A cluster " + "file is required.\n"); + printf( + " -f, --force For expire operations, force expiration even if minimum restorability would be violated.\n"); + printf(" -s, --snapshot-interval DURATION\n" + " For start or modify operations, specifies the backup's default target snapshot interval " + "as DURATION seconds. Defaults to %d for start operations.\n", + CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC); + printf(" --mode MODE Snapshot mechanism to use: bulkdump, rangefile (default, legacy), or both.\n" + " bulkdump: Uses BulkDump SST files for faster restore performance\n" + " rangefile: Traditional range files for backward compatibility\n" + " both: Generate both formats for validation (increases backup size)\n"); + printf(" --active-snapshot-interval DURATION\n" + " For modify operations, sets the desired interval for the backup's currently active " + "snapshot, relative to the start of the snapshot.\n"); + printf(" --verify-uid UID\n" + " Specifies a UID to verify against the BackupUID of the running backup. If provided, the " + "UID is verified in the same transaction\n" + " which sets the new backup parameters (if the UID matches).\n"); + printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); + printf(" -k KEYS List of key ranges to backup or to filter the backup in query operations.\n" + " If not specified, the entire database will be backed up or no filter will be applied.\n"); + printf(" --keys-file FILE\n" + " Same as -k option, except keys are specified in the input file.\n"); + printf(" --mutation-log-type TYPE\n" + " Specifies the mutation log type. Valid values are: " + "partitioned-log-experimental, range-partitioned-log-experimental.\n" + "If not specified, default log type is used.\n"); + printf(" -n, --dryrun For backup start or restore start, performs a trial run with no actual changes made.\n"); + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifies the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --loggroup LOG_GROUP\n" + " Sets the LogGroup field with the specified value for all\n" + " events in the trace output (defaults to `default').\n"); + printf(" --trace-format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" --max-cleanup-seconds SECONDS\n" + " Specifies the amount of time a backup or DR needs to be stale before cleanup will\n" + " remove mutations for it. By default this is set to one hour.\n"); + printf(" --delete-data\n" + " This flag will cause cleanup to remove mutations for the most stale backup or DR.\n"); + printf(" --incremental\n" + " Performs incremental backup without the base backup.\n" + " This option indicates to the backup agent that it will only need to record the log files, " + "and ignore the range files.\n"); + printf(" --encryption-key-file" + " The AES-256-GCM key in the provided file is used for encrypting backup files.\n" + " For modify operations, need to pass encryption key file only if Backup container URL is " + "changed to " + "re-encrypt all future backup files. \n"); + printf(" --encryption-block-size" + " Block size in bytes for file encryption. Only used with fdbbackup start command. Default " + "is 1048576 (1MB).\n"); + + printf(TLS_HELP); + printf(" -w, --wait Wait for the backup to complete (allowed with `start' and `discontinue').\n"); + printf(" -z, --no-stop-when-done\n" + " Do not stop backup when restorable.\n"); + printf(" -h, --help Display this help and exit.\n"); + + if (devhelp) { +#ifdef _WIN32 + printf(" -n Create a new console.\n"); + printf(" -q Disable error dialog on crash.\n"); + printf(" --parentpid PID\n"); + printf(" Specify a process after whose termination to exit.\n"); +#endif + printf(" --deep For describe operations, do not use cached metadata. Warning: Very slow\n"); + } + printf("\n" + " KEYS FORMAT: \" \" [...]\n"); + printf("\n"); + puts(BlobCredentialInfo); + + return; +} + +static void printRestoreUsage(bool devhelp) { + printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); + printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | abort | wait) [OPTIONS]\n\n", + exeRestore.toString().c_str()); + + printf(" TOP LEVEL OPTIONS:\n"); + printf(" --build-flags Print build information and exit.\n"); + printf(" -v, --version Print version information and exit.\n"); + printf(" -h, --help Display this help and exit.\n"); + printf("\n"); + + printf(" ACTION OPTIONS:\n"); + // printf(" FOLDERS Paths to folders containing the backup files.\n"); + printf(" Options for all commands:\n\n"); + printf(" --dest-cluster-file CONNFILE\n"); + printf(" The cluster file to restore data into.\n"); + printf(" -t, --tagname TAGNAME\n"); + printf(" The restore tag to act on. Default is 'default'\n"); + printf("\n"); + printf(" Options for start:\n\n"); + printf(" -r URL The Backup URL for the restore to read from.\n"); + printBackupContainerInfo(); + printf(" -w, --waitfordone\n"); + printf(" Wait for the restore to complete before exiting. Prints progress updates.\n"); + printf(" -k KEYS List of key ranges from the backup to restore.\n"); + printf(" --keys-file FILE\n" + " Same as -k option, except keys are specified in the input file.\n"); + printf(" --remove-prefix PREFIX\n"); + printf(" Prefix to remove from the restored keys.\n"); + printf(" --add-prefix PREFIX\n"); + printf(" Prefix to add to the restored keys\n"); + printf(" -n, --dryrun Perform a trial run with no changes made.\n"); + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifies the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --loggroup LOG_GROUP\n" + " Sets the LogGroup field with the specified value for all\n" + " events in the trace output (defaults to `default').\n"); + printf(" --trace-format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" --incremental\n" + " Performs incremental restore without the base backup.\n" + " This tells the backup agent to only replay the log files from the backup source.\n" + " This also allows a restore to be performed into a non-empty destination database.\n"); + printf(" --begin-version\n" + " To be used in conjunction with incremental restore.\n" + " Indicates to the backup agent to only begin replaying log files from a certain version, " + "instead of the entire set.\n"); + printf( + " --mode MODE Restore mechanism to use: rangefile (default), bulkload.\n" + " rangefile: Traditional range file restore from kvranges/\n" + " bulkload: Use BulkLoad for faster range data restoration if BulkDump dataset is available\n" + " If incomplete dataset: restore returns error with clear message directing user to retry with " + "--mode rangefile.\n"); + printf(" --encryption-key-file" + " The AES-256-GCM key in the provided file is used for decrypting backup files.\n"); + printf(TLS_HELP); + printf(" -v DBVERSION The version at which the database will be restored.\n"); + printf(" --timestamp Instead of a numeric version, use this to specify a timestamp in %s\n", + BackupAgentBase::timeFormat().c_str()); + printf( + " and it will be converted to a version from that time using metadata in orig_cluster_file.\n"); + printf(" --orig-cluster-file CONNFILE\n"); + printf(" The cluster file for the original database from which the backup was created. The " + "original database\n"); + printf(" is only needed to convert a --timestamp argument to a database version.\n"); + printf(" --user-data\n" + " Restore only the user keyspace. This option should NOT be used alongside " + "--system-metadata (below) and CANNOT be used alongside other specified key ranges.\n"); + printf( + " --system-metadata\n" + " Restore only the relevant system keyspace. This option " + "should NOT be used alongside --user-data (above) and CANNOT be used alongside other specified key ranges.\n"); + + if (devhelp) { +#ifdef _WIN32 + printf(" -q Disable error dialog on crash.\n"); + printf(" --parentpid PID\n"); + printf(" Specify a process after whose termination to exit.\n"); +#endif + } + + printf("\n" + " KEYS FORMAT: \" \" [...]\n"); + printf("\n"); + puts(BlobCredentialInfo); + + return; +} + +static void printDBAgentUsage(bool devhelp) { + printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); + printf("Usage: %s [OPTIONS]\n\n", exeDatabaseAgent.toString().c_str()); + printf(" -d, --destination CONNFILE\n" + " The path of a file containing the connection string for the\n" + " destination FoundationDB cluster.\n"); + printf(" -s, --source CONNFILE\n" + " The path of a file containing the connection string for the\n" + " source FoundationDB cluster.\n"); + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifies the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --loggroup LOG_GROUP\n" + " Sets the LogGroup field with the specified value for all\n" + " events in the trace output (defaults to `default').\n"); + printf(" --trace-format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" -m, --memory SIZE\n" + " Memory limit. The default value is 8GiB. When specified\n" + " without a unit, MiB is assumed.\n"); + printf(TLS_HELP); + printf(" --build-flags Print build information and exit.\n"); + printf(" -v, --version Print version information and exit.\n"); + printf(" -h, --help Display this help and exit.\n"); + if (devhelp) { +#ifdef _WIN32 + printf(" -n Create a new console.\n"); + printf(" -q Disable error dialog on crash.\n"); + printf(" --parentpid PID\n"); + printf(" Specify a process after whose termination to exit.\n"); +#endif + } + + return; +} + +static void printDBBackupUsage(bool devhelp) { + printf("FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); + printf("Usage: %s [TOP_LEVEL_OPTIONS] (start | status | switch | abort | pause | resume) [OPTIONS]\n\n", + exeDatabaseBackup.toString().c_str()); + + printf(" TOP LEVEL OPTIONS:\n"); + printf(" --build-flags Print build information and exit.\n"); + printf(" -v, --version Print version information and exit.\n"); + printf(" -h, --help Display this help and exit.\n"); + printf("\n"); + + printf(" ACTION OPTIONS:\n"); + printf(" -d, --destination CONNFILE\n" + " The path of a file containing the connection string for the\n"); + printf(" destination FoundationDB cluster.\n"); + printf(" -s, --source CONNFILE\n" + " The path of a file containing the connection string for the\n" + " source FoundationDB cluster.\n"); + printf(" -e ERRORLIMIT The maximum number of errors printed by status (default is 10).\n"); + printf(" -k KEYS List of key ranges to backup.\n" + " If not specified, the entire database will be backed up.\n"); + printf(" --keys-file FILE\n" + " Same as -k option, except keys are specified in the input file.\n"); + printf(" --cleanup Abort will attempt to stop mutation logging on the source cluster.\n"); + printf(" --dstonly Abort will not make any changes on the source cluster.\n"); + printf(TLS_HELP); + printf(" --log Enables trace file logging for the CLI session.\n" + " --logdir PATH Specifies the output directory for trace files. If\n" + " unspecified, defaults to the current directory. Has\n" + " no effect unless --log is specified.\n"); + printf(" --loggroup LOG_GROUP\n" + " Sets the LogGroup field with the specified value for all\n" + " events in the trace output (defaults to `default').\n"); + printf(" --trace-format FORMAT\n" + " Select the format of the trace files. xml (the default) and json are supported.\n" + " Has no effect unless --log is specified.\n"); + printf(" -h, --help Display this help and exit.\n"); + printf("\n" + " KEYS FORMAT: \" \" [...]\n"); + + if (devhelp) { +#ifdef _WIN32 + printf(" -n Create a new console.\n"); + printf(" -q Disable error dialog on crash.\n"); + printf(" --parentpid PID\n"); + printf(" Specify a process after whose termination to exit.\n"); +#endif + } + + return; +} + +static void printUsage(ProgramExe programExe, bool devhelp) { + + switch (programExe) { + case ProgramExe::AGENT: + printAgentUsage(devhelp); + break; + case ProgramExe::BACKUP: + printBackupUsage(devhelp); + break; + case ProgramExe::RESTORE: + printRestoreUsage(devhelp); + break; + case ProgramExe::DR_AGENT: + printDBAgentUsage(devhelp); + break; + case ProgramExe::DB_BACKUP: + printDBBackupUsage(devhelp); + break; + case ProgramExe::UNDEFINED: + default: + break; + } + + return; +} + +extern bool g_crashOnError; + +// Return the type of program executable based on the name of executable file +ProgramExe getProgramType(std::string programExe) { + ProgramExe enProgramExe = ProgramExe::UNDEFINED; + + std::transform(programExe.begin(), programExe.end(), programExe.begin(), ::tolower); + + // Remove the extension, if Windows +#ifdef _WIN32 + size_t lastDot = programExe.find_last_of("."); + if (lastDot != std::string::npos) { + size_t lastSlash = programExe.find_last_of("\\"); + + // Ensure last dot is after last slash, if present + if ((lastSlash == std::string::npos) || (lastSlash < lastDot)) { + programExe = programExe.substr(0, lastDot); + } + } +#endif + // For debugging convenience, remove .debug suffix if present. + if (StringRef(programExe).endsWith(".debug"_sr)) + programExe = programExe.substr(0, programExe.size() - 6); + + // Check if backup agent + if ((programExe.length() >= exeAgent.size()) && + (programExe.compare(programExe.length() - exeAgent.size(), exeAgent.size(), (const char*)exeAgent.begin()) == + 0)) { + enProgramExe = ProgramExe::AGENT; + } + + // Check if backup + else if ((programExe.length() >= exeBackup.size()) && + (programExe.compare( + programExe.length() - exeBackup.size(), exeBackup.size(), (const char*)exeBackup.begin()) == 0)) { + enProgramExe = ProgramExe::BACKUP; + } + + // Check if restore + else if ((programExe.length() >= exeRestore.size()) && + (programExe.compare( + programExe.length() - exeRestore.size(), exeRestore.size(), (const char*)exeRestore.begin()) == 0)) { + enProgramExe = ProgramExe::RESTORE; + } + + // Check if db agent + else if ((programExe.length() >= exeDatabaseAgent.size()) && + (programExe.compare(programExe.length() - exeDatabaseAgent.size(), + exeDatabaseAgent.size(), + (const char*)exeDatabaseAgent.begin()) == 0)) { + enProgramExe = ProgramExe::DR_AGENT; + } + + // Check if db backup + else if ((programExe.length() >= exeDatabaseBackup.size()) && + (programExe.compare(programExe.length() - exeDatabaseBackup.size(), + exeDatabaseBackup.size(), + (const char*)exeDatabaseBackup.begin()) == 0)) { + enProgramExe = ProgramExe::DB_BACKUP; + } + + return enProgramExe; +} + +BackupType getBackupType(std::string backupType) { + BackupType enBackupType = BackupType::UNDEFINED; + + std::transform(backupType.begin(), backupType.end(), backupType.begin(), ::tolower); + + static std::map values; + if (values.empty()) { + values["start"] = BackupType::START; + values["status"] = BackupType::STATUS; + values["abort"] = BackupType::ABORT; + values["cleanup"] = BackupType::CLEANUP; + values["wait"] = BackupType::WAIT; + values["discontinue"] = BackupType::DISCONTINUE; + values["pause"] = BackupType::PAUSE; + values["resume"] = BackupType::RESUME; + values["expire"] = BackupType::EXPIRE; + values["delete"] = BackupType::DELETE_BACKUP; + values["describe"] = BackupType::DESCRIBE; + values["list"] = BackupType::LIST; + values["query"] = BackupType::QUERY; + values["dump"] = BackupType::DUMP; + values["modify"] = BackupType::MODIFY; + values["tags"] = BackupType::TAGS; + } + + auto i = values.find(backupType); + if (i != values.end()) + enBackupType = i->second; + + return enBackupType; +} + +Optional getSnapshotMode(std::string mode) { + std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); + + if (mode == "rangefile") + return SnapshotMode::RANGEFILE; + if (mode == "bulkdump") + return SnapshotMode::BULKDUMP; + if (mode == "both") + return SnapshotMode::BOTH; + return Optional(); +} + +Optional getRestoreMode(std::string mode) { + std::transform(mode.begin(), mode.end(), mode.begin(), ::tolower); + + if (mode == "rangefile") + return RestoreMode::RANGEFILE; + if (mode == "bulkload") + return RestoreMode::BULKLOAD; + return Optional(); +} + +Optional getMutationLogType(std::string type) { + std::transform(type.begin(), type.end(), type.begin(), ::tolower); + + if (type == "partitioned-log-experimental") + return MutationLogType::PARTITIONED_LOG; + if (type == "range-partitioned-log-experimental") + return MutationLogType::RANGE_PARTITIONED_LOG; + return Optional(); +} + +RestoreType getRestoreType(std::string name) { + if (name == "start") + return RestoreType::START; + if (name == "abort") + return RestoreType::ABORT; + if (name == "status") + return RestoreType::STATUS; + if (name == "wait") + return RestoreType::WAIT; + return RestoreType::UNKNOWN; +} + +DBType getDBType(std::string dbType) { + DBType enBackupType = DBType::UNDEFINED; + + std::transform(dbType.begin(), dbType.end(), dbType.begin(), ::tolower); + + static std::map values; + if (values.empty()) { + values["start"] = DBType::START; + values["status"] = DBType::STATUS; + values["switch"] = DBType::SWITCH; + values["abort"] = DBType::ABORT; + values["pause"] = DBType::PAUSE; + values["resume"] = DBType::RESUME; + } + + auto i = values.find(dbType); + if (i != values.end()) + enBackupType = i->second; + + return enBackupType; +} + +AsyncResult getLayerStatus(Reference tr, + IPAddress localIP, + std::string name, + std::string id, + ProgramExe exe, + Database dest, + Snapshot snapshot = Snapshot::False) { + // This process will write a document that looks like this: + // { backup : { $expires : {}, version: } + // so that the value under 'backup' will eventually expire to null and thus be ignored by + // readers of status. This is because if all agents die then they can no longer clean up old + // status docs from other dead agents. + + Version readVer = co_await tr->getReadVersion(); + + json_spirit::mValue layersRootValue; // Will contain stuff that goes into the doc at the layers status root + JSONDoc layersRoot(layersRootValue); // Convenient mutator / accessor for the layers root + JSONDoc op = layersRoot.subDoc(name); // Operator object for the $expires operation + // Create the $expires key which is where the rest of the status output will go + + JSONDoc layerRoot = op.subDoc("$expires"); + // Set the version argument in the $expires operator object. + op.create("version") = readVer + 120 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + + layerRoot.create("instances_running.$sum") = 1; + layerRoot.create("last_updated.$max") = now(); + + JSONDoc o = layerRoot.subDoc("instances." + id); + + o.create("version") = FDB_VT_VERSION; + o.create("id") = id; + o.create("last_updated") = now(); + o.create("memory_usage") = (int64_t)getMemoryUsage(); + o.create("resident_size") = (int64_t)getResidentMemoryUsage(); + o.create("main_thread_cpu_seconds") = getProcessorTimeThread(); + o.create("process_cpu_seconds") = getProcessorTimeProcess(); + o.create("configured_workers") = CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; + o.create("processID") = ::getpid(); + o.create("locality") = tr->getDatabase()->clientLocality.toJSON(); + o.create("networkAddress") = localIP.toString(); + + if (exe == ProgramExe::AGENT) { + static IBlobStoreEndpoint::Stats last_stats; + static double last_ts = 0; + IBlobStoreEndpoint::Stats current_stats = IBlobStoreEndpoint::s_stats; + JSONDoc blobstats = o.create("blob_stats"); + blobstats.create("total") = current_stats.getJSON(); + IBlobStoreEndpoint::Stats diff = current_stats - last_stats; + json_spirit::mObject diffObj = diff.getJSON(); + if (last_ts > 0) + diffObj["bytes_per_second"] = double(current_stats.bytes_sent - last_stats.bytes_sent) / (now() - last_ts); + blobstats.create("recent") = diffObj; + last_stats = current_stats; + last_ts = now(); + + JSONDoc totalBlobStats = layerRoot.subDoc("blob_recent_io"); + for (auto& p : diffObj) + totalBlobStats.create(p.first + ".$sum") = p.second; + + FileBackupAgent fba; + std::vector backupTags = co_await getAllBackupTags(tr, snapshot); + std::vector>> tagLastRestorableVersions; + std::vector> tagStates; + std::vector>> tagContainers; + std::vector> tagRangeBytes; + std::vector> tagLogBytes; + Future> fBackupPaused = tr->get(fba.taskBucket->getPauseKey(), snapshot); + + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + std::vector::iterator tag; + std::vector backupTagUids; + for (tag = backupTags.begin(); tag != backupTags.end(); tag++) { + UidAndAbortedFlagT uidAndAbortedFlag = co_await tag->getOrThrow(tr, snapshot); + BackupConfig config(uidAndAbortedFlag.first); + backupTagUids.push_back(config.getUid()); + + tagStates.push_back(config.stateEnum().getOrThrow(tr, snapshot)); + tagRangeBytes.push_back(config.rangeBytesWritten().getD(tr, snapshot, 0)); + tagLogBytes.push_back(config.logBytesWritten().getD(tr, snapshot, 0)); + tagContainers.push_back(config.backupContainer().getOrThrow(tr, snapshot)); + tagLastRestorableVersions.push_back(fba.getLastRestorable(tr, StringRef(tag->tagName), snapshot)); + } + + co_await (waitForAll(tagLastRestorableVersions) && waitForAll(tagStates) && waitForAll(tagContainers) && + waitForAll(tagRangeBytes) && waitForAll(tagLogBytes) && success(fBackupPaused)); + + std::vector> encryptionSetupResults; + std::vector encryptionContainerIndices; + + for (int i = 0; i < tagContainers.size(); i++) { + if (tagContainers[i].get()->getEncryptionKeyFileName().present()) { + encryptionSetupResults.push_back(tagContainers[i].get()->encryptionSetupComplete()); + encryptionContainerIndices.push_back(i); + } + } + co_await waitForAllReady(encryptionSetupResults); + json_spirit::mArray keysArr; + std::unordered_set seenKeyPaths; + for (int j = 0; j < encryptionContainerIndices.size() && j < 1e6; j++) { + int i = encryptionContainerIndices[j]; + std::string keyPath = tagContainers[i].get()->getEncryptionKeyFileName().get(); + + if (!seenKeyPaths.contains(keyPath)) { + seenKeyPaths.insert(keyPath); + json_spirit::mObject keyObj; + keyObj["path"] = tagContainers[i].get()->getEncryptionKeyFileName().get(); + keyObj["success"] = !encryptionSetupResults[j].isError(); + keysArr.push_back(keyObj); + } + } + o.create("encryption_keys") = keysArr; + + JSONDoc tagsRoot = layerRoot.subDoc("tags"); + layerRoot.create("tags.timestamp") = now(); + layerRoot.create("total_workers.$sum") = + fBackupPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; + layerRoot.create("paused.$latest") = fBackupPaused.get().present(); + + int j = 0; + for (const KeyBackedTag& eachTag : backupTags) { + EBackupState status = tagStates[j].get(); + const char* statusText = fba.getStateText(status); + + // The object for this backup tag inside this instance's subdocument + JSONDoc tagRoot = tagsRoot.subDoc(eachTag.tagName).subDoc("$latest"); + tagRoot.create("current_container") = tagContainers[j].get()->getURL(); + tagRoot.create("current_status") = statusText; + if (tagLastRestorableVersions[j].get().present()) { + Version last_restorable_version = tagLastRestorableVersions[j].get().get(); + double last_restorable_seconds_behind = + ((double)readVer - last_restorable_version) / CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + tagRoot.create("last_restorable_version") = last_restorable_version; + tagRoot.create("last_restorable_seconds_behind") = last_restorable_seconds_behind; + } + tagRoot.create("running_backup") = + (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING); + tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL); + tagRoot.create("range_bytes_written") = tagRangeBytes[j].get(); + tagRoot.create("mutation_log_bytes_written") = tagLogBytes[j].get(); + tagRoot.create("mutation_stream_id") = backupTagUids[j].toString(); + tagRoot.create("file_level_encryption") = + tagContainers[j].get()->getEncryptionKeyFileName().present() ? true : false; + if (tagContainers[j].get()->getEncryptionKeyFileName().present()) { + tagRoot.create("encryption_key_file") = tagContainers[j].get()->getEncryptionKeyFileName().get(); + } + j++; + } + } else if (exe == ProgramExe::DR_AGENT) { + DatabaseBackupAgent dba; + Reference tr2(new ReadYourWritesTransaction(dest)); + tr2->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr2->setOption(FDBTransactionOptions::LOCK_AWARE); + RangeResult tagNames = co_await tr2->getRange(dba.tagNames.range(), 10000, snapshot); + std::vector>> backupVersion; + std::vector> backupStatus; + std::vector> tagRangeBytesDR; + std::vector> tagLogBytesDR; + Future> fDRPaused = tr->get(dba.taskBucket->getPauseKey(), snapshot); + + std::vector drTagUids; + for (int i = 0; i < tagNames.size(); i++) { + backupVersion.push_back(tr2->get(tagNames[i].value.withPrefix(applyMutationsBeginRange.begin), snapshot)); + UID tagUID = BinaryReader::fromStringRef(tagNames[i].value, Unversioned()); + drTagUids.push_back(tagUID); + backupStatus.push_back(dba.getStateValue(tr2, tagUID, snapshot)); + tagRangeBytesDR.push_back(dba.getRangeBytesWritten(tr2, tagUID, snapshot)); + tagLogBytesDR.push_back(dba.getLogBytesWritten(tr2, tagUID, snapshot)); + } + + co_await (waitForAll(backupStatus) && waitForAll(backupVersion) && waitForAll(tagRangeBytesDR) && + waitForAll(tagLogBytesDR) && success(fDRPaused)); + + JSONDoc tagsRoot = layerRoot.subDoc("tags"); + layerRoot.create("tags.timestamp") = now(); + layerRoot.create("total_workers.$sum") = fDRPaused.get().present() ? 0 : CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT; + layerRoot.create("paused.$latest") = fDRPaused.get().present(); + + for (int i = 0; i < tagNames.size(); i++) { + std::string tagName = dba.sourceTagNames.unpack(tagNames[i].key).getString(0).toString(); + + auto status = backupStatus[i].get(); + + JSONDoc tagRoot = tagsRoot.subDoc(tagName).subDoc("$latest"); + tagRoot.create("running_backup") = + (status == EBackupState::STATE_RUNNING_DIFFERENTIAL || status == EBackupState::STATE_RUNNING); + tagRoot.create("running_backup_is_restorable") = (status == EBackupState::STATE_RUNNING_DIFFERENTIAL); + tagRoot.create("range_bytes_written") = tagRangeBytesDR[i].get(); + tagRoot.create("mutation_log_bytes_written") = tagLogBytesDR[i].get(); + tagRoot.create("mutation_stream_id") = drTagUids[i].toString(); + + if (backupVersion[i].get().present()) { + double seconds_behind = ((double)readVer - BinaryReader::fromStringRef( + backupVersion[i].get().get(), Unversioned())) / + CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + tagRoot.create("seconds_behind") = seconds_behind; + //TraceEvent("BackupMetrics").detail("SecondsBehind", seconds_behind); + } + + tagRoot.create("backup_state") = BackupAgentBase::getStateText(status); + } + } + + std::string json = json_spirit::write_string(layersRootValue); + co_return json; +} + +// Check for unparsable or expired statuses and delete them. +// First checks the first doc in the key range, and if it is valid, alive and not "me" then +// returns. Otherwise, checks the rest of the range as well. +Future cleanupStatus(Reference tr, + std::string rootKey, + std::string name, + std::string id, + int limit = 1) { + RangeResult docs = co_await tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True); + bool readMore = false; + for (int i = 0; i < docs.size(); ++i) { + json_spirit::mValue docValue; + try { + json_spirit::read_string(docs[i].value.toString(), docValue); + JSONDoc doc(docValue); + // Update the reference version for $expires + JSONDoc::expires_reference_version = tr->getReadVersion().get(); + // Evaluate the operators in the document, which will reduce to nothing if it is expired. + doc.cleanOps(); + if (!doc.has(name + ".last_updated")) + throw Error(); + + // Alive and valid. + // If limit == 1 and id is present then read more + if (limit == 1 && doc.has(name + ".instances." + id)) + readMore = true; + } catch (Error& e) { + // If doc can't be parsed or isn't alive, delete it. + TraceEvent(SevWarn, "RemovedDeadBackupLayerStatus").errorUnsuppressed(e).detail("Key", docs[i].key); + tr->clear(docs[i].key); + // If limit is 1 then read more. + if (limit == 1) + readMore = true; + } + if (readMore) { + limit = 10000; + RangeResult docs2 = co_await tr->getRange(KeyRangeRef(rootKey, strinc(rootKey)), limit, Snapshot::True); + docs = std::move(docs2); + readMore = false; + } + } +} + +// Get layer status document for just this layer +AsyncResult getLayerStatus(Database src, std::string rootKey) { + Transaction tr(src); + + while (true) { + Error err; + try { + tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + RangeResult kvPairs = + co_await tr.getRange(KeyRangeRef(rootKey, strinc(rootKey)), GetRangeLimits::ROW_LIMIT_UNLIMITED); + json_spirit::mObject statusDoc; + JSONDoc modifier(statusDoc); + for (auto& kv : kvPairs) { + json_spirit::mValue docValue; + json_spirit::read_string(kv.value.toString(), docValue); + co_await yield(); + modifier.absorb(docValue); + co_await yield(); + } + JSONDoc::expires_reference_version = (uint64_t)tr.getReadVersion().get(); + modifier.cleanOps(); + co_return statusDoc; + } catch (Error& e) { + err = e; + } + co_await tr.onError(err); + } +} + +// Read layer status for this layer and get the total count of agent processes (instances) then adjust the poll delay +// based on that and BACKUP_AGGREGATE_POLL_RATE +Future updateAgentPollRate(Database src, + std::string rootKey, + std::string name, + std::shared_ptr pollDelay) { + while (true) { + try { + json_spirit::mObject status = co_await getLayerStatus(src, rootKey); + int64_t processes = 0; + // If instances count is present and greater than 0 then update pollDelay + if (JSONDoc(status).tryGet(name + ".instances_running", processes) && processes > 0) { + // The aggregate poll rate is the target poll rate for all agent processes in the cluster + // The poll rate (polls/sec) for a single processes is aggregate poll rate / processes, and pollDelay is + // the inverse of that + *pollDelay = (double)processes / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE; + } + } catch (Error& e) { + TraceEvent(SevWarn, "BackupAgentPollRateUpdateError").error(e); + } + co_await delay(CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE_UPDATE_INTERVAL); + } +} + +Future statusUpdateActor(Database statusUpdateDest, + std::string name, + ProgramExe exe, + std::shared_ptr pollDelay, + Database taskDest = Database(), + std::string id = nondeterministicRandom()->randomUniqueID().toString()) { + std::string metaKey = layerStatusMetaPrefixRange.begin.toString() + "json/" + name; + std::string rootKey = backupStatusPrefixRange.begin.toString() + name + "/json"; + std::string instanceKey = rootKey + "/" + "agent-" + id; + Reference tr(new ReadYourWritesTransaction(statusUpdateDest)); + Future pollRateUpdater; + + // In order to report a useful networkAddress to the cluster's layer status JSON object, determine which local + // network interface IP will be used to talk to the cluster. This is a blocking call, so it is only done once, + // and in a retry loop because if we can't connect to the cluster we can't do any work anyway. + IPAddress localIP; + + while (true) { + Error err; + try { + localIP = statusUpdateDest->getConnectionRecord()->getConnectionString().determineLocalSourceIP(); + break; + } catch (Error& e) { + err = e; + } + TraceEvent(SevWarn, "AgentCouldNotDetermineLocalIP").error(err); + co_await delay(1.0); + } + + // Register the existence of this layer in the meta key space + while (true) { + tr->reset(); + Error err; + try { + while (true) { + Error innerErr; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + tr->set(metaKey, rootKey); + co_await tr->commit(); + break; + } catch (Error& e) { + innerErr = e; + } + TraceEvent(SevWarnAlways, "LayerStatusMetaKeyUpdateError").errorUnsuppressed(innerErr); + co_await tr->onError(innerErr); // Non-retryable txns throws back the error. + } + break; + } catch (Error& e) { + err = e; + } + // For non-retryable txns, do delay, reset txn and retry + TraceEvent(SevWarnAlways, "UnableToWriteLayerStatusMetaKey").errorUnsuppressed(err); + co_await delay(5.0); + } + + // Write status periodically + while (true) { + tr->reset(); + Error err; + try { + while (true) { + Error innerErr; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + AsyncResult asyncResultStatusDoc = + getLayerStatus(tr, localIP, name, id, exe, taskDest, Snapshot::True); + co_await cleanupStatus(tr, rootKey, name, id); + std::string statusdoc = co_await std::move(asyncResultStatusDoc); + tr->set(instanceKey, statusdoc); + co_await tr->commit(); + break; + } catch (Error& e) { + innerErr = e; + } + TraceEvent(SevWarnAlways, "LayerBackupStatusUpdateError").errorUnsuppressed(innerErr); + co_await tr->onError(innerErr); + } + + co_await delay(CLIENT_KNOBS->BACKUP_STATUS_DELAY * + ((1.0 - CLIENT_KNOBS->BACKUP_STATUS_JITTER) + + 2 * deterministicRandom()->random01() * CLIENT_KNOBS->BACKUP_STATUS_JITTER)); + + // Now that status was written at least once by this process (and hopefully others), start the poll rate + // control updater if it wasn't started yet + if (!pollRateUpdater.isValid()) + pollRateUpdater = updateAgentPollRate(statusUpdateDest, rootKey, name, pollDelay); + continue; + } catch (Error& e) { + err = e; + } + TraceEvent(SevWarnAlways, "UnableToWriteBackupStatus").error(err); + co_await delay(10.0); + } +} + +Future runDBAgent(Database src, Database dest) { + std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); + std::string id = nondeterministicRandom()->randomUniqueID().toString(); + Future status = statusUpdateActor(src, "dr_backup", ProgramExe::DR_AGENT, pollDelay, dest, id); + Future status_other = statusUpdateActor(dest, "dr_backup_dest", ProgramExe::DR_AGENT, pollDelay, dest, id); + + DatabaseBackupAgent backupAgent(src); + + while (true) { + Error err; + try { + co_await backupAgent.run(dest, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT); + break; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_operation_cancelled) + throw err; + + TraceEvent(SevError, "DA_runAgent").error(err); + fprintf(stderr, "ERROR: DR agent encountered fatal error `%s'\n", err.what()); + + co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY); + } +} + +Future runAgent(Database db) { + std::shared_ptr pollDelay = std::make_shared(1.0 / CLIENT_KNOBS->BACKUP_AGGREGATE_POLL_RATE); + Future status = statusUpdateActor(db, "backup", ProgramExe::AGENT, pollDelay); + + FileBackupAgent backupAgent; + + while (true) { + Error err; + try { + co_await backupAgent.run(db, pollDelay, CLIENT_KNOBS->BACKUP_TASKS_PER_AGENT); + break; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_operation_cancelled) + throw err; + + TraceEvent(SevError, "BA_runAgent").error(err); + fprintf(stderr, "ERROR: backup agent encountered fatal error `%s'\n", err.what()); + + co_await delay(FLOW_KNOBS->PREVENT_FAST_SPIN_DELAY); + } +} + +Future submitDBBackup(Database src, + Database dest, + Standalone> backupRanges, + std::string tagName) { + try { + DatabaseBackupAgent backupAgent(src); + ASSERT(!backupRanges.empty()); + + co_await backupAgent.submitBackup( + dest, KeyRef(tagName), backupRanges, StopWhenDone::False, StringRef(), StringRef(), LockDB::True); + + // Check if a backup agent is running + bool agentRunning = co_await backupAgent.checkActive(dest); + + if (!agentRunning) { + printf("The DR on tag `%s' was successfully submitted but no DR agents are responding.\n", + printable(StringRef(tagName)).c_str()); + + // Throw an error that will not display any additional information + throw actor_cancelled(); + } else { + printf("The DR on tag `%s' was successfully submitted.\n", printable(StringRef(tagName)).c_str()); + } + } + + catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); + break; + case error_code_backup_duplicate: + fprintf(stderr, "ERROR: A DR is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + break; + default: + fprintf(stderr, "ERROR: %s\n", e.what()); + break; + } + + throw backup_error(); + } +} + +Future submitBackup(Database db, + std::string url, + Optional proxy, + int initialSnapshotIntervalSeconds, + int snapshotIntervalSeconds, + Standalone> backupRanges, + std::string tagName, + bool dryRun, + WaitForComplete waitForCompletion, + StopWhenDone stopWhenDone, + MutationLogType mutationLogType, + IncrementalBackupOnly incrementalBackupOnly, + Optional encryptionKeyFile, + int encryptionBlockSize, + SnapshotMode snapshotMode = SnapshotMode::RANGEFILE) { + try { + FileBackupAgent backupAgent; + ASSERT(!backupRanges.empty()); + + if (dryRun) { + KeyBackedTag tag = makeBackupTag(tagName); + Optional uidFlag = co_await tag.get(db.getReference()); + + if (uidFlag.present()) { + BackupConfig config(uidFlag.get().first); + EBackupState backupStatus = co_await config.stateEnum().getOrThrow(db.getReference()); + + // Throw error if a backup is currently running until we support parallel backups + if (BackupAgentBase::isRunnable(backupStatus)) { + throw backup_duplicate(); + } + } + + if (waitForCompletion) { + printf("Submitted and now waiting for the backup on tag `%s' to complete. (DRY RUN)\n", + printable(StringRef(tagName)).c_str()); + } + + else { + // Check if a backup agent is running + bool agentRunning = co_await backupAgent.checkActive(db); + + if (!agentRunning) { + printf("The backup on tag `%s' was successfully submitted but no backup agents are responding. " + "(DRY RUN)\n", + printable(StringRef(tagName)).c_str()); + + // Throw an error that will not display any additional information + throw actor_cancelled(); + } else { + printf("The backup on tag `%s' was successfully submitted. (DRY RUN)\n", + printable(StringRef(tagName)).c_str()); + } + } + } + + else { + co_await backupAgent.submitBackup(db, + KeyRef(url), + proxy, + initialSnapshotIntervalSeconds, + snapshotIntervalSeconds, + tagName, + backupRanges, + stopWhenDone, + mutationLogType, + incrementalBackupOnly, + encryptionKeyFile, + encryptionBlockSize, + static_cast(snapshotMode)); + + // Wait for the backup to complete, if requested + if (waitForCompletion) { + printf("Submitted and now waiting for the backup on tag `%s' to complete.\n", + printable(StringRef(tagName)).c_str()); + co_await backupAgent.waitBackup(db, tagName); + } else { + // Check if a backup agent is running + bool agentRunning = co_await backupAgent.checkActive(db); + + if (!agentRunning) { + printf("The backup on tag `%s' was successfully submitted but no backup agents are responding.\n", + printable(StringRef(tagName)).c_str()); + + // Throw an error that will not display any additional information + throw actor_cancelled(); + } else { + printf("The backup on tag `%s' was successfully submitted.\n", + printable(StringRef(tagName)).c_str()); + } + } + } + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); + break; + case error_code_backup_duplicate: + fprintf(stderr, "ERROR: A backup is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + break; + default: + fprintf(stderr, "ERROR: %s\n", e.what()); + break; + } + + throw backup_error(); + } +} + +Future switchDBBackup(Database src, + Database dest, + Standalone> backupRanges, + std::string tagName, + ForceAction forceAction) { + try { + DatabaseBackupAgent backupAgent(src); + ASSERT(!backupRanges.empty()); + + co_await backupAgent.atomicSwitchover( + dest, KeyRef(tagName), backupRanges, StringRef(), StringRef(), forceAction); + printf("The DR on tag `%s' was successfully switched.\n", printable(StringRef(tagName)).c_str()); + } + + catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); + break; + case error_code_backup_duplicate: + fprintf(stderr, "ERROR: A DR is already running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + break; + default: + fprintf(stderr, "ERROR: %s\n", e.what()); + break; + } + + throw backup_error(); + } +} + +Future statusDBBackup(Database src, Database dest, std::string tagName, int errorLimit) { + try { + DatabaseBackupAgent backupAgent(src); + + std::string statusText = co_await backupAgent.getStatus(dest, errorLimit, StringRef(tagName)); + printf("%s\n", statusText.c_str()); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future statusBackup(Database db, std::string tagName, ShowErrors showErrors, bool json) { + try { + FileBackupAgent backupAgent; + + std::string statusText = + co_await (json ? backupAgent.getStatusJSON(db, tagName) : backupAgent.getStatus(db, showErrors, tagName)); + printf("%s\n", statusText.c_str()); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future abortDBBackup(Database src, Database dest, std::string tagName, PartialBackup partial, DstOnly dstOnly) { + try { + DatabaseBackupAgent backupAgent(src); + + co_await backupAgent.abortBackup(dest, Key(tagName), partial, AbortOldBackup::False, dstOnly); + co_await backupAgent.unlockBackup(dest, Key(tagName)); + + printf("The DR on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); + break; + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A DR was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + break; + default: + fprintf(stderr, "ERROR: %s\n", e.what()); + break; + } + throw; + } +} + +Future abortBackup(Database db, std::string tagName) { + try { + FileBackupAgent backupAgent; + + co_await backupAgent.abortBackup(db, tagName); + + printf("The backup on tag `%s' was successfully aborted.\n", printable(StringRef(tagName)).c_str()); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An error was encountered during submission\n"); + break; + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A backup was not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + break; + default: + fprintf(stderr, "ERROR: %s\n", e.what()); + break; + } + throw; + } +} + +Future cleanupMutations(Database db, DeleteData deleteData) { + try { + co_await cleanupBackup(db, deleteData); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future waitBackup(Database db, std::string tagName, StopWhenDone stopWhenDone) { + try { + FileBackupAgent backupAgent; + + EBackupState status = co_await backupAgent.waitBackup(db, tagName, stopWhenDone); + + printf("The backup on tag `%s' %s.\n", + printable(StringRef(tagName)).c_str(), + BackupAgentBase::getStateText(status)); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future discontinueBackup(Database db, std::string tagName, WaitForComplete waitForCompletion) { + try { + FileBackupAgent backupAgent; + + co_await backupAgent.discontinueBackup(db, StringRef(tagName)); + + // Wait for the backup to complete, if requested + if (waitForCompletion) { + printf("Discontinued and now waiting for the backup on tag `%s' to complete.\n", + printable(StringRef(tagName)).c_str()); + co_await backupAgent.waitBackup(db, tagName); + } else { + printf("The backup on tag `%s' was successfully discontinued.\n", printable(StringRef(tagName)).c_str()); + } + + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + switch (e.code()) { + case error_code_backup_error: + fprintf(stderr, "ERROR: An encounter was error during submission\n"); + break; + case error_code_backup_unneeded: + fprintf(stderr, "ERROR: A backup in not running on tag `%s'\n", printable(StringRef(tagName)).c_str()); + break; + case error_code_backup_duplicate: + fprintf(stderr, + "ERROR: The backup on tag `%s' is already discontinued\n", + printable(StringRef(tagName)).c_str()); + break; + default: + fprintf(stderr, "ERROR: %s\n", e.what()); + break; + } + throw; + } +} + +Future changeBackupResumed(Database db, bool pause) { + try { + FileBackupAgent backupAgent; + co_await backupAgent.changePause(db, pause); + printf("All backup agents have been %s.\n", pause ? "paused" : "resumed"); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future changeDBBackupResumed(Database src, Database dest, bool pause) { + try { + DatabaseBackupAgent backupAgent(src); + co_await backupAgent.taskBucket->changePause(dest, pause); + printf("All DR agents have been %s.\n", pause ? "paused" : "resumed"); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Reference openBackupContainer(const char* name, + const std::string& destinationContainer, + const Optional& proxy, + const Optional& encryptionKeyFile, + int encryptionBlockSize) { + // Error, if no dest container was specified + if (destinationContainer.empty()) { + fprintf(stderr, "ERROR: No backup destination was specified.\n"); + printHelpTeaser(name); + throw backup_error(); + } + + if (destinationContainer.find("../") != std::string::npos) { + fprintf( + stderr, "ERROR: Backup Container URL '%s' contains directory traversals\n", destinationContainer.c_str()); + printHelpTeaser(name); + throw backup_invalid_url(); + } + + Reference c; + try { + c = IBackupContainer::openContainer(destinationContainer, proxy, encryptionKeyFile, encryptionBlockSize); + } catch (Error& e) { + std::string msg = format("ERROR: '%s' on URL '%s'", e.what(), destinationContainer.c_str()); + if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { + msg += format(": %s", IBackupContainer::lastOpenError.c_str()); + } + fprintf(stderr, "%s\n", msg.c_str()); + printHelpTeaser(name); + throw; + } + + return c; +} + +// Submit the restore request to the database if "performRestore" is true. Otherwise, +// check if the restore can be performed. +Future runRestore(Database db, + std::string originalClusterFile, + std::string tagName, + std::string container, + Optional proxy, + Standalone> ranges, + Version beginVersion, + Version targetVersion, + std::string targetTimestamp, + bool performRestore, + Verbose verbose, + WaitForComplete waitForDone, + std::string addPrefix, + std::string removePrefix, + OnlyApplyMutationLogs onlyApplyMutationLogs, + InconsistentSnapshotOnly inconsistentSnapshotOnly, + Optional encryptionKeyFile, + RestoreMode restoreMode = RestoreMode::RANGEFILE) { + ASSERT(!ranges.empty()); + + if (targetVersion != invalidVersion && !targetTimestamp.empty()) { + fprintf(stderr, "Restore target version and target timestamp cannot both be specified\n"); + throw restore_error(); + } + + Optional origDb; + + // Resolve targetTimestamp if given + if (!targetTimestamp.empty()) { + if (originalClusterFile.empty()) { + fprintf(stderr, + "An original cluster file must be given in order to resolve restore target timestamp '%s'\n", + targetTimestamp.c_str()); + throw restore_error(); + } + + if (!fileExists(originalClusterFile)) { + fprintf( + stderr, "Original source database cluster file '%s' does not exist.\n", originalClusterFile.c_str()); + throw restore_error(); + } + + origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION); + Version v = co_await timeKeeperVersionFromDatetime(targetTimestamp, origDb.get()); + fmt::print("Timestamp '{0}' resolves to version {1}\n", targetTimestamp, v); + targetVersion = v; + } + + try { + FileBackupAgent backupAgent; + + // encryptionBlockSize is passed 0 because we don't know about the block size yet and it will be read in the + // describeBackup call after this. + Reference bc = openBackupContainer( + exeRestore.toString().c_str(), container, proxy, encryptionKeyFile, /*encryptionBlockSize=*/0); + // If targetVersion is unset then use the maximum restorable version from the backup description + if (targetVersion == invalidVersion) { + if (verbose) + printf( + "No restore target version given, will use maximum restorable version from backup description.\n"); + + // For blobstore:// URLs, use invalidVersion to allow describeBackup to write missing version properties + BackupDescription desc = co_await bc->describeBackup(false, isBlobstoreUrl(container) ? invalidVersion : 0); + + if (onlyApplyMutationLogs && desc.contiguousLogEnd.present()) { + targetVersion = desc.contiguousLogEnd.get() - 1; + } else if (desc.maxRestorableVersion.present()) { + targetVersion = desc.maxRestorableVersion.get(); + } else { + fprintf(stderr, "The specified backup is not restorable to any version.\n"); + throw restore_error(); + } + + if (verbose) { + fmt::print("Using target restore version {}\n", targetVersion); + } + } + + if (performRestore) { + Version restoredVersion = co_await backupAgent.restore(db, + origDb, + KeyRef(tagName), + KeyRef(container), + proxy, + ranges, + waitForDone, + targetVersion, + verbose, + KeyRef(addPrefix), + KeyRef(removePrefix), + LockDB::True, + UnlockDB::True, + onlyApplyMutationLogs, + inconsistentSnapshotOnly, + beginVersion, + encryptionKeyFile, + {}, + restoreMode == RestoreMode::RANGEFILE); + + if (waitForDone && verbose) { + // If restore is now complete then report version restored + fmt::print("Restored to version {}\n", restoredVersion); + } + } else { + Optional rset = co_await bc->getRestoreSet(targetVersion, ranges); + + if (!rset.present()) { + fmt::print(stderr, + "Insufficient data to restore to version {}. Describe backup for more information.\n", + targetVersion); + throw restore_invalid_version(); + } + + fmt::print("Backup can be used to restore to version {}\n", targetVersion); + } + + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future dumpBackupData(const char* name, + std::string destinationContainer, + Optional proxy, + Version beginVersion, + Version endVersion) { + Reference c = openBackupContainer(name, destinationContainer, proxy, {}, 0); + + if (beginVersion < 0 || endVersion < 0) { + BackupDescription desc = co_await c->describeBackup(); + + if (!desc.maxLogEnd.present()) { + fprintf(stderr, "ERROR: Backup must have log data in order to use relative begin/end versions.\n"); + throw backup_invalid_info(); + } + + if (beginVersion < 0) { + beginVersion += desc.maxLogEnd.get(); + } + + if (endVersion < 0) { + endVersion += desc.maxLogEnd.get(); + } + } + + fmt::print("Scanning version range {0} to {1}\n", beginVersion, endVersion); + BackupFileList files = co_await c->dumpFileList(beginVersion, endVersion); + files.toStream(stdout); +} + +Future expireBackupData(const char* name, + std::string destinationContainer, + Optional proxy, + Version endVersion, + std::string endDatetime, + Database db, + bool force, + Version restorableAfterVersion, + std::string restorableAfterDatetime) { + if (!endDatetime.empty()) { + Version v = co_await timeKeeperVersionFromDatetime(endDatetime, db); + endVersion = v; + } + + if (!restorableAfterDatetime.empty()) { + Version v = co_await timeKeeperVersionFromDatetime(restorableAfterDatetime, db); + restorableAfterVersion = v; + } + + if (endVersion == invalidVersion) { + fprintf(stderr, "ERROR: No version or date/time is specified.\n"); + printHelpTeaser(name); + throw backup_error(); + } + + try { + Reference c = openBackupContainer(name, destinationContainer, proxy, {}, 0); + + IBackupContainer::ExpireProgress progress; + std::string lastProgress; + Future expire = c->expireData(endVersion, force, &progress, restorableAfterVersion); + + while (true) { + if (auto const res = co_await timeout(expire, 5); res.present()) { + break; + } + + std::string p = progress.toString(); + if (p != lastProgress) { + int spaces = lastProgress.size() - p.size(); + printf("\r%s%s", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); + lastProgress = p; + } + } + + std::string p = progress.toString(); + int spaces = lastProgress.size() - p.size(); + printf("\r%s%s\n", p.c_str(), (spaces > 0 ? std::string(spaces, ' ').c_str() : "")); + + if (endVersion < 0) + fmt::print("All data before {0} versions ({1}" + " days) prior to latest backup log has been deleted.\n", + -endVersion, + -endVersion / ((int64_t)24 * 3600 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); + else + fmt::print("All data before version {} has been deleted.\n", endVersion); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + if (e.code() == error_code_backup_cannot_expire) + fprintf(stderr, + "ERROR: Requested expiration would be unsafe. Backup would not meet minimum restorability. Use " + "--force to delete data anyway.\n"); + else + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future deleteBackupContainer(const char* name, std::string destinationContainer, Optional proxy) { + try { + Reference c = openBackupContainer(name, destinationContainer, proxy, {}, 0); + int numDeleted = 0; + Future done = c->deleteContainer(&numDeleted); + + int lastUpdate = -1; + printf("Deleting %s...\n", destinationContainer.c_str()); + + while (true) { + if (auto const res = co_await timeout(done, 5); res.present()) { + break; + } + + if (numDeleted != lastUpdate) { + printf("\r%d...", numDeleted); + lastUpdate = numDeleted; + } + } + printf("\r%d objects deleted\n", numDeleted); + printf("The entire container has been deleted.\n"); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +Future describeBackup(const char* name, + std::string destinationContainer, + Optional proxy, + bool deep, + Optional cx, + bool json) { + try { + Reference c = openBackupContainer(name, destinationContainer, proxy, {}, 0); + BackupDescription desc = co_await c->describeBackup(deep); + if (cx.present()) + co_await desc.resolveVersionTimes(cx.get()); + printf("%s\n", (json ? desc.toJSON() : desc.toString()).c_str()); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, "ERROR: %s\n", e.what()); + throw; + } +} + +static void reportBackupQueryError(UID operationId, JsonBuilderObject& result, std::string errorMessage) { + result["error"] = errorMessage; + printf("%s\n", result.getJson().c_str()); + TraceEvent("BackupQueryFailure").detail("OperationId", operationId).detail("Reason", errorMessage); +} + +std::pair getMaxMinRestorableVersions(const BackupDescription& desc, bool mayOnlyApplyMutationLog) { + Version maxRestorableVersion = invalidVersion; + Version minRestorableVersion = invalidVersion; + if (desc.maxRestorableVersion.present()) { + maxRestorableVersion = desc.maxRestorableVersion.get(); + } else if (mayOnlyApplyMutationLog && desc.contiguousLogEnd.present()) { + maxRestorableVersion = desc.contiguousLogEnd.get(); + } + + if (desc.minRestorableVersion.present()) { + minRestorableVersion = desc.minRestorableVersion.get(); + } else if (mayOnlyApplyMutationLog && desc.minLogBegin.present()) { + minRestorableVersion = desc.minLogBegin.get(); + } + + return std::make_pair(maxRestorableVersion, minRestorableVersion); +} + +// If restoreVersion is invalidVersion or latestVersion, use the maximum or minimum restorable version respectively for +// selected key ranges. If restoreTimestamp is specified, any specified restoreVersion will be overridden to the version +// resolved to that timestamp. +Future queryBackup(const char* name, + std::string destinationContainer, + Optional proxy, + Standalone> keyRangesFilter, + Version restoreVersion, + Version snapshotVersion, + std::string originalClusterFile, + std::string restoreTimestamp, + Verbose verbose, + Optional cx) { + UID operationId = deterministicRandom()->randomUniqueID(); + JsonBuilderObject result; + std::string errorMessage; + result["key_ranges_filter"] = printable(keyRangesFilter); + result["destination_container"] = destinationContainer; + + TraceEvent("BackupQueryStart") + .detail("OperationId", operationId) + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("SpecifiedRestoreVersion", restoreVersion) + .detail("SpecifiedSnapshotVersion", snapshotVersion) + .detail("RestoreTimestamp", restoreTimestamp) + .detail("BackupClusterFile", originalClusterFile); + + // Resolve restoreTimestamp if given + if (!restoreTimestamp.empty()) { + if (originalClusterFile.empty()) { + reportBackupQueryError( + operationId, + result, + format("an original cluster file must be given in order to resolve restore target timestamp '%s'", + restoreTimestamp.c_str())); + co_return; + } + + if (!fileExists(originalClusterFile)) { + reportBackupQueryError(operationId, + result, + format("The specified original source database cluster file '%s' does not exist\n", + originalClusterFile.c_str())); + co_return; + } + + Database origDb = Database::createDatabase(originalClusterFile, ApiVersion::LATEST_VERSION); + Version v = co_await timeKeeperVersionFromDatetime(restoreTimestamp, origDb); + result["restore_timestamp"] = restoreTimestamp; + result["restore_timestamp_resolved_version"] = v; + restoreVersion = v; + } + + int64_t totalRangeFilesSize = 0; + int64_t totalLogFilesSize = 0; + JsonBuilderArray rangeFilesJson; + JsonBuilderArray logFilesJson; + try { + Reference bc = openBackupContainer(name, destinationContainer, proxy, {}, 0); + BackupDescription desc = co_await bc->describeBackup(); + // Use continuous log end version for the maximum restorable version for the key ranges when a restorable + // version doesn't exist. + auto [maxRestorableVersion, minRestorableVersion] = getMaxMinRestorableVersions(desc, !keyRangesFilter.empty()); + if (restoreVersion == invalidVersion) { + restoreVersion = maxRestorableVersion; + } + if (restoreVersion == earliestVersion) { + restoreVersion = minRestorableVersion; + } + if (snapshotVersion == earliestVersion) { + snapshotVersion = minRestorableVersion; + } + TraceEvent("BackupQueryResolveRestoreVersion") + .detail("RestoreVersion", restoreVersion) + .detail("SnapshotVersion", snapshotVersion); + if (restoreVersion < 0) { + reportBackupQueryError(operationId, + result, + errorMessage = + format("the specified restorable version %lld is not valid", restoreVersion)); + co_return; + } + + Optional fileSet; + if (snapshotVersion != invalidVersion) { + // When a snapshot version is specified, we will first get a restore set using the latest snapshot file to + // restore to the snapshot version. After snapshot version, we will only use mutation logs to restore. + fileSet = co_await bc->getRestoreSet(snapshotVersion, keyRangesFilter); + if (fileSet.present()) { + result["snapshot_version"] = fileSet.get().targetVersion; + for (const auto& rangeFile : fileSet.get().ranges) { + JsonBuilderObject object; + object["file_name"] = rangeFile.fileName; + object["file_size"] = rangeFile.fileSize; + object["version"] = rangeFile.version; + object["key_range"] = !fileSet.get().keyRanges.contains(rangeFile.fileName) + ? "none" + : fileSet.get().keyRanges.at(rangeFile.fileName).toString(); + rangeFilesJson.push_back(object); + totalRangeFilesSize += rangeFile.fileSize; + } + for (const auto& log : fileSet.get().logs) { + JsonBuilderObject object; + object["file_name"] = log.fileName; + object["file_size"] = log.fileSize; + object["begin_version"] = log.beginVersion; + object["end_version"] = log.endVersion; + logFilesJson.push_back(object); + totalLogFilesSize += log.fileSize; + } + + snapshotVersion = fileSet.get().targetVersion; + + TraceEvent("BackupQueryReceivedRestorableFilesSetFromSnapshot") + .detail("SnapshotVersion", snapshotVersion) + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("ActualRestoreVersion", fileSet.get().targetVersion) + .detail("NumRangeFiles", fileSet.get().ranges.size()) + .detail("NumLogFiles", fileSet.get().logs.size()) + .detail("RangeFilesBytes", totalRangeFilesSize) + .detail("LogFilesBytes", totalLogFilesSize); + } else { + reportBackupQueryError( + operationId, + result, + format("no restorable files set found for specified key ranges from snapshotVersion %lld", + snapshotVersion)); + co_return; + } + + // We only need to know all the mutation logs from `snapshotVersion` to `restoreVersion`. + fileSet = co_await bc->getRestoreSet(restoreVersion, keyRangesFilter, /*logOnly=*/true, snapshotVersion); + } else { + // When a snapshot version is not specified, we use the latest snapshot to restore to the `restoreVersion`. + fileSet = co_await bc->getRestoreSet(restoreVersion, keyRangesFilter); + } + + if (fileSet.present()) { + result["restore_version"] = fileSet.get().targetVersion; + for (const auto& rangeFile : fileSet.get().ranges) { + JsonBuilderObject object; + object["file_name"] = rangeFile.fileName; + object["file_size"] = rangeFile.fileSize; + object["version"] = rangeFile.version; + object["key_range"] = !fileSet.get().keyRanges.contains(rangeFile.fileName) + ? "none" + : fileSet.get().keyRanges.at(rangeFile.fileName).toString(); + rangeFilesJson.push_back(object); + totalRangeFilesSize += rangeFile.fileSize; + } + for (const auto& log : fileSet.get().logs) { + JsonBuilderObject object; + object["file_name"] = log.fileName; + object["file_size"] = log.fileSize; + object["begin_version"] = log.beginVersion; + object["end_version"] = log.endVersion; + logFilesJson.push_back(object); + totalLogFilesSize += log.fileSize; + } + + TraceEvent("BackupQueryReceivedRestorableFilesSet") + .detail("DestinationContainer", destinationContainer) + .detail("KeyRangesFilter", printable(keyRangesFilter)) + .detail("ActualRestoreVersion", fileSet.get().targetVersion) + .detail("NumRangeFiles", fileSet.get().ranges.size()) + .detail("NumLogFiles", fileSet.get().logs.size()) + .detail("RangeFilesBytes", totalRangeFilesSize) + .detail("LogFilesBytes", totalLogFilesSize); + } else if (snapshotVersion == invalidVersion) { + reportBackupQueryError(operationId, result, "no restorable files set found for specified key ranges"); + co_return; + } + + } catch (Error& e) { + reportBackupQueryError(operationId, result, e.what()); + co_return; + } + + result["total_range_files_size"] = totalRangeFilesSize; + result["total_log_files_size"] = totalLogFilesSize; + + if (verbose) { + result["ranges"] = rangeFilesJson; + result["logs"] = logFilesJson; + } + + printf("%s\n", result.getJson().c_str()); +} + +Future listBackup(std::string baseUrl, Optional proxy) { + try { + std::vector containers = co_await IBackupContainer::listContainers(baseUrl, proxy); + for (const std::string& container : containers) { + printf("%s\n", container.c_str()); + } + } catch (Error& e) { + std::string msg = format("ERROR: %s", e.what()); + if (e.code() == error_code_backup_invalid_url && !IBackupContainer::lastOpenError.empty()) { + msg += format(": %s", IBackupContainer::lastOpenError.c_str()); + } + fprintf(stderr, "%s\n", msg.c_str()); + throw; + } +} + +Future listBackupTags(Database cx) { + auto tr = makeReference(cx); + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + std::vector tags = co_await getAllBackupTags(tr); + for (const auto& tag : tags) { + printf("%s\n", tag.tagName.c_str()); + } + co_return; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } +} + +struct BackupModifyOptions { + Optional verifyUID; + Optional destURL; + Optional proxy; + Optional snapshotIntervalSeconds; + Optional activeSnapshotIntervalSeconds; + Optional encryptionKeyFile; + bool hasChanges() const { + return destURL.present() || snapshotIntervalSeconds.present() || activeSnapshotIntervalSeconds.present(); + } +}; + +Future modifyBackup(Database db, std::string tagName, BackupModifyOptions options) { + if (!options.hasChanges()) { + fprintf(stderr, "No changes were specified, nothing to do!\n"); + throw backup_error(); + } + + KeyBackedTag tag = makeBackupTag(tagName); + + Reference tr(new ReadYourWritesTransaction(db)); + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + Optional uidFlag = co_await tag.get(db.getReference()); + + if (!uidFlag.present()) { + fprintf(stderr, "No backup exists on tag '%s'\n", tagName.c_str()); + throw backup_error(); + } + + if (uidFlag.get().second) { + fprintf(stderr, "Cannot modify aborted backup on tag '%s'\n", tagName.c_str()); + throw backup_error(); + } + + BackupConfig config(uidFlag.get().first); + EBackupState s = co_await config.stateEnum().getOrThrow(tr, Snapshot::False, backup_invalid_info()); + if (!FileBackupAgent::isRunnable(s)) { + fprintf(stderr, "Backup on tag '%s' is not runnable.\n", tagName.c_str()); + throw backup_error(); + } + + if (options.verifyUID.present() && options.verifyUID.get() != uidFlag.get().first.toString()) { + fprintf(stderr, + "UID verification failed, backup on tag '%s' is '%s' but '%s' was specified.\n", + tagName.c_str(), + uidFlag.get().first.toString().c_str(), + options.verifyUID.get().c_str()); + throw backup_error(); + } + + if (options.destURL.present()) { + Reference prevContainer = + co_await config.backupContainer().getOrThrow(tr, Snapshot::False, backup_invalid_info()); + std::string prevURL = prevContainer->getURL(); + std::string newURL = options.destURL.get(); + if (!prevURL.empty() && prevURL.back() == '/') { + prevURL.pop_back(); + } + if (!newURL.empty() && newURL.back() == '/') { + newURL.pop_back(); + } + + if (prevURL == newURL) { + if ((options.encryptionKeyFile.present() && !prevContainer->getEncryptionKeyFileName().present()) || + (!options.encryptionKeyFile.present() && prevContainer->getEncryptionKeyFileName().present()) || + (options.encryptionKeyFile.present() && prevContainer->getEncryptionKeyFileName().present() && + options.encryptionKeyFile.get() != prevContainer->getEncryptionKeyFileName().get())) { + fprintf(stderr, + "Destination URL matches the existing backup URL for tag '%s', " + "but the encryption key file does not match.\n", + tagName.c_str()); + throw backup_error(); + } + } + + Reference bc; + TraceEvent("ModifyBackupSetNewContainer") + .detail("TagName", tagName) + .detail("DestURL", options.destURL.get()) + .detail("EncryptionKeyFile", + options.encryptionKeyFile.present() ? options.encryptionKeyFile.get() : "None"); + bc = openBackupContainer(exeBackup.toString().c_str(), + options.destURL.get(), + options.proxy, + options.encryptionKeyFile, + prevContainer->getEncryptionBlockSize()); + try { + co_await timeoutError(bc->create(), 30); + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + fprintf(stderr, + "ERROR: Could not create backup container at '%s': %s\n", + options.destURL.get().c_str(), + e.what()); + throw backup_error(); + } + config.backupContainer().set(tr, bc); + co_await bc->writeEncryptionMetadata(bc->getEncryptionBlockSize()); + } else if (options.encryptionKeyFile.present()) { + fprintf(stdout, + " Encryption key file specified without a new destination URL." + " The encryption key will not be used.\n"); + } + + if (options.snapshotIntervalSeconds.present()) { + config.snapshotIntervalSeconds().set(tr, options.snapshotIntervalSeconds.get()); + } + + if (options.activeSnapshotIntervalSeconds.present()) { + Version begin = co_await config.snapshotBeginVersion().getOrThrow(tr, Snapshot::False, backup_error()); + config.snapshotTargetEndVersion().set(tr, + begin + ((int64_t)options.activeSnapshotIntervalSeconds.get() * + CLIENT_KNOBS->CORE_VERSIONSPERSECOND)); + } + + co_await tr->commit(); + break; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } +} + +// NOLINTBEGIN(bugprone-use-after-move): ignore clang-tidy false positives in parseLine's token buffer handling. +static std::vector> parseLine(std::string& line, bool& err, bool& partial) { + err = false; + partial = false; + + bool quoted = false; + std::vector buf; + std::vector> ret; + + size_t i = line.find_first_not_of(' '); + size_t offset = i; + + bool forcetoken = false; + + while (i <= line.length()) { + switch (line[i]) { + case ';': + if (!quoted) { + if (i > offset) + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); + ret.push_back(std::move(buf)); + offset = i = line.find_first_not_of(' ', i + 1); + } else { + i++; + } + break; + case '"': + quoted = !quoted; + line.erase(i, 1); + if (quoted) + forcetoken = true; + break; + case ' ': + if (!quoted) { + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); + offset = i = line.find_first_not_of(' ', i); + forcetoken = false; + } else { + i++; + } + break; + case '\\': + if (i + 2 > line.length()) { + err = true; + ret.push_back(std::move(buf)); + return ret; + } + switch (line[i + 1]) { + char ent, save; + case '"': + case '\\': + case ' ': + case ';': + line.erase(i, 1); + break; + case 'x': + if (i + 4 > line.length()) { + err = true; + ret.push_back(std::move(buf)); + return ret; + } + char* pEnd; + save = line[i + 4]; + line[i + 4] = 0; + ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); + if (*pEnd) { + err = true; + ret.push_back(std::move(buf)); + return ret; + } + line[i + 4] = save; + line.replace(i, 4, 1, ent); + break; + default: + err = true; + ret.push_back(std::move(buf)); + return ret; + } + default: + i++; + } + } + + i -= 1; + if (i > offset || forcetoken) + buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); + + ret.push_back(std::move(buf)); + + if (quoted) + partial = true; + + return ret; +} +// NOLINTEND(bugprone-use-after-move) + +static void addKeyRange(std::string optionValue, Standalone>& keyRanges) { + bool err = false, partial = false; + [[maybe_unused]] int tokenArray = 0; + + auto parsed = parseLine(optionValue, err, partial); + + for (auto tokens : parsed) { + tokenArray++; + + /* + int tokenIndex = 0; + for (auto token : tokens) + { + tokenIndex++; + + printf("%4d token #%2d: %s\n", tokenArray, tokenIndex, printable(token).c_str()); + } + */ + + // Process the keys + // [end] + switch (tokens.size()) { + // empty + case 0: + break; + + // single key range + case 1: + keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), strinc(tokens.at(0)))); + break; + + // full key range + case 2: + try { + keyRanges.push_back_deep(keyRanges.arena(), KeyRangeRef(tokens.at(0), tokens.at(1))); + } catch (Error& e) { + fprintf(stderr, + "ERROR: Invalid key range `%s %s' reported error %s\n", + tokens.at(0).toString().c_str(), + tokens.at(1).toString().c_str(), + e.what()); + throw invalid_option_value(); + } + break; + + // Too many keys + default: + fmt::print(stderr, "ERROR: Invalid key range identified with {} keys", tokens.size()); + throw invalid_option_value(); + break; + } + } + + return; +} + +Version parseVersion(const char* str) { + StringRef s((const uint8_t*)str, strlen(str)); + + if (s.endsWith("days"_sr) || s.endsWith("d"_sr)) { + float days; + if (sscanf(str, "%f", &days) != 1) { + fprintf(stderr, "Could not parse version: %s\n", str); + flushAndExit(FDB_EXIT_ERROR); + } + return (double)CLIENT_KNOBS->CORE_VERSIONSPERSECOND * 24 * 3600 * -days; + } + + Version ver; + if (sscanf(str, "%" SCNd64, &ver) != 1) { + fprintf(stderr, "Could not parse version: %s\n", str); + flushAndExit(FDB_EXIT_ERROR); + } + return ver; +} + +// Creates a connection to a cluster. Optionally prints an error if the connection fails. +Optional connectToCluster(std::string const& clusterFile, + LocalityData const& localities, + bool quiet = false) { + auto resolvedClusterFile = ClusterConnectionFile::lookupClusterFileName(clusterFile); + Reference ccf; + + Optional db; + + try { + ccf = makeReference(resolvedClusterFile.first); + } catch (Error& e) { + if (!quiet) + fprintf(stderr, "%s\n", ClusterConnectionFile::getErrorString(resolvedClusterFile, e).c_str()); + return db; + } + + try { + db = Database::createDatabase(ccf, ApiVersion::LATEST_VERSION, IsInternal::True, localities); + } catch (Error& e) { + if (!quiet) { + fprintf(stderr, "ERROR: %s\n", e.what()); + fprintf(stderr, "ERROR: Unable to connect to cluster from `%s'\n", ccf->getLocation().c_str()); + } + return db; + } + + return db; +}; + +static constexpr CSimpleOpt::SOption* const allOptionArrays[] = { g_rgOptions, + g_rgAgentOptions, + g_rgBackupStartOptions, + g_rgBackupModifyOptions, + g_rgBackupStatusOptions, + g_rgBackupAbortOptions, + g_rgBackupCleanupOptions, + g_rgBackupDiscontinueOptions, + g_rgBackupWaitOptions, + g_rgBackupPauseOptions, + g_rgBackupExpireOptions, + g_rgBackupDeleteOptions, + g_rgBackupDescribeOptions, + g_rgBackupDumpOptions, + g_rgBackupTagsOptions, + g_rgBackupListOptions, + g_rgBackupQueryOptions, + g_rgRestoreOptions, + g_rgDBAgentOptions, + g_rgDBStartOptions, + g_rgDBStatusOptions, + g_rgDBSwitchOptions, + g_rgDBAbortOptions, + g_rgDBPauseOptions }; + +// The last element in SOption arrays is always END_MARKER = SO_END_OF_OPTIONS. +constexpr CSimpleOpt::SOption END_MARKER = SO_END_OF_OPTIONS; + +/** + * Validates and processes a command-line option. + * + * This function checks if the current argument (argv[i]) matches any known option. + * If the option requires a parameter, it consumes the next argument as the parameter. + * The processed option (and its parameter, if any) are added to the 'options' vector. + * + * Parameters: + * argc - Total number of command-line arguments. + * argv - Array of command-line argument strings. + * i - Current index in argv; incremented if a parameter is consumed. + * options - Vector to which valid options (and their parameters, if any) are appended. + * + * Returns: + * true if the option is recognized and valid (and its parameter, if required, is present); + * false otherwise. + * + * This function is used to reorder and validate command-line arguments. + */ +static bool processOption(int argc, char* argv[], int& i, std::vector& options) { + std::string_view option = argv[i]; + + options.emplace_back(argv[i]); + size_t equalPos = option.find('='); + + if (equalPos != std::string_view::npos) { + option = option.substr(0, equalPos); + } + + for (auto* opt : allOptionArrays) { + for (int j = 0; opt[j].nId != END_MARKER.nId; ++j) { + const char* knownOpt = opt[j].pszArg; + size_t knownOptLen = strlen(knownOpt); + bool isPrefixOpt = knownOptLen > 1 && knownOpt[knownOptLen - 1] == '-'; + + // Create normalized versions for hyphen-underscore equivalence + std::string optNorm(option); + std::replace(optNorm.begin(), optNorm.end(), '-', '_'); + std::string knownNorm(knownOpt, isPrefixOpt ? knownOptLen - 1 : knownOptLen); + std::replace(knownNorm.begin(), knownNorm.end(), '-', '_'); + + if (optNorm == knownNorm || (isPrefixOpt && optNorm.size() >= knownNorm.size() && + optNorm.compare(0, knownNorm.size(), knownNorm) == 0)) { + if (opt[j].nArgType == SO_REQ_SEP && equalPos == std::string_view::npos) { + ++i; + if (i >= argc) { + fmt::print(stderr, "ERROR: Option {} requires a parameter\n", option); + return false; + } + options.emplace_back(argv[i]); + } + return true; + } + } + } + fmt::print(stderr, "ERROR: unknown option '{}'\n", option); + return false; +} + +/** + * Reorders command-line arguments so that all non-option parameters (subcommands) are placed before options. + * Validates all options using processOption. Returns the reordered arguments via the output parameters + * newArgC (argument count) and newArgV (argument vector). + * + * Note: The returned newArgV pointer points to static storage (argvStorage) + */ +static bool reorderArguments(int argc, char* argv[], int& newArgC, char**& newArgV) { + static std::vector argvStorage; + std::vector parameters; + std::vector options; + + parameters.push_back(argv[0]); // program name + auto isOptions = [](const char* arg) -> bool { return arg && *arg == '-'; }; + + for (int i = 1; i < argc; ++i) { + char* arg = argv[i]; + + if (isOptions(arg)) { + if (!processOption(argc, argv, i, options)) + return false; + } else { + parameters.emplace_back(arg); + } + } + + argvStorage.reserve(parameters.size() + options.size() + 1); // +1 for null terminator + argvStorage = parameters; + argvStorage.insert(argvStorage.end(), options.begin(), options.end()); + + newArgC = static_cast(argvStorage.size()); + + argvStorage.push_back(nullptr); // Null-terminate the argv array + newArgV = argvStorage.data(); + + return true; +} + +#ifndef EXCLUDE_MAIN_FUNCTION + +int main(int argc, char* argv[]) { + platformInit(); + + int status = FDB_EXIT_SUCCESS; + + std::string commandLine; + for (int a = 0; a < argc; a++) { + if (a) + commandLine += ' '; + commandLine += argv[a]; + } + + try { + registerCrashHandler(); + + // Set default of line buffering standard out and error + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + + ProgramExe programExe = getProgramType(argv[0]); + BackupType backupType = BackupType::UNDEFINED; + RestoreType restoreType = RestoreType::UNKNOWN; + DBType dbType = DBType::UNDEFINED; + + std::unique_ptr args; + + char** newArgV{}; + int newArgC{}; + + if (!reorderArguments(argc, argv, newArgC, newArgV)) { + return FDB_EXIT_ERROR; + } + + switch (programExe) { + case ProgramExe::AGENT: + args = std::make_unique( + newArgC, newArgV, g_rgAgentOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case ProgramExe::DR_AGENT: + args = std::make_unique( + newArgC, newArgV, g_rgDBAgentOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case ProgramExe::BACKUP: + // Display backup help, if no arguments + if (newArgC < 2) { + printBackupUsage(false); + return FDB_EXIT_ERROR; + } else { + // Get the backup type + backupType = getBackupType(newArgV[1]); + + // Create the appropriate simple opt + switch (backupType) { + case BackupType::START: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupStartOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::STATUS: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupStatusOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::ABORT: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupAbortOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::CLEANUP: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupCleanupOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::WAIT: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupWaitOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::DISCONTINUE: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupDiscontinueOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::PAUSE: + case BackupType::RESUME: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::EXPIRE: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupExpireOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::DELETE_BACKUP: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupDeleteOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::DESCRIBE: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupDescribeOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::DUMP: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupDumpOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::LIST: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupListOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::QUERY: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupQueryOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::MODIFY: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupModifyOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::TAGS: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgBackupTagsOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case BackupType::UNDEFINED: + default: + args = std::make_unique( + newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + } + } + break; + case ProgramExe::DB_BACKUP: + // Display backup help, if no arguments + if (newArgC < 2) { + printDBBackupUsage(false); + return FDB_EXIT_ERROR; + } else { + // Get the backup type + dbType = getDBType(newArgV[1]); + + // Create the appropriate simple opt + switch (dbType) { + case DBType::START: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgDBStartOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case DBType::STATUS: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgDBStatusOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case DBType::SWITCH: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgDBSwitchOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case DBType::ABORT: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgDBAbortOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case DBType::PAUSE: + case DBType::RESUME: + args = std::make_unique( + newArgC - 1, &newArgV[1], g_rgDBPauseOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + case DBType::UNDEFINED: + default: + args = std::make_unique( + newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + break; + } + } + break; + case ProgramExe::RESTORE: + if (newArgC < 2) { + printRestoreUsage(false); + return FDB_EXIT_ERROR; + } + // Get the restore operation type + restoreType = getRestoreType(newArgV[1]); + if (restoreType == RestoreType::UNKNOWN) { + args = + std::make_unique(newArgC, newArgV, g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + } else { + args = std::make_unique( + newArgC - 1, newArgV + 1, g_rgRestoreOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + } + break; + case ProgramExe::UNDEFINED: + default: + fprintf(stderr, "FoundationDB " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n"); + fprintf(stderr, "ERROR: Unable to determine program type based on executable `%s'\n", newArgV[0]); + return FDB_EXIT_ERROR; + break; + } + + Optional proxy; + std::string p; + if (platform::getEnvironmentVar("HTTP_PROXY", p) || platform::getEnvironmentVar("HTTPS_PROXY", p)) { + proxy = p; + } + std::string destinationContainer; + bool describeDeep = false; + bool describeTimestamps = false; + int initialSnapshotIntervalSeconds = + 0; // The initial snapshot has a desired duration of 0, meaning go as fast as possible. + int snapshotIntervalSeconds = CLIENT_KNOBS->BACKUP_DEFAULT_SNAPSHOT_INTERVAL_SEC; + std::string clusterFile; + std::string sourceClusterFile; + std::string baseUrl; + std::string expireDatetime; + Version expireVersion = invalidVersion; + std::string expireRestorableAfterDatetime; + Version expireRestorableAfterVersion = std::numeric_limits::max(); + std::vector> knobs; + std::string tagName = BackupAgentBase::getDefaultTag().toString(); + bool tagProvided = false; + std::string restoreContainer; + std::string addPrefix; + std::string removePrefix; + Standalone> backupKeys; + Standalone> backupKeysFilter; + int maxErrors = 20; + Version beginVersion = invalidVersion; + Version restoreVersion = invalidVersion; + Version snapshotVersion = invalidVersion; + std::string restoreTimestamp; + WaitForComplete waitForDone{ false }; + StopWhenDone stopWhenDone{ true }; + MutationLogType mutationLogType{ MutationLogType::DEFAULT }; + IncrementalBackupOnly incrementalBackupOnly{ false }; + OnlyApplyMutationLogs onlyApplyMutationLogs{ false }; + InconsistentSnapshotOnly inconsistentSnapshotOnly{ false }; + ForceAction forceAction{ false }; + bool trace = false; + bool quietDisplay = false; + bool dryRun = false; + bool restoreSystemKeys = false; + bool restoreUserKeys = false; + RestoreMode restoreMode = RestoreMode::RANGEFILE; // Default to traditional range file restore + std::string traceDir = ""; + std::string traceFormat = ""; + std::string traceLogGroup; + uint64_t traceRollSize = TRACE_DEFAULT_ROLL_SIZE; + uint64_t traceMaxLogsSize = TRACE_DEFAULT_MAX_LOGS_SIZE; + ESOError lastError; + PartialBackup partial{ true }; + DstOnly dstOnly{ false }; + LocalityData localities; + uint64_t memLimit = 8LL << 30; + uint64_t virtualMemLimit = 0; // unlimited + Optional ti; + BackupTLSConfig tlsConfig; + Version dumpBegin = 0; + Version dumpEnd = std::numeric_limits::max(); + std::string restoreClusterFileDest; + std::string restoreClusterFileOrig; + bool jsonOutput = false; + DeleteData deleteData{ false }; + Optional encryptionKeyFile; + int encryptionBlockSize = 0; + Optional blobManifestUrl; + SnapshotMode snapshotMode = SnapshotMode::RANGEFILE; // Default to legacy rangefile mode + + BackupModifyOptions modifyOptions; + + if (newArgC == 1) { + printUsage(programExe, false); + return FDB_EXIT_ERROR; + } + +#ifdef _WIN32 + // Windows needs a gentle nudge to format floats correctly + //_set_output_format(_TWO_DIGIT_EXPONENT); +#endif + + while (args->Next()) { + lastError = args->LastError(); + + switch (lastError) { + case SO_SUCCESS: + break; + + case SO_ARG_INVALID_DATA: + fprintf(stderr, "ERROR: invalid argument to option `%s'\n", args->OptionText()); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + + case SO_ARG_INVALID: + fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText()); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + + case SO_ARG_MISSING: + fprintf(stderr, "ERROR: missing argument for option `%s'\n", args->OptionText()); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + + case SO_OPT_INVALID: + fprintf(stderr, "ERROR: unknown option `%s'\n", args->OptionText()); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + + default: + fprintf(stderr, "ERROR: argument given for option `%s'\n", args->OptionText()); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + } + + int optId = args->OptionId(); + switch (optId) { + case OPT_HELP: + printUsage(programExe, false); + return FDB_EXIT_SUCCESS; + break; + case OPT_DEVHELP: + printUsage(programExe, true); + return FDB_EXIT_SUCCESS; + break; + case OPT_VERSION: + printVersion(); + return FDB_EXIT_SUCCESS; + break; + case OPT_BUILD_FLAGS: + printBuildInformation(); + return FDB_EXIT_SUCCESS; + break; + case OPT_NOBUFSTDOUT: + setvbuf(stdout, nullptr, _IONBF, 0); + setvbuf(stderr, nullptr, _IONBF, 0); + break; + case OPT_BUFSTDOUTERR: + setvbuf(stdout, nullptr, _IOFBF, BUFSIZ); + setvbuf(stderr, nullptr, _IOFBF, BUFSIZ); + break; + case OPT_QUIET: + quietDisplay = true; + break; + case OPT_DRYRUN: + dryRun = true; + break; + case OPT_DELETE_DATA: + deleteData.set(true); + break; + case OPT_MIN_CLEANUP_SECONDS: + knobs.emplace_back("min_cleanup_seconds", args->OptionArg()); + break; + case OPT_FORCE: + forceAction.set(true); + break; + case OPT_TRACE: + trace = true; + break; + case OPT_TRACE_DIR: + trace = true; + traceDir = args->OptionArg(); + break; + case OPT_TRACE_FORMAT: + if (!validateTraceFormat(args->OptionArg())) { + fprintf(stderr, "WARNING: Unrecognized trace format `%s'\n", args->OptionArg()); + } + traceFormat = args->OptionArg(); + break; + case OPT_TRACE_LOG_GROUP: + traceLogGroup = args->OptionArg(); + break; + case OPT_LOCALITY: { + Optional localityKey = extractPrefixedArgument("--locality", args->OptionSyntax()); + if (!localityKey.present()) { + fprintf(stderr, "ERROR: unable to parse locality key '%s'\n", args->OptionSyntax()); + return FDB_EXIT_ERROR; + } + Standalone key = StringRef(localityKey.get()); + std::transform(key.begin(), key.end(), mutateString(key), ::tolower); + localities.set(key, Standalone(std::string(args->OptionArg()))); + break; + } + case OPT_EXPIRE_BEFORE_DATETIME: + expireDatetime = args->OptionArg(); + break; + case OPT_EXPIRE_RESTORABLE_AFTER_DATETIME: + expireRestorableAfterDatetime = args->OptionArg(); + break; + case OPT_EXPIRE_BEFORE_VERSION: + case OPT_EXPIRE_RESTORABLE_AFTER_VERSION: + case OPT_EXPIRE_MIN_RESTORABLE_DAYS: + case OPT_EXPIRE_DELETE_BEFORE_DAYS: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse expiration version `%s'\n", a); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + + // Interpret the value as days worth of versions relative to now (negative) + if (optId == OPT_EXPIRE_MIN_RESTORABLE_DAYS || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) { + ver = -ver * 24 * 60 * 60 * CLIENT_KNOBS->CORE_VERSIONSPERSECOND; + } + + if (optId == OPT_EXPIRE_BEFORE_VERSION || optId == OPT_EXPIRE_DELETE_BEFORE_DAYS) + expireVersion = ver; + else + expireRestorableAfterVersion = ver; + break; + } + case OPT_RESTORE_TIMESTAMP: + restoreTimestamp = args->OptionArg(); + break; + case OPT_BASEURL: + baseUrl = args->OptionArg(); + break; + case OPT_RESTORE_CLUSTERFILE_DEST: + restoreClusterFileDest = args->OptionArg(); + break; + case OPT_RESTORE_CLUSTERFILE_ORIG: + restoreClusterFileOrig = args->OptionArg(); + break; + case OPT_CLUSTERFILE: + case OPT_DEST_CLUSTER: + clusterFile = args->OptionArg(); + break; + case OPT_SOURCE_CLUSTER: + sourceClusterFile = args->OptionArg(); + break; + case OPT_CLEANUP: + partial.set(false); + break; + case OPT_DSTONLY: + dstOnly.set(true); + break; + case OPT_KNOB: { + Optional knobName = extractPrefixedArgument("--knob", args->OptionSyntax()); + if (!knobName.present()) { + fprintf(stderr, "ERROR: unable to parse knob option '%s'\n", args->OptionSyntax()); + return FDB_EXIT_ERROR; + } + knobs.emplace_back(knobName.get(), args->OptionArg()); + break; + } + case OPT_BACKUPKEYS: + try { + addKeyRange(args->OptionArg(), backupKeys); + } catch (Error&) { + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + break; + case OPT_BACKUPKEYS_FILE: + try { + std::string line = readFileBytes(args->OptionArg(), 64 * 1024 * 1024); + addKeyRange(line, backupKeys); + } catch (Error&) { + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + break; + case OPT_BACKUPKEYS_FILTER: + try { + addKeyRange(args->OptionArg(), backupKeysFilter); + } catch (Error&) { + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + break; + case OPT_PROXY: + proxy = args->OptionArg(); + if (!Hostname::isHostname(proxy.get()) && !NetworkAddress::parseOptional(proxy.get()).present()) { + fprintf(stderr, "ERROR: Proxy format should be either IP:port or host:port\n"); + return FDB_EXIT_ERROR; + } + modifyOptions.proxy = proxy; + break; + case OPT_DESTCONTAINER: + destinationContainer = args->OptionArg(); + // If the url starts with '/' then prepend "file://" for backwards compatibility + if (StringRef(destinationContainer).startsWith("/"_sr)) + destinationContainer = std::string("file://") + destinationContainer; + modifyOptions.destURL = destinationContainer; + break; + case OPT_SNAPSHOTINTERVAL: + case OPT_INITIAL_SNAPSHOT_INTERVAL: + case OPT_MOD_ACTIVE_INTERVAL: { + const char* a = args->OptionArg(); + int seconds; + if (!sscanf(a, "%d", &seconds)) { + fprintf(stderr, "ERROR: Could not parse snapshot interval `%s'\n", a); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + if (optId == OPT_SNAPSHOTINTERVAL) { + snapshotIntervalSeconds = seconds; + modifyOptions.snapshotIntervalSeconds = seconds; + } else if (optId == OPT_INITIAL_SNAPSHOT_INTERVAL) { + initialSnapshotIntervalSeconds = seconds; + } else if (optId == OPT_MOD_ACTIVE_INTERVAL) { + modifyOptions.activeSnapshotIntervalSeconds = seconds; + } + break; + } + case OPT_MOD_VERIFY_UID: + modifyOptions.verifyUID = args->OptionArg(); + break; + case OPT_WAITFORDONE: + waitForDone.set(true); + break; + case OPT_NOSTOPWHENDONE: + stopWhenDone.set(false); + break; + case OPT_MUTATION_LOG_TYPE: { + auto parsedType = getMutationLogType(args->OptionArg()); + if (!parsedType.present()) { + fprintf(stderr, + "ERROR: Unknown mutation log type '%s'. Valid modes are: partitioned-log-experimental, " + "range-partitioned-log-experimental\n", + args->OptionArg()); + return FDB_EXIT_ERROR; + } + mutationLogType = parsedType.get(); + break; + } + case OPT_INCREMENTALONLY: + incrementalBackupOnly.set(true); + onlyApplyMutationLogs.set(true); + break; + case OPT_ENCRYPTION_KEY_FILE: + encryptionKeyFile = args->OptionArg(); + modifyOptions.encryptionKeyFile = encryptionKeyFile; + break; + case OPT_ENCRYPTION_BLOCK_SIZE: + try { + encryptionBlockSize = std::stoi(args->OptionArg()); + } catch (std::exception&) { + fprintf(stderr, "ERROR: Invalid encryption block size `%s'\n", args->OptionArg()); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + if (encryptionBlockSize <= 0) { + fprintf(stderr, "ERROR: Invalid encryption block size `%s'\n", args->OptionArg()); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + break; + case OPT_RESTORECONTAINER: + restoreContainer = args->OptionArg(); + // If the url starts with '/' then prepend "file://" for backwards compatibility + if (StringRef(restoreContainer).startsWith("/"_sr)) + restoreContainer = std::string("file://") + restoreContainer; + break; + case OPT_DESCRIBE_DEEP: + describeDeep = true; + break; + case OPT_DESCRIBE_TIMESTAMPS: + describeTimestamps = true; + break; + case OPT_PREFIX_ADD: { + bool err = false; + addPrefix = decode_hex_string(args->OptionArg(), err); + if (err) { + fprintf(stderr, "ERROR: Could not parse add prefix\n"); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + break; + } + case OPT_PREFIX_REMOVE: { + bool err = false; + removePrefix = decode_hex_string(args->OptionArg(), err); + if (err) { + fprintf(stderr, "ERROR: Could not parse remove prefix\n"); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + break; + } + case OPT_ERRORLIMIT: { + const char* a = args->OptionArg(); + if (!sscanf(a, "%d", &maxErrors)) { + fprintf(stderr, "ERROR: Could not parse max number of errors `%s'\n", a); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + break; + } + case OPT_RESTORE_BEGIN_VERSION: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse database beginVersion `%s'\n", a); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + beginVersion = ver; + break; + } + case OPT_RESTORE_VERSION: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + restoreVersion = ver; + break; + } + case OPT_RESTORE_SNAPSHOT_VERSION: { + const char* a = args->OptionArg(); + long long ver = 0; + if (!sscanf(a, "%lld", &ver)) { + fprintf(stderr, "ERROR: Could not parse database version `%s'\n", a); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + snapshotVersion = ver; + break; + } + case OPT_RESTORE_USER_DATA: { + restoreUserKeys = true; + break; + } + case OPT_RESTORE_SYSTEM_DATA: { + restoreSystemKeys = true; + break; + } + case OPT_RESTORE_INCONSISTENT_SNAPSHOT_ONLY: { + inconsistentSnapshotOnly.set(true); + break; + } +#ifdef _WIN32 + case OPT_PARENTPID: { + auto pid_str = args->OptionArg(); + int parent_pid = atoi(pid_str); + auto pHandle = OpenProcess(SYNCHRONIZE, FALSE, parent_pid); + if (!pHandle) { + TraceEvent("ParentProcessOpenError").GetLastError(); + fprintf(stderr, "Could not open parent process at pid %d (error %d)", parent_pid, GetLastError()); + throw platform_error(); + } + startThread(&parentWatcher, pHandle); + break; + } +#endif + case OPT_TAGNAME: + tagName = args->OptionArg(); + tagProvided = true; + break; + case OPT_CRASHONERROR: + g_crashOnError = true; + break; + case OPT_MEMLIMIT: + ti = parse_with_suffix(args->OptionArg(), "MiB"); + if (!ti.present()) { + fprintf(stderr, "ERROR: Could not parse memory limit from `%s'\n", args->OptionArg()); + printHelpTeaser(newArgV[0]); + flushAndExit(FDB_EXIT_ERROR); + } + memLimit = ti.get(); + break; + case OPT_VMEMLIMIT: + ti = parse_with_suffix(args->OptionArg(), "MiB"); + if (!ti.present()) { + fprintf(stderr, "ERROR: Could not parse virtual memory limit from `%s'\n", args->OptionArg()); + printHelpTeaser(newArgV[0]); + flushAndExit(FDB_EXIT_ERROR); + } + virtualMemLimit = ti.get(); + break; + case OPT_BLOB_CREDENTIALS: + tlsConfig.blobCredentials.push_back(args->OptionArg()); + break; + case TLSConfig::OPT_TLS_PLUGIN: + args->OptionArg(); + break; + case TLSConfig::OPT_TLS_CERTIFICATES: + tlsConfig.tlsCertPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_PASSWORD: + tlsConfig.tlsPassword = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_CA_FILE: + tlsConfig.tlsCAPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_KEY: + tlsConfig.tlsKeyPath = args->OptionArg(); + break; + case TLSConfig::OPT_TLS_VERIFY_PEERS: + tlsConfig.tlsVerifyPeers = args->OptionArg(); + break; + case OPT_DUMP_BEGIN: + dumpBegin = parseVersion(args->OptionArg()); + break; + case OPT_DUMP_END: + dumpEnd = parseVersion(args->OptionArg()); + break; + case OPT_JSON: + jsonOutput = true; + break; + case OPT_MODE: + // Handle mode parameter for both backup and restore + if (programExe == ProgramExe::BACKUP) { + // Validate and store mode parameter for snapshot generation + auto parsedMode = getSnapshotMode(args->OptionArg()); + if (!parsedMode.present()) { + fprintf(stderr, + "ERROR: Unknown snapshot mode '%s'. Valid modes are: rangefile, bulkdump, both\n", + args->OptionArg()); + return FDB_EXIT_ERROR; + } + snapshotMode = parsedMode.get(); + } else if (programExe == ProgramExe::RESTORE) { + // Validate and store mode parameter for restore mechanism + auto parsedMode = getRestoreMode(args->OptionArg()); + if (!parsedMode.present()) { + fprintf(stderr, + "ERROR: Unknown restore mode '%s'. Valid modes are: rangefile, bulkload\n", + args->OptionArg()); + return FDB_EXIT_ERROR; + } + restoreMode = parsedMode.get(); + } + break; + } + } + + // Process the extra arguments + for (int argLoop = 0; argLoop < args->FileCount(); argLoop++) { + switch (programExe) { + case ProgramExe::AGENT: + fprintf(stderr, "ERROR: Backup Agent does not support argument value `%s'\n", args->File(argLoop)); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + + // Add the backup key range + case ProgramExe::BACKUP: + // Error, if the keys option was not specified + if (backupKeys.empty()) { + fprintf(stderr, "ERROR: Unknown backup option value `%s'\n", args->File(argLoop)); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + // Otherwise, assume the item is a key range + else { + try { + addKeyRange(args->File(argLoop), backupKeys); + } catch (Error&) { + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + } + break; + + case ProgramExe::RESTORE: + fprintf(stderr, "ERROR: FDB Restore does not support argument value `%s'\n", args->File(argLoop)); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + + case ProgramExe::DR_AGENT: + fprintf(stderr, "ERROR: DR Agent does not support argument value `%s'\n", args->File(argLoop)); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + + case ProgramExe::DB_BACKUP: + // Error, if the keys option was not specified + if (backupKeys.empty()) { + fprintf(stderr, "ERROR: Unknown DR option value `%s'\n", args->File(argLoop)); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + // Otherwise, assume the item is a key range + else { + try { + addKeyRange(args->File(argLoop), backupKeys); + } catch (Error&) { + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + } + } + break; + + case ProgramExe::UNDEFINED: + default: + return FDB_EXIT_ERROR; + } + } + + if (restoreSystemKeys && restoreUserKeys) { + fprintf(stderr, "ERROR: Please only specify one of --user-data or --system-metadata, not both\n"); + return FDB_EXIT_ERROR; + } + + if (trace) { + if (!traceLogGroup.empty()) + setNetworkOption(FDBNetworkOptions::TRACE_LOG_GROUP, StringRef(traceLogGroup)); + + if (traceDir.empty()) + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE); + else + setNetworkOption(FDBNetworkOptions::TRACE_ENABLE, StringRef(traceDir)); + if (!traceFormat.empty()) { + setNetworkOption(FDBNetworkOptions::TRACE_FORMAT, StringRef(traceFormat)); + } + + setNetworkOption(FDBNetworkOptions::ENABLE_SLOW_TASK_PROFILING); + } + setNetworkOption(FDBNetworkOptions::DISABLE_CLIENT_STATISTICS_LOGGING); + + // deferred TLS options + if (!tlsConfig.setupTLS()) { + return 1; + } + + Error::init(); + std::set_new_handler(&platform::outOfMemory); + setMemoryQuota(virtualMemLimit); + + Database db; + Database sourceDb; + FileBackupAgent ba; + Key tag; + Future> f; + Future> fstatus; + Reference c; + + try { + setupNetwork(0, UseMetrics::True); + } catch (Error& e) { + fprintf(stderr, "ERROR: %s\n", e.what()); + return FDB_EXIT_ERROR; + } + + Future memoryUsageMonitor = startMemoryUsageMonitor(memLimit); + + setupClientKnobs(knobs); + // Reinitialize knobs in order to update knobs that are dependent on explicitly set knobs + initializeClientKnobs(Randomize::False, IsSimulated::False); + + TraceEvent("ProgramStart") + .setMaxEventLength(12000) + .detail("SourceVersion", getSourceVersion()) + .detail("Version", FDB_VT_VERSION) + .detail("PackageName", FDB_VT_PACKAGE_NAME) + .detailf("ActualTime", "%lld", DEBUG_DETERMINISM ? 0 : time(nullptr)) + .setMaxFieldLength(10000) + .detail("CommandLine", commandLine) + .setMaxFieldLength(0) + .detail("MemoryLimit", memLimit) + .detail("Proxy", proxy.orDefault("")) + .trackLatest("ProgramStart"); + + // Ordinarily, this is done when the network is run. However, network thread should be set before + // TraceEvents are logged. This thread will eventually run the network, so call it now. + TraceEvent::setNetworkThread(); + + // Sets up blob credentials, including one from the environment FDB_BLOB_CREDENTIALS. + tlsConfig.setupBlobCredentials(); + + // Opens a trace file if trace is set (and if a trace file isn't already open) + // For most modes, initCluster() will open a trace file, but some fdbbackup operations do not require + // a cluster so they should use this instead. + auto initTraceFile = [&]() { + if (trace) + openTraceFile({}, traceRollSize, traceMaxLogsSize, traceDir, "trace", traceLogGroup); + }; + + auto initCluster = [&](bool quiet = false) { + Optional result = connectToCluster(clusterFile, localities, quiet); + if (result.present()) { + db = result.get(); + // Make sure we are setting a transaction timeout and retry limit to prevent cases + // where the fdbbackup command hangs infinitely. 60 seconds should be more than + // enough for all cases to finish and 5 retries should also be good enough for + // most cases. + int64_t timeout = 60000; + db->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, sizeof(timeout)))); + int64_t retryLimit = 5; + db->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit)))); + } + + return result.present(); + }; + + auto initSourceCluster = [&](bool required, bool quiet = false) { + if (sourceClusterFile.empty() && required) { + if (!quiet) { + fprintf(stderr, "ERROR: source cluster file is required\n"); + } + return false; + } + + Optional result = connectToCluster(sourceClusterFile, localities, quiet); + if (result.present()) { + sourceDb = result.get(); + // Make sure we are setting a transaction timeout and retry limit to prevent cases + // where the fdbbackup command hangs infinitely. 60 seconds should be more than + // enough for all cases to finish and 5 retries should also be good enough for + // most cases. + int64_t timeout = 60000; + sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT, + Optional(StringRef((const uint8_t*)&timeout, sizeof(timeout)))); + int64_t retryLimit = 5; + sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT, + Optional(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit)))); + } + + return result.present(); + }; + + if (encryptionKeyFile.present() && encryptionBlockSize == 0) { + encryptionBlockSize = DEFAULT_ENCRYPTION_BLOCK_SIZE; + } + + if (encryptionBlockSize > 0 && !encryptionKeyFile.present()) { + fprintf(stderr, "ERROR: --encryption-block-size option requires --encryption-key-file to be set\n"); + return FDB_EXIT_ERROR; + } + + if (!restoreSystemKeys && !restoreUserKeys && backupKeys.empty()) { + addDefaultBackupRanges(backupKeys); + } + + if ((restoreUserKeys || restoreSystemKeys) && !backupKeys.empty()) { + fprintf(stderr, + "ERROR: Cannot specify additional ranges when using --user-data or --system-metadata " + "options\n"); + return FDB_EXIT_ERROR; + } + if (restoreUserKeys) { + backupKeys.push_back_deep(backupKeys.arena(), normalKeys); + } else if (restoreSystemKeys) { + for (const auto& r : getSystemBackupRanges()) { + backupKeys.push_back_deep(backupKeys.arena(), r); + } + } + + switch (programExe) { + case ProgramExe::AGENT: + if (!initCluster()) + return FDB_EXIT_ERROR; + fileBackupAgentProxy = proxy; + f = stopAfter(runAgent(db)); + break; + case ProgramExe::BACKUP: + switch (backupType) { + case BackupType::START: { + if (!initCluster()) + return FDB_EXIT_ERROR; + // Test out the backup url to make sure it parses. Doesn't test to make sure it's actually + // writeable. + openBackupContainer(newArgV[0], destinationContainer, proxy, encryptionKeyFile, encryptionBlockSize); + f = stopAfter(submitBackup(db, + destinationContainer, + proxy, + initialSnapshotIntervalSeconds, + snapshotIntervalSeconds, + backupKeys, + tagName, + dryRun, + waitForDone, + stopWhenDone, + mutationLogType, + incrementalBackupOnly, + encryptionKeyFile, + encryptionBlockSize, + snapshotMode)); + break; + } + + case BackupType::MODIFY: { + if (!initCluster()) + return FDB_EXIT_ERROR; + + f = stopAfter(modifyBackup(db, tagName, modifyOptions)); + break; + } + + case BackupType::STATUS: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(statusBackup(db, tagName, ShowErrors::True, jsonOutput)); + break; + + case BackupType::ABORT: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(abortBackup(db, tagName)); + break; + + case BackupType::CLEANUP: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(cleanupMutations(db, deleteData)); + break; + + case BackupType::WAIT: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(waitBackup(db, tagName, stopWhenDone)); + break; + + case BackupType::DISCONTINUE: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(discontinueBackup(db, tagName, waitForDone)); + break; + + case BackupType::PAUSE: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(changeBackupResumed(db, true)); + break; + + case BackupType::RESUME: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(changeBackupResumed(db, false)); + break; + + case BackupType::EXPIRE: + initTraceFile(); + // Must have a usable cluster if either expire DateTime options were used + if (!expireDatetime.empty() || !expireRestorableAfterDatetime.empty()) { + if (!initCluster()) + return FDB_EXIT_ERROR; + } + f = stopAfter(expireBackupData(newArgV[0], + destinationContainer, + proxy, + expireVersion, + expireDatetime, + db, + forceAction, + expireRestorableAfterVersion, + expireRestorableAfterDatetime)); + break; + + case BackupType::DELETE_BACKUP: + initTraceFile(); + f = stopAfter(deleteBackupContainer(newArgV[0], destinationContainer, proxy)); + break; + + case BackupType::DESCRIBE: + initTraceFile(); + // If timestamp lookups are desired, require a cluster file + if (describeTimestamps && !initCluster()) + return FDB_EXIT_ERROR; + + // Only pass database optionDatabase Describe will lookup version timestamps if a cluster file was + // given, but quietly skip them if not. + f = stopAfter(describeBackup(newArgV[0], + destinationContainer, + proxy, + describeDeep, + describeTimestamps ? Optional(db) : Optional(), + jsonOutput)); + break; + + case BackupType::LIST: + initTraceFile(); + f = stopAfter(listBackup(baseUrl, proxy)); + break; + + case BackupType::TAGS: + if (!initCluster()) + return FDB_EXIT_ERROR; + f = stopAfter(listBackupTags(db)); + break; + + case BackupType::QUERY: + initTraceFile(); + f = stopAfter(queryBackup(newArgV[0], + destinationContainer, + proxy, + backupKeysFilter, + restoreVersion, + snapshotVersion, + restoreClusterFileOrig, + restoreTimestamp, + Verbose{ !quietDisplay }, + db)); + break; + + case BackupType::DUMP: + initTraceFile(); + f = stopAfter(dumpBackupData(newArgV[0], destinationContainer, proxy, dumpBegin, dumpEnd)); + break; + + case BackupType::UNDEFINED: + default: + fprintf(stderr, "ERROR: Unsupported backup action %s\n", newArgV[1]); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + } + + break; + case ProgramExe::RESTORE: + if (dryRun) { + if (restoreType != RestoreType::START) { + fprintf(stderr, "Restore dry run only works for 'start' command\n"); + return FDB_EXIT_ERROR; + } + + // Must explicitly call trace file options handling if not calling Database::createDatabase() + initTraceFile(); + } else { + if (restoreClusterFileDest.empty()) { + fprintf(stderr, "Restore destination cluster file must be specified explicitly.\n"); + return FDB_EXIT_ERROR; + } + + if (!fileExists(restoreClusterFileDest)) { + fprintf(stderr, + "Restore destination cluster file '%s' does not exist.\n", + restoreClusterFileDest.c_str()); + return FDB_EXIT_ERROR; + } + + try { + db = Database::createDatabase(restoreClusterFileDest, ApiVersion::LATEST_VERSION); + } catch (Error& e) { + fprintf(stderr, + "Restore destination cluster file '%s' invalid: %s\n", + restoreClusterFileDest.c_str(), + e.what()); + return FDB_EXIT_ERROR; + } + } + + switch (restoreType) { + case RestoreType::START: + f = stopAfter(runRestore(db, + restoreClusterFileOrig, + tagName, + restoreContainer, + proxy, + backupKeys, + beginVersion, + restoreVersion, + restoreTimestamp, + !dryRun, + Verbose{ !quietDisplay }, + waitForDone, + addPrefix, + removePrefix, + onlyApplyMutationLogs, + inconsistentSnapshotOnly, + encryptionKeyFile, + restoreMode)); // Pass RestoreMode directly + break; + case RestoreType::WAIT: + f = stopAfter(success(ba.waitRestore(db, KeyRef(tagName), Verbose::True))); + break; + case RestoreType::ABORT: + f = stopAfter( + map(ba.abortRestore(db, KeyRef(tagName)), [tagName](FileBackupAgent::ERestoreState s) -> Void { + printf("RESTORE_ABORT Tag: %s State: %s\n", + tagName.c_str(), + FileBackupAgent::restoreStateText(s).toString().c_str()); + return Void(); + })); + break; + case RestoreType::STATUS: + // If no tag is specifically provided then print all tag status, don't just use "default" + if (tagProvided) + tag = tagName; + f = stopAfter(map(ba.restoreStatus(db, KeyRef(tag)), [](std::string s) -> Void { + printf("%s\n", s.c_str()); + return Void(); + })); + break; + default: + throw restore_error(); + } + break; + case ProgramExe::DR_AGENT: + if (!initCluster() || !initSourceCluster(true)) { + return FDB_EXIT_ERROR; + } + f = stopAfter(runDBAgent(sourceDb, db)); + break; + case ProgramExe::DB_BACKUP: + if (!initCluster() || !initSourceCluster(dbType != DBType::ABORT || !dstOnly)) { + return FDB_EXIT_ERROR; + } + switch (dbType) { + case DBType::START: + f = stopAfter(submitDBBackup(sourceDb, db, backupKeys, tagName)); + break; + case DBType::STATUS: + f = stopAfter(statusDBBackup(sourceDb, db, tagName, maxErrors)); + break; + case DBType::SWITCH: + f = stopAfter(switchDBBackup(sourceDb, db, backupKeys, tagName, forceAction)); + break; + case DBType::ABORT: + f = stopAfter(abortDBBackup(sourceDb, db, tagName, partial, dstOnly)); + break; + case DBType::PAUSE: + f = stopAfter(changeDBBackupResumed(sourceDb, db, true)); + break; + case DBType::RESUME: + f = stopAfter(changeDBBackupResumed(sourceDb, db, false)); + break; + case DBType::UNDEFINED: + default: + fprintf(stderr, "ERROR: Unsupported DR action %s\n", newArgV[1]); + printHelpTeaser(newArgV[0]); + return FDB_EXIT_ERROR; + break; + } + break; + case ProgramExe::UNDEFINED: + default: + return FDB_EXIT_ERROR; + } + + runNetwork(); + + if (f.isValid() && f.isReady() && !f.isError() && !f.get().present()) { + status = FDB_EXIT_ERROR; + } + + if (fstatus.isValid() && fstatus.isReady() && !fstatus.isError() && fstatus.get().present()) { + status = fstatus.get().get(); + } + +#ifdef ALLOC_INSTRUMENTATION + { + std::cout << "Page Counts: " << FastAllocator<16>::pageCount << " " << FastAllocator<32>::pageCount << " " + << FastAllocator<64>::pageCount << " " << FastAllocator<128>::pageCount << " " + << FastAllocator<256>::pageCount << " " << FastAllocator<512>::pageCount << " " + << FastAllocator<1024>::pageCount << " " << FastAllocator<2048>::pageCount << " " + << FastAllocator<4096>::pageCount << " " << FastAllocator<8192>::pageCount << " " + << FastAllocator<16384>::pageCount << std::endl; + + std::vector> typeNames; + for (auto i = allocInstr.begin(); i != allocInstr.end(); ++i) { + std::string s; + +#ifdef __linux__ + char* demangled = abi::__cxa_demangle(i->first, nullptr, nullptr, nullptr); + if (demangled) { + s = demangled; + if (StringRef(s).startsWith("(anonymous namespace)::"_sr)) + s = s.substr("(anonymous namespace)::"_sr.size()); + free(demangled); + } else + s = i->first; +#else + s = i->first; + if (StringRef(s).startsWith("class `anonymous namespace'::"_sr)) + s = s.substr("class `anonymous namespace'::"_sr.size()); + else if (StringRef(s).startsWith("class "_sr)) + s = s.substr("class "_sr.size()); + else if (StringRef(s).startsWith("struct "_sr)) + s = s.substr("struct "_sr.size()); +#endif + + typeNames.emplace_back(s, i->first); + } + std::sort(typeNames.begin(), typeNames.end()); + for (int i = 0; i < typeNames.size(); i++) { + const char* n = typeNames[i].second; + auto& f = allocInstr[n]; + printf("%+d\t%+d\t%d\t%d\t%s\n", + f.allocCount, + -f.deallocCount, + f.allocCount - f.deallocCount, + f.maxAllocated, + typeNames[i].first.c_str()); + } + + // We're about to exit and clean up data structures, this will wreak havoc on allocation recording + memSample_entered = true; + } +#endif + } catch (Error& e) { + TraceEvent(SevError, "MainError").error(e); + status = FDB_EXIT_MAIN_ERROR; + } catch (boost::system::system_error& e) { + if (g_network) { + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + } else { + fprintf(stderr, "ERROR: %s (%d)\n", e.what(), e.code().value()); + } + status = FDB_EXIT_MAIN_EXCEPTION; + } catch (std::exception& e) { + TraceEvent(SevError, "MainError").error(unknown_error()).detail("RootException", e.what()); + status = FDB_EXIT_MAIN_EXCEPTION; + } + + flushAndExit(status); +} + +#else // EXCLUDE_MAIN_FUNCTION + +int main() { + + printf("=== Running ParsedArgs Tests ===\n"); + + auto testOptionParsing = [](std::initializer_list args, + const std::vector& expectedOptions = {}, + bool shouldSucceed = true, + const char* testName = "", + bool expectCSimpleOptions = false) -> bool { + printf("\n--- Test: %s ---\n", testName); + static std::vector persistentArgs; + persistentArgs.clear(); + persistentArgs.reserve(args.size()); + for (const char* arg : args) { + persistentArgs.emplace_back(arg); + } + int argc = static_cast(persistentArgs.size()); + + std::vector argv; + for (auto& arg : persistentArgs) { + argv.push_back(arg.data()); + } + argv.push_back(nullptr); + + int argcNew{}; + char** argvNew{}; + + printf("DEBUG: argc: %d\n", argc); + for (int i = 0; i < argv.size(); ++i) { + printf("DEBUG: argv[%d]: %s\n", i, argv[i]); + } + + bool success = reorderArguments(argc, argv.data(), argcNew, argvNew); + + printf("DEBUG: argcNew: %d\n", argcNew); + for (int i = 0; i < argcNew; ++i) { + printf("DEBUG: argvNew[%d]: %s\n", i, argvNew[i]); + } + + if (success != shouldSucceed) { + printf("%s: FAIL - Expected %s but got %s\n", + testName, + shouldSucceed ? "success" : "failure", + success ? "success" : "failure"); + return false; + } + if (!shouldSucceed) { + printf("\n\t--- Test PASSED: %s (expected failure)---\n", testName); + return true; + } + + // Test CSimpleOpt conversion + std::vector actualOptions; + for (int i = 1; i < argcNew; i++) { + actualOptions.push_back(argvNew[i]); + } + + if (actualOptions != expectedOptions) { + printf("%s: FAIL - Options mismatch\n", testName); + printf(" Expected options (%zu): ", expectedOptions.size()); + for (const auto& opt : expectedOptions) + printf("'%s' ", opt.c_str()); + printf("\n Actual options (%zu): ", actualOptions.size()); + for (const auto& opt : actualOptions) + printf("'%s' ", opt.c_str()); + printf("\n"); + return false; + } + + // Test with actual CSimpleOpt if expected + if (expectCSimpleOptions && !expectedOptions.empty()) { + try { + std::unique_ptr simpleOpt = std::make_unique( + argcNew, const_cast(argvNew), g_rgOptions, SO_O_EXACT | SO_O_HYPHEN_TO_UNDERSCORE); + + ESOError lastError = SO_SUCCESS; + bool foundExpectedOptions = true; + + while (simpleOpt->Next()) { + lastError = simpleOpt->LastError(); + if (lastError != SO_SUCCESS) { + printf("CSimpleOpt parsing error: %d\n", lastError); + foundExpectedOptions = false; + break; + } + + int optId = simpleOpt->OptionId(); + printf("CSimpleOpt found option: id=%d, text='%s', arg='%s'\n", + optId, + simpleOpt->OptionText(), + simpleOpt->OptionArg() ? simpleOpt->OptionArg() : "null"); + } + + if (!foundExpectedOptions) { + printf("%s: FAIL - CSimpleOpt parsing failed\n", testName); + return false; + } + } catch (const std::exception& e) { + printf("%s: FAIL - CSimpleOpt exception: %s\n", testName, e.what()); + return false; + } + } + + printf("\n\t--- Test PASSED: %s ---\n", testName); + return true; + }; + + printf("\n1) Basic Command Tests:\n"); + bool allPassed = true; + allPassed &= testOptionParsing({ "fdbbackup", "status" }, { "status" }, true, "1.1 Single command"); + allPassed &= testOptionParsing({ "fdbbackup" }, {}, true, "1.2 No commands"); + allPassed &= testOptionParsing({ "fdbbackup", "unknown" }, { "unknown" }, true, "1.3 Unknown command"); + allPassed &= testOptionParsing( + { "fdbbackup", "unknown1", "unknown2" }, { "unknown1", "unknown2" }, true, "1.4 Several unknown commands"); + + printf("\n2) Command Positioning Tests:\n"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file", "/cluster" }, + { "start", "--cluster-file", "/cluster" }, + true, + "2.1 Command before options"); + allPassed &= testOptionParsing({ "fdbbackup", "--cluster-file", "/cluster", "start" }, + { "start", "--cluster-file", "/cluster" }, + true, + "2.2 Command after options"); + allPassed &= testOptionParsing({ "fdbbackup", "--cluster-file", "/cluster", "list", "--json" }, + { "list", "--cluster-file", "/cluster", "--json" }, + true, + "2.3 Options before and after command"); + + printf("\n3) Option Parameter Tests:\n"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "-C", "/cluster" }, + { "start", "-C", "/cluster" }, + true, + "3.1 Short option with parameter"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--snapshot-interval", "30" }, + { "start", "--snapshot-interval", "30" }, + true, + "3.2 Option with parameter"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--logdir", "/logs", "--trace-format", "json" }, + { "start", "--logdir", "/logs", "--trace-format", "json" }, + true, + "3.3 Multiple options with parameters"); + + printf("\n4) Equal Sign Parameter Tests:\n"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file=/cluster" }, + { "start", "--cluster-file=/cluster" }, + true, + "4.1 Option with equals"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--snapshot-interval", "30", "--cluster-file=/cluster" }, + { "start", "--snapshot-interval", "30", "--cluster-file=/cluster" }, + true, + "4.2 Multiple options using both equals and space separators"); + + printf("\n5) Prefix Option Tests:\n"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--knob-max_workers", "10" }, + { "start", "--knob-max_workers", "10" }, + true, + "5.1 Knob option with parameter"); + + printf("\n6) Global flag options and CSimpleOpt Tests:\n"); + allPassed &= + testOptionParsing({ "fdbbackup", "--version", "-h" }, { "--version", "-h" }, true, "6.1 Version flag", true); + + printf("\n7) Error Tests:\n"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--unknown-option" }, {}, false, "7.1 Unknown option"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file" }, {}, false, "7.2 Missing parameter"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file", "--help" }, + { "start", "--cluster-file", "--help" }, + true, + "7.3 Option as parameter"); + allPassed &= testOptionParsing({ "fdbbackup", "start", "--cluster-file=/cluster", "-C=" }, + { "start", "--cluster-file=/cluster", "-C=" }, + true, + "7.4 Option with empty parameter value using equals"); + allPassed &= testOptionParsing( + { "fdbbackup", "start", "-C=" }, { "start", "-C=" }, true, "7.5 Empty parameter value with equals"); + + printf("\n=== %s ===\n", allPassed ? "All tests PASSED!" : "Some tests FAILED!"); + return allPassed ? 0 : 1; +} + +#endif // EXCLUDE_MAIN_FUNCTION diff --git a/fdbbackup/include/fdbbackup/FileConverter.h b/fdbbackup/include/fdbbackup/FileConverter.h index 62be298d96c..f858c8d3101 100644 --- a/fdbbackup/include/fdbbackup/FileConverter.h +++ b/fdbbackup/include/fdbbackup/FileConverter.h @@ -24,7 +24,7 @@ #include #include "SimpleOpt/SimpleOpt.h" -#include "flow/TLSConfig.actor.h" +#include "flow/TLSConfig.h" namespace file_converter { @@ -52,6 +52,7 @@ enum { OPT_END_VERSION_FILTER, OPT_KNOB, OPT_SAVE_FILE, + OPT_ENCRYPTION_KEY_FILE, OPT_HELP }; @@ -84,6 +85,7 @@ CSimpleOpt::SOption gConverterOptions[] = { { OPT_CONTAINER, "-r", SO_REQ_SEP }, { OPT_KNOB, "--knob-", SO_REQ_SEP }, { OPT_SAVE_FILE, "-s", SO_NONE }, { OPT_SAVE_FILE, "--save", SO_NONE }, + { OPT_ENCRYPTION_KEY_FILE, "--encryption-key-file", SO_REQ_SEP }, { OPT_HELP, "-?", SO_NONE }, { OPT_HELP, "-h", SO_NONE }, { OPT_HELP, "--help", SO_NONE }, diff --git a/fdbbackup/tests/backup_tests_common.sh b/fdbbackup/tests/backup_tests_common.sh index 81dac39fce6..1ab34b62503 100644 --- a/fdbbackup/tests/backup_tests_common.sh +++ b/fdbbackup/tests/backup_tests_common.sh @@ -19,9 +19,15 @@ # limitations under the License. # # Common backup test functions -# Shared between s3_backup_test.sh, s3_backup_bulkdump_bulkload.sh, dir_backup_test.sh, etc. +# Shared between blob_backup_restore_test.sh, s3_backup_bulkdump_bulkload.sh, dir_backup_test.sh, etc. # These functions work with both S3/blobstore and file-based backup testing +# Source shared test utilities (output_contains, output_matches, etc.) +_BACKUP_COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +if [[ -f "${_BACKUP_COMMON_DIR}/../../fdbclient/tests/tests_common.sh" ]]; then + source "${_BACKUP_COMMON_DIR}/../../fdbclient/tests/tests_common.sh" +fi + # Helper function to add base arguments (cluster file and logging) # Uses bash nameref (requires bash 4.3+) to modify the array in place # $1 name of the array variable to modify (passed by name, not value) @@ -106,7 +112,7 @@ function s3_cleanup_url { local local_scratch_dir="${2}" local local_url="${3}" local credentials="${4}" - + local cmd=("${local_build_dir}/bin/s3client") cmd+=("${KNOBS[@]}") @@ -151,7 +157,11 @@ function run_backup { add_common_optional_args cmd_args "${blob_credentials}" "${backup_mode}" "${local_encryption_key_file}" if [[ "${USE_PARTITIONED_LOG:-false}" == "true" ]]; then - cmd_args+=("--partitioned-log-experimental") + cmd_args+=("--mutation-log-type" "partitioned-log-experimental") + fi + + if [[ "${USE_ENCRYPTION_BLOCK_SIZE:-false}" == "true" ]]; then + cmd_args+=("--encryption-block-size" "4096") fi # Start backup without -w flag to avoid hanging @@ -182,12 +192,12 @@ function run_backup { set -e # Check if backup is restorable (differential state) or completed - if echo "${status_output}" | grep -q "is restorable"; then + if output_contains "${status_output}" "is restorable"; then log "Backup is now restorable after ${elapsed}s" break fi - if echo "${status_output}" | grep -q "completed"; then + if output_contains "${status_output}" "completed"; then log "Backup completed after ${elapsed}s" break fi @@ -196,17 +206,17 @@ function run_backup { if [[ $((elapsed % 30)) -eq 0 ]]; then log "Still waiting for backup to become restorable (${elapsed}s elapsed)..." # Show current state for debugging - if echo "${status_output}" | grep -q "is restorable"; then + if output_contains "${status_output}" "is restorable"; then log " Status: backup is restorable (should have exited loop)" - elif echo "${status_output}" | grep -q "in progress to"; then + elif output_contains "${status_output}" "in progress to"; then log " Status: backup running, waiting for snapshot to complete" - elif echo "${status_output}" | grep -q "just started"; then + elif output_contains "${status_output}" "just started"; then log " Status: backup submitted, tasks starting up" fi # Check snapshot mode for debugging - if echo "${status_output}" | grep -q "Snapshot Mode: bulkdump"; then + if output_contains "${status_output}" "Snapshot Mode: bulkdump"; then log " Snapshot Mode: bulkdump (using BulkDump for snapshots)" - elif echo "${status_output}" | grep -q "Snapshot Mode: both"; then + elif output_contains "${status_output}" "Snapshot Mode: both"; then log " Snapshot Mode: both (generating both formats)" fi fi @@ -218,13 +228,48 @@ function run_backup { echo "${status_output}" return 1 fi - + + # For 'both' or 'bulkdump' mode, wait for BulkDump snapshot to be written + # BulkDump writes the snapshot file AFTER DD job completes, which is async from rangefile snapshot + if [[ "${backup_mode}" == "both" || "${backup_mode}" == "bulkdump" ]]; then + log "Waiting for BulkDump snapshot to be written (mode: ${backup_mode})..." + local bulkdump_timeout=120 # Additional timeout for BulkDump + local bulkdump_elapsed=0 + local bulkdump_poll=5 + + while [[ $bulkdump_elapsed -lt $bulkdump_timeout ]]; do + sleep $bulkdump_poll + bulkdump_elapsed=$((bulkdump_elapsed + bulkdump_poll)) + + # Check for BulkDump snapshot file with ",bulk" suffix (used in 'both' mode) + # or check if any snapshot has bulkDumpJobId field + set +e + snapshot_list=$("${local_build_dir}"/bin/fdbbackup describe -d "${local_url}" -C "${local_scratch_dir}/loopback_cluster/fdb.cluster" --log --logdir="${local_scratch_dir}" 2>&1) + set -e + + # Check if BulkDump snapshot exists (look for bulkDumpJobId in describe output) + if output_matches "${snapshot_list}" "bulkDumpJobId\|,bulk"; then + log "BulkDump snapshot found after ${bulkdump_elapsed}s" + break + fi + + if [[ $((bulkdump_elapsed % 30)) -eq 0 ]]; then + log "Still waiting for BulkDump snapshot (${bulkdump_elapsed}s elapsed)..." + fi + done + + if [[ $bulkdump_elapsed -ge $bulkdump_timeout ]]; then + log "Warning: Timeout waiting for BulkDump snapshot after ${bulkdump_timeout}s" + log "BulkDump may still be running - proceeding anyway" + fi + fi + # Check if backup already completed (no need to discontinue) - if echo "${status_output}" | grep -q "completed"; then + if output_contains "${status_output}" "completed"; then log "Backup already completed - no need to discontinue" return 0 fi - + # Stop the backup to finalize it (only if still running) log "Stopping backup to finalize restorable state" set +e @@ -233,7 +278,7 @@ function run_backup { set -e if [[ $stop_exit_code -ne 0 ]]; then - if echo "${stop_output}" | grep -q "already discontinued\|not running\|unneeded"; then + if output_matches "${stop_output}" "already discontinued\|not running\|unneeded"; then log "Backup already completed and finalized - this is success!" else err "Failed to stop backup: ${stop_output}" @@ -299,13 +344,13 @@ function run_restore { # Check if restore completed # Status output contains "State: completed" or "Phase: Complete" when done # Also check "No restore" for when restore tag doesn't exist (completed and cleaned up) - if echo "${status_output}" | grep -qi "State:.*completed\|Phase:.*Complete\|No restore"; then + if output_matches_i "${status_output}" "State:.*completed\|Phase:.*Complete\|No restore"; then log "Restore completed after ${elapsed}s" return 0 fi # Check if restore failed (be specific - "LastError: None" contains "Error" so avoid false positives) - if echo "${status_output}" | grep -qi "State:.*aborted"; then + if output_matches_i "${status_output}" "State:.*aborted"; then err "Restore aborted after ${elapsed}s" log "Status output:" echo "${status_output}" @@ -324,7 +369,7 @@ function run_restore { if [[ $((elapsed % 30)) -eq 0 ]]; then log "Still waiting for restore to complete (${elapsed}s elapsed)..." # Show phase info for debugging - if echo "${status_output}" | grep -qi "Phase:"; then + if output_matches_i "${status_output}" "Phase:"; then phase_info=$(echo "${status_output}" | grep -i "Phase:" | head -1) log " ${phase_info}" fi @@ -380,7 +425,9 @@ function test_encryption_mismatches { if [[ ${exit_code1} -eq 0 ]]; then err "Restore without encryption on encrypted backup succeeded when it should have failed!" - rm -rf "${mismatch_logdir}" + if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then + rm -rf "${mismatch_logdir}" + fi return 1 fi log "SUCCESS: Restore without encryption on encrypted backup failed as expected (exit code: ${exit_code1})" @@ -402,7 +449,9 @@ function test_encryption_mismatches { if [[ ${exit_code2} -eq 0 ]]; then err "Restore with wrong encryption key succeeded when it should have failed!" - rm -rf "${mismatch_logdir}" + if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then + rm -rf "${mismatch_logdir}" + fi return 1 fi log "SUCCESS: Restore with wrong encryption key failed as expected (exit code: ${exit_code2})" @@ -428,14 +477,18 @@ function test_encryption_mismatches { if [[ ${exit_code} -eq 0 ]]; then err "Restore with encryption on unencrypted backup succeeded when it should have failed!" - rm -rf "${mismatch_logdir}" + if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then + rm -rf "${mismatch_logdir}" + fi return 1 fi log "SUCCESS: Restore with encryption on unencrypted backup failed as expected (exit code: ${exit_code})" fi # Clean up separate log directory - rm -rf "${mismatch_logdir}" + if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then + rm -rf "${mismatch_logdir}" + fi log "All encryption mismatch tests completed successfully" return 0 @@ -461,11 +514,11 @@ function run_restore_wait { status_output=$("${local_build_dir}"/bin/fdbrestore status -t "${local_tag}" --dest-cluster-file "${local_scratch_dir}/loopback_cluster/fdb.cluster" --log --logdir="${local_scratch_dir}" 2>&1) set -e - if echo "${status_output}" | grep -qi "State:.*completed\|Phase:.*Complete\|No restore"; then + if output_matches_i "${status_output}" "State:.*completed\|Phase:.*Complete\|No restore"; then return 0 fi - if echo "${status_output}" | grep -qi "State:.*aborted"; then + if output_matches_i "${status_output}" "State:.*aborted"; then return 1 fi @@ -485,15 +538,18 @@ function run_restore_wait { function setup_backup_test_environment { local http_verbose_level="${1}" local additional_knobs=("${@:2}") - + # Clear proxy environment variables unset HTTP_PROXY unset HTTPS_PROXY - + # Set USE_S3 based on environment readonly USE_S3="${USE_S3:-$( if [[ -n "${OKTETO_NAMESPACE+x}" ]]; then echo "true" ; else echo "false"; fi )}" - - # Set KNOBS based on whether we're using real S3 or MockS3Server + + # Detect GCS from environment variables + detect_blobstore_provider + + # Set KNOBS based on which provider we're using if [[ "${USE_S3}" == "true" ]]; then # Use AWS KMS encryption for real S3 KNOBS=("--knob_blobstore_encryption_type=aws:kms" "--knob_http_verbose_level=${http_verbose_level}") @@ -501,13 +557,13 @@ function setup_backup_test_environment { # No encryption for MockS3Server KNOBS=("--knob_http_verbose_level=${http_verbose_level}") fi - + # Add any additional knobs (handle empty array when set -u is enabled) if [[ ${#additional_knobs[@]} -gt 0 ]]; then KNOBS+=("${additional_knobs[@]}") fi readonly KNOBS - + setup_tls_ca_file } diff --git a/fdbbackup/tests/blob_backup_restore_test.sh b/fdbbackup/tests/blob_backup_restore_test.sh new file mode 100755 index 00000000000..b2dbc3dc003 --- /dev/null +++ b/fdbbackup/tests/blob_backup_restore_test.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# +# Test backup and restore from blob storage (S3, GCS, Azure, or MockS3Server). +# +# In the below we start a small FDB cluster, populate it with +# some data and then start up MockS3Server or use S3/GCS/Azure +# if configured. We then run a backup to blob storage and then +# a restore. We verify the restore is the same as the original. +# +# Debugging: +# - Run with -x flag: bash -x blob_backup_restore_test.sh... +# - Preserve test data: PRESERVE_TEST_DATA=1 ./blob_backup_restore_test.sh ... +# This will leave all test data including MockS3 persistence files +# in the test scratch directory for analysis after the test completes. +# +# Usage: +# blob_backup_restore_test.sh [scratch_dir] [--encrypt] +# +# See https://apple.github.io/foundationdb/backups.html + +# Install signal traps. Depends on globals being set. +# Calls the cleanup function. +trap "exit 1" HUP INT PIPE QUIT TERM +trap cleanup EXIT + +# Cleanup. Called from signal trap. +# Has a hard 30-second timeout to prevent CTest timeouts. +function cleanup { + echo "$(date -Iseconds) cleanup: starting (with 30s hard timeout)" + start_cleanup_watchdog 30 + + # Check if test data should be preserved (common function from tests_common.sh) + if cleanup_with_preserve_check; then + echo "$(date -Iseconds) cleanup: preserving test data, skipping cleanup" + cancel_cleanup_watchdog + return 0 + fi + + echo "$(date -Iseconds) cleanup: shutting down FDB cluster" + if type shutdown_fdb_cluster &> /dev/null; then + shutdown_fdb_cluster + else + echo "$(date -Iseconds) cleanup: shutdown_fdb_cluster not available" + fi + + echo "$(date -Iseconds) cleanup: shutting down MockS3" + if type shutdown_mocks3 &> /dev/null; then + shutdown_mocks3 + else + echo "$(date -Iseconds) cleanup: shutdown_mocks3 not available" + fi + + echo "$(date -Iseconds) cleanup: shutting down AWS" + if type shutdown_aws &> /dev/null; then + shutdown_aws "${TEST_SCRATCH_DIR}" + else + echo "$(date -Iseconds) cleanup: shutdown_aws not available" + fi + + # Clean up encryption key file + if [[ -n "${ENCRYPTION_KEY_FILE:-}" ]] && [[ -f "${ENCRYPTION_KEY_FILE}" ]]; then + echo "$(date -Iseconds) cleanup: removing encryption key file: ${ENCRYPTION_KEY_FILE}" + rm -f "${ENCRYPTION_KEY_FILE}" + fi + + echo "$(date -Iseconds) cleanup: complete" + cancel_cleanup_watchdog +} + +# Resolve passed in reference to an absolute path. +# e.g. /tmp on mac is actually /private/tmp. +# $1 path to resolve +function resolve_to_absolute_path { + local p="${1}" + while [[ -h "${p}" ]]; do + dir=$( cd -P "$( dirname "${p}" )" >/dev/null 2>&1 && pwd ) + p=$(readlink "${p}") + [[ ${p} != /* ]] && p="${dir}/${p}" + done + realpath "${p}" +} + +# Run a backup to s3 and then a restore. +# $1 The url to use +# $2 the scratch directory +# $3 The credentials file. +# $4 build directory +# $5 encryption key file (optional) +function test_s3_backup_and_restore { + local local_url="${1}" + local local_scratch_dir="${2}" + local credentials="${3}" + local local_build_dir="${4}" + local local_encryption_key_file="${5:-}" + + # Edit the url. Backup adds 'data' to the path. Need this url for cleanup. + local edited_url=$(echo "${local_url}" | sed -e "s/ctest/data\/ctest/" ) + readonly edited_url + if ! s3_preclear_url "${local_build_dir}" "${local_scratch_dir}" "${edited_url}" "${credentials}"; then + return 1 + fi + log "Load data" + if ! load_data "${local_build_dir}" "${local_scratch_dir}"; then + err "Failed loading data into fdb" + return 1 + fi + log "Run blob storage backup" + if ! run_backup "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "" "${credentials}"; then + err "Failed backup" + return 1 + fi + + test_fdbcli_status_json_for_bkup "${local_build_dir}" "${local_scratch_dir}" + + log "Clear fdb data" + if ! clear_data "${local_build_dir}" "${local_scratch_dir}"; then + err "Failed clear data in fdb" + return 1 + fi + # Test encryption mismatches (always run to test both encrypted and unencrypted scenarios) + log "Testing encryption mismatches" + test_encryption_mismatches "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "${credentials}" + + log "Restore from blob storage" + if ! run_restore "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "" "${credentials}"; then + err "Failed restore" + return 1 + fi + log "Verify restore" + if ! verify_data "${local_build_dir}" "${local_scratch_dir}"; then + err "Failed verification of data in fdb" + return 1 + fi + + # Cleanup test data (skip if preserving test data for debugging). + if [[ "${PRESERVE_TEST_DATA:-0}" != "1" ]]; then + if ! s3_cleanup_url "${local_build_dir}" "${local_scratch_dir}" "${edited_url}" "${credentials}"; then + return 1 + fi + fi + log "Check for Severity=40 errors" + if ! grep_for_severity40 "${local_scratch_dir}"; then + err "Found Severity=40 errors in logs" + return 1 + fi +} + +# set -o xtrace # a.k.a set -x # Set this one when debugging (or 'bash -x THIS_SCRIPT'). +set -o errexit # a.k.a. set -e +set -o nounset # a.k.a. set -u +set -o pipefail +set -o noclobber + +# Parse command line arguments +USE_ENCRYPTION=$(((RANDOM % 2)) && echo true || echo false ) +USE_PARTITIONED_LOG=$(((RANDOM % 2)) && echo true || echo false ) + +# Set USE_ENCRYPTION_BLOCK_SIZE only if encryption is enabled +USE_ENCRYPTION_BLOCK_SIZE=false +if [[ "${USE_ENCRYPTION}" == "true" ]]; then + USE_ENCRYPTION_BLOCK_SIZE=$(((RANDOM % 2)) && echo true || echo false) +fi + +# Get the working directory for this script. +if ! path=$(resolve_to_absolute_path "${BASH_SOURCE[0]}"); then + echo "Failed resolve_to_absolute_path" >&2 + exit 1 +fi +if ! cwd=$( cd -P "$( dirname "${path}" )" >/dev/null 2>&1 && pwd ); then + echo "Failed dirname on ${path}" >&2 + exit 1 +fi +readonly cwd + +# Source common test functions first (needed for setup_backup_test_environment) +# shellcheck source=/dev/null +if ! source "${cwd}/../../fdbclient/tests/tests_common.sh"; then + echo "Failed to source tests_common.sh" >&2 + exit 1 +fi +# shellcheck source=/dev/null +if ! source "${cwd}/backup_tests_common.sh"; then + echo "Failed to source backup_tests_common.sh" >&2 + exit 1 +fi + +# Globals +TEST_SCRATCH_DIR= +readonly TAG="test_backup" + +# Setup common environment (USE_S3, KNOBS, TLS_CA_FILE, clears HTTP_PROXY/HTTPS_PROXY) +setup_backup_test_environment 10 +# Process command-line options. +if (( $# < 2 )) || (( $# > 3 )); then + echo "ERROR: ${0} requires the fdb src and build directories --" + echo "CMAKE_SOURCE_DIR and CMAKE_BINARY_DIR -- and then, optionally," + echo "a directory into which we write scratch test data and logs" + echo "(otherwise we will write to subdirs under $TMPDIR). We will" + echo "leave the download of seaweed this directory for other" + echo "tests to find if they need it. Otherwise, we clean everything" + echo "else up on our way out." + echo "Example: ${0} ./foundationdb ./build_output ./scratch_dir [--encrypt]" + exit 1 +fi +if ! source_dir=$(is_fdb_source_dir "${1}"); then + err "${1} is not an fdb source directory" + exit 1 +fi +readonly source_dir +readonly build_dir="${2}" +if [[ ! -d "${build_dir}" ]]; then + err "${build_dir} is not a directory" + exit 1 +fi +scratch_dir="${TMPDIR:-/tmp}" +if (( $# == 3 )); then + scratch_dir="${3}" +fi +readonly scratch_dir + +# Create encryption key file if needed +ENCRYPTION_KEY_FILE="" +if [[ "${USE_ENCRYPTION}" == "true" ]]; then + log "Enabling encryption for backups" + ENCRYPTION_KEY_FILE="${scratch_dir}/test_encryption_key_file" + create_encryption_key_file "${ENCRYPTION_KEY_FILE}" + log "Created encryption key file at ${ENCRYPTION_KEY_FILE}" +else + log "Using plaintext for backups" +fi +readonly ENCRYPTION_KEY_FILE +readonly USE_PARTITIONED_LOG +readonly USE_ENCRYPTION_BLOCK_SIZE + +# Setup S3/MockS3 environment using common function +readonly temp_dir_prefix="mocks3_backup_test" +readonly url_path_prefix="ctests" +setup_s3_environment "${build_dir}" "${scratch_dir}" "${temp_dir_prefix}" + +# Startup fdb cluster and backup agent +setup_fdb_cluster_with_backup "${source_dir}" "${build_dir}" "${TEST_SCRATCH_DIR}" 1 + +# Run tests. +test="test_s3_backup_and_restore" +url="blobstore://${host}/${url_path_prefix}/${test}?${query_str}" +test_s3_backup_and_restore "${url}" "${TEST_SCRATCH_DIR}" "${blob_credentials_file}" "${build_dir}" "${ENCRYPTION_KEY_FILE}" +log_test_result $? "test_s3_backup_and_restore" diff --git a/fdbbackup/tests/dir_backup_test.sh b/fdbbackup/tests/dir_backup_test.sh index 27523b56c53..a4e3b65f17f 100755 --- a/fdbbackup/tests/dir_backup_test.sh +++ b/fdbbackup/tests/dir_backup_test.sh @@ -19,6 +19,10 @@ SCRATCH_DIR= readonly TAG="test_backup" USE_PARTITIONED_LOG=$(((RANDOM % 2)) && echo true || echo false ) USE_ENCRYPTION=$(((RANDOM % 2)) && echo true || echo false ) +USE_ENCRYPTION_BLOCK_SIZE=false +if [[ "${USE_ENCRYPTION}" == "true" ]]; then + USE_ENCRYPTION_BLOCK_SIZE=$(((RANDOM % 2)) && echo true || echo false) +fi ENCRYPTION_KEY_FILE= # Install signal traps. Calls the cleanup function. @@ -30,21 +34,27 @@ trap cleanup EXIT function cleanup { echo "$(date -Iseconds) cleanup: starting (with 30s hard timeout)" start_cleanup_watchdog 30 - + + if cleanup_with_preserve_check; then + echo "$(date -Iseconds) cleanup: preserving test data, skipping cleanup" + cancel_cleanup_watchdog + return 0 + fi + echo "$(date -Iseconds) cleanup: shutting down FDB cluster" shutdown_fdb_cluster - + if [[ -d "${SCRATCH_DIR}" ]]; then echo "$(date -Iseconds) cleanup: removing scratch dir: ${SCRATCH_DIR}" rm -rf "${SCRATCH_DIR}" fi - + # Clean up encryption key file if [[ -n "${ENCRYPTION_KEY_FILE:-}" ]] && [[ -f "${ENCRYPTION_KEY_FILE}" ]]; then echo "$(date -Iseconds) cleanup: removing encryption key file: ${ENCRYPTION_KEY_FILE}" rm -f "${ENCRYPTION_KEY_FILE}" fi - + echo "$(date -Iseconds) cleanup: complete" cancel_cleanup_watchdog } @@ -82,8 +92,12 @@ function backup { cmd_args+=("--encryption-key-file" "${local_encryption_key_file}") fi + if [[ "${USE_ENCRYPTION_BLOCK_SIZE}" == "true" ]]; then + cmd_args+=("--encryption-block-size" "4096") + fi + if [[ "${USE_PARTITIONED_LOG}" == "true" ]]; then - cmd_args+=("--partitioned-log-experimental") + cmd_args+=("--mutation-log-type" "partitioned-log-experimental") fi if ! "${local_build_dir}"/bin/fdbbackup start "${cmd_args[@]}"; then @@ -157,6 +171,17 @@ function test_dir_backup_and_restore { err "Failed clear data in fdb" return 1 fi + + log "Testing encryption mismatches" + local backup + local backup_name + if ! backup=$(ls -dt "${scratch_dir}"/backups/backup-* | head -1); then + err "Failed to list backups under ${scratch_dir}/backups/" + return 1 + fi + backup_name=$(basename "${backup}") + test_encryption_mismatches "${local_build_dir}" "${scratch_dir}" "file://${scratch_dir}/backups/${backup_name}" "${TAG}" "${local_encryption_key_file}" + log "Restore" if ! restore "${local_build_dir}" "${scratch_dir}" "${local_encryption_key_file}"; then err "Failed restore" @@ -197,6 +222,11 @@ if ! source "${cwd}/../../fdbclient/tests/tests_common.sh"; then err "Failed to source tests_common.sh" exit 1 fi +# shellcheck source=/dev/null +if ! source "${cwd}/backup_tests_common.sh"; then + err "Failed to source backup_tests_common.sh" + exit 1 +fi # Process command-line options. if (( $# < 2 )) || (( $# > 3 )); then @@ -238,6 +268,7 @@ fi readonly USE_PARTITIONED_LOG readonly USE_ENCRYPTION +readonly USE_ENCRYPTION_BLOCK_SIZE readonly ENCRYPTION_KEY_FILE # Startup fdb cluster and backup agent. diff --git a/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh b/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh index f460e903eb3..334d200e070 100755 --- a/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh +++ b/fdbbackup/tests/s3_backup_bulkdump_bulkload.sh @@ -164,7 +164,7 @@ function run_validate_restore_audit { break fi - if echo "${audit_output}" | grep -qE "1221|1230|1010"; then + if output_matches_E "${audit_output}" "1221|1230|1010"; then log "Transient error detected, retrying in ${retry_delay}s..." sleep $retry_delay continue @@ -193,12 +193,12 @@ function run_validate_restore_audit { local status_output status_output=$("${fdbcli}" -C "${cluster_file}" --exec "get_audit_status validate_restore id ${audit_id}" 2>&1) - if echo "${status_output}" | grep -q "Phase.*2"; then + if output_matches "${status_output}" "Phase.*2"; then log "Audit completed successfully after ${elapsed}s" return 0 fi - if echo "${status_output}" | grep -q "Phase.*[34]"; then + if output_matches "${status_output}" "Phase.*[34]"; then err "Audit failed with status: ${status_output}" return 1 fi diff --git a/fdbbackup/tests/s3_backup_test.sh b/fdbbackup/tests/s3_backup_test.sh deleted file mode 100755 index 2664c55c0d5..00000000000 --- a/fdbbackup/tests/s3_backup_test.sh +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env bash -# -# Test backup and restore from s3. -# -# In the below we start a small FDB cluster, populate it with -# some data and then start up MockS3Server or use S3 -# if it is available. We then run a backup to 'S3' and then -# a restore. We verify the restore is the same as the original. -# -# Debugging: -# - Run with -x flag: bash -x s3_backup_test.sh... -# - Preserve test data: PRESERVE_TEST_DATA=1 ./s3_backup_test.sh ... -# This will leave all test data including MockS3 persistence files -# in the test scratch directory for analysis after the test completes. -# -# Usage: -# s3_backup_unified.sh [scratch_dir] [--encrypt] -# -# See https://apple.github.io/foundationdb/backups.html - -# Install signal traps. Depends on globals being set. -# Calls the cleanup function. -trap "exit 1" HUP INT PIPE QUIT TERM -trap cleanup EXIT - -# Cleanup. Called from signal trap. -# Has a hard 30-second timeout to prevent CTest timeouts. -function cleanup { - echo "$(date -Iseconds) cleanup: starting (with 30s hard timeout)" - start_cleanup_watchdog 30 - - # Check if test data should be preserved (common function from tests_common.sh) - if cleanup_with_preserve_check; then - echo "$(date -Iseconds) cleanup: preserving test data, skipping cleanup" - cancel_cleanup_watchdog - return 0 - fi - - echo "$(date -Iseconds) cleanup: shutting down FDB cluster" - if type shutdown_fdb_cluster &> /dev/null; then - shutdown_fdb_cluster - else - echo "$(date -Iseconds) cleanup: shutdown_fdb_cluster not available" - fi - - echo "$(date -Iseconds) cleanup: shutting down MockS3" - if type shutdown_mocks3 &> /dev/null; then - shutdown_mocks3 - else - echo "$(date -Iseconds) cleanup: shutdown_mocks3 not available" - fi - - echo "$(date -Iseconds) cleanup: shutting down AWS" - if type shutdown_aws &> /dev/null; then - shutdown_aws "${TEST_SCRATCH_DIR}" - else - echo "$(date -Iseconds) cleanup: shutdown_aws not available" - fi - - # Clean up encryption key file - if [[ -n "${ENCRYPTION_KEY_FILE:-}" ]] && [[ -f "${ENCRYPTION_KEY_FILE}" ]]; then - echo "$(date -Iseconds) cleanup: removing encryption key file: ${ENCRYPTION_KEY_FILE}" - rm -f "${ENCRYPTION_KEY_FILE}" - fi - - echo "$(date -Iseconds) cleanup: complete" - cancel_cleanup_watchdog -} - -# Resolve passed in reference to an absolute path. -# e.g. /tmp on mac is actually /private/tmp. -# $1 path to resolve -function resolve_to_absolute_path { - local p="${1}" - while [[ -h "${p}" ]]; do - dir=$( cd -P "$( dirname "${p}" )" >/dev/null 2>&1 && pwd ) - p=$(readlink "${p}") - [[ ${p} != /* ]] && p="${dir}/${p}" - done - realpath "${p}" -} - -# Run a backup to s3 and then a restore. -# $1 The url to use -# $2 the scratch directory -# $3 The credentials file. -# $4 build directory -# $5 encryption key file (optional) -function test_s3_backup_and_restore { - local local_url="${1}" - local local_scratch_dir="${2}" - local credentials="${3}" - local local_build_dir="${4}" - local local_encryption_key_file="${5:-}" - - # Edit the url. Backup adds 'data' to the path. Need this url for cleanup. - local edited_url=$(echo "${local_url}" | sed -e "s/ctest/data\/ctest/" ) - readonly edited_url - if ! s3_preclear_url "${local_build_dir}" "${local_scratch_dir}" "${edited_url}" "${credentials}"; then - return 1 - fi - log "Load data" - if ! load_data "${local_build_dir}" "${local_scratch_dir}"; then - err "Failed loading data into fdb" - return 1 - fi - log "Run s3 backup" - if ! run_backup "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "" "${credentials}"; then - err "Failed backup" - return 1 - fi - - test_fdbcli_status_json_for_bkup "${local_build_dir}" "${local_scratch_dir}" - - log "Clear fdb data" - if ! clear_data "${local_build_dir}" "${local_scratch_dir}"; then - err "Failed clear data in fdb" - return 1 - fi - # Test encryption mismatches (always run to test both encrypted and unencrypted scenarios) - log "Testing encryption mismatches" - test_encryption_mismatches "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "${credentials}" - - log "Restore from s3" - if ! run_restore "${local_build_dir}" "${local_scratch_dir}" "${local_url}" "${TAG}" "${local_encryption_key_file}" "" "${credentials}"; then - err "Failed restore" - return 1 - fi - log "Verify restore" - if ! verify_data "${local_build_dir}" "${local_scratch_dir}"; then - err "Failed verification of data in fdb" - return 1 - fi - - # Cleanup test data. - if ! s3_cleanup_url "${local_build_dir}" "${local_scratch_dir}" "${edited_url}" "${credentials}"; then - return 1 - fi - log "Check for Severity=40 errors" - if ! grep_for_severity40 "${local_scratch_dir}"; then - err "Found Severity=40 errors in logs" - return 1 - fi -} - -# set -o xtrace # a.k.a set -x # Set this one when debugging (or 'bash -x THIS_SCRIPT'). -set -o errexit # a.k.a. set -e -set -o nounset # a.k.a. set -u -set -o pipefail -set -o noclobber - -# Parse command line arguments -USE_ENCRYPTION=false -USE_PARTITIONED_LOG=$(((RANDOM % 2)) && echo true || echo false ) -PARAMS=() - -while (( "$#" )); do - case "$1" in - --encrypt) - USE_ENCRYPTION=true - shift - ;; - --encrypt-at-random) - USE_ENCRYPTION=$(((RANDOM % 2)) && echo true || echo false ) - shift - ;; - --partitioned-log-experimental) - USE_PARTITIONED_LOG=true - shift - ;; - --partitioned-log-experimental-at-random) - USE_PARTITIONED_LOG=$(((RANDOM % 2)) && echo true || echo false ) - shift - ;; - -*|--*=) # unsupported flags - err "Error: Unsupported flag $1" >&2 - exit 1 - ;; - *) # preserve positional arguments - PARAMS+=("$1") - shift - ;; - esac -done - -# Set positional arguments in their proper place -if [ ${#PARAMS[@]} -ne 0 ]; then - set -- "${PARAMS[@]}" -fi - -# Get the working directory for this script. -if ! path=$(resolve_to_absolute_path "${BASH_SOURCE[0]}"); then - echo "Failed resolve_to_absolute_path" >&2 - exit 1 -fi -if ! cwd=$( cd -P "$( dirname "${path}" )" >/dev/null 2>&1 && pwd ); then - echo "Failed dirname on ${path}" >&2 - exit 1 -fi -readonly cwd - -# Source common test functions first (needed for setup_backup_test_environment) -# shellcheck source=/dev/null -if ! source "${cwd}/../../fdbclient/tests/tests_common.sh"; then - echo "Failed to source tests_common.sh" >&2 - exit 1 -fi -# shellcheck source=/dev/null -if ! source "${cwd}/backup_tests_common.sh"; then - echo "Failed to source backup_tests_common.sh" >&2 - exit 1 -fi - -# Globals -TEST_SCRATCH_DIR= -readonly TAG="test_backup" - -# Setup common environment (USE_S3, KNOBS, TLS_CA_FILE, clears HTTP_PROXY/HTTPS_PROXY) -setup_backup_test_environment 10 -# Process command-line options. -if (( $# < 2 )) || (( $# > 3 )); then - echo "ERROR: ${0} requires the fdb src and build directories --" - echo "CMAKE_SOURCE_DIR and CMAKE_BINARY_DIR -- and then, optionally," - echo "a directory into which we write scratch test data and logs" - echo "(otherwise we will write to subdirs under $TMPDIR). We will" - echo "leave the download of seaweed this directory for other" - echo "tests to find if they need it. Otherwise, we clean everything" - echo "else up on our way out." - echo "Example: ${0} ./foundationdb ./build_output ./scratch_dir [--encrypt]" - exit 1 -fi -if ! source_dir=$(is_fdb_source_dir "${1}"); then - err "${1} is not an fdb source directory" - exit 1 -fi -readonly source_dir -readonly build_dir="${2}" -if [[ ! -d "${build_dir}" ]]; then - err "${build_dir} is not a directory" - exit 1 -fi -scratch_dir="${TMPDIR:-/tmp}" -if (( $# == 3 )); then - scratch_dir="${3}" -fi -readonly scratch_dir - -# Create encryption key file if needed -ENCRYPTION_KEY_FILE="" -if [[ "${USE_ENCRYPTION}" == "true" ]]; then - log "Enabling encryption for backups" - ENCRYPTION_KEY_FILE="${scratch_dir}/test_encryption_key_file" - create_encryption_key_file "${ENCRYPTION_KEY_FILE}" - log "Created encryption key file at ${ENCRYPTION_KEY_FILE}" -else - log "Using plaintext for backups" -fi -readonly ENCRYPTION_KEY_FILE -readonly USE_PARTITIONED_LOG - -# Setup S3/MockS3 environment using common function -readonly temp_dir_prefix="mocks3_backup_test" -readonly url_path_prefix="ctests" -setup_s3_environment "${build_dir}" "${scratch_dir}" "${temp_dir_prefix}" - -# Startup fdb cluster and backup agent -setup_fdb_cluster_with_backup "${source_dir}" "${build_dir}" "${TEST_SCRATCH_DIR}" 1 - -# Run tests. -test="test_s3_backup_and_restore" -url="blobstore://${host}/${url_path_prefix}/${test}?${query_str}" -test_s3_backup_and_restore "${url}" "${TEST_SCRATCH_DIR}" "${blob_credentials_file}" "${build_dir}" "${ENCRYPTION_KEY_FILE}" -log_test_result $? "test_s3_backup_and_restore" diff --git a/fdbcli/AdvanceVersionCommand.actor.cpp b/fdbcli/AdvanceVersionCommand.actor.cpp deleted file mode 100644 index 656f06daeb6..00000000000 --- a/fdbcli/AdvanceVersionCommand.actor.cpp +++ /dev/null @@ -1,75 +0,0 @@ -/* - * AdvanceVersionCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/lexical_cast.hpp" -#include "fmt/format.h" -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/IClientApi.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -const KeyRef advanceVersionSpecialKey = "\xff\xff/management/min_required_commit_version"_sr; - -ACTOR Future advanceVersionCommandActor(Reference db, std::vector tokens) { - if (tokens.size() != 2) { - printUsage(tokens[0]); - return false; - } else { - state Version v; - int n = 0; - if (sscanf(tokens[1].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[1].size()) { - printUsage(tokens[0]); - return false; - } else { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - Version rv = wait(safeThreadFutureToFuture(tr->getReadVersion())); - if (rv <= v) { - tr->set(advanceVersionSpecialKey, boost::lexical_cast(v)); - wait(safeThreadFutureToFuture(tr->commit())); - } else { - fmt::print("Current read version is {}\n", rv); - return true; - } - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } - } - } -} - -CommandFactory advanceVersionFactory( - "advanceversion", - CommandHelp( - "advanceversion ", - "Force the cluster to recover at the specified version", - "Forces the cluster to recover at the specified version. If the specified version is larger than the current " - "version of the cluster, the cluster version is advanced " - "to the specified version via a forced recovery.")); -} // namespace fdb_cli diff --git a/fdbcli/AdvanceVersionCommand.cpp b/fdbcli/AdvanceVersionCommand.cpp new file mode 100644 index 00000000000..7be9acc3916 --- /dev/null +++ b/fdbcli/AdvanceVersionCommand.cpp @@ -0,0 +1,77 @@ +/* + * AdvanceVersionCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/lexical_cast.hpp" +#include "fmt/format.h" +#include "fdbcli/fdbcli.h" + +#include "fdbclient/IClientApi.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace fdb_cli { + +const KeyRef advanceVersionSpecialKey = "\xff\xff/management/min_required_commit_version"_sr; + +Future advanceVersionCommandActor(Reference db, std::vector tokens) { + if (tokens.size() != 2) { + printUsage(tokens[0]); + co_return false; + } else { + Version v; + int n = 0; + if (sscanf(tokens[1].toString().c_str(), "%" PRId64 "%n", &v, &n) != 1 || n != tokens[1].size()) { + printUsage(tokens[0]); + co_return false; + } else { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + Version rv = co_await safeThreadFutureToFuture(tr->getReadVersion()); + if (rv <= v) { + tr->set(advanceVersionSpecialKey, boost::lexical_cast(v)); + co_await safeThreadFutureToFuture(tr->commit()); + continue; + } else { + fmt::print("Current read version is {}\n", rv); + co_return true; + } + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } + } + } +} + +CommandFactory advanceVersionFactory( + "advanceversion", + CommandHelp( + "advanceversion ", + "Force the cluster to recover at the specified version", + "Forces the cluster to recover at the specified version. If the specified version is larger than the current " + "version of the cluster, the cluster version is advanced " + "to the specified version via a forced recovery.")); +} // namespace fdb_cli diff --git a/fdbcli/AuditStorageCommand.actor.cpp b/fdbcli/AuditStorageCommand.actor.cpp deleted file mode 100644 index 3d63a58912c..00000000000 --- a/fdbcli/AuditStorageCommand.actor.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/* - * AuditStorageCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * ============================================================================ - * AUDIT STORAGE COMMANDS - * ============================================================================ - * - * This file implements CLI commands for various storage audit operations: - * - audit_storage ha : Validate high availability - * - audit_storage replica : Validate replica consistency - * - audit_storage locationmetadata : Validate location metadata - * - audit_storage ssshard : Validate storage server shards - * - audit_storage validate_restore : Validate restored backup data - * - * ============================================================================ - * RESTORE VALIDATION (validate_restore) - Quick Reference - * ============================================================================ - * - * USAGE: audit_storage validate_restore - * - * Validates that restored backup data matches original source data. - * - * EXAMPLE WORKFLOW: - * 1. Backup: fdbbackup start -d file:///backup -z - * 2. Stop: fdbbackup discontinue -C - * 3. Lock: fdb> lock (save the returned UID) - * 4. Restore: fdbrestore start -r file:///backup --add-prefix "\xff\x02/rlog/" - * 5. Validate: fdb> audit_storage validate_restore "" "\xff" - * 6. Check: fdb> get_audit_status validate_restore id - * 7. Unlock: fdb> unlock - * 8. Cleanup: fdb> clearrange "\xff\x02/rlog/" "\xff\x02/rlog0" - * - * NOTE: Steps 2-3 (stop backup and lock) prevent writes during validation, avoiding - * false positive audit errors. The --add-prefix parameter in step 4 allows the restore - * to run on a non-empty database, enabling validation by comparing restored data - * against the source. Both restore and audit use LOCK_AWARE transactions, so they - * work on a locked database. - * - * See fdbserver/storageserver.actor.cpp for detailed implementation docs. - * ============================================================================ - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/IClientApi.h" - -#include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/Audit.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future auditStorageCommandActor(Reference clusterFile, - std::vector tokens) { - if (tokens.size() < 2) { - printUsage(tokens[0]); - return UID(); - } - - state UID resAuditId; - if (tokencmp(tokens[1], "cancel")) { - if (tokens.size() != 4) { - printUsage(tokens[0]); - return UID(); - } - AuditType type = AuditType::Invalid; - if (tokencmp(tokens[2], "ha")) { - type = AuditType::ValidateHA; - } else if (tokencmp(tokens[2], "replica")) { - type = AuditType::ValidateReplica; - } else if (tokencmp(tokens[2], "locationmetadata")) { - type = AuditType::ValidateLocationMetadata; - } else if (tokencmp(tokens[2], "ssshard")) { - type = AuditType::ValidateStorageServerShard; - } else if (tokencmp(tokens[2], "validate_restore")) { - type = AuditType::ValidateRestore; - } else { - printUsage(tokens[0]); - return UID(); - } - const UID auditId = UID::fromString(tokens[3].toString()); - UID cancelledAuditId = wait(cancelAuditStorage(clusterFile, type, auditId, /*timeoutSeconds=*/60)); - resAuditId = cancelledAuditId; - - } else { - AuditType type = AuditType::Invalid; - if (tokencmp(tokens[1], "ha")) { - type = AuditType::ValidateHA; - } else if (tokencmp(tokens[1], "replica")) { - type = AuditType::ValidateReplica; - } else if (tokencmp(tokens[1], "locationmetadata")) { - type = AuditType::ValidateLocationMetadata; - } else if (tokencmp(tokens[1], "ssshard")) { - type = AuditType::ValidateStorageServerShard; - } else if (tokencmp(tokens[1], "validate_restore")) { - type = AuditType::ValidateRestore; - } else { - printUsage(tokens[0]); - return UID(); - } - - Key begin = allKeys.begin, end = allKeys.end; - if (tokens.size() == 3) { - begin = tokens[2]; - } else if (tokens.size() == 4 || tokens.size() == 5) { - begin = tokens[2]; - end = tokens[3]; - } else { - printUsage(tokens[0]); - return UID(); - } - if (end > allKeys.end) { - printUsage(tokens[0]); - return UID(); - } - if (begin >= end) { - printUsage(tokens[0]); - return UID(); - } - - KeyValueStoreType engineType = KeyValueStoreType::END; - if (tokens.size() == 5 && (type == AuditType::ValidateHA || type == AuditType::ValidateReplica)) { - engineType = KeyValueStoreType::fromString(tokens[4].toString()); - if (engineType != KeyValueStoreType::SSD_BTREE_V2 && engineType != KeyValueStoreType::SSD_ROCKSDB_V1 && - engineType != KeyValueStoreType::SSD_SHARDED_ROCKSDB) { - printUsage(tokens[0]); - return UID(); - } - } - // For KeyValueStoreType::END: do not specify any storage engine - // Every storage engine will be audited - UID startedAuditId = - wait(auditStorage(clusterFile, KeyRangeRef(begin, end), type, engineType, /*timeoutSeconds=*/60)); - resAuditId = startedAuditId; - } - return resAuditId; -} - -CommandFactory auditStorageFactory( - "audit_storage", - CommandHelp("audit_storage [BeginKey EndKey] ", - "Start an audit storage", - "Specify audit `Type' (only `ha' and `replica' and `locationmetadata' and " - "`ssshard' and `validate_restore' `Type' are supported currently), and\n" - "optionally a sub-range with `BeginKey' and `EndKey'.\n" - "Specify audit `EngineType' when auditType is `ha' or `replica'\n" - "(only `ssd-rocksdb-v1' and `ssd-sharded-rocksdb' and `ssd-2' are supported).\n" - "If no EngineType is specified, every storage engine will be audited.\n" - "For example, to audit the full key range: `audit_storage ha'\n" - "To audit a sub-range only: `audit_storage ha \\xa \\xb'\n" - "Returns an audit `ID'. See also `get_audit_status' command.\n" - "Note that BeginKey should not equal to EndKey and EndKey is at most \\xff.\n" - "To cancel an audit: audit_storage cancel [ID]")); -} // namespace fdb_cli diff --git a/fdbcli/AuditStorageCommand.cpp b/fdbcli/AuditStorageCommand.cpp new file mode 100644 index 00000000000..b2f18ebc374 --- /dev/null +++ b/fdbcli/AuditStorageCommand.cpp @@ -0,0 +1,174 @@ +/* + * AuditStorageCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * ============================================================================ + * AUDIT STORAGE COMMANDS + * ============================================================================ + * + * This file implements CLI commands for various storage audit operations: + * - audit_storage ha : Validate high availability + * - audit_storage replica : Validate replica consistency + * - audit_storage locationmetadata : Validate location metadata + * - audit_storage ssshard : Validate storage server shards + * - audit_storage validate_restore : Validate restored backup data + * + * ============================================================================ + * RESTORE VALIDATION (validate_restore) - Quick Reference + * ============================================================================ + * + * USAGE: audit_storage validate_restore + * + * Validates that restored backup data matches original source data. + * + * EXAMPLE WORKFLOW: + * 1. Backup: fdbbackup start -d file:///backup -z + * 2. Stop: fdbbackup discontinue -C + * 3. Lock: fdb> lock (save the returned UID) + * 4. Restore: fdbrestore start -r file:///backup --add-prefix "\xff\x02/rlog/" + * 5. Validate: fdb> audit_storage validate_restore "" "\xff" + * 6. Check: fdb> get_audit_status validate_restore id + * 7. Unlock: fdb> unlock + * 8. Cleanup: fdb> clearrange "\xff\x02/rlog/" "\xff\x02/rlog0" + * + * NOTE: Steps 2-3 (stop backup and lock) prevent writes during validation, avoiding + * false positive audit errors. The --add-prefix parameter in step 4 allows the restore + * to run on a non-empty database, enabling validation by comparing restored data + * against the source. Both restore and audit use LOCK_AWARE transactions, so they + * work on a locked database. + * + * See fdbserver/storageserver.actor.cpp for detailed implementation docs. + * ============================================================================ + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/IClientApi.h" + +#include "fdbclient/ManagementAPI.h" +#include "fdbclient/Audit.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace fdb_cli { + +Future auditStorageCommandActor(Reference clusterFile, std::vector tokens) { + if (tokens.size() < 2) { + printUsage(tokens[0]); + co_return UID(); + } + + UID resAuditId; + if (tokencmp(tokens[1], "cancel")) { + if (tokens.size() != 4) { + printUsage(tokens[0]); + co_return UID(); + } + AuditType type = AuditType::Invalid; + if (tokencmp(tokens[2], "ha")) { + type = AuditType::ValidateHA; + } else if (tokencmp(tokens[2], "replica")) { + type = AuditType::ValidateReplica; + } else if (tokencmp(tokens[2], "locationmetadata")) { + type = AuditType::ValidateLocationMetadata; + } else if (tokencmp(tokens[2], "ssshard")) { + type = AuditType::ValidateStorageServerShard; + } else if (tokencmp(tokens[2], "validate_restore")) { + type = AuditType::ValidateRestore; + } else { + printUsage(tokens[0]); + co_return UID(); + } + const UID auditId = UID::fromString(tokens[3].toString()); + UID cancelledAuditId = co_await cancelAuditStorage(clusterFile, type, auditId, /*timeoutSeconds=*/60); + resAuditId = cancelledAuditId; + + } else { + AuditType type = AuditType::Invalid; + if (tokencmp(tokens[1], "ha")) { + type = AuditType::ValidateHA; + } else if (tokencmp(tokens[1], "replica")) { + type = AuditType::ValidateReplica; + } else if (tokencmp(tokens[1], "locationmetadata")) { + type = AuditType::ValidateLocationMetadata; + } else if (tokencmp(tokens[1], "ssshard")) { + type = AuditType::ValidateStorageServerShard; + } else if (tokencmp(tokens[1], "validate_restore")) { + type = AuditType::ValidateRestore; + } else { + printUsage(tokens[0]); + co_return UID(); + } + + Key begin = allKeys.begin, end = allKeys.end; + if (tokens.size() == 3) { + begin = tokens[2]; + } else if (tokens.size() == 4 || tokens.size() == 5) { + begin = tokens[2]; + end = tokens[3]; + } else { + printUsage(tokens[0]); + co_return UID(); + } + if (end > allKeys.end) { + printUsage(tokens[0]); + co_return UID(); + } + if (begin >= end) { + printUsage(tokens[0]); + co_return UID(); + } + + KeyValueStoreType engineType = KeyValueStoreType::END; + if (tokens.size() == 5 && (type == AuditType::ValidateHA || type == AuditType::ValidateReplica)) { + engineType = KeyValueStoreType::fromString(tokens[4].toString()); + if (engineType != KeyValueStoreType::SSD_BTREE_V2 && engineType != KeyValueStoreType::SSD_ROCKSDB_V1 && + engineType != KeyValueStoreType::SSD_SHARDED_ROCKSDB) { + printUsage(tokens[0]); + co_return UID(); + } + } + // For KeyValueStoreType::END: do not specify any storage engine + // Every storage engine will be audited + UID startedAuditId = + co_await auditStorage(clusterFile, KeyRangeRef(begin, end), type, engineType, /*timeoutSeconds=*/60); + resAuditId = startedAuditId; + } + co_return resAuditId; +} + +CommandFactory auditStorageFactory( + "audit_storage", + CommandHelp("audit_storage [BeginKey EndKey] ", + "Start an audit storage", + "Specify audit `Type' (only `ha' and `replica' and `locationmetadata' and " + "`ssshard' and `validate_restore' `Type' are supported currently), and\n" + "optionally a sub-range with `BeginKey' and `EndKey'.\n" + "Specify audit `EngineType' when auditType is `ha' or `replica'\n" + "(only `ssd-rocksdb-v1' and `ssd-sharded-rocksdb' and `ssd-2' are supported).\n" + "If no EngineType is specified, every storage engine will be audited.\n" + "For example, to audit the full key range: `audit_storage ha'\n" + "To audit a sub-range only: `audit_storage ha \\xa \\xb'\n" + "Returns an audit `ID'. See also `get_audit_status' command.\n" + "Note that BeginKey should not equal to EndKey and EndKey is at most \\xff.\n" + "To cancel an audit: audit_storage cancel [ID]")); +} // namespace fdb_cli diff --git a/fdbcli/BulkDumpCommand.actor.cpp b/fdbcli/BulkDumpCommand.actor.cpp deleted file mode 100644 index fa95a1e1ea1..00000000000 --- a/fdbcli/BulkDumpCommand.actor.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * BulkDumpCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/BulkDumping.h" -#include "fdbclient/BulkLoading.h" -#include "fdbclient/ManagementAPI.actor.h" -#include "flow/Arena.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -static const char* BULK_DUMP_MODE_USAGE = "To set bulkdump mode: bulkdump mode [on|off]\n"; -static const char* BULK_DUMP_DUMP_USAGE = "To dump a range of key/values: bulkdump dump

\n" - " where to denotes the key/value range and is\n" - " a local directory OR blobstore url to dump SST files to.\n"; -static const char* BULK_DUMP_STATUS_USAGE = "To get status: bulkdump status\n"; -static const char* BULK_DUMP_CANCEL_USAGE = "To cancel current bulkdump job: bulkdump cancel \n"; - -static const std::string BULK_DUMP_HELP_MESSAGE = - std::string(BULK_DUMP_MODE_USAGE) + std::string(BULK_DUMP_DUMP_USAGE) + std::string(BULK_DUMP_STATUS_USAGE) + - std::string(BULK_DUMP_CANCEL_USAGE); - -ACTOR Future getOngoingBulkDumpJob(Database cx) { - state Transaction tr(cx); - loop { - try { - Optional job = wait(getSubmittedBulkDumpJob(&tr)); - if (job.present()) { - fmt::println("Running bulk dumping job: {}", job.get().getJobId().toString()); - return true; - } else { - fmt::println("No bulk dumping job is running"); - return false; - } - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - -ACTOR Future getBulkDumpCompleteRanges(Database cx, KeyRange rangeToRead) { - try { - size_t finishCount = wait(getBulkDumpCompleteTaskCount(cx, rangeToRead)); - fmt::println("Finished {} tasks", finishCount); - } catch (Error& e) { - if (e.code() == error_code_timed_out) { - fmt::println("timed out"); - } - } - return Void(); -} - -ACTOR Future bulkDumpCommandActor(Database cx, std::vector tokens) { - state BulkDumpState bulkDumpJob; - if (tokencmp(tokens[1], "mode")) { - if (tokens.size() != 2 && tokens.size() != 3) { - fmt::println("{}", BULK_DUMP_MODE_USAGE); - return UID(); - } - if (tokens.size() == 2) { - int mode = wait(getBulkDumpMode(cx)); - if (mode == 0) { - fmt::println("Bulkdump mode is disabled"); - } else if (mode == 1) { - fmt::println("Bulkdump mode is enabled"); - } else { - fmt::println("Invalid bulkload mode value {}", mode); - } - return UID(); - } - ASSERT(tokens.size() == 3); - if (tokencmp(tokens[2], "on")) { - int old = wait(setBulkDumpMode(cx, 1)); - TraceEvent("SetBulkDumpModeCommand").detail("OldValue", old).detail("NewValue", 1); - return UID(); - } else if (tokencmp(tokens[2], "off")) { - int old = wait(setBulkDumpMode(cx, 0)); - TraceEvent("SetBulkDumpModeCommand").detail("OldValue", old).detail("NewValue", 0); - return UID(); - } else { - fmt::println("ERROR: Invalid bulkdump mode value {}", tokens[2].toString()); - fmt::println("{}", BULK_DUMP_MODE_USAGE); - return UID(); - } - - } else if (tokencmp(tokens[1], "dump")) { - int mode = wait(getBulkDumpMode(cx)); - if (mode == 0) { - fmt::println("ERROR: Bulkdump mode must be enabled to dump data"); - return UID(); - } - if (tokens.size() != 5) { - fmt::println("{}", BULK_DUMP_DUMP_USAGE); - return UID(); - } - Key rangeBegin = tokens[2]; - Key rangeEnd = tokens[3]; - // Bulk load can only inject data to normal key space, aka "" ~ \xff - if (rangeBegin >= rangeEnd || rangeEnd > normalKeys.end) { - fmt::println( - "ERROR: Invalid range: {} to {}, normal key space only", rangeBegin.toString(), rangeEnd.toString()); - fmt::println("{}", BULK_DUMP_DUMP_USAGE); - return UID(); - } - KeyRange range = Standalone(KeyRangeRef(rangeBegin, rangeEnd)); - std::string jobRoot = tokens[4].toString(); - bulkDumpJob = createBulkDumpJob(range, - jobRoot, - BulkLoadType::SST, - jobRoot.find("blobstore://") == 0 ? BulkLoadTransportMethod::BLOBSTORE - : BulkLoadTransportMethod::CP); - wait(submitBulkDumpJob(cx, bulkDumpJob)); - return bulkDumpJob.getJobId(); - } else if (tokencmp(tokens[1], "cancel")) { - if (tokens.size() != 3) { - fmt::println("{}", BULK_DUMP_CANCEL_USAGE); - return UID(); - } - state UID jobId = UID::fromString(tokens[2].toString()); - wait(cancelBulkDumpJob(cx, jobId)); - fmt::println("Job {} has been cancelled. No new tasks will be spawned.", jobId.toString()); - return UID(); - - } else if (tokencmp(tokens[1], "status")) { - if (tokens.size() != 2) { - fmt::println("{}", BULK_DUMP_STATUS_USAGE); - return UID(); - } - bool anyJob = wait(getOngoingBulkDumpJob(cx)); - if (!anyJob) { - return UID(); - } - KeyRange range = Standalone(KeyRangeRef(normalKeys.begin, normalKeys.end)); - wait(getBulkDumpCompleteRanges(cx, range)); - return UID(); - - } else { - printUsage(tokens[0]); - printLongDesc(tokens[0]); - return UID(); - } -} - -CommandFactory bulkDumpFactory("bulkdump", - CommandHelp("bulkdump [mode|dump|status|cancel] [ARGs]", - "bulkdump commands", - BULK_DUMP_HELP_MESSAGE.c_str())); -} // namespace fdb_cli diff --git a/fdbcli/BulkDumpCommand.cpp b/fdbcli/BulkDumpCommand.cpp new file mode 100644 index 00000000000..3776a7394ad --- /dev/null +++ b/fdbcli/BulkDumpCommand.cpp @@ -0,0 +1,218 @@ +/* + * BulkDumpCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "fdbcli/fdbcli.h" +#include "fdbclient/BulkDumping.h" +#include "fdbclient/BulkLoading.h" +#include "fdbclient/ManagementAPI.h" +#include "flow/Arena.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/Util.h" + +namespace fdb_cli { + +static const char* BULK_DUMP_MODE_USAGE = "To set bulkdump mode: bulkdump mode [on|off]\n"; +static const char* BULK_DUMP_DUMP_USAGE = "To dump a range of key/values: bulkdump dump \n" + " where to denotes the key/value range and is\n" + " a local directory OR blobstore url to dump SST files to.\n"; +static const char* BULK_DUMP_STATUS_USAGE = "To get status: bulkdump status\n"; +static const char* BULK_DUMP_CANCEL_USAGE = "To cancel current bulkdump job: bulkdump cancel \n"; + +static const std::string BULK_DUMP_HELP_MESSAGE = + std::string(BULK_DUMP_MODE_USAGE) + std::string(BULK_DUMP_DUMP_USAGE) + std::string(BULK_DUMP_STATUS_USAGE) + + std::string(BULK_DUMP_CANCEL_USAGE); + +Future getOngoingBulkDumpJob(Database cx) { + Transaction tr(cx); + while (true) { + Error err; + try { + Optional job = co_await getSubmittedBulkDumpJob(&tr); + if (job.present()) { + fmt::println("Running bulk dumping job: {}", job.get().getJobId().toString()); + co_return true; + } else { + fmt::println("No bulk dumping job is running"); + co_return false; + } + } catch (Error& e) { + err = e; + } + co_await tr.onError(err); + } +} + +Future getBulkDumpCompleteRanges(Database cx, KeyRange rangeToRead) { + try { + size_t finishCount = co_await getBulkDumpCompleteTaskCount(cx, rangeToRead); + fmt::println("Finished {} tasks", finishCount); + } catch (Error& e) { + if (e.code() == error_code_timed_out) { + fmt::println("timed out"); + } + } +} + +Future bulkDumpCommandActor(Database cx, std::vector tokens) { + BulkDumpState bulkDumpJob; + if (tokencmp(tokens[1], "mode")) { + if (tokens.size() != 2 && tokens.size() != 3) { + fmt::println("{}", BULK_DUMP_MODE_USAGE); + co_return UID(); + } + if (tokens.size() == 2) { + int mode = co_await getBulkDumpMode(cx); + if (mode == 0) { + fmt::println("Bulkdump mode is disabled"); + } else if (mode == 1) { + fmt::println("Bulkdump mode is enabled"); + } else { + fmt::println("Invalid bulkload mode value {}", mode); + } + co_return UID(); + } + ASSERT(tokens.size() == 3); + if (tokencmp(tokens[2], "on")) { + int old = co_await setBulkDumpMode(cx, 1); + TraceEvent("SetBulkDumpModeCommand").detail("OldValue", old).detail("NewValue", 1); + co_return UID(); + } else if (tokencmp(tokens[2], "off")) { + int old = co_await setBulkDumpMode(cx, 0); + TraceEvent("SetBulkDumpModeCommand").detail("OldValue", old).detail("NewValue", 0); + co_return UID(); + } else { + fmt::println("ERROR: Invalid bulkdump mode value {}", tokens[2].toString()); + fmt::println("{}", BULK_DUMP_MODE_USAGE); + co_return UID(); + } + + } else if (tokencmp(tokens[1], "dump")) { + int mode = co_await getBulkDumpMode(cx); + if (mode == 0) { + fmt::println("ERROR: Bulkdump mode must be enabled to dump data"); + co_return UID(); + } + if (tokens.size() != 5) { + fmt::println("{}", BULK_DUMP_DUMP_USAGE); + co_return UID(); + } + Key rangeBegin = tokens[2]; + Key rangeEnd = tokens[3]; + // Bulk load can only inject data to normal key space, aka "" ~ \xff + if (rangeBegin >= rangeEnd || rangeEnd > normalKeys.end) { + fmt::println( + "ERROR: Invalid range: {} to {}, normal key space only", rangeBegin.toString(), rangeEnd.toString()); + fmt::println("{}", BULK_DUMP_DUMP_USAGE); + co_return UID(); + } + KeyRange range = Standalone(KeyRangeRef(rangeBegin, rangeEnd)); + std::string jobRoot = tokens[4].toString(); + bulkDumpJob = createBulkDumpJob(range, + jobRoot, + BulkLoadType::SST, + jobRoot.find("blobstore://") == 0 ? BulkLoadTransportMethod::BLOBSTORE + : BulkLoadTransportMethod::CP); + co_await submitBulkDumpJob(cx, bulkDumpJob); + co_return bulkDumpJob.getJobId(); + } else if (tokencmp(tokens[1], "cancel")) { + if (tokens.size() != 3) { + fmt::println("{}", BULK_DUMP_CANCEL_USAGE); + co_return UID(); + } + UID jobId = validateBulkJobId(tokens[2], BULK_DUMP_CANCEL_USAGE); + co_await cancelBulkDumpJob(cx, jobId); + fmt::println("Job {} has been cancelled. No new tasks will be spawned.", jobId.toString()); + co_return UID(); + + } else if (tokencmp(tokens[1], "status")) { + if (tokens.size() != 2) { + fmt::println("{}", BULK_DUMP_STATUS_USAGE); + co_return UID(); + } + + // Get aggregated progress + Optional progressOpt = co_await getBulkDumpProgress(cx); + if (!progressOpt.present()) { + fmt::println("No bulk dumping job is running"); + co_return UID(); + } + + BulkDumpProgress progress = progressOpt.get(); + + // ctest is dependent on this output; don't change it. + fmt::println("Running bulk dumping job: {}", progress.jobId.toString()); + + // Check if this bulkdump is owned by another operation + Transaction tr(cx); + Optional jobState; + while (true) { + Error err; + try { + tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + Optional job = co_await getSubmittedBulkDumpJob(&tr); + jobState = job; + break; + } catch (Error& e) { + err = e; + } + co_await tr.onError(err); + } + + std::string ownerSuffix = co_await getBulkOwnerSuffix(cx, progress.jobId, true); + + fmt::println(" Range: {}{}", progress.jobRange.toString(), ownerSuffix); + fmt::println(""); + fmt::println("Progress:"); + int runningTasks = progress.totalTasks - progress.completeTasks - progress.errorTasks; + if (progress.totalTasks > 0) { + fmt::println(" Tasks: {}/{} complete ({:.1f}%) | {} running{}", + progress.completeTasks, + progress.totalTasks, + progress.progressPercent(), + runningTasks > 0 ? runningTasks : 0, + progress.errorTasks > 0 ? fmt::format(", {} error", progress.errorTasks) : ""); + } + + fmt::println(" Bytes: {}", formatBytesProgress(progress.completedBytes, progress.totalBytes)); + + printProgressMetrics(progress.avgBytesPerSecond(), progress.etaSeconds(), progress.elapsedSeconds); + + if (progress.errorTasks > 0) { + fmt::println(""); + fmt::println("WARNING: {} tasks in error state", progress.errorTasks); + } + + co_return UID(); + + } else { + printUsage(tokens[0]); + printLongDesc(tokens[0]); + co_return UID(); + } +} + +CommandFactory bulkDumpFactory("bulkdump", + CommandHelp("bulkdump [mode|dump|status|cancel] [ARGs]", + "bulkdump commands", + BULK_DUMP_HELP_MESSAGE.c_str())); +} // namespace fdb_cli diff --git a/fdbcli/BulkLoadCommand.actor.cpp b/fdbcli/BulkLoadCommand.actor.cpp deleted file mode 100644 index 63aa9af9b37..00000000000 --- a/fdbcli/BulkLoadCommand.actor.cpp +++ /dev/null @@ -1,362 +0,0 @@ -/* - * BulkLoadCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/BulkLoading.h" -#include "flow/Arena.h" -#include "flow/IRandom.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -static const std::string BULK_LOAD_MODE_USAGE = "To set bulkload mode: bulkload mode [on|off]\n"; -static const std::string BULK_LOAD_LOAD_USAGE = - "To load a range of key/values: bulkload load \n" - " where is the id of the bulkdumped job to load, to \n" - " denotes the key/value range to load, and is a local directory OR \n" - " blobstore url to load SST files from.\n"; -static const std::string BULK_LOAD_STATUS_USAGE = "To get status: bulkload status\n"; -static const std::string BULK_LOAD_CANCEL_USAGE = "To cancel current bulkload job: bulkload cancel \n"; -static const std::string BULK_LOAD_HISTORY_USAGE = "To print bulkload job history: bulkload history\n"; -static const std::string BULK_LOAD_HISTORY_CLEAR_USAGE = "To clear history: bulkload history clear [all|id]\n"; - -static const std::string BULKLOAD_ADD_LOCK_OWNER_USAGE = - "To add a range lock owner: bulkload addlockowner \n"; -static const std::string BULKLOAD_PRINT_LOCK_USAGE = "To print locked ranges: bulkload printlock\n"; -static const std::string BULKLOAD_PRINT_LOCK_OWNER_USAGE = "To print range lock owners: bulkload printlockowner\n"; -static const std::string BULKLOAD_CLEAR_LOCK_USAGE = "To clear a range lock: bulkload clearlock \n"; - -static const std::string BULK_LOAD_HELP_MESSAGE = - BULK_LOAD_MODE_USAGE + BULK_LOAD_LOAD_USAGE + BULK_LOAD_STATUS_USAGE + BULK_LOAD_CANCEL_USAGE + - BULK_LOAD_HISTORY_USAGE + BULK_LOAD_HISTORY_CLEAR_USAGE + BULKLOAD_ADD_LOCK_OWNER_USAGE + - BULKLOAD_PRINT_LOCK_USAGE + BULKLOAD_PRINT_LOCK_OWNER_USAGE + BULKLOAD_CLEAR_LOCK_USAGE; - -ACTOR Future printPastBulkLoadJob(Database cx) { - std::vector jobs = wait(getBulkLoadJobFromHistory(cx)); - if (jobs.empty()) { - fmt::println("No bulk loading job in the history"); - return Void(); - } - for (const auto& job : jobs) { - ASSERT(job.getPhase() == BulkLoadJobPhase::Complete || job.getPhase() == BulkLoadJobPhase::Error || - job.getPhase() == BulkLoadJobPhase::Cancelled); - if (!job.getTaskCount().present()) { - fmt::println("Job {} submitted at {} for range {}. The job has not initialized for {} mins and exited " - "with status {}.", - job.getJobId().toString(), - std::to_string(job.getSubmitTime()), - job.getJobRange().toString(), - std::to_string((job.getEndTime() - job.getSubmitTime()) / 60.0), - convertBulkLoadJobPhaseToString(job.getPhase())); - } else { - fmt::println( - "Job {} submitted at {} for range {}. The job has {} tasks. The job ran for {} mins and exited " - "with status {}.", - job.getJobId().toString(), - std::to_string(job.getSubmitTime()), - job.getJobRange().toString(), - job.getTaskCount().get(), - std::to_string((job.getEndTime() - job.getSubmitTime()) / 60.0), - convertBulkLoadJobPhaseToString(job.getPhase())); - } - if (job.getPhase() == BulkLoadJobPhase::Error) { - Optional errorMessage = job.getErrorMessage(); - fmt::println("Error message: {}", errorMessage.present() ? errorMessage.get() : "Not provided."); - } - } - return Void(); -} - -void printBulkLoadJobTotalTaskCount(Optional count) { - if (count.present()) { - fmt::println("Total {} tasks", count.get()); - } else { - fmt::println("Total task count is unknown"); - } - return; -} - -ACTOR Future printBulkLoadJobProgress(Database cx, BulkLoadJobState job) { - state Transaction tr(cx); - state Key readBegin = job.getJobRange().begin; - state Key readEnd = job.getJobRange().end; - state UID jobId = job.getJobId(); - state RangeResult rangeResult; - state size_t completeTaskCount = 0; - state size_t submitTaskCount = 0; - state size_t errorTaskCount = 0; - state Optional totalTaskCount = job.getTaskCount(); - while (readBegin < readEnd) { - try { - rangeResult.clear(); - tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - wait(store(rangeResult, krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(readBegin, readEnd)))); - for (int i = 0; i < rangeResult.size() - 1; ++i) { - if (rangeResult[i].value.empty()) { - continue; - } - BulkLoadTaskState bulkLoadTask = decodeBulkLoadTaskState(rangeResult[i].value); - if (bulkLoadTask.getJobId() != jobId) { - fmt::println("Submitted {} tasks", submitTaskCount); - fmt::println("Finished {} tasks", completeTaskCount); - fmt::println("Error {} tasks", errorTaskCount); - printBulkLoadJobTotalTaskCount(totalTaskCount); - if (bulkLoadTask.phase == BulkLoadPhase::Submitted && - bulkLoadTask.getJobId() != UID::fromString("00000000-0000-0000-0000-000000000000")) { - fmt::println("Job {} has been cancelled or has completed", jobId.toString()); - } - return Void(); - } - if (bulkLoadTask.phase == BulkLoadPhase::Complete) { - completeTaskCount = completeTaskCount + bulkLoadTask.getManifests().size(); - } else if (bulkLoadTask.phase == BulkLoadPhase::Error) { - errorTaskCount = errorTaskCount + bulkLoadTask.getManifests().size(); - } - submitTaskCount = submitTaskCount + bulkLoadTask.getManifests().size(); - } - readBegin = rangeResult.back().key; - } catch (Error& e) { - wait(tr.onError(e)); - } - } - fmt::println("Submitted {} tasks", submitTaskCount); - fmt::println("Finished {} tasks", completeTaskCount); - fmt::println("Error {} tasks", errorTaskCount); - printBulkLoadJobTotalTaskCount(totalTaskCount); - return Void(); -} - -ACTOR Future bulkLoadCommandActor(Database cx, std::vector tokens) { - if (tokencmp(tokens[1], "mode")) { - if (tokens.size() == 2) { - int mode = wait(getBulkLoadMode(cx)); - if (mode == 0) { - fmt::println("Bulkload mode is disabled"); - } else if (mode == 1) { - fmt::println("Bulkload mode is enabled"); - } else { - fmt::println("Invalid bulkload mode value {}", mode); - } - return UID(); - } - // Set bulk loading mode - if (tokens.size() != 3) { - fmt::println("{}", BULK_LOAD_MODE_USAGE); - return UID(); - } - if (tokencmp(tokens[2], "on")) { - int old = wait(setBulkLoadMode(cx, 1)); - TraceEvent("SetBulkLoadModeCommand").detail("OldValue", old).detail("NewValue", 1); - return UID(); - } else if (tokencmp(tokens[2], "off")) { - int old = wait(setBulkLoadMode(cx, 0)); - TraceEvent("SetBulkLoadModeCommand").detail("OldValue", old).detail("NewValue", 0); - return UID(); - } else { - fmt::println("ERROR: Invalid bulkload mode value {}", tokens[2].toString()); - fmt::println("{}", BULK_LOAD_MODE_USAGE); - return UID(); - } - } else if (tokencmp(tokens[1], "load")) { - int mode = wait(getBulkLoadMode(cx)); - if (mode == 0) { - fmt::println("ERROR: Bulkload mode must be enabled to load data"); - return UID(); - } - if (tokens.size() != 6) { - fmt::println("{}", BULK_LOAD_LOAD_USAGE); - return UID(); - } - UID jobId = UID::fromString(tokens[2].toString()); - if (!jobId.isValid()) { - fmt::println("ERROR: Invalid job id {}", tokens[2].toString()); - fmt::println("{}", BULK_LOAD_LOAD_USAGE); - return UID(); - } - Key rangeBegin = tokens[3]; - Key rangeEnd = tokens[4]; - // Bulk load can only inject data to normal key space, aka "" ~ \xff - if (rangeBegin >= rangeEnd || rangeEnd > normalKeys.end) { - fmt::println( - "ERROR: Invalid range: {} to {}, normal key space only", rangeBegin.toString(), rangeEnd.toString()); - fmt::println("{}", BULK_LOAD_LOAD_USAGE); - return UID(); - } - std::string jobRoot = tokens[5].toString(); - KeyRange range = Standalone(KeyRangeRef(rangeBegin, rangeEnd)); - state BulkLoadJobState bulkLoadJob = createBulkLoadJob( - jobId, - range, - jobRoot, - jobRoot.find("blobstore://") == 0 ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP); - wait(submitBulkLoadJob(cx, bulkLoadJob)); - return bulkLoadJob.getJobId(); - } else if (tokencmp(tokens[1], "cancel")) { - if (tokens.size() != 3) { - fmt::println("{}", BULK_LOAD_CANCEL_USAGE); - return UID(); - } - state UID jobId = UID::fromString(tokens[2].toString()); - if (!jobId.isValid()) { - fmt::println("ERROR: Invalid job id {}", tokens[2].toString()); - fmt::println("{}", BULK_LOAD_CANCEL_USAGE); - return UID(); - } - wait(cancelBulkLoadJob(cx, jobId)); - fmt::println("Job {} has been cancelled. The job range lock has been cleared", jobId.toString()); - return UID(); - - } else if (tokencmp(tokens[1], "status")) { - if (tokens.size() != 2) { - fmt::println("{}", BULK_LOAD_STATUS_USAGE); - return UID(); - } - Optional job = wait(getRunningBulkLoadJob(cx)); - if (!job.present()) { - fmt::println("No bulk loading job is running"); - return UID(); - } - fmt::println("Running bulk loading job: {}", job.get().getJobId().toString()); - fmt::println("Job information: {}", job.get().toString()); - wait(printBulkLoadJobProgress(cx, job.get())); - return UID(); - - } else if (tokencmp(tokens[1], "history")) { - if (tokens.size() == 2) { - wait(printPastBulkLoadJob(cx)); - return UID(); - } - if (tokens.size() == 3) { - if (tokencmp(tokens[2], "clear")) { - fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); - return UID(); - } else { - fmt::println("ERROR: Invalid history option {}", tokens[2].toString()); - fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); - return UID(); - } - } - if (tokens.size() == 4) { - if (tokencmp(tokens[2], "clear")) { - if (tokencmp(tokens[3], "all")) { - wait(clearBulkLoadJobHistory(cx)); - fmt::println("All bulkload job history has been cleared"); - return UID(); - } else { - fmt::println("ERROR: Invalid history clear option {}", tokens[3].toString()); - fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); - return UID(); - } - } else { - fmt::println("ERROR: Invalid history clear option {}", tokens[2].toString()); - fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); - return UID(); - } - } - if (tokens.size() == 5) { - if (tokencmp(tokens[2], "clear") && tokencmp(tokens[3], "id")) { - UID jobId = UID::fromString(tokens[4].toString()); - if (!jobId.isValid()) { - fmt::println("ERROR: Invalid job id {}", tokens[4].toString()); - fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); - return UID(); - } - wait(clearBulkLoadJobHistory(cx, jobId)); - fmt::println("Bulkload job {} has been cleared from history", jobId.toString()); - return jobId; - } - } - printLongDesc(tokens[0]); - return UID(); - - } else if (tokencmp(tokens[1], "addlockowner")) { - // For debugging purposes and invisible to users. - if (tokens.size() != 3) { - fmt::println("{}", BULK_LOAD_STATUS_USAGE); - return UID(); - } - std::string ownerUniqueID = tokens[2].toString(); - if (ownerUniqueID.empty()) { - fmt::println("ERROR: Owner unique id cannot be empty"); - fmt::println("{}", BULKLOAD_ADD_LOCK_OWNER_USAGE); - return UID(); - } - wait(registerRangeLockOwner(cx, ownerUniqueID, ownerUniqueID)); - return UID(); - - } else if (tokencmp(tokens[1], "printlock")) { - // For debugging purposes and invisible to users. - if (tokens.size() != 2) { - fmt::println("{}", BULKLOAD_PRINT_LOCK_USAGE); - return UID(); - } - std::vector> lockedRanges = - wait(findExclusiveReadLockOnRange(cx, normalKeys)); - fmt::println("Total {} locked ranges", lockedRanges.size()); - if (lockedRanges.size() > 10) { - fmt::println("First 10 locks are:"); - } - int count = 1; - for (const auto& lock : lockedRanges) { - if (count > 10) { - break; - } - fmt::println("Lock {} on {} for {}", count, lock.first.toString(), lock.second.toString()); - count++; - } - return UID(); - - } else if (tokencmp(tokens[1], "printlockowner")) { - // For debugging purposes and invisible to users. - if (tokens.size() != 2) { - fmt::println("{}", BULKLOAD_PRINT_LOCK_OWNER_USAGE); - return UID(); - } - std::vector owners = wait(getAllRangeLockOwners(cx)); - for (const auto owner : owners) { - fmt::println("{}", owner.toString()); - } - return UID(); - - } else if (tokencmp(tokens[1], "clearlock")) { - // For debugging purposes and invisible to users. - if (tokens.size() != 3) { - fmt::println("{}", BULKLOAD_CLEAR_LOCK_USAGE); - return UID(); - } - std::string ownerUniqueID = tokens[2].toString(); - wait(releaseExclusiveReadLockByUser(cx, ownerUniqueID)); - return UID(); - - } else { - printUsage(tokens[0]); - printLongDesc(tokens[0]); - return UID(); - } -} - -CommandFactory bulkLoadFactory("bulkload", - CommandHelp("bulkload [mode|load|status|cancel|history] [ARGs]", - "bulkload commands", - BULK_LOAD_HELP_MESSAGE.c_str())); -} // namespace fdb_cli diff --git a/fdbcli/BulkLoadCommand.cpp b/fdbcli/BulkLoadCommand.cpp new file mode 100644 index 00000000000..3bfb1a64f8a --- /dev/null +++ b/fdbcli/BulkLoadCommand.cpp @@ -0,0 +1,384 @@ +/* + * BulkLoadCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" +#include "fdbclient/ManagementAPI.h" +#include "fdbclient/BulkLoading.h" +#include "flow/Arena.h" +#include "flow/IRandom.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/Util.h" + +namespace fdb_cli { + +static const std::string BULK_LOAD_MODE_USAGE = "To set bulkload mode: bulkload mode [on|off]\n"; +static const std::string BULK_LOAD_LOAD_USAGE = + "To load a range of key/values: bulkload load \n" + " where is the id of the bulkdumped job to load, to \n" + " denotes the key/value range to load, and is a local directory OR \n" + " blobstore url to load SST files from.\n"; +static const std::string BULK_LOAD_STATUS_USAGE = "To get status: bulkload status\n"; +static const std::string BULK_LOAD_CANCEL_USAGE = "To cancel current bulkload job: bulkload cancel \n"; +static const std::string BULK_LOAD_HISTORY_USAGE = "To print bulkload job history: bulkload history\n"; +static const std::string BULK_LOAD_HISTORY_CLEAR_USAGE = "To clear history: bulkload history clear [all|id]\n"; + +static const std::string BULKLOAD_ADD_LOCK_OWNER_USAGE = + "To add a range lock owner: bulkload addlockowner \n"; +static const std::string BULKLOAD_PRINT_LOCK_USAGE = "To print locked ranges: bulkload printlock\n"; +static const std::string BULKLOAD_PRINT_LOCK_OWNER_USAGE = "To print range lock owners: bulkload printlockowner\n"; +static const std::string BULKLOAD_CLEAR_LOCK_USAGE = "To clear a range lock: bulkload clearlock \n"; + +static const std::string BULK_LOAD_HELP_MESSAGE = + BULK_LOAD_MODE_USAGE + BULK_LOAD_LOAD_USAGE + BULK_LOAD_STATUS_USAGE + BULK_LOAD_CANCEL_USAGE + + BULK_LOAD_HISTORY_USAGE + BULK_LOAD_HISTORY_CLEAR_USAGE + BULKLOAD_ADD_LOCK_OWNER_USAGE + + BULKLOAD_PRINT_LOCK_USAGE + BULKLOAD_PRINT_LOCK_OWNER_USAGE + BULKLOAD_CLEAR_LOCK_USAGE; + +Future printPastBulkLoadJob(Database cx) { + std::vector jobs = co_await getBulkLoadJobFromHistory(cx); + if (jobs.empty()) { + fmt::println("No bulk loading job in the history"); + co_return; + } + for (const auto& job : jobs) { + ASSERT(job.getPhase() == BulkLoadJobPhase::Complete || job.getPhase() == BulkLoadJobPhase::Error || + job.getPhase() == BulkLoadJobPhase::Cancelled); + if (!job.getTaskCount().present()) { + fmt::println("Job {} submitted at {} for range {}. The job has not initialized for {:.1f} mins and exited " + "with status {}.", + job.getJobId().toString(), + formatTimeISO8601(job.getSubmitTime()), + job.getJobRange().toString(), + (job.getEndTime() - job.getSubmitTime()) / 60.0, + convertBulkLoadJobPhaseToString(job.getPhase())); + } else { + fmt::println( + "Job {} submitted at {} for range {}. The job has {} tasks. The job ran for {:.1f} mins and exited " + "with status {}.", + job.getJobId().toString(), + formatTimeISO8601(job.getSubmitTime()), + job.getJobRange().toString(), + job.getTaskCount().get(), + (job.getEndTime() - job.getSubmitTime()) / 60.0, + convertBulkLoadJobPhaseToString(job.getPhase())); + } + if (job.getPhase() == BulkLoadJobPhase::Error) { + Optional errorMessage = job.getErrorMessage(); + fmt::println("Error message: {}", errorMessage.present() ? errorMessage.get() : "Not provided."); + } + } +} + +void printBulkLoadJobTotalTaskCount(Optional count) { + if (count.present()) { + fmt::println("Total {} tasks", count.get()); + } else { + fmt::println("Total task count is unknown"); + } + return; +} + +Future printBulkLoadJobProgress(Database cx, BulkLoadJobState job) { + Transaction tr(cx); + Key readBegin = job.getJobRange().begin; + Key readEnd = job.getJobRange().end; + UID jobId = job.getJobId(); + RangeResult rangeResult; + size_t completeTaskCount = 0; + size_t submitTaskCount = 0; + size_t errorTaskCount = 0; + Optional totalTaskCount = job.getTaskCount(); + while (readBegin < readEnd) { + Error err; + bool hasErr = false; + try { + rangeResult.clear(); + tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + rangeResult = co_await krmGetRanges(&tr, bulkLoadTaskPrefix, KeyRangeRef(readBegin, readEnd)); + for (int i = 0; i < rangeResult.size() - 1; ++i) { + if (rangeResult[i].value.empty()) { + continue; + } + BulkLoadTaskState bulkLoadTask = decodeBulkLoadTaskState(rangeResult[i].value); + if (bulkLoadTask.getJobId() != jobId) { + fmt::println("Submitted {} tasks", submitTaskCount); + fmt::println("Finished {} tasks", completeTaskCount); + fmt::println("Error {} tasks", errorTaskCount); + printBulkLoadJobTotalTaskCount(totalTaskCount); + if (bulkLoadTask.phase == BulkLoadPhase::Submitted && bulkLoadTask.getJobId().isValid()) { + fmt::println("Job {} has been cancelled or has completed", jobId.toString()); + } + co_return; + } + if (bulkLoadTask.phase == BulkLoadPhase::Complete) { + completeTaskCount = completeTaskCount + bulkLoadTask.getManifests().size(); + } else if (bulkLoadTask.phase == BulkLoadPhase::Error) { + errorTaskCount = errorTaskCount + bulkLoadTask.getManifests().size(); + } + submitTaskCount = submitTaskCount + bulkLoadTask.getManifests().size(); + } + readBegin = rangeResult.back().key; + } catch (Error& e) { + err = e; + hasErr = true; + } + if (hasErr) { + co_await tr.onError(err); + } + } + fmt::println("Submitted {} tasks", submitTaskCount); + fmt::println("Finished {} tasks", completeTaskCount); + fmt::println("Error {} tasks", errorTaskCount); + printBulkLoadJobTotalTaskCount(totalTaskCount); +} + +Future bulkLoadCommandActor(Database cx, std::vector tokens) { + if (tokencmp(tokens[1], "mode")) { + if (tokens.size() == 2) { + int mode = co_await getBulkLoadMode(cx); + if (mode == 0) { + fmt::println("Bulkload mode is disabled"); + } else if (mode == 1) { + fmt::println("Bulkload mode is enabled"); + } else { + fmt::println("Invalid bulkload mode value {}", mode); + } + co_return UID(); + } + // Set bulk loading mode + if (tokens.size() != 3) { + fmt::println("{}", BULK_LOAD_MODE_USAGE); + co_return UID(); + } + if (tokencmp(tokens[2], "on")) { + int old = co_await setBulkLoadMode(cx, 1); + TraceEvent("SetBulkLoadModeCommand").detail("OldValue", old).detail("NewValue", 1); + co_return UID(); + } else if (tokencmp(tokens[2], "off")) { + int old = co_await setBulkLoadMode(cx, 0); + TraceEvent("SetBulkLoadModeCommand").detail("OldValue", old).detail("NewValue", 0); + co_return UID(); + } else { + fmt::println("ERROR: Invalid bulkload mode value {}", tokens[2].toString()); + fmt::println("{}", BULK_LOAD_MODE_USAGE); + co_return UID(); + } + } else if (tokencmp(tokens[1], "load")) { + int mode = co_await getBulkLoadMode(cx); + if (mode == 0) { + fmt::println("ERROR: Bulkload mode must be enabled to load data"); + co_return UID(); + } + if (tokens.size() != 6) { + fmt::println("{}", BULK_LOAD_LOAD_USAGE); + co_return UID(); + } + UID jobId = validateBulkJobId(tokens[2], BULK_LOAD_LOAD_USAGE.c_str()); + Key rangeBegin = tokens[3]; + Key rangeEnd = tokens[4]; + // Bulk load can only inject data to normal key space, aka "" ~ \xff + if (rangeBegin >= rangeEnd || rangeEnd > normalKeys.end) { + fmt::println( + "ERROR: Invalid range: {} to {}, normal key space only", rangeBegin.toString(), rangeEnd.toString()); + fmt::println("{}", BULK_LOAD_LOAD_USAGE); + co_return UID(); + } + std::string jobRoot = tokens[5].toString(); + KeyRange range = Standalone(KeyRangeRef(rangeBegin, rangeEnd)); + BulkLoadJobState bulkLoadJob = createBulkLoadJob( + jobId, + range, + jobRoot, + jobRoot.find("blobstore://") == 0 ? BulkLoadTransportMethod::BLOBSTORE : BulkLoadTransportMethod::CP); + co_await submitBulkLoadJob(cx, bulkLoadJob); + co_return bulkLoadJob.getJobId(); + } else if (tokencmp(tokens[1], "cancel")) { + if (tokens.size() != 3) { + fmt::println("{}", BULK_LOAD_CANCEL_USAGE); + co_return UID(); + } + UID cancelJobId = validateBulkJobId(tokens[2], BULK_LOAD_CANCEL_USAGE.c_str()); + co_await cancelBulkLoadJob(cx, cancelJobId); + fmt::println("Job {} has been cancelled. The job range lock has been cleared", cancelJobId.toString()); + co_return UID(); + + } else if (tokencmp(tokens[1], "status")) { + if (tokens.size() != 2) { + fmt::println("{}", BULK_LOAD_STATUS_USAGE); + co_return UID(); + } + + // Get aggregated progress + Optional progressOpt = co_await getBulkLoadProgress(cx); + if (!progressOpt.present()) { + fmt::println("No bulk loading job is running"); + co_return UID(); + } + + BulkLoadProgress progress = progressOpt.get(); + + fmt::println("BulkLoad job {} is in progress.", progress.jobId.toString()); + + std::string ownerSuffix = co_await getBulkOwnerSuffix(cx, progress.jobId, false); + + fmt::println(" Range: {}{}", progress.jobRange.toString(), ownerSuffix); + fmt::println(""); + fmt::println("Progress:"); + + int totalTasks = progress.submittedTasks + progress.triggeredTasks + progress.runningTasks + + progress.completeTasks + progress.errorTasks; + if (totalTasks > 0) { + fmt::println(" Tasks: {}/{} complete ({:.1f}%) | {} submitted, {} triggered, {} running{}", + progress.completeTasks, + totalTasks, + 100.0 * progress.completeTasks / totalTasks, + progress.submittedTasks, + progress.triggeredTasks, + progress.runningTasks, + progress.errorTasks > 0 ? fmt::format(", {} error", progress.errorTasks) : ""); + } + + fmt::println(" Bytes: {}", formatBytesProgress(progress.completedBytes, progress.totalBytes)); + + printProgressMetrics(progress.avgBytesPerSecond(), progress.etaSeconds(), progress.elapsedSeconds); + + // Only show warnings for actual problems (stalled tasks, errors) + printStalledTasks(progress.stalledTasks); + if (progress.errorTasks > 0) { + fmt::println(""); + fmt::println("WARNING: {} tasks in error state", progress.errorTasks); + } + + co_return UID(); + + } else if (tokencmp(tokens[1], "history")) { + if (tokens.size() == 2) { + co_await printPastBulkLoadJob(cx); + co_return UID(); + } + if (tokens.size() == 3) { + if (tokencmp(tokens[2], "clear")) { + fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); + co_return UID(); + } else { + fmt::println("ERROR: Invalid history option {}", tokens[2].toString()); + fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); + co_return UID(); + } + } + if (tokens.size() == 4) { + if (tokencmp(tokens[2], "clear")) { + if (tokencmp(tokens[3], "all")) { + co_await clearBulkLoadJobHistory(cx); + fmt::println("All bulkload job history has been cleared"); + co_return UID(); + } else { + fmt::println("ERROR: Invalid history clear option {}", tokens[3].toString()); + fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); + co_return UID(); + } + } else { + fmt::println("ERROR: Invalid history clear option {}", tokens[2].toString()); + fmt::println("{}", BULK_LOAD_HISTORY_CLEAR_USAGE); + co_return UID(); + } + } + if (tokens.size() == 5) { + if (tokencmp(tokens[2], "clear") && tokencmp(tokens[3], "id")) { + UID jobId = validateBulkJobId(tokens[4], BULK_LOAD_HISTORY_CLEAR_USAGE.c_str()); + co_await clearBulkLoadJobHistory(cx, jobId); + fmt::println("Bulkload job {} has been cleared from history", jobId.toString()); + co_return jobId; + } + } + printLongDesc(tokens[0]); + co_return UID(); + + } else if (tokencmp(tokens[1], "addlockowner")) { + // For debugging purposes and invisible to users. + if (tokens.size() != 3) { + fmt::println("{}", BULK_LOAD_STATUS_USAGE); + co_return UID(); + } + std::string ownerUniqueID = tokens[2].toString(); + if (ownerUniqueID.empty()) { + fmt::println("ERROR: Owner unique id cannot be empty"); + fmt::println("{}", BULKLOAD_ADD_LOCK_OWNER_USAGE); + co_return UID(); + } + co_await registerRangeLockOwner(cx, ownerUniqueID, ownerUniqueID); + co_return UID(); + + } else if (tokencmp(tokens[1], "printlock")) { + // For debugging purposes and invisible to users. + if (tokens.size() != 2) { + fmt::println("{}", BULKLOAD_PRINT_LOCK_USAGE); + co_return UID(); + } + std::vector> lockedRanges = + co_await findExclusiveReadLockOnRange(cx, normalKeys); + fmt::println("Total {} locked ranges", lockedRanges.size()); + if (lockedRanges.size() > 10) { + fmt::println("First 10 locks are:"); + } + int count = 1; + for (const auto& lock : lockedRanges) { + if (count > 10) { + break; + } + fmt::println("Lock {} on {} for {}", count, lock.first.toString(), lock.second.toString()); + count++; + } + co_return UID(); + + } else if (tokencmp(tokens[1], "printlockowner")) { + // For debugging purposes and invisible to users. + if (tokens.size() != 2) { + fmt::println("{}", BULKLOAD_PRINT_LOCK_OWNER_USAGE); + co_return UID(); + } + std::vector owners = co_await getAllRangeLockOwners(cx); + for (const auto& owner : owners) { + fmt::println("{}", owner.toString()); + } + co_return UID(); + + } else if (tokencmp(tokens[1], "clearlock")) { + // For debugging purposes and invisible to users. + if (tokens.size() != 3) { + fmt::println("{}", BULKLOAD_CLEAR_LOCK_USAGE); + co_return UID(); + } + std::string ownerUniqueID = tokens[2].toString(); + co_await releaseExclusiveReadLockByUser(cx, ownerUniqueID); + co_return UID(); + + } else { + printUsage(tokens[0]); + printLongDesc(tokens[0]); + co_return UID(); + } +} + +CommandFactory bulkLoadFactory("bulkload", + CommandHelp("bulkload [mode|load|status|cancel|history] [ARGs]", + "bulkload commands", + BULK_LOAD_HELP_MESSAGE.c_str())); +} // namespace fdb_cli diff --git a/fdbcli/CMakeLists.txt b/fdbcli/CMakeLists.txt index 090649d080f..ad28477312c 100644 --- a/fdbcli/CMakeLists.txt +++ b/fdbcli/CMakeLists.txt @@ -2,7 +2,7 @@ include(AddFdbTest) fdb_find_sources(FDBCLI_SRCS) add_flow_target(EXECUTABLE NAME fdbcli SRCS ${FDBCLI_SRCS}) -target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_BINARY_DIR}/include") +target_include_directories(fdbcli PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/include") target_link_libraries(fdbcli PRIVATE fdbctl fdbclient SimpleOpt) if (USE_UBSAN) diff --git a/fdbcli/ConfigureCommand.actor.cpp b/fdbcli/ConfigureCommand.actor.cpp deleted file mode 100644 index 3f0a4164abe..00000000000 --- a/fdbcli/ConfigureCommand.actor.cpp +++ /dev/null @@ -1,374 +0,0 @@ -/* - * ConfigureCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/FlowLineNoise.h" -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/ManagementAPI.actor.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future configureCommandActor(Reference db, - Database localDb, - std::vector tokens, - LineNoise* linenoise, - Future warn) { - state ConfigurationResult result; - state StatusObject s; - state int startToken = 1; - state bool force = false; - if (tokens.size() < 2) - result = ConfigurationResult::NO_OPTIONS_PROVIDED; - else { - if (tokens[startToken] == "FORCE"_sr) { - force = true; - startToken = 2; - } - - state Optional conf; - if (tokens[startToken] == "auto"_sr) { - // get cluster status - state Reference tr = db->createTransaction(); - if (!tr->isValid()) { - StatusObject _s = wait(StatusClient::statusFetcher(localDb)); - s = _s; - } else { - state ThreadFuture> statusValueF = tr->get("\xff\xff/status/json"_sr); - Optional statusValue = wait(safeThreadFutureToFuture(statusValueF)); - if (!statusValue.present()) { - fprintf(stderr, "ERROR: Failed to get status json from the cluster\n"); - return false; - } - json_spirit::mValue mv; - json_spirit::read_string(statusValue.get().toString(), mv); - s = StatusObject(mv.get_obj()); - } - - if (warn.isValid()) - warn.cancel(); - - conf = parseConfig(s); - - if (!conf.get().isValid()) { - printf("Unable to provide advice for the current configuration.\n"); - return false; - } - - bool noChanges = conf.get().old_replication == conf.get().auto_replication && - conf.get().old_logs == conf.get().auto_logs && - conf.get().old_commit_proxies == conf.get().auto_commit_proxies && - conf.get().old_grv_proxies == conf.get().auto_grv_proxies && - conf.get().old_resolvers == conf.get().auto_resolvers && - conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction && - conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction; - - bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs && - conf.get().old_commit_proxies == conf.get().desired_commit_proxies && - conf.get().old_grv_proxies == conf.get().desired_grv_proxies && - conf.get().old_resolvers == conf.get().desired_resolvers; - - std::string outputString; - - outputString += "\nYour cluster has:\n\n"; - outputString += format(" processes %d\n", conf.get().processes); - outputString += format(" machines %d\n", conf.get().machines); - - if (noDesiredChanges) - outputString += "\nConfigure recommends keeping your current configuration:\n\n"; - else if (noChanges) - outputString += - "\nConfigure cannot modify the configuration because some parameters have been set manually:\n\n"; - else - outputString += "\nConfigure recommends the following changes:\n\n"; - outputString += " ------------------------------------------------------------------- \n"; - outputString += "| parameter | old | new |\n"; - outputString += " ------------------------------------------------------------------- \n"; - outputString += format("| replication | %16s | %16s |\n", - conf.get().old_replication.c_str(), - conf.get().auto_replication.c_str()); - outputString += - format("| logs | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs); - outputString += conf.get().auto_logs != conf.get().desired_logs - ? format(" (manually set; would be %d)\n", conf.get().desired_logs) - : "\n"; - outputString += format("| commit_proxies | %16d | %16d |", - conf.get().old_commit_proxies, - conf.get().auto_commit_proxies); - outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies - ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies) - : "\n"; - outputString += format("| grv_proxies | %16d | %16d |", - conf.get().old_grv_proxies, - conf.get().auto_grv_proxies); - outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies - ? format(" (manually set; would be %d)\n", conf.get().desired_grv_proxies) - : "\n"; - outputString += format( - "| resolvers | %16d | %16d |", conf.get().old_resolvers, conf.get().auto_resolvers); - outputString += conf.get().auto_resolvers != conf.get().desired_resolvers - ? format(" (manually set; would be %d)\n", conf.get().desired_resolvers) - : "\n"; - outputString += format("| transaction-class processes | %16d | %16d |\n", - conf.get().old_processes_with_transaction, - conf.get().auto_processes_with_transaction); - outputString += format("| transaction-class machines | %16d | %16d |\n", - conf.get().old_machines_with_transaction, - conf.get().auto_machines_with_transaction); - outputString += " ------------------------------------------------------------------- \n\n"; - - std::printf("%s", outputString.c_str()); - - if (noChanges) - return true; - - // TODO: disable completion - Optional line = wait(linenoise->read("Would you like to make these changes? [y/n]> ")); - - if (!line.present() || (line.get() != "y" && line.get() != "Y")) { - return true; - } - } - - // Check for backup_worker_enabled configuration and reject it. - // This setting is now managed automatically by the backup system. - for (auto it = tokens.begin() + startToken; it != tokens.end(); ++it) { - if (it->startsWith("backup_worker_enabled:="_sr)) { - result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED; - break; - } - } - - if (result != ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED) { - ConfigurationResult r = wait(ManagementAPI::changeConfig( - db, std::vector(tokens.begin() + startToken, tokens.end()), conf, force)); - result = r; - } - } - - // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but - // there are various results specific to changeConfig() that we need to report: - bool ret = true; - switch (result) { - case ConfigurationResult::NO_OPTIONS_PROVIDED: - case ConfigurationResult::CONFLICTING_OPTIONS: - case ConfigurationResult::UNKNOWN_OPTION: - case ConfigurationResult::INCOMPLETE_CONFIGURATION: - printUsage("configure"_sr); - ret = false; - break; - case ConfigurationResult::INVALID_CONFIGURATION: - fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); - ret = false; - break; - case ConfigurationResult::STORAGE_MIGRATION_DISABLED: - fprintf(stderr, - "ERROR: Storage engine type cannot be changed because " - "storage_migration_type=disabled.\n"); - fprintf(stderr, - "Type `configure perpetual_storage_wiggle=1 storage_migration_type=gradual' to enable gradual " - "migration with the perpetual wiggle, or `configure " - "storage_migration_type=aggressive' for aggressive migration.\n"); - ret = false; - break; - case ConfigurationResult::DATABASE_ALREADY_CREATED: - fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); - ret = false; - break; - case ConfigurationResult::DATABASE_CREATED: - printf("Database created\n"); - break; - case ConfigurationResult::DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL: - printf("Database created\n"); - fprintf( - stderr, - "WARN: Sharded RocksDB storage engine type is still in experimental stage, not yet production tested.\n"); - break; - case ConfigurationResult::DATABASE_UNAVAILABLE: - fprintf(stderr, "ERROR: The database is unavailable\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: - fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: - fprintf(stderr, - "ERROR: When usable_regions > 1, all regions with priority >= 0 must be fully replicated " - "before changing the configuration\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: - fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::REGIONS_CHANGED: - fprintf(stderr, - "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::NOT_ENOUGH_WORKERS: - fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::REGION_REPLICATION_MISMATCH: - fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::DCID_MISSING: - fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); - fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::SUCCESS: - printf("Configuration changed\n"); - break; - case ConfigurationResult::LOCKED_NOT_NEW: - fprintf(stderr, "ERROR: `only new databases can be configured as locked`\n"); - ret = false; - break; - case ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL: - printf("Configuration changed, with warnings\n"); - fprintf(stderr, - "WARN: To make progress toward the desired storage type with storage_migration_type=gradual, the " - "Perpetual Wiggle must be enabled.\n"); - fprintf(stderr, - "Type `configure perpetual_storage_wiggle=1' to enable the perpetual wiggle, or `configure " - "storage_migration_type=gradual' to set the gradual migration type.\n"); - break; - case ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL: - printf("Configuration changed\n"); - fprintf( - stderr, - "WARN: Sharded RocksDB storage engine type is still in experimental stage, not yet production tested.\n"); - break; - case ConfigurationResult::DATABASE_IS_REGISTERED: - fprintf(stderr, - "ERROR: a result of type `ConfigurationResult::DATABASE_IS_REGISTERED` was unexpectedly seen.\n"); - ret = false; - break; - case ConfigurationResult::INVALID_STORAGE_TYPE: - fprintf(stderr, "ERROR: Invalid storage type for storage or TLog.\n"); - ret = false; - break; - case ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED: - fprintf(stderr, - "ERROR: backup_worker_enabled configuration is restricted in fdbcli and managed automatically by the " - "backup system.\n"); - ret = false; - break; - default: - ASSERT(false); - ret = false; - }; - return ret; -} - -void configureGenerator(const char* text, - const char* line, - std::vector& lc, - std::vector const& tokens) { - const char* opts[] = { "new", - "single", - "double", - "triple", - "three_data_hall", - "three_datacenter", - "ssd", - "ssd-1", - "ssd-2", - "memory", - "memory-1", - "memory-2", - "memory-radixtree", - "commit_proxies=", - "grv_proxies=", - "logs=", - "resolvers=", - "perpetual_storage_wiggle=", - "perpetual_storage_wiggle_locality=", - // TODO(zhewu): update fdbcli command documentation. - "perpetual_storage_wiggle_engine=", - "storage_migration_type=", - nullptr }; - arrayGenerator(text, line, opts, lc); -} - -CommandFactory configureFactory( - "configure", - CommandHelp( - "configure [new|tss]" - "|" - "commit_proxies=|grv_proxies=|logs=|resolvers=>*|" - "count=|perpetual_storage_wiggle=|perpetual_storage_wiggle_locality=" - "<:|0>|storage_migration_type={disabled|gradual|aggressive}" - "|exclude=", - "change the database configuration", - "The `new' option, if present, initializes a new database with the given configuration rather than changing " - "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " - "specified.\n\ntss: when enabled, configures the testing storage server for the cluster instead." - "When used with new to set up tss for the first time, it requires both a count and a storage engine." - "To disable the testing storage server, run \"configure tss count=0\"\n\n" - "Redundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies " - "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " - "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " - "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " - "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. The proxy role is being " - "deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' " - "separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and " - "'grv_proxies' " - "should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and " - "'commit_proxies'. " - "Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the " - "default value.\n\ncommit_proxies=: Sets the desired number of commit proxies in the cluster. " - "Must be at least 1, or set to -1 which restores the number of commit proxies to the default " - "value.\n\ngrv_proxies=: Sets the desired number of GRV proxies in the cluster. Must be at least " - "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=: Sets the " - "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of " - "logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. " - "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\n" - "perpetual_storage_wiggle=: Set the value speed (a.k.a., the number of processes that the Data " - "Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the " - "perpetual storage wiggle.\n\n" - "perpetual_storage_wiggle_locality=<:|0>: Set the process filter for wiggling. " - "The processes that match the given locality key and locality value are only wiggled. The value 0 will disable " - "the locality filter and matches all the processes for wiggling.\n\n" - "exclude=: Sets the addresses in the format of IP1:port1,IP2:port2 pairs to be excluded during " - "recruitment. Note this should be only used when the database is unavailable because of the faulty processes " - "that are blocking the recovery from completion. The number of addresses should be less than the replication " - "factor to avoid data loss.\n\n" - - "See the FoundationDB Administration Guide for more information."), - &configureGenerator); - -} // namespace fdb_cli diff --git a/fdbcli/ConfigureCommand.cpp b/fdbcli/ConfigureCommand.cpp new file mode 100644 index 00000000000..13566965a28 --- /dev/null +++ b/fdbcli/ConfigureCommand.cpp @@ -0,0 +1,385 @@ +/* + * ConfigureCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/FlowLineNoise.h" +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/ManagementAPI.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace fdb_cli { + +Future configureCommandActor(Reference db, + Database localDb, + std::vector tokens, + LineNoise* linenoise, + Future _warn) { + auto warn = std::move(_warn); + ConfigurationResult result = ConfigurationResult::SUCCESS; + StatusObject s; + int startToken = 1; + bool force = false; + if (tokens.size() < 2) + result = ConfigurationResult::NO_OPTIONS_PROVIDED; + else { + if (tokens[startToken] == "FORCE"_sr) { + force = true; + startToken = 2; + } + + Optional conf; + if (tokens[startToken] == "auto"_sr) { + // get cluster status + Reference tr = db->createTransaction(); + if (!tr->isValid()) { + StatusObject _s = co_await StatusClient::statusFetcher(localDb); + s = _s; + } else { + ThreadFuture> statusValueF = tr->get("\xff\xff/status/json"_sr); + Optional statusValue = co_await safeThreadFutureToFuture(statusValueF); + if (!statusValue.present()) { + fprintf(stderr, "ERROR: Failed to get status json from the cluster\n"); + co_return false; + } + json_spirit::mValue mv; + json_spirit::read_string(statusValue.get().toString(), mv); + s = StatusObject(mv.get_obj()); + } + + if (warn.isValid()) + warn.cancel(); + + conf = parseConfig(s); + + if (!conf.get().isValid()) { + printf("Unable to provide advice for the current configuration.\n"); + co_return false; + } + + bool noChanges = conf.get().old_replication == conf.get().auto_replication && + conf.get().old_logs == conf.get().auto_logs && + conf.get().old_commit_proxies == conf.get().auto_commit_proxies && + conf.get().old_grv_proxies == conf.get().auto_grv_proxies && + conf.get().old_resolvers == conf.get().auto_resolvers && + conf.get().old_processes_with_transaction == conf.get().auto_processes_with_transaction && + conf.get().old_machines_with_transaction == conf.get().auto_machines_with_transaction; + + bool noDesiredChanges = noChanges && conf.get().old_logs == conf.get().desired_logs && + conf.get().old_commit_proxies == conf.get().desired_commit_proxies && + conf.get().old_grv_proxies == conf.get().desired_grv_proxies && + conf.get().old_resolvers == conf.get().desired_resolvers; + + std::string outputString; + + outputString += "\nYour cluster has:\n\n"; + outputString += format(" processes %d\n", conf.get().processes); + outputString += format(" machines %d\n", conf.get().machines); + + if (noDesiredChanges) + outputString += "\nConfigure recommends keeping your current configuration:\n\n"; + else if (noChanges) + outputString += + "\nConfigure cannot modify the configuration because some parameters have been set manually:\n\n"; + else + outputString += "\nConfigure recommends the following changes:\n\n"; + outputString += " ------------------------------------------------------------------- \n"; + outputString += "| parameter | old | new |\n"; + outputString += " ------------------------------------------------------------------- \n"; + outputString += format("| replication | %16s | %16s |\n", + conf.get().old_replication.c_str(), + conf.get().auto_replication.c_str()); + outputString += + format("| logs | %16d | %16d |", conf.get().old_logs, conf.get().auto_logs); + outputString += conf.get().auto_logs != conf.get().desired_logs + ? format(" (manually set; would be %d)\n", conf.get().desired_logs) + : "\n"; + outputString += format("| commit_proxies | %16d | %16d |", + conf.get().old_commit_proxies, + conf.get().auto_commit_proxies); + outputString += conf.get().auto_commit_proxies != conf.get().desired_commit_proxies + ? format(" (manually set; would be %d)\n", conf.get().desired_commit_proxies) + : "\n"; + outputString += format("| grv_proxies | %16d | %16d |", + conf.get().old_grv_proxies, + conf.get().auto_grv_proxies); + outputString += conf.get().auto_grv_proxies != conf.get().desired_grv_proxies + ? format(" (manually set; would be %d)\n", conf.get().desired_grv_proxies) + : "\n"; + outputString += format( + "| resolvers | %16d | %16d |", conf.get().old_resolvers, conf.get().auto_resolvers); + outputString += conf.get().auto_resolvers != conf.get().desired_resolvers + ? format(" (manually set; would be %d)\n", conf.get().desired_resolvers) + : "\n"; + outputString += format("| transaction-class processes | %16d | %16d |\n", + conf.get().old_processes_with_transaction, + conf.get().auto_processes_with_transaction); + outputString += format("| transaction-class machines | %16d | %16d |\n", + conf.get().old_machines_with_transaction, + conf.get().auto_machines_with_transaction); + outputString += " ------------------------------------------------------------------- \n\n"; + + std::printf("%s", outputString.c_str()); + + if (noChanges) + co_return true; + + // TODO: disable completion + Optional line = co_await linenoise->read("Would you like to make these changes? [y/n]> "); + + if (!line.present() || (line.get() != "y" && line.get() != "Y")) { + co_return true; + } + } + + // Check for backup_worker_enabled configuration and reject it. + // This setting is now managed automatically by the backup system. + for (auto it = tokens.begin() + startToken; it != tokens.end(); ++it) { + if (it->startsWith("backup_worker_enabled:="_sr)) { + result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED; + break; + } + if (it->startsWith("range_backup_worker_enabled:="_sr)) { + result = ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED; + break; + } + } + + if (result != ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED && + result != ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED) { + ConfigurationResult r = co_await ManagementAPI::changeConfig( + db, std::vector(tokens.begin() + startToken, tokens.end()), conf, force); + result = r; + } + } + + // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but + // there are various results specific to changeConfig() that we need to report: + bool ret = true; + switch (result) { + case ConfigurationResult::NO_OPTIONS_PROVIDED: + case ConfigurationResult::CONFLICTING_OPTIONS: + case ConfigurationResult::UNKNOWN_OPTION: + case ConfigurationResult::INCOMPLETE_CONFIGURATION: + printUsage("configure"_sr); + ret = false; + break; + case ConfigurationResult::INVALID_CONFIGURATION: + fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); + ret = false; + break; + case ConfigurationResult::STORAGE_MIGRATION_DISABLED: + fprintf(stderr, + "ERROR: Storage engine type cannot be changed because " + "storage_migration_type=disabled.\n"); + fprintf(stderr, + "Type `configure perpetual_storage_wiggle=1 storage_migration_type=gradual' to enable gradual " + "migration with the perpetual wiggle, or `configure " + "storage_migration_type=aggressive' for aggressive migration.\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_ALREADY_CREATED: + fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_CREATED: + printf("Database created\n"); + break; + case ConfigurationResult::DATABASE_CREATED_WARN_SHARDED_ROCKSDB_EXPERIMENTAL: + printf("Database created\n"); + fprintf( + stderr, + "WARN: Sharded RocksDB storage engine type is still in experimental stage, not yet production tested.\n"); + break; + case ConfigurationResult::DATABASE_UNAVAILABLE: + fprintf(stderr, "ERROR: The database is unavailable\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: + fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: + fprintf(stderr, + "ERROR: When usable_regions > 1, all regions with priority >= 0 must be fully replicated " + "before changing the configuration\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: + fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGIONS_CHANGED: + fprintf(stderr, + "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::NOT_ENOUGH_WORKERS: + fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_REPLICATION_MISMATCH: + fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::DCID_MISSING: + fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); + fprintf(stderr, "Type `configure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::SUCCESS: + printf("Configuration changed\n"); + break; + case ConfigurationResult::LOCKED_NOT_NEW: + fprintf(stderr, "ERROR: `only new databases can be configured as locked`\n"); + ret = false; + break; + case ConfigurationResult::SUCCESS_WARN_PPW_GRADUAL: + printf("Configuration changed, with warnings\n"); + fprintf(stderr, + "WARN: To make progress toward the desired storage type with storage_migration_type=gradual, the " + "Perpetual Wiggle must be enabled.\n"); + fprintf(stderr, + "Type `configure perpetual_storage_wiggle=1' to enable the perpetual wiggle, or `configure " + "storage_migration_type=gradual' to set the gradual migration type.\n"); + break; + case ConfigurationResult::SUCCESS_WARN_SHARDED_ROCKSDB_EXPERIMENTAL: + printf("Configuration changed\n"); + fprintf( + stderr, + "WARN: Sharded RocksDB storage engine type is still in experimental stage, not yet production tested.\n"); + break; + case ConfigurationResult::DATABASE_IS_REGISTERED: + fprintf(stderr, + "ERROR: a result of type `ConfigurationResult::DATABASE_IS_REGISTERED` was unexpectedly seen.\n"); + ret = false; + break; + case ConfigurationResult::INVALID_STORAGE_TYPE: + fprintf(stderr, "ERROR: Invalid storage type for storage or TLog.\n"); + ret = false; + break; + case ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED: + fprintf(stderr, + "ERROR: backup_worker_enabled configuration is restricted in fdbcli and managed automatically by the " + "backup system.\n"); + ret = false; + break; + case ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED: + fprintf(stderr, + "ERROR: range_backup_worker_enabled configuration is restricted in fdbcli and managed " + "automatically by the backup system.\n"); + ret = false; + break; + default: + ASSERT(false); + ret = false; + }; + co_return ret; +} + +void configureGenerator(const char* text, + const char* line, + std::vector& lc, + std::vector const& tokens) { + const char* opts[] = { "new", + "single", + "double", + "triple", + "three_data_hall", + "three_datacenter", + "ssd", + "ssd-1", + "ssd-2", + "memory", + "memory-1", + "memory-2", + "memory-radixtree", + "commit_proxies=", + "grv_proxies=", + "logs=", + "resolvers=", + "perpetual_storage_wiggle=", + "perpetual_storage_wiggle_locality=", + "perpetual_storage_wiggle_engine=", + "storage_migration_type=", + nullptr }; + arrayGenerator(text, line, opts, lc); +} + +CommandFactory configureFactory( + "configure", + CommandHelp( + "configure [new|tss]" + "|" + "commit_proxies=|grv_proxies=|logs=|resolvers=>*|" + "count=|perpetual_storage_wiggle=|perpetual_storage_wiggle_locality=" + "<:|0>|perpetual_storage_wiggle_engine=|" + "storage_migration_type={disabled|gradual|aggressive}" + "|exclude=", + "change the database configuration", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. When used, both a redundancy mode and a storage engine must be " + "specified.\n\ntss: when enabled, configures the testing storage server for the cluster instead." + "When used with new to set up tss for the first time, it requires both a count and a storage engine." + "To disable the testing storage server, run \"configure tss count=0\"\n\n" + "Redundancy mode:\n single - one copy of the data. Not fault tolerant.\n double - two copies " + "of data (survive one failure).\n triple - three copies of data (survive two failures).\n three_data_hall - " + "See the Admin Guide.\n three_datacenter - See the Admin Guide.\n\nStorage engine:\n ssd - B-Tree storage " + "engine optimized for solid state disks.\n memory - Durable in-memory storage engine for small " + "datasets.\n\nproxies=: Sets the desired number of proxies in the cluster. The proxy role is being " + "deprecated and split into GRV proxy and Commit proxy, now prefer configure 'grv_proxies' and 'commit_proxies' " + "separately. Generally we should follow that 'commit_proxies' is three times of 'grv_proxies' and " + "'grv_proxies' " + "should be not more than 4. If 'proxies' is specified, it will be converted to 'grv_proxies' and " + "'commit_proxies'. " + "Must be at least 2 (1 GRV proxy, 1 Commit proxy), or set to -1 which restores the number of proxies to the " + "default value.\n\ncommit_proxies=: Sets the desired number of commit proxies in the cluster. " + "Must be at least 1, or set to -1 which restores the number of commit proxies to the default " + "value.\n\ngrv_proxies=: Sets the desired number of GRV proxies in the cluster. Must be at least " + "1, or set to -1 which restores the number of GRV proxies to the default value.\n\nlogs=: Sets the " + "desired number of log servers in the cluster. Must be at least 1, or set to -1 which restores the number of " + "logs to the default value.\n\nresolvers=: Sets the desired number of resolvers in the cluster. " + "Must be at least 1, or set to -1 which restores the number of resolvers to the default value.\n\n" + "perpetual_storage_wiggle=: Set the value speed (a.k.a., the number of processes that the Data " + "Distributor should wiggle at a time). Currently, only 0 and 1 are supported. The value 0 means to disable the " + "perpetual storage wiggle.\n\n" + "perpetual_storage_wiggle_locality=<:|0>: Set the process filter for wiggling. " + "The processes that match the given locality key and locality value are only wiggled. The value 0 will disable " + "the locality filter and matches all the processes for wiggling.\n\n" + "exclude=: Sets the addresses in the format of IP1:port1,IP2:port2 pairs to be excluded during " + "recruitment. Note this should be only used when the database is unavailable because of the faulty processes " + "that are blocking the recovery from completion. The number of addresses should be less than the replication " + "factor to avoid data loss.\n\n" + + "See the FoundationDB Administration Guide for more information."), + &configureGenerator); + +} // namespace fdb_cli diff --git a/fdbcli/ConsistencyCheckCommand.actor.cpp b/fdbcli/ConsistencyCheckCommand.actor.cpp deleted file mode 100644 index 6745e3b830b..00000000000 --- a/fdbcli/ConsistencyCheckCommand.actor.cpp +++ /dev/null @@ -1,69 +0,0 @@ -/* - * ConsistencyCheckCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -const KeyRef consistencyCheckSpecialKey = "\xff\xff/management/consistency_check_suspended"_sr; - -ACTOR Future consistencyCheckCommandActor(Reference tr, - std::vector tokens, - bool intrans) { - // Here we do not proceed in a try-catch loop since the transaction is always supposed to succeed. - // If not, the outer loop catch block(fdbcli.actor.cpp) will handle the error and print out the error message - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - if (tokens.size() == 1) { - // hold the returned standalone object's memory - state ThreadFuture> suspendedF = tr->get(consistencyCheckSpecialKey); - Optional suspended = wait(safeThreadFutureToFuture(suspendedF)); - printf("ConsistencyCheck is %s\n", suspended.present() ? "off" : "on"); - } else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) { - tr->set(consistencyCheckSpecialKey, Value()); - if (!intrans) - wait(safeThreadFutureToFuture(tr->commit())); - } else if (tokens.size() == 2 && tokencmp(tokens[1], "on")) { - tr->clear(consistencyCheckSpecialKey); - if (!intrans) - wait(safeThreadFutureToFuture(tr->commit())); - } else { - printUsage(tokens[0]); - return false; - } - return true; -} - -CommandFactory consistencyCheckFactory( - "consistencycheck", - CommandHelp( - "consistencycheck [on|off]", - "permits or prevents consistency checking", - "Calling this command with `on' permits consistency check processes to run and `off' will halt their checking. " - "Calling this command with no arguments will display if consistency checking is currently allowed.\n")); - -} // namespace fdb_cli diff --git a/fdbcli/ConsistencyCheckCommand.cpp b/fdbcli/ConsistencyCheckCommand.cpp new file mode 100644 index 00000000000..3540fe6c181 --- /dev/null +++ b/fdbcli/ConsistencyCheckCommand.cpp @@ -0,0 +1,67 @@ +/* + * ConsistencyCheckCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +namespace fdb_cli { + +const KeyRef consistencyCheckSpecialKey = "\xff\xff/management/consistency_check_suspended"_sr; + +Future consistencyCheckCommandActor(Reference tr, + std::vector const& tokens, + bool intrans) { + // Here we do not proceed in a try-catch loop since the transaction is always supposed to succeed. + // If not, the outer loop catch block(fdbcli.cpp) will handle the error and print out the error message + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + if (tokens.size() == 1) { + // hold the returned standalone object's memory + ThreadFuture> suspendedF = tr->get(consistencyCheckSpecialKey); + Optional suspended = co_await safeThreadFutureToFuture(suspendedF); + printf("ConsistencyCheck is %s\n", suspended.present() ? "off" : "on"); + } else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) { + tr->set(consistencyCheckSpecialKey, Value()); + if (!intrans) + co_await safeThreadFutureToFuture(tr->commit()); + } else if (tokens.size() == 2 && tokencmp(tokens[1], "on")) { + tr->clear(consistencyCheckSpecialKey); + if (!intrans) + co_await safeThreadFutureToFuture(tr->commit()); + } else { + printUsage(tokens[0]); + co_return false; + } + co_return true; +} + +CommandFactory consistencyCheckFactory( + "consistencycheck", + CommandHelp( + "consistencycheck [on|off]", + "permits or prevents consistency checking", + "Calling this command with `on' permits consistency check processes to run and `off' will halt their checking. " + "Calling this command with no arguments will display if consistency checking is currently allowed.\n")); + +} // namespace fdb_cli diff --git a/fdbcli/ConsistencyScanCommand.actor.cpp b/fdbcli/ConsistencyScanCommand.actor.cpp deleted file mode 100644 index 1b215dd8d1d..00000000000 --- a/fdbcli/ConsistencyScanCommand.actor.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/* - * ConsistencyScanCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/ReadYourWrites.h" -#include "fdbclient/RunTransaction.actor.h" -#include "fdbclient/ConsistencyScanInterface.actor.h" - -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future dumpStats(ConsistencyScanState* cs, Reference tr) { - state ConsistencyScanState::LifetimeStats statsLifetime; - state ConsistencyScanState::RoundStats statsCurrentRound; - wait(store(statsLifetime, cs->lifetimeStats().getD(tr)) && - store(statsCurrentRound, cs->currentRoundStats().getD(tr))); - printf( - "Current Round:\n%s\n", - json_spirit::write_string(json_spirit::mValue(statsCurrentRound.toJSON()), json_spirit::pretty_print).c_str()); - printf("Lifetime:\n%s\n", - json_spirit::write_string(json_spirit::mValue(statsLifetime.toJSON()), json_spirit::pretty_print).c_str()); - return Void(); -} - -ACTOR Future consistencyScanCommandActor(Database db, std::vector tokens) { - // Skip the command token so start at begin+1 - state std::list args(tokens.begin() + 1, tokens.end()); - - state ConsistencyScanState cs = ConsistencyScanState(); - state Reference tr = makeReference(db); - state bool error = false; - - loop { - try { - SystemDBWriteLockedNow(db.getReference())->setOptions(tr); - - state ConsistencyScanState::Config config = wait(ConsistencyScanState().config().getD(tr)); - - if (args.empty()) { - printf( - "%s\n", - json_spirit::write_string(json_spirit::mValue(config.toJSON()), json_spirit::pretty_print).c_str()); - break; - } - - // TODO: Expose/document additional configuration options - // TODO: Range configuration. - while (!error && !args.empty()) { - auto next = args.front(); - args.pop_front(); - if (next == "on") { - config.enabled = true; - } else if (next == "off") { - config.enabled = false; - } else if (next == "restart") { - config.minStartVersion = tr->getReadVersion().get(); - } else if (next == "stats") { - wait(dumpStats(&cs, tr)); - } else if (next == "clearstats") { - wait(cs.clearStats(tr)); - } else if (next == "maxRate") { - error = args.empty(); - if (!error) { - config.maxReadByteRate = boost::lexical_cast(args.front().toString()); - args.pop_front(); - } - } else if (next == "targetInterval") { - error = args.empty(); - if (!error) { - config.targetRoundTimeSeconds = boost::lexical_cast(args.front().toString()); - args.pop_front(); - } - } - } - - if (error) { - break; - } - cs.config().set(tr, config); - wait(tr->commit()); - break; - } catch (Error& e) { - wait(tr->onError(e)); - } - } - - if (error) { - printUsage(tokens[0]); - return false; - } - - return true; -} - -CommandFactory consistencyScanFactory( - "consistencyscan", - CommandHelp( - // TODO: Expose/document additional configuration options - "consistencyscan [on|off] [restart] [stats] [clearstats] [maxRate ] [targetInterval " - "]", - "Enables, disables, or sets options for the Consistency Scan role which repeatedly scans " - "shard replicas for consistency.", - "`on' enables the scan.\n\n" - "`off' disables the scan but keeps the current cycle's progress so it will resume later if enabled again.\n\n" - "`restart' will end the current scan cycle. A new cycle will start if the scan is enabled, or later when " - "it is enabled.\n\n" - "`maxRate ' sets the maximum scan read speed rate to BYTES_PER_SECOND, post-replication.\n\n" - "`targetInterval ' sets the target interval for the scan to SECONDS. The scan will adjust speed " - "to attempt to complete in that amount of time but it will not exceed BYTES_PER_SECOND\n\n" - "`stats` dumps the current round and lifetime stats of the consistency scan. It is a convenience method to " - "expose the stats which are also in status json.\n\n" - "`clearstats` will clear all of the stats for the consistency scan but otherwise leave the configuration as " - "is. This can be used to clear errors or reset stat counts, for example.\n\n" - "The consistency scan role publishes its configuration and metrics in Status JSON under the path " - "`.cluster.consistency_scan'\n" - // TODO: Syntax hint generator - )); - -} // namespace fdb_cli \ No newline at end of file diff --git a/fdbcli/ConsistencyScanCommand.cpp b/fdbcli/ConsistencyScanCommand.cpp new file mode 100644 index 00000000000..0d56b7a0c35 --- /dev/null +++ b/fdbcli/ConsistencyScanCommand.cpp @@ -0,0 +1,140 @@ +/* + * ConsistencyScanCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "fdbcli/fdbcli.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/ReadYourWrites.h" +#include "fdbclient/RunTransaction.h" +#include "fdbclient/ConsistencyScanInterface.h" + +namespace fdb_cli { + +Future dumpStats(ConsistencyScanState* cs, Reference tr) { + ConsistencyScanState::LifetimeStats statsLifetime; + ConsistencyScanState::RoundStats statsCurrentRound; + co_await (store(statsLifetime, cs->lifetimeStats().getD(tr)) && + store(statsCurrentRound, cs->currentRoundStats().getD(tr))); + printf( + "Current Round:\n%s\n", + json_spirit::write_string(json_spirit::mValue(statsCurrentRound.toJSON()), json_spirit::pretty_print).c_str()); + printf("Lifetime:\n%s\n", + json_spirit::write_string(json_spirit::mValue(statsLifetime.toJSON()), json_spirit::pretty_print).c_str()); + co_return; +} + +Future consistencyScanCommandActor(Database db, std::vector const& tokens) { + // Skip the command token so start at begin+1 + std::list args(tokens.begin() + 1, tokens.end()); + + ConsistencyScanState cs = ConsistencyScanState(); + auto tr = makeReference(db); + bool error = false; + + while (true) { + Error err; + try { + SystemDBWriteLockedNow(db.getReference())->setOptions(tr); + + ConsistencyScanState::Config config = co_await ConsistencyScanState().config().getD(tr); + + if (args.empty()) { + printf( + "%s\n", + json_spirit::write_string(json_spirit::mValue(config.toJSON()), json_spirit::pretty_print).c_str()); + break; + } + + // TODO: Expose/document additional configuration options + // TODO: Range configuration. + while (!error && !args.empty()) { + auto next = args.front(); + args.pop_front(); + if (next == "on") { + config.enabled = true; + } else if (next == "off") { + config.enabled = false; + } else if (next == "restart") { + config.minStartVersion = tr->getReadVersion().get(); + } else if (next == "stats") { + co_await dumpStats(&cs, tr); + } else if (next == "clearstats") { + co_await cs.clearStats(tr); + } else if (next == "maxRate") { + error = args.empty(); + if (!error) { + config.maxReadByteRate = boost::lexical_cast(args.front().toString()); + args.pop_front(); + } + } else if (next == "targetInterval") { + error = args.empty(); + if (!error) { + config.targetRoundTimeSeconds = boost::lexical_cast(args.front().toString()); + args.pop_front(); + } + } + } + + if (error) { + break; + } + cs.config().set(tr, config); + co_await tr->commit(); + break; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } + + if (error) { + printUsage(tokens[0]); + co_return false; + } + + co_return true; +} + +CommandFactory consistencyScanFactory( + "consistencyscan", + CommandHelp( + // TODO: Expose/document additional configuration options + "consistencyscan [on|off] [restart] [stats] [clearstats] [maxRate ] [targetInterval " + "]", + "Enables, disables, or sets options for the Consistency Scan role which repeatedly scans " + "shard replicas for consistency.", + "`on' enables the scan.\n\n" + "`off' disables the scan but keeps the current cycle's progress so it will resume later if enabled again.\n\n" + "`restart' will end the current scan cycle. A new cycle will start if the scan is enabled, or later when " + "it is enabled.\n\n" + "`maxRate ' sets the maximum scan read speed rate to BYTES_PER_SECOND, post-replication.\n\n" + "`targetInterval ' sets the target interval for the scan to SECONDS. The scan will adjust speed " + "to attempt to complete in that amount of time but it will not exceed BYTES_PER_SECOND\n\n" + "`stats` dumps the current round and lifetime stats of the consistency scan. It is a convenience method to " + "expose the stats which are also in status json.\n\n" + "`clearstats` will clear all of the stats for the consistency scan but otherwise leave the configuration as " + "is. This can be used to clear errors or reset stat counts, for example.\n\n" + "The consistency scan role publishes its configuration and metrics in Status JSON under the path " + "`.cluster.consistency_scan'\n" + // TODO: Syntax hint generator + )); + +} // namespace fdb_cli diff --git a/fdbcli/CoordinatorsCommand.actor.cpp b/fdbcli/CoordinatorsCommand.actor.cpp deleted file mode 100644 index d7b6ef947cc..00000000000 --- a/fdbcli/CoordinatorsCommand.actor.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * CoordinatorsCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/algorithm/string.hpp" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" -#include "fdbclient/Schemas.h" -#include "fdbclient/ManagementAPI.actor.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -ACTOR Future printCoordinatorsInfo(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - // Hold the reference to the standalone's memory - state ThreadFuture> descriptionF = tr->get(fdb_cli::clusterDescriptionSpecialKey); - Optional description = wait(safeThreadFutureToFuture(descriptionF)); - ASSERT(description.present()); - printf("Cluster description: %s\n", description.get().toString().c_str()); - // Hold the reference to the standalone's memory - state ThreadFuture> processesF = tr->get(fdb_cli::coordinatorsProcessSpecialKey); - Optional processes = wait(safeThreadFutureToFuture(processesF)); - ASSERT(processes.present()); - std::vector process_addresses; - boost::split(process_addresses, processes.get().toString(), [](char c) { return c == ','; }); - printf("Cluster coordinators (%zu): %s\n", process_addresses.size(), processes.get().toString().c_str()); - printf("Type `help coordinators' to learn how to change this information.\n"); - return Void(); - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future changeCoordinators(Reference db, std::vector tokens) { - state int retries = 0; - state int notEnoughMachineResults = 0; - state StringRef new_cluster_description; - state std::string auto_coordinators_str; - StringRef nameTokenBegin = "description="_sr; - for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok) { - if (tok->startsWith(nameTokenBegin) && new_cluster_description.empty()) { - new_cluster_description = tok->substr(nameTokenBegin.size()); - auto next = tok - 1; - std::copy(tok + 1, tokens.end(), tok); - tokens.resize(tokens.size() - 1); - tok = next; - } - } - - state bool automatic = tokens.size() == 2 && tokens[1] == "auto"_sr; - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - // update cluster description - if (new_cluster_description.size()) { - tr->set(fdb_cli::clusterDescriptionSpecialKey, new_cluster_description); - } - // if auto change, read the special key to retrieve the recommended config - if (automatic) { - // if previous read failed, retry, otherwise, use the same recommended config - if (!auto_coordinators_str.size()) { - // Hold the reference to the standalone's memory - state ThreadFuture> auto_coordinatorsF = - tr->get(fdb_cli::coordinatorsAutoSpecialKey); - Optional auto_coordinators = wait(safeThreadFutureToFuture(auto_coordinatorsF)); - ASSERT(auto_coordinators.present()); - auto_coordinators_str = auto_coordinators.get().toString(); - } - tr->set(fdb_cli::coordinatorsProcessSpecialKey, auto_coordinators_str); - } else if (tokens.size() > 1) { - state std::set new_coordinators_addresses; - state std::set new_coordinators_hostnames; - state std::vector newCoordinatorslist; - state std::vector::iterator t; - for (t = tokens.begin() + 1; t != tokens.end(); ++t) { - try { - if (Hostname::isHostname(t->toString())) { - // We do not resolve hostnames here. We commit them as is. - const auto& hostname = Hostname::parse(t->toString()); - if (new_coordinators_hostnames.count(hostname)) { - fprintf(stderr, - "ERROR: passed redundant coordinators: `%s'\n", - hostname.toString().c_str()); - return true; - } - new_coordinators_hostnames.insert(hostname); - newCoordinatorslist.push_back(hostname.toString()); - } else { - const auto& addr = NetworkAddress::parse(t->toString()); - if (new_coordinators_addresses.count(addr)) { - fprintf( - stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str()); - return true; - } - new_coordinators_addresses.insert(addr); - newCoordinatorslist.push_back(addr.toString()); - } - } catch (Error& e) { - if (e.code() == error_code_connection_string_invalid) { - fprintf( - stderr, "ERROR: '%s' is not a valid network endpoint address\n", t->toString().c_str()); - return true; - } - throw; - } - } - std::string new_coordinators_str = boost::algorithm::join(newCoordinatorslist, ","); - tr->set(fdb_cli::coordinatorsProcessSpecialKey, new_coordinators_str); - } - wait(safeThreadFutureToFuture(tr->commit())); - // commit should always fail here - // If the commit succeeds, the coordinators change and the commit will fail with commit_unknown_result(). - ASSERT(false); - } catch (Error& e) { - state Error err(e); - if (e.code() == error_code_special_keys_api_failure) { - std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); - if (errorMsgStr == ManagementAPI::generateErrorMessage(CoordinatorsResult::NOT_ENOUGH_MACHINES) && - notEnoughMachineResults < 1) { - // we could get not_enough_machines if we happen to see the database while the cluster controller is - // updating the worker list, so make sure it happens twice before returning a failure - notEnoughMachineResults++; - wait(delay(1.0)); - tr->reset(); - continue; - } else if (errorMsgStr == - ManagementAPI::generateErrorMessage(CoordinatorsResult::SAME_NETWORK_ADDRESSES)) { - if (retries) - printf("Coordination state changed\n"); - else - printf("No change (existing configuration satisfies request)\n"); - return true; - } else { - fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str()); - return false; - } - } - wait(safeThreadFutureToFuture(tr->onError(err))); - ++retries; - } - } -} - -} // namespace - -namespace fdb_cli { - -const KeyRef clusterDescriptionSpecialKey = "\xff\xff/configuration/coordinators/cluster_description"_sr; -const KeyRef coordinatorsAutoSpecialKey = "\xff\xff/management/auto_coordinators"_sr; -const KeyRef coordinatorsProcessSpecialKey = "\xff\xff/configuration/coordinators/processes"_sr; - -ACTOR Future coordinatorsCommandActor(Reference db, std::vector tokens) { - if (tokens.size() < 2) { - wait(printCoordinatorsInfo(db)); - return true; - } else { - bool result = wait(changeCoordinators(db, tokens)); - return result; - } -} - -CommandFactory coordinatorsFactory( - "coordinators", - CommandHelp( - "coordinators auto|
+ [description=new_cluster_description]", - "change cluster coordinators or description", - "If 'auto' is specified, coordinator addresses will be chosen automatically to support the configured " - "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, " - "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by " - "
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators " - "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in " - "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+.")); -} // namespace fdb_cli diff --git a/fdbcli/CoordinatorsCommand.cpp b/fdbcli/CoordinatorsCommand.cpp new file mode 100644 index 00000000000..319f5144c93 --- /dev/null +++ b/fdbcli/CoordinatorsCommand.cpp @@ -0,0 +1,210 @@ +/* + * CoordinatorsCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/algorithm/string.hpp" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/Schemas.h" +#include "fdbclient/ManagementAPI.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +Future printCoordinatorsInfo(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + // Hold the reference to the standalone's memory + ThreadFuture> descriptionF = tr->get(fdb_cli::clusterDescriptionSpecialKey); + Optional description = co_await safeThreadFutureToFuture(descriptionF); + ASSERT(description.present()); + printf("Cluster description: %s\n", description.get().toString().c_str()); + // Hold the reference to the standalone's memory + ThreadFuture> processesF = tr->get(fdb_cli::coordinatorsProcessSpecialKey); + Optional processes = co_await safeThreadFutureToFuture(processesF); + ASSERT(processes.present()); + std::vector process_addresses; + boost::split(process_addresses, processes.get().toString(), [](char c) { return c == ','; }); + printf("Cluster coordinators (%zu): %s\n", process_addresses.size(), processes.get().toString().c_str()); + printf("Type `help coordinators' to learn how to change this information.\n"); + co_return; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future changeCoordinators(Reference db, std::vector tokens) { + int retries = 0; + int notEnoughMachineResults = 0; + StringRef new_cluster_description; + std::string auto_coordinators_str; + StringRef nameTokenBegin = "description="_sr; + for (auto tok = tokens.begin() + 1; tok != tokens.end(); ++tok) { + if (tok->startsWith(nameTokenBegin) && new_cluster_description.empty()) { + new_cluster_description = tok->substr(nameTokenBegin.size()); + auto next = tok - 1; + std::copy(tok + 1, tokens.end(), tok); + tokens.resize(tokens.size() - 1); + tok = next; + } + } + + bool automatic = tokens.size() == 2 && tokens[1] == "auto"_sr; + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error caughtErr; + bool hasCaughtErr = false; + try { + // update cluster description + if (!new_cluster_description.empty()) { + tr->set(fdb_cli::clusterDescriptionSpecialKey, new_cluster_description); + } + // if auto change, read the special key to retrieve the recommended config + if (automatic) { + // if previous read failed, retry, otherwise, use the same recommended config + if (auto_coordinators_str.empty()) { + // Hold the reference to the standalone's memory + ThreadFuture> auto_coordinatorsF = tr->get(fdb_cli::coordinatorsAutoSpecialKey); + Optional auto_coordinators = co_await safeThreadFutureToFuture(auto_coordinatorsF); + ASSERT(auto_coordinators.present()); + auto_coordinators_str = auto_coordinators.get().toString(); + } + tr->set(fdb_cli::coordinatorsProcessSpecialKey, auto_coordinators_str); + } else if (tokens.size() > 1) { + std::set new_coordinators_addresses; + std::set new_coordinators_hostnames; + std::vector newCoordinatorslist; + std::vector::iterator t; + for (t = tokens.begin() + 1; t != tokens.end(); ++t) { + try { + if (Hostname::isHostname(t->toString())) { + // We do not resolve hostnames here. We commit them as is. + const auto& hostname = Hostname::parse(t->toString()); + if (new_coordinators_hostnames.contains(hostname)) { + fprintf(stderr, + "ERROR: passed redundant coordinators: `%s'\n", + hostname.toString().c_str()); + co_return true; + } + new_coordinators_hostnames.insert(hostname); + newCoordinatorslist.push_back(hostname.toString()); + } else { + const auto& addr = NetworkAddress::parse(t->toString()); + if (new_coordinators_addresses.contains(addr)) { + fprintf( + stderr, "ERROR: passed redundant coordinators: `%s'\n", addr.toString().c_str()); + co_return true; + } + new_coordinators_addresses.insert(addr); + newCoordinatorslist.push_back(addr.toString()); + } + } catch (Error& e) { + if (e.code() == error_code_connection_string_invalid) { + fprintf( + stderr, "ERROR: '%s' is not a valid network endpoint address\n", t->toString().c_str()); + co_return true; + } + throw; + } + } + std::string new_coordinators_str = boost::algorithm::join(newCoordinatorslist, ","); + tr->set(fdb_cli::coordinatorsProcessSpecialKey, new_coordinators_str); + } + co_await safeThreadFutureToFuture(tr->commit()); + // commit should always fail here + // If the commit succeeds, the coordinators change and the commit will fail with + // commit_unknown_result(). + ASSERT(false); + } catch (Error& e) { + caughtErr = e; + hasCaughtErr = true; + } + if (hasCaughtErr) { + Error err(caughtErr); + if (caughtErr.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr); + if (errorMsgStr == ManagementAPI::generateErrorMessage(CoordinatorsResult::NOT_ENOUGH_MACHINES) && + notEnoughMachineResults < 1) { + // we could get not_enough_machines if we happen to see the database while the cluster + // controller is updating the worker list, so make sure it happens twice before returning a + // failure + notEnoughMachineResults++; + co_await delay(1.0); + tr->reset(); + continue; + } else if (errorMsgStr == + ManagementAPI::generateErrorMessage(CoordinatorsResult::SAME_NETWORK_ADDRESSES)) { + if (retries) + printf("Coordination state changed\n"); + else + printf("No change (existing configuration satisfies request)\n"); + co_return true; + } else { + fprintf(stderr, "ERROR: %s\n", errorMsgStr.c_str()); + co_return false; + } + } + co_await safeThreadFutureToFuture(tr->onError(err)); + ++retries; + } + } +} + +} // namespace + +namespace fdb_cli { + +const KeyRef clusterDescriptionSpecialKey = "\xff\xff/configuration/coordinators/cluster_description"_sr; +const KeyRef coordinatorsAutoSpecialKey = "\xff\xff/management/auto_coordinators"_sr; +const KeyRef coordinatorsProcessSpecialKey = "\xff\xff/configuration/coordinators/processes"_sr; + +Future coordinatorsCommandActor(Reference db, std::vector tokens) { + if (tokens.size() < 2) { + co_await printCoordinatorsInfo(db); + co_return true; + } else { + bool result = co_await changeCoordinators(db, tokens); + co_return result; + } +} + +CommandFactory coordinatorsFactory( + "coordinators", + CommandHelp( + "coordinators auto|
+ [description=new_cluster_description]", + "change cluster coordinators or description", + "If 'auto' is specified, coordinator addresses will be chosen automatically to support the configured " + "redundancy level. (If the current set of coordinators are healthy and already support the redundancy level, " + "nothing will be changed.)\n\nOtherwise, sets the coordinators to the list of IP:port pairs specified by " + "
+. An fdbserver process must be running on each of the specified addresses.\n\ne.g. coordinators " + "10.0.0.1:4000 10.0.0.2:4000 10.0.0.3:4000\n\nIf 'description=desc' is specified then the description field in " + "the cluster\nfile is changed to desc, which must match [A-Za-z0-9_]+.")); +} // namespace fdb_cli diff --git a/fdbcli/DataDistributionCommand.actor.cpp b/fdbcli/DataDistributionCommand.actor.cpp deleted file mode 100644 index 498cd1a580c..00000000000 --- a/fdbcli/DataDistributionCommand.actor.cpp +++ /dev/null @@ -1,172 +0,0 @@ -/* - * DataDistributionCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/lexical_cast.hpp" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBTypes.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -ACTOR Future setDDMode(Reference db, int mode) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - tr->set(fdb_cli::ddModeSpecialKey, boost::lexical_cast(mode)); - if (mode) { - // set DDMode to 1 will enable all disabled parts, for instance the SS failure monitors. - // hold the returned standalone object's memory - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - RangeResult res = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(res.size() <= 1); - if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { - // only clear the key if it is currently being used to disable all SS failure data movement - tr->clear(fdb_cli::maintenanceSpecialKeyRange); - } - tr->clear(fdb_cli::ddIgnoreRebalanceSpecialKey); - } - wait(safeThreadFutureToFuture(tr->commit())); - return Void(); - } catch (Error& e) { - TraceEvent("SetDDModeRetrying").error(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future setDDIgnoreRebalanceSwitch(Reference db, uint8_t DDIgnoreOptionMask, bool setMaskedBit) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - try { - state ThreadFuture> resultFuture = tr->get(rebalanceDDIgnoreKey); - Optional v = wait(safeThreadFutureToFuture(resultFuture)); - uint8_t oldValue = DDIgnore::NONE; // nothing is disabled - if (v.present()) { - if (v.get().size() > 0) { - oldValue = BinaryReader::fromStringRef(v.get(), Unversioned()); - } else { - // In old version (<= 7.1), the value is an empty string, which means all DD rebalance functions are - // disabled - oldValue = DDIgnore::ALL; - } - // printf("oldValue: %d Mask: %d V:%d\n", oldValue, DDIgnoreOptionMask, v.get().size()); - } - uint8_t newValue = setMaskedBit ? (oldValue | DDIgnoreOptionMask) : (oldValue & ~DDIgnoreOptionMask); - if (newValue > 0) { - tr->set(fdb_cli::ddIgnoreRebalanceSpecialKey, BinaryWriter::toValue(newValue, Unversioned())); - } else { - tr->clear(fdb_cli::ddIgnoreRebalanceSpecialKey); - } - wait(safeThreadFutureToFuture(tr->commit())); - return Void(); - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -// set masked bit -Future setDDIgnoreRebalanceOn(Reference db, uint8_t DDIgnoreOptionMask) { - return setDDIgnoreRebalanceSwitch(db, DDIgnoreOptionMask, true); -} - -// reset masked bit -Future setDDIgnoreRebalanceOff(Reference db, uint8_t DDIgnoreOptionMask) { - return setDDIgnoreRebalanceSwitch(db, DDIgnoreOptionMask, false); -} - -} // namespace - -namespace fdb_cli { - -const KeyRef ddModeSpecialKey = "\xff\xff/management/data_distribution/mode"_sr; -const KeyRef ddIgnoreRebalanceSpecialKey = "\xff\xff/management/data_distribution/rebalance_ignored"_sr; -constexpr auto usage = - "Usage: datadistribution |enable " - ">\n"; -ACTOR Future dataDistributionCommandActor(Reference db, std::vector tokens) { - state bool result = true; - if (tokens.size() != 2 && tokens.size() != 3) { - printf(usage); - result = false; - } else { - if (tokencmp(tokens[1], "on")) { - wait(success(setDDMode(db, 1))); - printf("Data distribution is turned on.\n"); - } else if (tokencmp(tokens[1], "off")) { - wait(success(setDDMode(db, 0))); - printf("Data distribution is turned off.\n"); - } else if (tokencmp(tokens[1], "disable")) { - if (tokencmp(tokens[2], "ssfailure")) { - wait(success((setHealthyZone(db, "IgnoreSSFailures"_sr, 0)))); - printf("Data distribution is disabled for storage server failures.\n"); - } else if (tokencmp(tokens[2], "rebalance")) { - wait(setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ)); - printf("Data distribution is disabled for rebalance.\n"); - } else if (tokencmp(tokens[2], "rebalance_disk")) { - wait(setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK)); - printf("Data distribution is disabled for rebalance_disk.\n"); - } else if (tokencmp(tokens[2], "rebalance_read")) { - wait(setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_READ)); - printf("Data distribution is disabled for rebalance_read.\n"); - } else { - printf(usage); - result = false; - } - } else if (tokencmp(tokens[1], "enable")) { - if (tokencmp(tokens[2], "ssfailure")) { - wait(success((clearHealthyZone(db, false, true)))); - printf("Data distribution is enabled for storage server failures.\n"); - } else if (tokencmp(tokens[2], "rebalance")) { - wait(setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ)); - printf("Data distribution is enabled for rebalance.\n"); - } else if (tokencmp(tokens[2], "rebalance_disk")) { - wait(setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK)); - printf("Data distribution is enabled for rebalance_disk.\n"); - } else if (tokencmp(tokens[2], "rebalance_read")) { - wait(setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_READ)); - printf("Data distribution is enabled for rebalance_read.\n"); - } else { - printf(usage); - result = false; - } - } else { - printf(usage); - result = false; - } - } - return result; -} - -// hidden commands, no help text for now -CommandFactory dataDistributionFactory("datadistribution"); -} // namespace fdb_cli diff --git a/fdbcli/DataDistributionCommand.cpp b/fdbcli/DataDistributionCommand.cpp new file mode 100644 index 00000000000..acc0457d72e --- /dev/null +++ b/fdbcli/DataDistributionCommand.cpp @@ -0,0 +1,175 @@ +/* + * DataDistributionCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/lexical_cast.hpp" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBTypes.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +Future setDDMode(Reference db, int mode) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + tr->set(fdb_cli::ddModeSpecialKey, boost::lexical_cast(mode)); + if (mode) { + // set DDMode to 1 will enable all disabled parts, for instance the SS failure monitors. + // hold the returned standalone object's memory + ThreadFuture resultFuture = + tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult res = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(res.size() <= 1); + if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { + // only clear the key if it is currently being used to disable all SS failure data movement + tr->clear(fdb_cli::maintenanceSpecialKeyRange); + } + tr->clear(fdb_cli::ddIgnoreRebalanceSpecialKey); + } + co_await safeThreadFutureToFuture(tr->commit()); + co_return; + } catch (Error& e) { + err = e; + } + TraceEvent("SetDDModeRetrying").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future setDDIgnoreRebalanceSwitch(Reference db, uint8_t DDIgnoreOptionMask, bool setMaskedBit) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + Error err; + try { + ThreadFuture> resultFuture = tr->get(rebalanceDDIgnoreKey); + Optional v = co_await safeThreadFutureToFuture(resultFuture); + uint8_t oldValue = DDIgnore::NONE; // nothing is disabled + if (v.present()) { + if (!v.get().empty()) { + oldValue = BinaryReader::fromStringRef(v.get(), Unversioned()); + } else { + // In old version (<= 7.1), the value is an empty string, which means all DD rebalance functions + // are disabled + oldValue = DDIgnore::ALL; + } + // printf("oldValue: %d Mask: %d V:%d\n", oldValue, DDIgnoreOptionMask, v.get().size()); + } + uint8_t newValue = setMaskedBit ? (oldValue | DDIgnoreOptionMask) : (oldValue & ~DDIgnoreOptionMask); + if (newValue > 0) { + tr->set(fdb_cli::ddIgnoreRebalanceSpecialKey, BinaryWriter::toValue(newValue, Unversioned())); + } else { + tr->clear(fdb_cli::ddIgnoreRebalanceSpecialKey); + } + co_await safeThreadFutureToFuture(tr->commit()); + co_return; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +// set masked bit +Future setDDIgnoreRebalanceOn(Reference db, uint8_t DDIgnoreOptionMask) { + return setDDIgnoreRebalanceSwitch(db, DDIgnoreOptionMask, true); +} + +// reset masked bit +Future setDDIgnoreRebalanceOff(Reference db, uint8_t DDIgnoreOptionMask) { + return setDDIgnoreRebalanceSwitch(db, DDIgnoreOptionMask, false); +} + +} // namespace + +namespace fdb_cli { + +const KeyRef ddModeSpecialKey = "\xff\xff/management/data_distribution/mode"_sr; +const KeyRef ddIgnoreRebalanceSpecialKey = "\xff\xff/management/data_distribution/rebalance_ignored"_sr; +constexpr auto usage = + "Usage: datadistribution |enable " + ">\n"; +Future dataDistributionCommandActor(Reference db, std::vector tokens) { + bool result = true; + if (tokens.size() != 2 && tokens.size() != 3) { + printf(usage); + result = false; + } else { + if (tokencmp(tokens[1], "on")) { + co_await setDDMode(db, 1); + printf("Data distribution is turned on.\n"); + } else if (tokencmp(tokens[1], "off")) { + co_await setDDMode(db, 0); + printf("Data distribution is turned off.\n"); + } else if (tokencmp(tokens[1], "disable")) { + if (tokencmp(tokens[2], "ssfailure")) { + co_await (setHealthyZone(db, "IgnoreSSFailures"_sr, 0)); + printf("Data distribution is disabled for storage server failures.\n"); + } else if (tokencmp(tokens[2], "rebalance")) { + co_await setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ); + printf("Data distribution is disabled for rebalance.\n"); + } else if (tokencmp(tokens[2], "rebalance_disk")) { + co_await setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_DISK); + printf("Data distribution is disabled for rebalance_disk.\n"); + } else if (tokencmp(tokens[2], "rebalance_read")) { + co_await setDDIgnoreRebalanceOn(db, DDIgnore::REBALANCE_READ); + printf("Data distribution is disabled for rebalance_read.\n"); + } else { + printf(usage); + result = false; + } + } else if (tokencmp(tokens[1], "enable")) { + if (tokencmp(tokens[2], "ssfailure")) { + co_await (clearHealthyZone(db, false, true)); + printf("Data distribution is enabled for storage server failures.\n"); + } else if (tokencmp(tokens[2], "rebalance")) { + co_await setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK | DDIgnore::REBALANCE_READ); + printf("Data distribution is enabled for rebalance.\n"); + } else if (tokencmp(tokens[2], "rebalance_disk")) { + co_await setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_DISK); + printf("Data distribution is enabled for rebalance_disk.\n"); + } else if (tokencmp(tokens[2], "rebalance_read")) { + co_await setDDIgnoreRebalanceOff(db, DDIgnore::REBALANCE_READ); + printf("Data distribution is enabled for rebalance_read.\n"); + } else { + printf(usage); + result = false; + } + } else { + printf(usage); + result = false; + } + } + co_return result; +} + +// hidden commands, no help text for now +CommandFactory dataDistributionFactory("datadistribution"); +} // namespace fdb_cli diff --git a/fdbcli/DebugCommands.actor.cpp b/fdbcli/DebugCommands.actor.cpp deleted file mode 100644 index 2403a650ab4..00000000000 --- a/fdbcli/DebugCommands.actor.cpp +++ /dev/null @@ -1,491 +0,0 @@ -/* - * DebugCommands.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBTypes.h" -#include "fdbclient/NativeAPI.actor.h" - -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -std::string toHex(StringRef v) { - std::string result; - result.reserve(v.size() * 4); - for (int i = 0; i < v.size(); i++) { - result.append(format("\\x%02x", v[i])); - } - return result; -} - -// Gets a version at which to read from the storage servers -ACTOR Future getVersion(Database cx) { - loop { - state Transaction tr(cx); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - try { - Version version = wait(tr.getReadVersion()); - return version; - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - -// Get a list of storage servers that persist keys within range "kr" from the -// first commit proxy. Returns false if there is a failure (in this case, -// keyServersPromise will never be set). -// If dcid is set, only return the storage servers in the given datacenter. -// Ignore the input dcid if the cluster has not set dcid. -ACTOR Future getKeyServers( - Database cx, - Promise>>> keyServersPromise, - KeyRangeRef kr, - Optional dcid) { - state std::vector>> keyServers; - - // Try getting key server locations from the first commit proxy - state Future> keyServerLocationFuture; - state Key begin = kr.begin; - state Key end = kr.end; - state int limitKeyServers = 100; - - while (begin < end) { - state Reference commitProxyInfo = - wait(cx->getCommitProxiesFuture(UseProvisionalProxies::False)); - keyServerLocationFuture = - commitProxyInfo->get(0, &CommitProxyInterface::getKeyServersLocations) - .getReplyUnlessFailedFor( - GetKeyServerLocationsRequest({}, begin, end, limitKeyServers, false, latestVersion, Arena()), 2, 0); - - state bool keyServersInsertedForThisIteration = false; - choose { - when(ErrorOr shards = wait(keyServerLocationFuture)) { - // Get the list of shards if one was returned. - if (shards.present() && !keyServersInsertedForThisIteration) { - std::vector>> shardResultList; - for (auto& result : shards.get().results) { - std::vector servers; - for (auto& server : result.second) { - // Filter out storage servers that are not in the given datacenter - Optional> serverDcId = server.locality.dcId(); - if (dcid.present() && serverDcId.present() && serverDcId.get() != dcid.get()) { - continue; - } - servers.push_back(server); - } - shardResultList.push_back({ result.first, servers }); - } - keyServers.insert(keyServers.end(), shardResultList.begin(), shardResultList.end()); - keyServersInsertedForThisIteration = true; - begin = shards.get().results.back().first.end; - break; - } - } - when(wait(cx->onProxiesChanged())) {} - } - - if (!keyServersInsertedForThisIteration) // Retry the entire workflow - wait(delay(1.0)); - } - - keyServersPromise.send(keyServers); - return true; -} - -// The command is used to get all storage server addresses for a given key. -ACTOR Future getLocationCommandActor(Database cx, std::vector tokens) { - if (tokens.size() != 2 && tokens.size() != 3) { - fmt::println("getlocation []\n" - "fetch the storage server address for a given key or range.\n" - "Displays the addresses of storage servers, or `not found' if location is not found."); - return false; - } - - state KeyRange kr = KeyRangeRef(tokens[1], tokens.size() == 3 ? tokens[2] : keyAfter(tokens[1])); - // find key range locations without GRV - state Promise>>> keyServersPromise; - bool found = wait(getKeyServers(cx, keyServersPromise, kr, Optional())); - if (!found) { - fmt::println("{} locations not found", printable(kr)); - return false; - } - std::vector>> keyServers = - keyServersPromise.getFuture().get(); - for (const auto& [range, servers] : keyServers) { - fmt::println("Key range: {}", printable(range)); - for (const auto& server : servers) { - fmt::println(" {}", server.address().toString()); - } - } - return true; -} -// hidden commands, no help text for now -CommandFactory getLocationCommandFactory("getlocation"); - -// The command is used to get values from all storage servers that have the given key. -ACTOR Future getallCommandActor(Database cx, std::vector tokens, Version version) { - if (tokens.size() != 2) { - fmt::println("getall \n" - "fetch values from all storage servers that have the given key.\n" - "Displays the value and the addresses of storage servers, or `not found' if key is not found."); - return false; - } - - KeyRangeLocationInfo loc = wait(getKeyLocation_internal( - cx, tokens[1], SpanContext(), Optional(), UseProvisionalProxies::False, Reverse::False, version)); - - if (loc.locations) { - fmt::println("version is {}", version); - fmt::println("`{}' is at:", printable(tokens[1])); - state Reference locations = loc.locations->locations(); - state std::vector> replies; - for (int i = 0; locations && i < locations->size(); i++) { - GetValueRequest req(/*spanContext=*/{}, tokens[1], version, {}, {}, {}); - replies.push_back(locations->get(i, &StorageServerInterface::getValue).getReply(req)); - } - wait(waitForAll(replies)); - for (int i = 0; i < replies.size(); i++) { - std::string ssi = locations->getInterface(i).address().toString(); - if (replies[i].isError()) { - fmt::println(stderr, "ERROR: {} {}", ssi, replies[i].getError().what()); - } else { - Optional v = replies[i].get().value; - fmt::println(" {} {}", ssi, v.present() ? printable(v.get()) : "(not found)"); - } - } - } else { - fmt::println("`{}': location not found", printable(tokens[1])); - } - return true; -} -// hidden commands, no help text for now -CommandFactory getallCommandFactory("getall"); - -std::string printStorageServerMachineInfo(const StorageServerInterface& server) { - std::string serverIp = server.address().toString(); - std::string serverLocality = server.locality.toString(); - return serverLocality + " " + serverIp; -} - -std::string printAllStorageServerMachineInfo(const std::vector& servers) { - std::string res; - for (int i = 0; i < servers.size(); i++) { - if (i == 0) { - res = printStorageServerMachineInfo(servers[i]); - } else { - res = res + "; " + printStorageServerMachineInfo(servers[i]); - } - } - return res; -} - -// check that all replies are the same. Update begin to the next key to check -// checkResults keeps invariants: -// (1) hasMore = true if any server has more data not read yet -// (2) nextBeginKey is the minimal key returned from all servers -// (3) checkResults reports inconsistency of keys only before the nextBeginKey if hasMore=true -// Therefore, whether to proceed to the next round depends on hasMore -// If there is a next round, it starts from the minimal key returned from all servers -bool checkResults(Version version, - bool hasMore, - Key claimEndKey, - const std::vector& servers, - const std::vector& replies) { - // Compare servers - bool allSame = true; - int firstValidServer = -1; - for (int j = 0; j < replies.size(); j++) { - if (firstValidServer == -1) { - firstValidServer = j; - // Print full list of comparing servers and the reference server - // Used to check server info which does not produce an inconsistency log - fmt::println("CheckResult: servers: {}, reference server: {}", - printAllStorageServerMachineInfo(servers), - printStorageServerMachineInfo(servers[firstValidServer])); - continue; // always select the first server as reference - } - // compare reference and current - GetKeyValuesReply current = replies[j]; - GetKeyValuesReply reference = replies[firstValidServer]; - if (current.data == reference.data && current.more == reference.more) { - continue; - } - // Detecting corrupted keys for any mismatching replies between current and reference servers - allSame = false; - size_t currentI = 0, referenceI = 0; - while (currentI < current.data.size() || referenceI < reference.data.size()) { - if (hasMore && ((referenceI < reference.data.size() && reference.data[referenceI].key >= claimEndKey) || - (currentI < current.data.size() && current.data[currentI].key >= claimEndKey))) { - // If there will be a next round and the key is out of claimEndKey - // We will delay the detection to the next round - break; - } - if (currentI >= current.data.size()) { - // ServerA(1), ServerB(0): 1 indicates that ServerA has the key while 0 indicates that ServerB does not - // have the key - fmt::println( - "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, Key {}", - printStorageServerMachineInfo(servers[firstValidServer]), - printStorageServerMachineInfo(servers[j]), - currentI, - referenceI, - version, - toHex(reference.data[referenceI].key)); - referenceI++; - } else if (referenceI >= reference.data.size()) { - fmt::println( - "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, Key {}", - printStorageServerMachineInfo(servers[j]), - printStorageServerMachineInfo(servers[firstValidServer]), - currentI, - referenceI, - version, - toHex(current.data[currentI].key)); - currentI++; - } else { - KeyValueRef currentKV = current.data[currentI]; - KeyValueRef referenceKV = reference.data[referenceI]; - if (currentKV.key == referenceKV.key) { - if (currentKV.value != referenceKV.value) { - fmt::println("Inconsistency: MismatchValue, {}(1), {}(1), CurrentIndex {}, ReferenceIndex {}, " - "Version {}, Key {}", - printStorageServerMachineInfo(servers[firstValidServer]), - printStorageServerMachineInfo(servers[j]), - currentI, - referenceI, - version, - toHex(currentKV.key)); - } - currentI++; - referenceI++; - } else if (currentKV.key < referenceKV.key) { - fmt::println( - "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, " - "Key {}", - printStorageServerMachineInfo(servers[j]), - printStorageServerMachineInfo(servers[firstValidServer]), - currentI, - referenceI, - version, - toHex(currentKV.key)); - currentI++; - } else { - fmt::println( - "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, " - "Key {}", - printStorageServerMachineInfo(servers[firstValidServer]), - printStorageServerMachineInfo(servers[j]), - currentI, - referenceI, - version, - toHex(referenceKV.key)); - referenceI++; - } - } - } - } - - return allSame; -} - -ACTOR Future doCheckAll(Database cx, KeyRange inputRange, Optional dcid, bool checkAll); -// Return whether inconsistency is detected in the inputRange -ACTOR Future doCheckAll(Database cx, KeyRange inputRange, Optional dcid, bool checkAll) { - state Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior - state bool consistent = true; - loop { - try { - fmt::println("Start checking for range: {}", printable(inputRange)); - // Get SS interface for each shard of the inputRange - state Promise>>> keyServerPromise; - bool foundKeyServers = wait(getKeyServers(cx, keyServerPromise, inputRange, dcid)); - if (!foundKeyServers) { - fmt::println("key server locations for {} not found, retrying in 1s...", printable(inputRange)); - wait(delay(1.0)); - continue; - } - state std::vector>> keyServers = - keyServerPromise.getFuture().get(); - // We partition the entire input range into shards - // and we conduct comparison shard by shard - state int i = 0; - for (; i < keyServers.size(); i++) { // for each shard - state KeyRange rangeToCheck = keyServers[i].first; - rangeToCheck = rangeToCheck & inputRange; // Only check the shard part within the inputRange - if (rangeToCheck.empty()) { - continue; // Skip the shard if it is outside of the inputRange - } - const auto& servers = keyServers[i].second; - state Key beginKeyToCheck = rangeToCheck.begin; - fmt::println("Key range to check: {}", printable(rangeToCheck)); - for (const auto& server : servers) { - fmt::println("\t{}", server.address().toString()); - } - state std::vector>> replies; - state bool hasMore = true; - state int round = 0; - state Version version; - while (hasMore) { - wait(store(version, getVersion(cx))); - replies.clear(); - fmt::println("Round {}: {} - {}", round, toHex(beginKeyToCheck), toHex(rangeToCheck.end)); - for (const auto& s : keyServers[i].second) { // for each storage server - GetKeyValuesRequest req; - req.begin = firstGreaterOrEqual(beginKeyToCheck); - req.end = firstGreaterOrEqual(rangeToCheck.end); - req.limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT; - req.limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES; - req.version = version; // all replica should read at the same version - req.tags = TagSet(); - replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0)); - } - wait(waitForAll(replies)); - - // Decide comparison scope - Key claimEndKey; // used for the next round if hasMore == true - Key maxEndKey; - hasMore = false; // re-calculate hasMore according to replies - for (int j = 0; j < replies.size(); j++) { - auto reply = replies[j].get(); - if (reply.isError()) { - fmt::println("checkResults error: {}", reply.getError().what()); - throw reply.getError(); - } else if (reply.get().error.present()) { - fmt::println("checkResults error: {}", reply.get().error.get().what()); - throw reply.get().error.get(); - } - GetKeyValuesReply current = reply.get(); - if (current.data.size() == 0) { - continue; // Ignore if no data has replied - } - if (claimEndKey.empty() || current.data[current.data.size() - 1].key < claimEndKey) { - claimEndKey = current.data[current.data.size() - 1].key; - } - if (maxEndKey.empty() || current.data[current.data.size() - 1].key > maxEndKey) { - maxEndKey = current.data[current.data.size() - 1].key; - } - hasMore = hasMore || current.more; - } - fmt::println("Compare scope has been decided\n\tBeginKey: {}\n\tEndKey: {}\n\tHasMore: {}", - toHex(beginKeyToCheck), - toHex(claimEndKey), - hasMore); - if (claimEndKey.empty()) { - // It is possible that there is clear operation between the prev round and the current round - // which result in empty claimEndKey --- nothing to compare - // In this case, we simply skip the current shard - ASSERT(hasMore == false); - continue; - } else if ((beginKeyToCheck == claimEndKey) && hasMore) { - // This is a special case: rangeBegin == claimEndKey == next beginKeyToCheck - // We separate this case and the third case to solve a corner issue led by the - // third code path: the progress will get stuck on repeatedly checking beginKeyToCheck. - // In the third code path, if hasMore == true and beginKeyToCheck == claimEndKey, - // The next round of beginKeyToCheck (aka claimEndKey) will always be beginKeyToCheck of the - // current round. To avoid this issue, we spawn a child checkall on a smaller range - // (beginKeyToCheck ~ maxEndKey). This smaller range guarantees that the hasMore is always false - // and the child checkall will complete and the global progress will move forward. Once the - // child checkall is done, we move to the range: maxEndKey ~ rangeToCheck.end - state KeyRange spawnedRangeToCheck = Standalone(KeyRangeRef(beginKeyToCheck, maxEndKey)); - fmt::println("Spawn new checkall for range {}", printable(spawnedRangeToCheck)); - bool allSame = wait(doCheckAll(cx, spawnedRangeToCheck, dcid, checkAll)); - beginKeyToCheck = spawnedRangeToCheck.end; - consistent = consistent && allSame; // !allSame of any subrange results in !consistent - } else { - std::vector keyValueReplies; - for (int j = 0; j < replies.size(); j++) { - auto reply = replies[j].get(); - ASSERT(reply.present() && !reply.get().error.present()); // has thrown eariler of error - keyValueReplies.push_back(reply.get()); - } - // keyServers and keyValueReplies must follow the same order - bool allSame = - checkResults(version, hasMore, claimEndKey, keyServers[i].second, keyValueReplies); - // Using claimEndKey of the current round as the nextBeginKey for the next round - // Note that claimEndKey is not compared in the current round - // This key will be compared in the next round - fmt::println("Result: compared {} - {}", toHex(beginKeyToCheck), toHex(claimEndKey)); - beginKeyToCheck = claimEndKey; - fmt::println("allSame {}, hasMore {}, checkAll {}", allSame, hasMore, checkAll); - consistent = consistent && allSame; // !allSame of any subrange results in !consistent - } - if (!consistent && !checkAll) { - return false; - } - round++; - } - } - break; - - } catch (Error& e) { - fmt::print("Error: {}", e.what()); - wait(onErrorTr.onError(e)); - fmt::println(", retrying in 1s..."); - } - wait(delay(1.0)); - } - return consistent; -} - -// The command is used to check the data inconsistency of the user input range -ACTOR Future checkallCommandActor(Database cx, std::vector tokens) { - state bool checkAll = false; // If set, do not return on first error, continue checking all keys - state Optional dcid; - state KeyRange inputRange; - if (tokens.size() == 3) { - inputRange = KeyRangeRef(tokens[1], tokens[2]); - } else if (tokens.size() == 4 && tokens[3] == "all"_sr) { - inputRange = KeyRangeRef(tokens[1], tokens[2]); - checkAll = true; - } else if (tokens.size() == 4 && tokens[3] != "all"_sr) { - inputRange = KeyRangeRef(tokens[1], tokens[2]); - dcid = tokens[3]; - } else if (tokens.size() == 5 && tokens[4] == "all"_sr) { - inputRange = KeyRangeRef(tokens[1], tokens[2]); - checkAll = true; - dcid = tokens[3]; - } else { - fmt::println( - "checkall [ ] (all)\n" - "Check inconsistency of the input range by comparing all replicas and print any corruptions.\n" - "The default behavior is to stop on the first subrange where corruption is found\n" - "DCID is optional. If set, the tool only check storage server of the specified data center.\n" - "DCID is ignored if the cluster has not set dcid.\n" - "`all` is optional. When `all` is appended, the checker does not stop until all subranges have checked.\n" - "Note this is intended to check a small range of keys, not the entire database (consider consistencycheck " - "for that purpose)."); - return false; - } - if (inputRange.empty()) { - fmt::println("Input empty range: {}.\nImmediately exit.", printable(inputRange)); - return false; - } - // At this point, we have a non-empty inputRange to check - bool res = wait(doCheckAll(cx, inputRange, dcid, checkAll)); - fmt::println("Checking complete. AllSame: {}", res); - return true; -} - -CommandFactory checkallCommandFactory("checkall"); -} // namespace fdb_cli diff --git a/fdbcli/DebugCommands.cpp b/fdbcli/DebugCommands.cpp new file mode 100644 index 00000000000..571a221427c --- /dev/null +++ b/fdbcli/DebugCommands.cpp @@ -0,0 +1,494 @@ +/* + * DebugCommands.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBTypes.h" +#include "fdbclient/NativeAPI.actor.h" + +#include "flow/CoroUtils.h" + +namespace fdb_cli { + +std::string toHex(StringRef v) { + std::string result; + result.reserve(v.size() * 4); + for (int i = 0; i < v.size(); i++) { + result.append(format("\\x%02x", v[i])); + } + return result; +} + +// Gets a version at which to read from the storage servers +Future getVersion(Database cx) { + while (true) { + Transaction tr(cx); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + Error err; + try { + Version version = co_await tr.getReadVersion(); + co_return version; + } catch (Error& e) { + err = e; + } + co_await tr.onError(err); + } +} + +// Get a list of storage servers that persist keys within range "kr" from the +// first commit proxy. Returns false if there is a failure (in this case, +// keyServersPromise will never be set). +// If dcid is set, only return the storage servers in the given datacenter. +// Ignore the input dcid if the cluster has not set dcid. +Future getKeyServers( + Database cx, + Promise>>> keyServersPromise, + KeyRangeRef kr, + Optional dcid) { + std::vector>> keyServers; + + // Try getting key server locations from the first commit proxy + Future> keyServerLocationFuture; + Key begin = kr.begin; + Key end = kr.end; + int limitKeyServers = 100; + + while (begin < end) { + Reference commitProxyInfo = co_await cx->getCommitProxiesFuture(UseProvisionalProxies::False); + keyServerLocationFuture = + commitProxyInfo->get(0, &CommitProxyInterface::getKeyServersLocations) + .getReplyUnlessFailedFor( + GetKeyServerLocationsRequest({}, begin, end, limitKeyServers, false, latestVersion, Arena()), 2, 0); + + bool keyServersInsertedForThisIteration = false; + auto choice = co_await race(keyServerLocationFuture, cx->onProxiesChanged()); + if (choice.index() == 0) { + ErrorOr shards = std::get<0>(std::move(choice)); + // Get the list of shards if one was returned. + if (shards.present() && !keyServersInsertedForThisIteration) { + std::vector>> shardResultList; + for (auto& result : shards.get().results) { + std::vector servers; + for (auto& server : result.second) { + // Filter out storage servers that are not in the given datacenter + Optional> serverDcId = server.locality.dcId(); + if (dcid.present() && serverDcId.present() && serverDcId.get() != dcid.get()) { + continue; + } + servers.push_back(server); + } + shardResultList.push_back({ result.first, servers }); + } + keyServers.insert(keyServers.end(), shardResultList.begin(), shardResultList.end()); + keyServersInsertedForThisIteration = true; + begin = shards.get().results.back().first.end; + } + } else if (choice.index() != 1) { + UNREACHABLE(); + } + + if (!keyServersInsertedForThisIteration) // Retry the entire workflow + co_await delay(1.0); + } + + keyServersPromise.send(keyServers); + co_return true; +} + +// The command is used to get all storage server addresses for a given key. +Future getLocationCommandActor(Database cx, std::vector tokens) { + if (tokens.size() != 2 && tokens.size() != 3) { + fmt::println("getlocation []\n" + "fetch the storage server address for a given key or range.\n" + "Displays the addresses of storage servers, or `not found' if location is not found."); + co_return false; + } + + KeyRange kr = KeyRangeRef(tokens[1], tokens.size() == 3 ? tokens[2] : keyAfter(tokens[1])); + // find key range locations without GRV + Promise>>> keyServersPromise; + bool found = co_await getKeyServers(cx, keyServersPromise, kr, Optional()); + if (!found) { + fmt::println("{} locations not found", printable(kr)); + co_return false; + } + std::vector>> keyServers = + keyServersPromise.getFuture().get(); + for (const auto& [range, servers] : keyServers) { + fmt::println("Key range: {}", printable(range)); + for (const auto& server : servers) { + fmt::println(" {}", server.address().toString()); + } + } + co_return true; +} +// hidden commands, no help text for now +CommandFactory getLocationCommandFactory("getlocation"); + +// The command is used to get values from all storage servers that have the given key. +Future getallCommandActor(Database cx, std::vector tokens, Version version) { + if (tokens.size() != 2) { + fmt::println("getall \n" + "fetch values from all storage servers that have the given key.\n" + "Displays the value and the addresses of storage servers, or `not found' if key is not found."); + co_return false; + } + + KeyRangeLocationInfo loc = co_await getKeyLocation_internal( + cx, tokens[1], SpanContext(), Optional(), UseProvisionalProxies::False, Reverse::False, version); + + if (loc.locations) { + fmt::println("version is {}", version); + fmt::println("`{}' is at:", printable(tokens[1])); + Reference locations = loc.locations->locations(); + std::vector> replies; + for (int i = 0; locations && i < locations->size(); i++) { + GetValueRequest req(/*spanContext=*/{}, tokens[1], version, {}, {}, {}); + replies.push_back(locations->get(i, &StorageServerInterface::getValue).getReply(req)); + } + co_await waitForAll(replies); + for (int i = 0; i < replies.size(); i++) { + std::string ssi = locations->getInterface(i).address().toString(); + if (replies[i].isError()) { + fmt::println(stderr, "ERROR: {} {}", ssi, replies[i].getError().what()); + } else { + Optional v = replies[i].get().value; + fmt::println(" {} {}", ssi, v.present() ? printable(v.get()) : "(not found)"); + } + } + } else { + fmt::println("`{}': location not found", printable(tokens[1])); + } + co_return true; +} +// hidden commands, no help text for now +CommandFactory getallCommandFactory("getall"); + +std::string printStorageServerMachineInfo(const StorageServerInterface& server) { + std::string serverIp = server.address().toString(); + std::string serverLocality = server.locality.toString(); + return serverLocality + " " + serverIp; +} + +std::string printAllStorageServerMachineInfo(const std::vector& servers) { + std::string res; + for (int i = 0; i < servers.size(); i++) { + if (i == 0) { + res = printStorageServerMachineInfo(servers[i]); + } else { + res = res + "; " + printStorageServerMachineInfo(servers[i]); + } + } + return res; +} + +// check that all replies are the same. Update begin to the next key to check +// checkResults keeps invariants: +// (1) hasMore = true if any server has more data not read yet +// (2) nextBeginKey is the minimal key returned from all servers +// (3) checkResults reports inconsistency of keys only before the nextBeginKey if hasMore=true +// Therefore, whether to proceed to the next round depends on hasMore +// If there is a next round, it starts from the minimal key returned from all servers +bool checkResults(Version version, + bool hasMore, + Key claimEndKey, + const std::vector& servers, + const std::vector& replies) { + // Compare servers + bool allSame = true; + int firstValidServer = -1; + for (int j = 0; j < replies.size(); j++) { + if (firstValidServer == -1) { + firstValidServer = j; + // Print full list of comparing servers and the reference server + // Used to check server info which does not produce an inconsistency log + fmt::println("CheckResult: servers: {}, reference server: {}", + printAllStorageServerMachineInfo(servers), + printStorageServerMachineInfo(servers[firstValidServer])); + continue; // always select the first server as reference + } + // compare reference and current + GetKeyValuesReply current = replies[j]; + GetKeyValuesReply reference = replies[firstValidServer]; + if (current.data == reference.data && current.more == reference.more) { + continue; + } + // Detecting corrupted keys for any mismatching replies between current and reference servers + allSame = false; + size_t currentI = 0, referenceI = 0; + while (currentI < current.data.size() || referenceI < reference.data.size()) { + if (hasMore && ((referenceI < reference.data.size() && reference.data[referenceI].key >= claimEndKey) || + (currentI < current.data.size() && current.data[currentI].key >= claimEndKey))) { + // If there will be a next round and the key is out of claimEndKey + // We will delay the detection to the next round + break; + } + if (currentI >= current.data.size()) { + // ServerA(1), ServerB(0): 1 indicates that ServerA has the key while 0 indicates that ServerB does not + // have the key + fmt::println( + "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, Key {}", + printStorageServerMachineInfo(servers[firstValidServer]), + printStorageServerMachineInfo(servers[j]), + currentI, + referenceI, + version, + toHex(reference.data[referenceI].key)); + referenceI++; + } else if (referenceI >= reference.data.size()) { + fmt::println( + "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, Key {}", + printStorageServerMachineInfo(servers[j]), + printStorageServerMachineInfo(servers[firstValidServer]), + currentI, + referenceI, + version, + toHex(current.data[currentI].key)); + currentI++; + } else { + KeyValueRef currentKV = current.data[currentI]; + KeyValueRef referenceKV = reference.data[referenceI]; + if (currentKV.key == referenceKV.key) { + if (currentKV.value != referenceKV.value) { + fmt::println("Inconsistency: MismatchValue, {}(1), {}(1), CurrentIndex {}, ReferenceIndex {}, " + "Version {}, Key {}", + printStorageServerMachineInfo(servers[firstValidServer]), + printStorageServerMachineInfo(servers[j]), + currentI, + referenceI, + version, + toHex(currentKV.key)); + } + currentI++; + referenceI++; + } else if (currentKV.key < referenceKV.key) { + fmt::println( + "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, " + "Key {}", + printStorageServerMachineInfo(servers[j]), + printStorageServerMachineInfo(servers[firstValidServer]), + currentI, + referenceI, + version, + toHex(currentKV.key)); + currentI++; + } else { + fmt::println( + "Inconsistency: UniqueKey, {}(1), {}(0), CurrentIndex {}, ReferenceIndex {}, Version {}, " + "Key {}", + printStorageServerMachineInfo(servers[firstValidServer]), + printStorageServerMachineInfo(servers[j]), + currentI, + referenceI, + version, + toHex(referenceKV.key)); + referenceI++; + } + } + } + } + + return allSame; +} + +Future doCheckAll(Database cx, KeyRange inputRange, Optional dcid, bool checkAll); +// Return whether inconsistency is detected in the inputRange +Future doCheckAll(Database cx, KeyRange inputRange, Optional dcid, bool checkAll) { + Transaction onErrorTr(cx); // This transaction exists only to access onError and its backoff behavior + bool consistent = true; + while (true) { + Error err; + try { + fmt::println("Start checking for range: {}", printable(inputRange)); + // Get SS interface for each shard of the inputRange + Promise>>> keyServerPromise; + bool foundKeyServers = co_await getKeyServers(cx, keyServerPromise, inputRange, dcid); + if (!foundKeyServers) { + fmt::println("key server locations for {} not found, retrying in 1s...", printable(inputRange)); + co_await delay(1.0); + continue; + } + std::vector>> keyServers = + keyServerPromise.getFuture().get(); + // We partition the entire input range into shards + // and we conduct comparison shard by shard + int i = 0; + for (; i < keyServers.size(); i++) { // for each shard + KeyRange rangeToCheck = keyServers[i].first; + rangeToCheck = rangeToCheck & inputRange; // Only check the shard part within the inputRange + if (rangeToCheck.empty()) { + continue; // Skip the shard if it is outside of the inputRange + } + const auto& servers = keyServers[i].second; + Key beginKeyToCheck = rangeToCheck.begin; + fmt::println("Key range to check: {}", printable(rangeToCheck)); + for (const auto& server : servers) { + fmt::println("\t{}", server.address().toString()); + } + std::vector>> replies; + bool hasMore = true; + int round = 0; + Version version{ 0 }; + while (hasMore) { + version = co_await getVersion(cx); + replies.clear(); + fmt::println("Round {}: {} - {}", round, toHex(beginKeyToCheck), toHex(rangeToCheck.end)); + for (const auto& s : keyServers[i].second) { // for each storage server + GetKeyValuesRequest req; + req.begin = firstGreaterOrEqual(beginKeyToCheck); + req.end = firstGreaterOrEqual(rangeToCheck.end); + req.limit = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT; + req.limitBytes = CLIENT_KNOBS->KRM_GET_RANGE_LIMIT_BYTES; + req.version = version; // all replica should read at the same version + req.tags = TagSet(); + replies.push_back(s.getKeyValues.getReplyUnlessFailedFor(req, 2, 0)); + } + co_await waitForAll(replies); + + // Decide comparison scope + Key claimEndKey; // used for the next round if hasMore == true + Key maxEndKey; + hasMore = false; // re-calculate hasMore according to replies + for (int j = 0; j < replies.size(); j++) { + auto reply = replies[j].get(); + if (reply.isError()) { + fmt::println("checkResults error: {}", reply.getError().what()); + throw reply.getError(); + } else if (reply.get().error.present()) { + fmt::println("checkResults error: {}", reply.get().error.get().what()); + throw reply.get().error.get(); + } + GetKeyValuesReply current = reply.get(); + if (current.data.empty()) { + continue; // Ignore if no data has replied + } + if (claimEndKey.empty() || current.data[current.data.size() - 1].key < claimEndKey) { + claimEndKey = current.data[current.data.size() - 1].key; + } + if (maxEndKey.empty() || current.data[current.data.size() - 1].key > maxEndKey) { + maxEndKey = current.data[current.data.size() - 1].key; + } + hasMore = hasMore || current.more; + } + fmt::println("Compare scope has been decided\n\tBeginKey: {}\n\tEndKey: {}\n\tHasMore: {}", + toHex(beginKeyToCheck), + toHex(claimEndKey), + hasMore); + if (claimEndKey.empty()) { + // It is possible that there is clear operation between the prev round and the current round + // which result in empty claimEndKey --- nothing to compare + // In this case, we simply skip the current shard + ASSERT(hasMore == false); + continue; + } else if ((beginKeyToCheck == claimEndKey) && hasMore) { + // This is a special case: rangeBegin == claimEndKey == next beginKeyToCheck + // We separate this case and the third case to solve a corner issue led by the + // third code path: the progress will get stuck on repeatedly checking beginKeyToCheck. + // In the third code path, if hasMore == true and beginKeyToCheck == claimEndKey, + // The next round of beginKeyToCheck (aka claimEndKey) will always be beginKeyToCheck of the + // current round. To avoid this issue, we spawn a child checkall on a smaller range + // (beginKeyToCheck ~ maxEndKey). This smaller range guarantees that the hasMore is always + // false and the child checkall will complete and the global progress will move forward. + // Once the child checkall is done, we move to the range: maxEndKey ~ rangeToCheck.end + KeyRange spawnedRangeToCheck = Standalone(KeyRangeRef(beginKeyToCheck, maxEndKey)); + fmt::println("Spawn new checkall for range {}", printable(spawnedRangeToCheck)); + bool allSame = co_await doCheckAll(cx, spawnedRangeToCheck, dcid, checkAll); + beginKeyToCheck = spawnedRangeToCheck.end; + consistent = consistent && allSame; // !allSame of any subrange results in !consistent + } else { + std::vector keyValueReplies; + for (int j = 0; j < replies.size(); j++) { + auto reply = replies[j].get(); + ASSERT(reply.present() && !reply.get().error.present()); // has thrown eariler of error + keyValueReplies.push_back(reply.get()); + } + // keyServers and keyValueReplies must follow the same order + bool allSame = + checkResults(version, hasMore, claimEndKey, keyServers[i].second, keyValueReplies); + // Using claimEndKey of the current round as the nextBeginKey for the next round + // Note that claimEndKey is not compared in the current round + // This key will be compared in the next round + fmt::println("Result: compared {} - {}", toHex(beginKeyToCheck), toHex(claimEndKey)); + beginKeyToCheck = claimEndKey; + fmt::println("allSame {}, hasMore {}, checkAll {}", allSame, hasMore, checkAll); + consistent = consistent && allSame; // !allSame of any subrange results in !consistent + } + if (!consistent && !checkAll) { + co_return false; + } + round++; + } + } + break; + + } catch (Error& e) { + err = e; + } + fmt::print("Error: {}", err.what()); + co_await onErrorTr.onError(err); + fmt::println(", retrying in 1s..."); + co_await delay(1.0); + } + co_return consistent; +} + +// The command is used to check the data inconsistency of the user input range +Future checkallCommandActor(Database cx, std::vector tokens) { + bool checkAll = false; // If set, do not return on first error, continue checking all keys + Optional dcid; + KeyRange inputRange; + if (tokens.size() == 3) { + inputRange = KeyRangeRef(tokens[1], tokens[2]); + } else if (tokens.size() == 4 && tokens[3] == "all"_sr) { + inputRange = KeyRangeRef(tokens[1], tokens[2]); + checkAll = true; + } else if (tokens.size() == 4 && tokens[3] != "all"_sr) { + inputRange = KeyRangeRef(tokens[1], tokens[2]); + dcid = tokens[3]; + } else if (tokens.size() == 5 && tokens[4] == "all"_sr) { + inputRange = KeyRangeRef(tokens[1], tokens[2]); + checkAll = true; + dcid = tokens[3]; + } else { + fmt::println( + "checkall [ ] (all)\n" + "Check inconsistency of the input range by comparing all replicas and print any corruptions.\n" + "The default behavior is to stop on the first subrange where corruption is found\n" + "DCID is optional. If set, the tool only check storage server of the specified data center.\n" + "DCID is ignored if the cluster has not set dcid.\n" + "`all` is optional. When `all` is appended, the checker does not stop until all subranges have checked.\n" + "Note this is intended to check a small range of keys, not the entire database (consider consistencycheck " + "for that purpose)."); + co_return false; + } + if (inputRange.empty()) { + fmt::println("Input empty range: {}.\nImmediately exit.", printable(inputRange)); + co_return false; + } + // At this point, we have a non-empty inputRange to check + bool res = co_await doCheckAll(cx, inputRange, dcid, checkAll); + fmt::println("Checking complete. AllSame: {}", res); + co_return true; +} + +CommandFactory checkallCommandFactory("checkall"); +} // namespace fdb_cli diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp deleted file mode 100644 index 1fde6c61de8..00000000000 --- a/fdbcli/ExcludeCommand.actor.cpp +++ /dev/null @@ -1,507 +0,0 @@ -/* - * ExcludeCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" -#include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/Schemas.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -// Exclude the given servers and localities -ACTOR Future excludeServersAndLocalities(Reference db, - std::vector servers, - std::unordered_set localities, - bool failed, - bool force) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - try { - if (force && servers.size()) - tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey, - ValueRef()); - for (const auto& s : servers) { - Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString()) - : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString()); - tr->set(addr, ValueRef()); - } - if (force && localities.size()) - tr->set(failed ? fdb_cli::failedLocalityForceOptionSpecialKey - : fdb_cli::excludedLocalityForceOptionSpecialKey, - ValueRef()); - for (const auto& l : localities) { - Key addr = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l) - : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l); - tr->set(addr, ValueRef()); - } - wait(safeThreadFutureToFuture(tr->commit())); - return true; - } catch (Error& e) { - state Error err(e); - if (e.code() == error_code_special_keys_api_failure) { - std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); - // last character is \n - auto pos = errorMsgStr.find_last_of("\n", errorMsgStr.size() - 2); - auto last_line = errorMsgStr.substr(pos + 1); - // customized the error message for fdbcli - fprintf(stderr, - "%s\n%s\n", - errorMsgStr.substr(0, pos).c_str(), - last_line.find("free space") != std::string::npos - ? "Type `exclude FORCE ' to exclude without checking free space." - : "Type `exclude FORCE failed ' to exclude without performing safety checks."); - return false; - } - TraceEvent(SevWarn, "ExcludeServersAndLocalitiesError").error(err); - wait(safeThreadFutureToFuture(tr->onError(err))); - } - } -} - -ACTOR Future> getExcludedServers(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::excludedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - state RangeResult r = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); - - std::vector exclusions; - for (const auto& i : r) { - auto addr = i.key.removePrefix(fdb_cli::excludedServersSpecialKeyRange.begin).toString(); - exclusions.push_back(addr); - } - return exclusions; - } catch (Error& e) { - TraceEvent(SevWarn, "GetExcludedServersError").error(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -// Get the list of excluded localities by reading the keys. -ACTOR Future> getExcludedLocalities(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::excludedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - state RangeResult r = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); - - std::vector excludedLocalities; - for (const auto& i : r) { - auto locality = i.key.removePrefix(fdb_cli::excludedLocalitySpecialKeyRange.begin).toString(); - excludedLocalities.push_back(locality); - } - return excludedLocalities; - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future> getFailedServers(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::failedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - state RangeResult r = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); - - std::vector exclusions; - for (const auto& i : r) { - auto addr = i.key.removePrefix(fdb_cli::failedServersSpecialKeyRange.begin).toString(); - exclusions.push_back(addr); - } - return exclusions; - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -// Get the list of failed localities by reading the keys. -ACTOR Future> getFailedLocalities(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::failedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - state RangeResult r = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); - - std::vector excludedLocalities; - for (const auto& i : r) { - auto locality = i.key.removePrefix(fdb_cli::failedLocalitySpecialKeyRange.begin).toString(); - excludedLocalities.push_back(locality); - } - return excludedLocalities; - } catch (Error& e) { - TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future> getInProgressExclusion(Reference tr) { - ThreadFuture resultFuture = - tr->getRange(fdb_cli::exclusionInProgressSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - RangeResult result = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); - std::set inProgressExclusion; - for (const auto& addr : result) { - inProgressExclusion.insert( - NetworkAddress::parse(addr.key.removePrefix(fdb_cli::exclusionInProgressSpecialKeyRange.begin).toString())); - } - return inProgressExclusion; -} - -ACTOR Future> checkForExcludingServers(Reference db, - std::set exclusions, - bool waitForAllExcluded) { - state std::set inProgressExclusion; - state Reference tr = db->createTransaction(); - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - loop { - inProgressExclusion.clear(); - try { - std::set result = wait(getInProgressExclusion(tr)); - if (result.empty()) - return inProgressExclusion; - inProgressExclusion = result; - - // Check if all of the specified exclusions are done. - bool allExcluded = true; - for (const auto& inProgressAddr : inProgressExclusion) { - if (!allExcluded) { - break; - } - - for (const auto& exclusion : exclusions) { - // We found an exclusion that is still in progress - if (exclusion.excludes(inProgressAddr)) { - allExcluded = false; - break; - } - } - } - - if (allExcluded) { - inProgressExclusion.clear(); - return inProgressExclusion; - } - - if (!waitForAllExcluded) - break; - - wait(delayJittered(1.0)); // SOMEDAY: watches! - } catch (Error& e) { - TraceEvent(SevWarn, "CheckForExcludingServersError").error(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } - return inProgressExclusion; -} - -ACTOR Future checkForCoordinators(Reference db, std::set exclusions) { - - state bool foundCoordinator = false; - state std::vector coordinatorList; - state Reference tr = db->createTransaction(); - loop { - try { - // Hold the reference to the standalone's memory - state ThreadFuture> coordinatorsF = tr->get(fdb_cli::coordinatorsProcessSpecialKey); - Optional coordinators = wait(safeThreadFutureToFuture(coordinatorsF)); - ASSERT(coordinators.present()); - coordinatorList = NetworkAddress::parseList(coordinators.get().toString()); - break; - } catch (Error& e) { - TraceEvent(SevWarn, "CheckForCoordinatorsError").error(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } - - for (const auto& c : coordinatorList) { - if (exclusions.find(AddressExclusion(c.ip, c.port)) != exclusions.end() || - exclusions.find(AddressExclusion(c.ip)) != exclusions.end()) { - fprintf(stderr, "WARNING: %s is a coordinator!\n", c.toString().c_str()); - foundCoordinator = true; - } - } - if (foundCoordinator) - printf("Type `help coordinators' for information on how to change the\n" - "cluster's coordination servers before removing them.\n"); - return Void(); -} - -} // namespace - -namespace fdb_cli { - -const KeyRangeRef excludedServersSpecialKeyRange("\xff\xff/management/excluded/"_sr, - "\xff\xff/management/excluded0"_sr); -const KeyRangeRef failedServersSpecialKeyRange("\xff\xff/management/failed/"_sr, "\xff\xff/management/failed0"_sr); -const KeyRangeRef excludedLocalitySpecialKeyRange("\xff\xff/management/excluded_locality/"_sr, - "\xff\xff/management/excluded_locality0"_sr); -const KeyRangeRef failedLocalitySpecialKeyRange("\xff\xff/management/failed_locality/"_sr, - "\xff\xff/management/failed_locality0"_sr); -const KeyRef excludedForceOptionSpecialKey = "\xff\xff/management/options/excluded/force"_sr; -const KeyRef failedForceOptionSpecialKey = "\xff\xff/management/options/failed/force"_sr; -const KeyRef excludedLocalityForceOptionSpecialKey = "\xff\xff/management/options/excluded_locality/force"_sr; -const KeyRef failedLocalityForceOptionSpecialKey = "\xff\xff/management/options/failed_locality/force"_sr; -const KeyRangeRef exclusionInProgressSpecialKeyRange("\xff\xff/management/in_progress_exclusion/"_sr, - "\xff\xff/management/in_progress_exclusion0"_sr); - -ACTOR Future excludeCommandActor(Reference db, std::vector tokens, Future warn) { - if (tokens.size() <= 1) { - state std::vector excludedAddresses = wait(getExcludedServers(db)); - state std::vector excludedLocalities = wait(getExcludedLocalities(db)); - state std::vector failedAddresses = wait(getFailedServers(db)); - state std::vector failedLocalities = wait(getFailedLocalities(db)); - - if (!excludedAddresses.size() && !excludedLocalities.size() && !failedAddresses.size() && - !failedLocalities.size()) { - printf("There are currently no servers or localities excluded from the database.\n" - "To learn how to exclude a server, type `help exclude'.\n"); - return true; - } - - printf("There are currently %zu servers or localities being excluded from the database:\n", - excludedAddresses.size() + excludedLocalities.size()); - for (const auto& e : excludedAddresses) - printf(" %s\n", e.c_str()); - for (const auto& e : excludedLocalities) - printf(" %s\n", e.c_str()); - - if (excludedAddresses.size() || excludedLocalities.size()) { - printf("To find out whether it is safe to remove one or more of these\n" - "servers from the cluster, type `exclude '.\n" - "To return one of these servers to the cluster, type `include '.\n"); - } - - printf("\n"); - - printf("There are currently %zu servers or localities marked as failed in the database:\n", - failedAddresses.size() + failedLocalities.size()); - for (const auto& f : failedAddresses) - printf(" %s\n", f.c_str()); - for (const auto& f : failedLocalities) - printf(" %s\n", f.c_str()); - - if (failedAddresses.size() || failedLocalities.size()) { - printf("To return one of these servers to the cluster, type `include failed '.\n"); - } - - printf("\n"); - - Reference tr = db->createTransaction(); - std::set inProgressExclusion = wait(getInProgressExclusion(tr)); - printf("There are currently %zu processes for which exclusion is in progress:\n", inProgressExclusion.size()); - for (const auto& addr : inProgressExclusion) { - printf("%s\n", addr.toString().c_str()); - } - - return true; - } else { - state std::set exclusionSet; - state std::vector exclusionAddresses; - state std::unordered_set exclusionLocalities; - state std::vector noMatchLocalities; - state bool force = false; - state bool waitForAllExcluded = true; - state bool markFailed = false; - state std::vector workers; - state std::map server_interfaces; - state Future future_workers = fdb_cli::getWorkers(db, &workers); - state Future future_server_interfaces = fdb_cli::getStorageServerInterfaces(db, &server_interfaces); - wait(success(future_workers) && success(future_server_interfaces)); - - for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) { - if (*t == "FORCE"_sr) { - force = true; - } else if (*t == "no_wait"_sr) { - waitForAllExcluded = false; - } else if (*t == "failed"_sr) { - markFailed = true; - } else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && - t->toString().find(':') != std::string::npos) { - exclusionLocalities.insert(t->toString()); - auto localityAddresses = getAddressesByLocality(workers, t->toString()); - auto localityServerAddresses = getServerAddressesByLocality(server_interfaces, t->toString()); - if (localityAddresses.empty() && localityServerAddresses.empty()) { - noMatchLocalities.push_back(t->toString()); - continue; - } - - if (!localityAddresses.empty()) { - exclusionSet.insert(localityAddresses.begin(), localityAddresses.end()); - } - - if (!localityServerAddresses.empty()) { - exclusionSet.insert(localityServerAddresses.begin(), localityServerAddresses.end()); - } - } else { - auto a = AddressExclusion::parse(*t); - if (!a.isValid()) { - fprintf(stderr, - "ERROR: '%s' is neither a valid network endpoint address nor a locality\n", - t->toString().c_str()); - if (t->toString().find(":tls") != std::string::npos) - printf(" Do not include the `:tls' suffix when naming a process\n"); - return true; - } - exclusionSet.insert(a); - exclusionAddresses.push_back(a); - } - } - - // The validation if a locality or address has no match is done below and will result in a warning. If we abort - // here the provided locality and/or address will not be excluded. - if (exclusionAddresses.empty() && exclusionLocalities.empty()) { - fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality must be provided\n"); - return false; - } - - bool res = wait(excludeServersAndLocalities(db, exclusionAddresses, exclusionLocalities, markFailed, force)); - if (!res) - return false; - - if (waitForAllExcluded) { - printf("Waiting for state to be removed from all excluded servers. This may take a while.\n"); - printf("(Interrupting this wait with CTRL+C will not cancel the data movement.)\n"); - } - - if (warn.isValid()) - warn.cancel(); - - state std::set notExcludedServers = - wait(checkForExcludingServers(db, exclusionSet, waitForAllExcluded)); - std::map> workerPorts; - for (auto addr : workers) - workerPorts[addr.address.ip].insert(addr.address.port); - - // Print a list of all excluded addresses that don't have a corresponding worker - std::set absentExclusions; - for (const auto& addr : exclusionSet) { - auto worker = workerPorts.find(addr.ip); - if (worker == workerPorts.end()) - absentExclusions.insert(addr); - else if (addr.port > 0 && worker->second.count(addr.port) == 0) - absentExclusions.insert(addr); - } - - for (const auto& exclusion : exclusionSet) { - if (absentExclusions.find(exclusion) != absentExclusions.end()) { - if (exclusion.port == 0) { - fprintf(stderr, - " %s(Whole machine) ---- WARNING: Missing from cluster!Be sure that you excluded the " - "correct machines before removing them from the cluster!\n", - exclusion.ip.toString().c_str()); - } else { - fprintf(stderr, - " %s ---- WARNING: Missing from cluster! Be sure that you excluded the correct processes " - "before removing them from the cluster!\n", - exclusion.toString().c_str()); - } - } else if (std::any_of(notExcludedServers.begin(), notExcludedServers.end(), [&](const NetworkAddress& a) { - return addressExcluded({ exclusion }, a); - })) { - if (exclusion.port == 0) { - fprintf(stderr, - " %s(Whole machine) ---- WARNING: Exclusion in progress! It is not safe to remove this " - "machine from the cluster\n", - exclusion.ip.toString().c_str()); - } else { - fprintf(stderr, - " %s ---- WARNING: Exclusion in progress! It is not safe to remove this process from the " - "cluster\n", - exclusion.toString().c_str()); - } - } else { - if (exclusion.port == 0) { - printf(" %s(Whole machine) ---- Successfully excluded. It is now safe to remove this machine " - "from the cluster.\n", - exclusion.ip.toString().c_str()); - } else { - printf( - " %s ---- Successfully excluded. It is now safe to remove this process from the cluster.\n", - exclusion.toString().c_str()); - } - } - } - - for (const auto& locality : noMatchLocalities) { - fprintf( - stderr, - " %s ---- WARNING: Currently no servers found with this locality match! Be sure that you excluded " - "the correct locality.\n", - locality.c_str()); - } - - wait(checkForCoordinators(db, exclusionSet)); - return true; - } -} - -CommandFactory excludeFactory( - "exclude", - CommandHelp( - "exclude [FORCE] [failed] [no_wait] [] [locality_dcid:]\n" - " [locality_zoneid:] [locality_machineid:]\n" - " [locality_processid:] [locality_:]", - "exclude servers from the database by IP address or locality", - "If no addresses or localities are specified, lists the set of excluded addresses and localities in addition " - "to addresses for which exclusion is in progress.\n" - "\n" - "For each IP address or IP:port pair in and/or each locality attribute (like dcid, " - "zoneid, machineid, processid), adds the address/locality to the set of exclusions and waits until all " - "database state has been safely moved away from affected servers.\n" - "\n" - "If 'FORCE' is set, the command does not perform safety checks before excluding.\n" - "\n" - "If 'no_wait' is set, the command returns immediately without checking if the exclusions have completed " - "successfully.\n" - "\n" - "If 'failed' is set, the cluster will immediately forget all data associated with the excluded processes. " - "Doing so can be helpful if the process is not expected to recover, as it will allow the cluster to delete " - "state that would be needed to catch the failed process up. Re-including a process excluded with 'failed' will " - "result in it joining as an empty process.\n" - "\n" - "If a cluster has failed storage servers that result in all replicas of some data being permanently gone, " - "'exclude failed' can be used to clean up the affected key ranges by restoring them to empty.\n" - "\n" - "WARNING: use of 'exclude failed' can result in data loss. If an excluded server contains the last replica of " - "some data, then using the 'failed' option will permanently remove that data from the cluster.")); -} // namespace fdb_cli diff --git a/fdbcli/ExcludeCommand.cpp b/fdbcli/ExcludeCommand.cpp new file mode 100644 index 00000000000..2590a802945 --- /dev/null +++ b/fdbcli/ExcludeCommand.cpp @@ -0,0 +1,523 @@ +/* + * ExcludeCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/ManagementAPI.h" +#include "fdbclient/Schemas.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +// Exclude the given servers and localities +Future excludeServersAndLocalities(Reference db, + std::vector servers, + std::unordered_set localities, + bool failed, + bool force) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + Error err; + try { + if (force && !servers.empty()) + tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey, + ValueRef()); + for (const auto& s : servers) { + Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString()) + : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString()); + tr->set(addr, ValueRef()); + } + if (force && !localities.empty()) + tr->set(failed ? fdb_cli::failedLocalityForceOptionSpecialKey + : fdb_cli::excludedLocalityForceOptionSpecialKey, + ValueRef()); + for (const auto& l : localities) { + Key addr = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l) + : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l); + tr->set(addr, ValueRef()); + } + co_await safeThreadFutureToFuture(tr->commit()); + co_return true; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr); + // last character is \n + auto pos = errorMsgStr.find_last_of("\n", errorMsgStr.size() - 2); + auto last_line = errorMsgStr.substr(pos + 1); + // customized the error message for fdbcli + fprintf(stderr, + "%s\n%s\n", + errorMsgStr.substr(0, pos).c_str(), + last_line.find("free space") != std::string::npos + ? "Type `exclude FORCE ' to exclude without checking free space." + : "Type `exclude FORCE failed ' to exclude without performing safety checks."); + co_return false; + } + TraceEvent(SevWarn, "ExcludeServersAndLocalitiesError").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +AsyncResult> getExcludedServers(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + ThreadFuture resultFuture = + tr->getRange(fdb_cli::excludedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult r = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); + + std::vector exclusions; + for (const auto& i : r) { + auto addr = i.key.removePrefix(fdb_cli::excludedServersSpecialKeyRange.begin).toString(); + exclusions.push_back(addr); + } + co_return exclusions; + } catch (Error& e) { + err = e; + } + TraceEvent(SevWarn, "GetExcludedServersError").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +// Get the list of excluded localities by reading the keys. +AsyncResult> getExcludedLocalities(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + ThreadFuture resultFuture = + tr->getRange(fdb_cli::excludedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult r = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); + + std::vector excludedLocalities; + for (const auto& i : r) { + auto locality = i.key.removePrefix(fdb_cli::excludedLocalitySpecialKeyRange.begin).toString(); + excludedLocalities.push_back(locality); + } + co_return excludedLocalities; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +AsyncResult> getFailedServers(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + ThreadFuture resultFuture = + tr->getRange(fdb_cli::failedServersSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult r = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); + + std::vector exclusions; + for (const auto& i : r) { + auto addr = i.key.removePrefix(fdb_cli::failedServersSpecialKeyRange.begin).toString(); + exclusions.push_back(addr); + } + co_return exclusions; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +// Get the list of failed localities by reading the keys. +AsyncResult> getFailedLocalities(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + ThreadFuture resultFuture = + tr->getRange(fdb_cli::failedLocalitySpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult r = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(!r.more && r.size() < CLIENT_KNOBS->TOO_MANY); + + std::vector excludedLocalities; + for (const auto& i : r) { + auto locality = i.key.removePrefix(fdb_cli::failedLocalitySpecialKeyRange.begin).toString(); + excludedLocalities.push_back(locality); + } + co_return excludedLocalities; + } catch (Error& e) { + err = e; + } + TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future> getInProgressExclusion(Reference tr) { + ThreadFuture resultFuture = + tr->getRange(fdb_cli::exclusionInProgressSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult result = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(!result.more && result.size() < CLIENT_KNOBS->TOO_MANY); + std::set inProgressExclusion; + for (const auto& addr : result) { + inProgressExclusion.insert( + NetworkAddress::parse(addr.key.removePrefix(fdb_cli::exclusionInProgressSpecialKeyRange.begin).toString())); + } + co_return inProgressExclusion; +} + +Future> checkForExcludingServers(Reference db, + std::set exclusions, + bool waitForAllExcluded) { + std::set inProgressExclusion; + Reference tr = db->createTransaction(); + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + while (true) { + inProgressExclusion.clear(); + Error err; + bool hasErr = false; + try { + std::set result = co_await getInProgressExclusion(tr); + if (result.empty()) + co_return inProgressExclusion; + inProgressExclusion = result; + + // Check if all of the specified exclusions are done. + bool allExcluded = true; + for (const auto& inProgressAddr : inProgressExclusion) { + if (!allExcluded) { + break; + } + + for (const auto& exclusion : exclusions) { + // We found an exclusion that is still in progress + if (exclusion.excludes(inProgressAddr)) { + allExcluded = false; + break; + } + } + } + + if (allExcluded) { + inProgressExclusion.clear(); + co_return inProgressExclusion; + } + + if (!waitForAllExcluded) + break; + + co_await delayJittered(1.0); // SOMEDAY: watches! + } catch (Error& e) { + err = e; + hasErr = true; + } + if (hasErr) { + TraceEvent(SevWarn, "CheckForExcludingServersError").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } + } + co_return inProgressExclusion; +} + +Future checkForCoordinators(Reference db, std::set exclusions) { + + bool foundCoordinator = false; + std::vector coordinatorList; + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + // Hold the reference to the standalone's memory + ThreadFuture> coordinatorsF = tr->get(fdb_cli::coordinatorsProcessSpecialKey); + Optional coordinators = co_await safeThreadFutureToFuture(coordinatorsF); + ASSERT(coordinators.present()); + coordinatorList = NetworkAddress::parseList(coordinators.get().toString()); + break; + } catch (Error& e) { + err = e; + } + TraceEvent(SevWarn, "CheckForCoordinatorsError").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } + + for (const auto& c : coordinatorList) { + if (exclusions.find(AddressExclusion(c.ip, c.port)) != exclusions.end() || + exclusions.find(AddressExclusion(c.ip)) != exclusions.end()) { + fprintf(stderr, "WARNING: %s is a coordinator!\n", c.toString().c_str()); + foundCoordinator = true; + } + } + if (foundCoordinator) + printf("Type `help coordinators' for information on how to change the\n" + "cluster's coordination servers before removing them.\n"); +} + +} // namespace + +namespace fdb_cli { + +const KeyRangeRef excludedServersSpecialKeyRange("\xff\xff/management/excluded/"_sr, + "\xff\xff/management/excluded0"_sr); +const KeyRangeRef failedServersSpecialKeyRange("\xff\xff/management/failed/"_sr, "\xff\xff/management/failed0"_sr); +const KeyRangeRef excludedLocalitySpecialKeyRange("\xff\xff/management/excluded_locality/"_sr, + "\xff\xff/management/excluded_locality0"_sr); +const KeyRangeRef failedLocalitySpecialKeyRange("\xff\xff/management/failed_locality/"_sr, + "\xff\xff/management/failed_locality0"_sr); +const KeyRef excludedForceOptionSpecialKey = "\xff\xff/management/options/excluded/force"_sr; +const KeyRef failedForceOptionSpecialKey = "\xff\xff/management/options/failed/force"_sr; +const KeyRef excludedLocalityForceOptionSpecialKey = "\xff\xff/management/options/excluded_locality/force"_sr; +const KeyRef failedLocalityForceOptionSpecialKey = "\xff\xff/management/options/failed_locality/force"_sr; +const KeyRangeRef exclusionInProgressSpecialKeyRange("\xff\xff/management/in_progress_exclusion/"_sr, + "\xff\xff/management/in_progress_exclusion0"_sr); + +Future excludeCommandActor(Reference db, std::vector tokens, Future _warn) { + auto warn = std::move(_warn); + if (tokens.size() <= 1) { + std::vector excludedAddresses = co_await getExcludedServers(db); + std::vector excludedLocalities = co_await getExcludedLocalities(db); + std::vector failedAddresses = co_await getFailedServers(db); + std::vector failedLocalities = co_await getFailedLocalities(db); + + if (excludedAddresses.empty() && excludedLocalities.empty() && failedAddresses.empty() && + failedLocalities.empty()) { + printf("There are currently no servers or localities excluded from the database.\n" + "To learn how to exclude a server, type `help exclude'.\n"); + co_return true; + } + + printf("There are currently %zu servers or localities being excluded from the database:\n", + excludedAddresses.size() + excludedLocalities.size()); + for (const auto& e : excludedAddresses) + printf(" %s\n", e.c_str()); + for (const auto& e : excludedLocalities) + printf(" %s\n", e.c_str()); + + if (!excludedAddresses.empty() || !excludedLocalities.empty()) { + printf("To find out whether it is safe to remove one or more of these\n" + "servers from the cluster, type `exclude '.\n" + "To return one of these servers to the cluster, type `include '.\n"); + } + + printf("\n"); + + printf("There are currently %zu servers or localities marked as failed in the database:\n", + failedAddresses.size() + failedLocalities.size()); + for (const auto& f : failedAddresses) + printf(" %s\n", f.c_str()); + for (const auto& f : failedLocalities) + printf(" %s\n", f.c_str()); + + if (!failedAddresses.empty() || !failedLocalities.empty()) { + printf("To return one of these servers to the cluster, type `include failed '.\n"); + } + + printf("\n"); + + Reference tr = db->createTransaction(); + std::set inProgressExclusion = co_await getInProgressExclusion(tr); + printf("There are currently %zu processes for which exclusion is in progress:\n", inProgressExclusion.size()); + for (const auto& addr : inProgressExclusion) { + printf("%s\n", addr.toString().c_str()); + } + + co_return true; + } else { + std::set exclusionSet; + std::vector exclusionAddresses; + std::unordered_set exclusionLocalities; + std::vector noMatchLocalities; + bool force = false; + bool waitForAllExcluded = true; + bool markFailed = false; + std::vector workers; + std::map server_interfaces; + Future future_workers = fdb_cli::getWorkers(db, &workers); + Future future_server_interfaces = fdb_cli::getStorageServerInterfaces(db, &server_interfaces); + co_await (success(future_workers) && success(future_server_interfaces)); + + for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) { + if (*t == "FORCE"_sr) { + force = true; + } else if (*t == "no_wait"_sr) { + waitForAllExcluded = false; + } else if (*t == "failed"_sr) { + markFailed = true; + } else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && + t->toString().find(':') != std::string::npos) { + exclusionLocalities.insert(t->toString()); + auto localityAddresses = getAddressesByLocality(workers, t->toString()); + auto localityServerAddresses = getServerAddressesByLocality(server_interfaces, t->toString()); + if (localityAddresses.empty() && localityServerAddresses.empty()) { + noMatchLocalities.push_back(t->toString()); + continue; + } + + if (!localityAddresses.empty()) { + exclusionSet.insert(localityAddresses.begin(), localityAddresses.end()); + } + + if (!localityServerAddresses.empty()) { + exclusionSet.insert(localityServerAddresses.begin(), localityServerAddresses.end()); + } + } else { + auto a = AddressExclusion::parse(*t); + if (!a.isValid()) { + fprintf(stderr, + "ERROR: '%s' is neither a valid network endpoint address nor a locality\n", + t->toString().c_str()); + if (t->toString().find(":tls") != std::string::npos) + printf(" Do not include the `:tls' suffix when naming a process\n"); + co_return true; + } + exclusionSet.insert(a); + exclusionAddresses.push_back(a); + } + } + + // The validation if a locality or address has no match is done below and will result in a warning. If we abort + // here the provided locality and/or address will not be excluded. + if (exclusionAddresses.empty() && exclusionLocalities.empty()) { + fprintf(stderr, "ERROR: At least one valid network endpoint address or a locality must be provided\n"); + co_return false; + } + + bool res = co_await excludeServersAndLocalities(db, exclusionAddresses, exclusionLocalities, markFailed, force); + if (!res) + co_return false; + + if (waitForAllExcluded) { + printf("Waiting for state to be removed from all excluded servers. This may take a while.\n"); + printf("(Interrupting this wait with CTRL+C will not cancel the data movement.)\n"); + } + + if (warn.isValid()) + warn.cancel(); + + std::set notExcludedServers = + co_await checkForExcludingServers(db, exclusionSet, waitForAllExcluded); + std::map> workerPorts; + for (const auto& addr : workers) + workerPorts[addr.address.ip].insert(addr.address.port); + + // Print a list of all excluded addresses that don't have a corresponding worker + std::set absentExclusions; + for (const auto& addr : exclusionSet) { + auto worker = workerPorts.find(addr.ip); + if (worker == workerPorts.end()) + absentExclusions.insert(addr); + else if (addr.port > 0 && !worker->second.contains(addr.port)) + absentExclusions.insert(addr); + } + + for (const auto& exclusion : exclusionSet) { + if (absentExclusions.find(exclusion) != absentExclusions.end()) { + if (exclusion.port == 0) { + fprintf(stderr, + " %s(Whole machine) ---- WARNING: Missing from cluster!Be sure that you excluded the " + "correct machines before removing them from the cluster!\n", + exclusion.ip.toString().c_str()); + } else { + fprintf(stderr, + " %s ---- WARNING: Missing from cluster! Be sure that you excluded the correct processes " + "before removing them from the cluster!\n", + exclusion.toString().c_str()); + } + } else if (std::any_of(notExcludedServers.begin(), notExcludedServers.end(), [&](const NetworkAddress& a) { + return addressExcluded({ exclusion }, a); + })) { + if (exclusion.port == 0) { + fprintf(stderr, + " %s(Whole machine) ---- WARNING: Exclusion in progress! It is not safe to remove this " + "machine from the cluster\n", + exclusion.ip.toString().c_str()); + } else { + fprintf(stderr, + " %s ---- WARNING: Exclusion in progress! It is not safe to remove this process from the " + "cluster\n", + exclusion.toString().c_str()); + } + } else { + if (exclusion.port == 0) { + printf(" %s(Whole machine) ---- Successfully excluded. It is now safe to remove this machine " + "from the cluster.\n", + exclusion.ip.toString().c_str()); + } else { + printf( + " %s ---- Successfully excluded. It is now safe to remove this process from the cluster.\n", + exclusion.toString().c_str()); + } + } + } + + for (const auto& locality : noMatchLocalities) { + fprintf( + stderr, + " %s ---- WARNING: Currently no servers found with this locality match! Be sure that you excluded " + "the correct locality.\n", + locality.c_str()); + } + + co_await checkForCoordinators(db, exclusionSet); + co_return true; + } +} + +CommandFactory excludeFactory( + "exclude", + CommandHelp( + "exclude [FORCE] [failed] [no_wait] [] [locality_dcid:]\n" + " [locality_zoneid:] [locality_machineid:]\n" + " [locality_processid:] [locality_:]", + "exclude servers from the database by IP address or locality", + "If no addresses or localities are specified, lists the set of excluded addresses and localities in addition " + "to addresses for which exclusion is in progress.\n" + "\n" + "For each IP address or IP:port pair in and/or each locality attribute (like dcid, " + "zoneid, machineid, processid), adds the address/locality to the set of exclusions and waits until all " + "database state has been safely moved away from affected servers.\n" + "\n" + "If 'FORCE' is set, the command does not perform safety checks before excluding.\n" + "\n" + "If 'no_wait' is set, the command returns immediately without checking if the exclusions have completed " + "successfully.\n" + "\n" + "If 'failed' is set, the cluster will immediately forget all data associated with the excluded processes. " + "Doing so can be helpful if the process is not expected to recover, as it will allow the cluster to delete " + "state that would be needed to catch the failed process up. Re-including a process excluded with 'failed' will " + "result in it joining as an empty process.\n" + "\n" + "If a cluster has failed storage servers that result in all replicas of some data being permanently gone, " + "'exclude failed' can be used to clean up the affected key ranges by restoring them to empty.\n" + "\n" + "WARNING: use of 'exclude failed' can result in data loss. If an excluded server contains the last replica of " + "some data, then using the 'failed' option will permanently remove that data from the cluster.")); +} // namespace fdb_cli diff --git a/fdbcli/ExpensiveDataCheckCommand.actor.cpp b/fdbcli/ExpensiveDataCheckCommand.actor.cpp deleted file mode 100644 index a9b02c144b4..00000000000 --- a/fdbcli/ExpensiveDataCheckCommand.actor.cpp +++ /dev/null @@ -1,117 +0,0 @@ -/* - * ExpensiveDataCheckCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/algorithm/string.hpp" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -// The command is used to send a data check request to the specified process -// The check request is accomplished by rebooting the process - -ACTOR Future expensiveDataCheckCommandActor( - Reference db, - Reference tr, - std::vector tokens, - std::map>* address_interface) { - state bool result = true; - state std::string addressesStr; - if (tokens.size() == 1) { - // initialize worker interfaces - address_interface->clear(); - wait(getWorkerInterfaces(tr, address_interface, true)); - } - if (tokens.size() == 1 || tokencmp(tokens[1], "list")) { - if (address_interface->size() == 0) { - printf("\nNo addresses can be checked.\n"); - } else if (address_interface->size() == 1) { - printf("\nThe following address can be checked:\n"); - } else { - printf("\nThe following %zu addresses can be checked:\n", address_interface->size()); - } - for (auto it : *address_interface) { - printf("%s\n", printable(it.first).c_str()); - } - printf("\n"); - } else if (tokencmp(tokens[1], "all")) { - if (address_interface->size() == 0) { - fprintf(stderr, - "ERROR: no processes to check. You must run the `expensive_data_check’ " - "command before running `expensive_data_check all’.\n"); - } else { - std::vector addressesVec; - for (const auto& [address, _] : *address_interface) { - addressesVec.push_back(address.toString()); - } - addressesStr = boost::algorithm::join(addressesVec, ","); - // make sure we only call the interface once to send requests in parallel - int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0))); - if (!checkRequestsSent) { - result = false; - fprintf(stderr, - "ERROR: failed to send requests to check all processes, please run the `expensive_data_check’ " - "command again to fetch latest addresses.\n"); - } else { - printf("Attempted to kill and check %zu processes\n", address_interface->size()); - } - } - } else { - state int i; - for (i = 1; i < tokens.size(); i++) { - if (!address_interface->count(tokens[i])) { - fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str()); - result = false; - break; - } - } - - if (result) { - std::vector addressesVec; - for (i = 1; i < tokens.size(); i++) { - addressesVec.push_back(tokens[i].toString()); - } - addressesStr = boost::algorithm::join(addressesVec, ","); - int64_t checkRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0))); - if (!checkRequestsSent) { - result = false; - fprintf(stderr, - "ERROR: failed to send requests to check processes `%s', please run the `expensive_data_check’ " - "command again to fetch latest addresses.\n", - addressesStr.c_str()); - } else { - printf("Attempted to kill and check %zu processes\n", tokens.size() - 1); - } - } - } - return result; -} -// hidden commands, no help text for now -CommandFactory expensiveDataCheckFactory("expensive_data_check"); -} // namespace fdb_cli diff --git a/fdbcli/ExpensiveDataCheckCommand.cpp b/fdbcli/ExpensiveDataCheckCommand.cpp new file mode 100644 index 00000000000..0b56e63b687 --- /dev/null +++ b/fdbcli/ExpensiveDataCheckCommand.cpp @@ -0,0 +1,116 @@ +/* + * ExpensiveDataCheckCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/algorithm/string.hpp" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace fdb_cli { + +// The command is used to send a data check request to the specified process +// The check request is accomplished by rebooting the process + +Future expensiveDataCheckCommandActor( + Reference db, + Reference tr, + std::vector tokens, + std::map>* address_interface) { + bool result = true; + std::string addressesStr; + if (tokens.size() == 1) { + // initialize worker interfaces + address_interface->clear(); + co_await getWorkerInterfaces(tr, address_interface, true); + } + if (tokens.size() == 1 || tokencmp(tokens[1], "list")) { + if (address_interface->empty()) { + printf("\nNo addresses can be checked.\n"); + } else if (address_interface->size() == 1) { + printf("\nThe following address can be checked:\n"); + } else { + printf("\nThe following %zu addresses can be checked:\n", address_interface->size()); + } + for (const auto& it : *address_interface) { + printf("%s\n", printable(it.first).c_str()); + } + printf("\n"); + } else if (tokencmp(tokens[1], "all")) { + if (address_interface->empty()) { + fprintf(stderr, + "ERROR: no processes to check. You must run the `expensive_data_check’ " + "command before running `expensive_data_check all’.\n"); + } else { + std::vector addressesVec; + for (const auto& [address, _] : *address_interface) { + addressesVec.push_back(address.toString()); + } + addressesStr = boost::algorithm::join(addressesVec, ","); + // make sure we only call the interface once to send requests in parallel + int64_t checkRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)); + if (!checkRequestsSent) { + result = false; + fprintf(stderr, + "ERROR: failed to send requests to check all processes, please run the `expensive_data_check’ " + "command again to fetch latest addresses.\n"); + } else { + printf("Attempted to kill and check %zu processes\n", address_interface->size()); + } + } + } else { + int i{ 0 }; + for (i = 1; i < tokens.size(); i++) { + if (!address_interface->count(tokens[i])) { + fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str()); + result = false; + break; + } + } + + if (result) { + std::vector addressesVec; + for (i = 1; i < tokens.size(); i++) { + addressesVec.push_back(tokens[i].toString()); + } + addressesStr = boost::algorithm::join(addressesVec, ","); + int64_t checkRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, true, 0)); + if (!checkRequestsSent) { + result = false; + fprintf(stderr, + "ERROR: failed to send requests to check processes `%s', please run the `expensive_data_check’ " + "command again to fetch latest addresses.\n", + addressesStr.c_str()); + } else { + printf("Attempted to kill and check %zu processes\n", tokens.size() - 1); + } + } + } + co_return result; +} +// hidden commands, no help text for now +CommandFactory expensiveDataCheckFactory("expensive_data_check"); +} // namespace fdb_cli diff --git a/fdbcli/FileConfigureCommand.actor.cpp b/fdbcli/FileConfigureCommand.actor.cpp deleted file mode 100644 index a521e70aa92..00000000000 --- a/fdbcli/FileConfigureCommand.actor.cpp +++ /dev/null @@ -1,190 +0,0 @@ -/* - * FileConfigureCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/FlowLineNoise.h" -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/DatabaseConfiguration.h" -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/Schemas.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future fileConfigureCommandActor(Reference db, - std::string filePath, - bool isNewDatabase, - bool force) { - state ConfigurationResult result; - std::string contents(readFileBytes(filePath, 100000)); - json_spirit::mValue config; - if (!json_spirit::read_string(contents, config)) { - fprintf(stderr, "ERROR: Invalid JSON\n"); - return false; - } - if (config.type() != json_spirit::obj_type) { - fprintf(stderr, "ERROR: Configuration file must contain a JSON object\n"); - return false; - } - StatusObject configJSON = config.get_obj(); - - json_spirit::mValue schema; - if (!json_spirit::read_string(JSONSchemas::clusterConfigurationSchema.toString(), schema)) { - ASSERT(false); - } - - std::string errorStr; - if (!schemaMatch(schema.get_obj(), configJSON, errorStr)) { - printf("%s", errorStr.c_str()); - return false; - } - - state std::string configString; - - try { - configString += DatabaseConfiguration::configureStringFromJSON(configJSON); - } catch (Error& e) { - fmt::print("ERROR: {}", e.what()); - printUsage("fileconfigure"_sr); - return false; - } - - if (isNewDatabase) { - configString = "new" + configString; - } else { - configString.erase(0, 1); // configureStringFromJSON returns a string with leading space. - } - - // Check for backup_worker_enabled configuration and reject it. - // This setting is now managed automatically by the backup system. - if (configString.find(" backup_worker_enabled:=") != std::string::npos) { - result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED; - } else { - ConfigurationResult r = wait(ManagementAPI::changeConfig(db, configString, force)); - result = r; - } - // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but - // there are various results specific to changeConfig() that we need to report: - bool ret = true; - switch (result) { - case ConfigurationResult::NO_OPTIONS_PROVIDED: - fprintf(stderr, "ERROR: No options provided\n"); - ret = false; - break; - case ConfigurationResult::CONFLICTING_OPTIONS: - fprintf(stderr, "ERROR: Conflicting options\n"); - ret = false; - break; - case ConfigurationResult::UNKNOWN_OPTION: - fprintf(stderr, "ERROR: Unknown option\n"); // This should not be possible because of schema match - ret = false; - break; - case ConfigurationResult::INCOMPLETE_CONFIGURATION: - fprintf(stderr, - "ERROR: Must specify both a replication level and a storage engine when creating a new database\n"); - ret = false; - break; - case ConfigurationResult::INVALID_CONFIGURATION: - fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); - ret = false; - break; - case ConfigurationResult::DATABASE_ALREADY_CREATED: - fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); - ret = false; - break; - case ConfigurationResult::DATABASE_CREATED: - printf("Database created\n"); - break; - case ConfigurationResult::DATABASE_UNAVAILABLE: - fprintf(stderr, "ERROR: The database is unavailable\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: - fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: - fprintf(stderr, - "ERROR: When usable_regions > 1, All regions with priority >= 0 must be fully replicated " - "before changing the configuration\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: - fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::REGIONS_CHANGED: - fprintf(stderr, - "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::NOT_ENOUGH_WORKERS: - fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::REGION_REPLICATION_MISMATCH: - fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::DCID_MISSING: - fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); - printf("Type `fileconfigure FORCE ' to configure without this check\n"); - ret = false; - break; - case ConfigurationResult::SUCCESS: - printf("Configuration changed\n"); - break; - case ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED: - fprintf(stderr, - "ERROR: backup_worker_enabled configuration is restricted in fdbcli and managed automatically by the " - "backup system\n"); - ret = false; - break; - default: - ASSERT(false); - ret = false; - }; - return ret; -} - -CommandFactory fileconfigureFactory( - "fileconfigure", - CommandHelp( - "fileconfigure [new] ", - "change the database configuration from a file", - "The `new' option, if present, initializes a new database with the given configuration rather than changing " - "the configuration of an existing one. Load a JSON document from the provided file, and change the database " - "configuration to match the contents of the JSON document. The format should be the same as the value of the " - "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\".")); - -} // namespace fdb_cli diff --git a/fdbcli/FileConfigureCommand.cpp b/fdbcli/FileConfigureCommand.cpp new file mode 100644 index 00000000000..b4535e53686 --- /dev/null +++ b/fdbcli/FileConfigureCommand.cpp @@ -0,0 +1,196 @@ +/* + * FileConfigureCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/FlowLineNoise.h" +#include "fdbcli/fdbcli.h" + +#include "fdbclient/DatabaseConfiguration.h" +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/ManagementAPI.h" +#include "fdbclient/Schemas.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +namespace fdb_cli { + +Future fileConfigureCommandActor(Reference db, + std::string const& filePath, + bool isNewDatabase, + bool force) { + ConfigurationResult result; + std::string contents(readFileBytes(filePath, 100000)); + json_spirit::mValue config; + if (!json_spirit::read_string(contents, config)) { + fprintf(stderr, "ERROR: Invalid JSON\n"); + co_return false; + } + if (config.type() != json_spirit::obj_type) { + fprintf(stderr, "ERROR: Configuration file must contain a JSON object\n"); + co_return false; + } + StatusObject configJSON = config.get_obj(); + + json_spirit::mValue schema; + if (!json_spirit::read_string(JSONSchemas::clusterConfigurationSchema.toString(), schema)) { + ASSERT(false); + } + + std::string errorStr; + if (!schemaMatch(schema.get_obj(), configJSON, errorStr)) { + printf("%s", errorStr.c_str()); + co_return false; + } + + std::string configString; + + try { + configString += DatabaseConfiguration::configureStringFromJSON(configJSON); + } catch (Error& e) { + fmt::print("ERROR: {}", e.what()); + printUsage("fileconfigure"_sr); + co_return false; + } + + if (isNewDatabase) { + configString = "new" + configString; + } else { + configString.erase(0, 1); // configureStringFromJSON returns a string with leading space. + } + + // Check for backup_worker_enabled configuration and reject it. + // This setting is now managed automatically by the backup system. + if (configString.find(" backup_worker_enabled:=") != std::string::npos) { + result = ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED; + } else if (configString.find(" range_backup_worker_enabled:=") != std::string::npos) { + result = ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED; + } else { + ConfigurationResult r = co_await ManagementAPI::changeConfig(db, configString, force); + result = r; + } + // Real errors get thrown from makeInterruptable and printed by the catch block in cli(), but + // there are various results specific to changeConfig() that we need to report: + bool ret = true; + switch (result) { + case ConfigurationResult::NO_OPTIONS_PROVIDED: + fprintf(stderr, "ERROR: No options provided\n"); + ret = false; + break; + case ConfigurationResult::CONFLICTING_OPTIONS: + fprintf(stderr, "ERROR: Conflicting options\n"); + ret = false; + break; + case ConfigurationResult::UNKNOWN_OPTION: + fprintf(stderr, "ERROR: Unknown option\n"); // This should not be possible because of schema match + ret = false; + break; + case ConfigurationResult::INCOMPLETE_CONFIGURATION: + fprintf(stderr, + "ERROR: Must specify both a replication level and a storage engine when creating a new database\n"); + ret = false; + break; + case ConfigurationResult::INVALID_CONFIGURATION: + fprintf(stderr, "ERROR: These changes would make the configuration invalid\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_ALREADY_CREATED: + fprintf(stderr, "ERROR: Database already exists! To change configuration, don't say `new'\n"); + ret = false; + break; + case ConfigurationResult::DATABASE_CREATED: + printf("Database created\n"); + break; + case ConfigurationResult::DATABASE_UNAVAILABLE: + fprintf(stderr, "ERROR: The database is unavailable\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::STORAGE_IN_UNKNOWN_DCID: + fprintf(stderr, "ERROR: All storage servers must be in one of the known regions\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_NOT_FULLY_REPLICATED: + fprintf(stderr, + "ERROR: When usable_regions > 1, All regions with priority >= 0 must be fully replicated " + "before changing the configuration\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::MULTIPLE_ACTIVE_REGIONS: + fprintf(stderr, "ERROR: When changing usable_regions, only one region can have priority >= 0\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGIONS_CHANGED: + fprintf(stderr, + "ERROR: The region configuration cannot be changed while simultaneously changing usable_regions\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::NOT_ENOUGH_WORKERS: + fprintf(stderr, "ERROR: Not enough processes exist to support the specified configuration\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::REGION_REPLICATION_MISMATCH: + fprintf(stderr, "ERROR: `three_datacenter' replication is incompatible with region configuration\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::DCID_MISSING: + fprintf(stderr, "ERROR: `No storage servers in one of the specified regions\n"); + printf("Type `fileconfigure FORCE ' to configure without this check\n"); + ret = false; + break; + case ConfigurationResult::SUCCESS: + printf("Configuration changed\n"); + break; + case ConfigurationResult::BACKUP_WORKER_ENABLED_RESTRICTED: + fprintf(stderr, + "ERROR: backup_worker_enabled configuration is restricted in fdbcli and managed automatically by the " + "backup system\n"); + ret = false; + break; + case ConfigurationResult::RANGE_BACKUP_WORKER_ENABLED_RESTRICTED: + fprintf(stderr, + "ERROR: range_backup_worker_enabled configuration is restricted in fdbcli and managed " + "automatically by the backup system\n"); + ret = false; + break; + default: + ASSERT(false); + ret = false; + }; + co_return ret; +} + +CommandFactory fileconfigureFactory( + "fileconfigure", + CommandHelp( + "fileconfigure [new] ", + "change the database configuration from a file", + "The `new' option, if present, initializes a new database with the given configuration rather than changing " + "the configuration of an existing one. Load a JSON document from the provided file, and change the database " + "configuration to match the contents of the JSON document. The format should be the same as the value of the " + "\"configuration\" entry in status JSON without \"excluded_servers\" or \"coordinators_count\".")); + +} // namespace fdb_cli diff --git a/fdbcli/FlowLineNoise.actor.cpp b/fdbcli/FlowLineNoise.actor.cpp deleted file mode 100644 index 91757e39267..00000000000 --- a/fdbcli/FlowLineNoise.actor.cpp +++ /dev/null @@ -1,186 +0,0 @@ -/* - * FlowLineNoise.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/FlowLineNoise.h" -#include "flow/IThreadPool.h" - -#ifndef BOOST_SYSTEM_NO_LIB -#define BOOST_SYSTEM_NO_LIB -#endif -#ifndef BOOST_DATE_TIME_NO_LIB -#define BOOST_DATE_TIME_NO_LIB -#endif -#ifndef BOOST_REGEX_NO_LIB -#define BOOST_REGEX_NO_LIB -#endif -#include "boost/asio.hpp" - -#include "flow/ThreadHelper.actor.h" - -#if __unixish__ -#define HAVE_LINENOISE 1 -#include "linenoise/linenoise.h" -#else -#define HAVE_LINENOISE 0 -#endif -#include "flow/actorcompiler.h" // This must be the last #include. - -struct LineNoiseReader final : IThreadPoolReceiver { - void init() override {} - - struct Read final : TypedAction { - std::string prompt; - ThreadReturnPromise> result; - - double getTimeEstimate() const override { return 0.0; } - explicit Read(std::string const& prompt) : prompt(prompt) {} - }; - - void action(Read& r) { - try { - r.result.send(read(r.prompt)); - } catch (Error& e) { - r.result.sendError(e); - } catch (...) { - r.result.sendError(unknown_error()); - } - } - -private: - Optional read(std::string const& prompt) { -#if HAVE_LINENOISE - errno = 0; - char* line = linenoise(prompt.c_str()); - if (line) { - std::string s(line); - free(line); - return s; - } else { - if (errno == EAGAIN) // Ctrl-C - return std::string(); - return Optional(); - } -#else - std::string line; - std::fputs(prompt.c_str(), stdout); - if (!std::getline(std::cin, line).eof()) { - return line; - } else - return Optional(); -#endif - } -}; - -LineNoise::LineNoise(std::function&)> _completion_callback, - std::function _hint_callback, - int maxHistoryLines, - bool multiline) - : threadPool(createGenericThreadPool()) { - reader = new LineNoiseReader(); - -#if HAVE_LINENOISE - // It should be OK to call these functions from this thread, since read() can't be called yet - // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely - // invoke the callbacks we've been given - - // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables - static std::function&)> completion_callback; - static std::function hint_callback; - completion_callback = _completion_callback; - hint_callback = _hint_callback; - - linenoiseHistorySetMaxLen(maxHistoryLines); - linenoiseSetMultiLine(multiline); - linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) { - // This code will run in the thread pool - std::vector completions; - onMainThread([line, &completions]() -> Future { - completion_callback(line, completions); - return Void(); - }).getBlocking(); - for (auto const& c : completions) - linenoiseAddCompletion(lc, c.c_str()); - }); - linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* { - Hint h = onMainThread([line]() -> Future { return hint_callback(line); }).getBlocking(); - if (!h.valid) - return nullptr; - *color = h.color; - *bold = h.bold; - return strdup(h.text.c_str()); - }); - linenoiseSetFreeHintsCallback(free); -#endif - - threadPool->addThread(reader, "fdb-linenoise"); -} - -LineNoise::~LineNoise() { - threadPool->stop(); -} - -Future> LineNoise::read(std::string const& prompt) { - auto r = new LineNoiseReader::Read(prompt); - auto f = r->result.getFuture(); - threadPool->post(r); - return f; -} - -ACTOR Future waitKeyboardInterrupt(boost::asio::io_service* ios) { - state boost::asio::signal_set signals(*ios, SIGINT); - Promise result; - signals.async_wait([result](const boost::system::error_code& error, int signal_number) { - if (error) { - result.sendError(io_error()); - } else { - result.send(Void()); - } - }); - - wait(result.getFuture()); - return Void(); -} - -Future LineNoise::onKeyboardInterrupt() { - boost::asio::io_service* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); - if (!ios) - return Never(); - return waitKeyboardInterrupt(ios); -} - -void LineNoise::historyAdd(std::string const& line) { -#if HAVE_LINENOISE - linenoiseHistoryAdd(line.c_str()); -#endif -} -void LineNoise::historyLoad(std::string const& filename) { -#if HAVE_LINENOISE - if (linenoiseHistoryLoad(filename.c_str()) != 0) { - throw io_error(); - } -#endif -} -void LineNoise::historySave(std::string const& filename) { -#if HAVE_LINENOISE - if (linenoiseHistorySave(filename.c_str()) != 0) { - throw io_error(); - } -#endif -} diff --git a/fdbcli/FlowLineNoise.cpp b/fdbcli/FlowLineNoise.cpp new file mode 100644 index 00000000000..05f52f57b4b --- /dev/null +++ b/fdbcli/FlowLineNoise.cpp @@ -0,0 +1,184 @@ +/* + * FlowLineNoise.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/FlowLineNoise.h" +#include "flow/IThreadPool.h" + +#ifndef BOOST_SYSTEM_NO_LIB +#define BOOST_SYSTEM_NO_LIB +#endif +#ifndef BOOST_DATE_TIME_NO_LIB +#define BOOST_DATE_TIME_NO_LIB +#endif +#ifndef BOOST_REGEX_NO_LIB +#define BOOST_REGEX_NO_LIB +#endif +#include "boost/asio.hpp" + +#include "flow/ThreadHelper.actor.h" + +#if __unixish__ +#define HAVE_LINENOISE 1 +#include "linenoise/linenoise.h" +#else +#define HAVE_LINENOISE 0 +#endif + +struct LineNoiseReader final : IThreadPoolReceiver { + void init() override {} + + struct Read final : TypedAction { + std::string prompt; + ThreadReturnPromise> result; + + double getTimeEstimate() const override { return 0.0; } + explicit Read(std::string const& prompt) : prompt(prompt) {} + }; + + void action(Read& r) { + try { + r.result.send(read(r.prompt)); + } catch (Error& e) { + r.result.sendError(e); + } catch (...) { + r.result.sendError(unknown_error()); + } + } + +private: + Optional read(std::string const& prompt) { +#if HAVE_LINENOISE + errno = 0; + char* line = linenoise(prompt.c_str()); + if (line) { + std::string s(line); + free(line); + return s; + } else { + if (errno == EAGAIN) // Ctrl-C + return std::string(); + return Optional(); + } +#else + std::string line; + std::fputs(prompt.c_str(), stdout); + if (!std::getline(std::cin, line).eof()) { + return line; + } else + return Optional(); +#endif + } +}; + +LineNoise::LineNoise(std::function&)> _completion_callback, + std::function _hint_callback, + int maxHistoryLines, + bool multiline) + : threadPool(createGenericThreadPool()) { + reader = new LineNoiseReader(); + +#if HAVE_LINENOISE + // It should be OK to call these functions from this thread, since read() can't be called yet + // The callbacks passed to linenoise*() will be invoked from the thread pool, and use onMainThread() to safely + // invoke the callbacks we've been given + + // linenoise doesn't provide any form of data parameter to callbacks, so we have to use static variables + static std::function&)> completion_callback; + static std::function hint_callback; + completion_callback = _completion_callback; + hint_callback = _hint_callback; + + linenoiseHistorySetMaxLen(maxHistoryLines); + linenoiseSetMultiLine(multiline); + linenoiseSetCompletionCallback([](const char* line, linenoiseCompletions* lc) { + // This code will run in the thread pool + std::vector completions; + onMainThread([line, &completions]() -> Future { + completion_callback(line, completions); + return Void(); + }).getBlocking(); + for (auto const& c : completions) + linenoiseAddCompletion(lc, c.c_str()); + }); + linenoiseSetHintsCallback([](const char* line, int* color, int* bold) -> char* { + Hint h = onMainThread([line]() -> Future { return hint_callback(line); }).getBlocking(); + if (!h.valid) + return nullptr; + *color = h.color; + *bold = h.bold; + return strdup(h.text.c_str()); + }); + linenoiseSetFreeHintsCallback(free); +#endif + + threadPool->addThread(reader, "fdb-linenoise"); +} + +LineNoise::~LineNoise() { + threadPool->stop(); +} + +Future> LineNoise::read(std::string const& prompt) { + auto r = new LineNoiseReader::Read(prompt); + auto f = r->result.getFuture(); + threadPool->post(r); + return f; +} + +Future waitKeyboardInterrupt(boost::asio::io_service* ios) { + boost::asio::signal_set signals(*ios, SIGINT); + Promise result; + signals.async_wait([result](const boost::system::error_code& error, int signal_number) { + if (error) { + result.sendError(io_error()); + } else { + result.send(Void()); + } + }); + + co_await result.getFuture(); +} + +Future LineNoise::onKeyboardInterrupt() { + auto* ios = (boost::asio::io_service*)g_network->global(INetwork::enASIOService); + if (!ios) + return Never(); + return waitKeyboardInterrupt(ios); +} + +void LineNoise::historyAdd(std::string const& line) { +#if HAVE_LINENOISE + linenoiseHistoryAdd(line.c_str()); +#endif +} +void LineNoise::historyLoad(std::string const& filename) { +#if HAVE_LINENOISE + if (linenoiseHistoryLoad(filename.c_str()) != 0) { + throw io_error(); + } +#endif +} +void LineNoise::historySave(std::string const& filename) { +#if HAVE_LINENOISE + if (linenoiseHistorySave(filename.c_str()) != 0) { + throw io_error(); + } +#endif +} diff --git a/fdbcli/ForceRecoveryWithDataLossCommand.actor.cpp b/fdbcli/ForceRecoveryWithDataLossCommand.actor.cpp deleted file mode 100644 index 7c5e35c152d..00000000000 --- a/fdbcli/ForceRecoveryWithDataLossCommand.actor.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/* - * ForceRecoveryWithDataLossCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/IClientApi.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future forceRecoveryWithDataLossCommandActor(Reference db, std::vector tokens) { - if (tokens.size() != 2) { - printUsage(tokens[0]); - return false; - } - wait(safeThreadFutureToFuture(db->forceRecoveryWithDataLoss(tokens[1]))); - return true; -} - -CommandFactory forceRecoveryWithDataLossFactory( - "force_recovery_with_data_loss", - CommandHelp("force_recovery_with_data_loss ", - "Force the database to recover into DCID", - "A forced recovery will cause the database to lose the most recently committed mutations. The " - "amount of mutations that will be lost depends on how far behind the remote datacenter is. This " - "command will change the region configuration to have a positive priority for the chosen DCID, and " - "a negative priority for all other DCIDs. This command will set usable_regions to 1. If the " - "database has already recovered, this command does nothing.\n")); -} // namespace fdb_cli diff --git a/fdbcli/ForceRecoveryWithDataLossCommand.cpp b/fdbcli/ForceRecoveryWithDataLossCommand.cpp new file mode 100644 index 00000000000..37e1d4a1ee6 --- /dev/null +++ b/fdbcli/ForceRecoveryWithDataLossCommand.cpp @@ -0,0 +1,48 @@ +/* + * ForceRecoveryWithDataLossCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/IClientApi.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +namespace fdb_cli { + +Future forceRecoveryWithDataLossCommandActor(Reference db, std::vector const& tokens) { + if (tokens.size() != 2) { + printUsage(tokens[0]); + co_return false; + } + co_await safeThreadFutureToFuture(db->forceRecoveryWithDataLoss(tokens[1])); + co_return true; +} + +CommandFactory forceRecoveryWithDataLossFactory( + "force_recovery_with_data_loss", + CommandHelp("force_recovery_with_data_loss ", + "Force the database to recover into DCID", + "A forced recovery will cause the database to lose the most recently committed mutations. The " + "amount of mutations that will be lost depends on how far behind the remote datacenter is. This " + "command will change the region configuration to have a positive priority for the chosen DCID, and " + "a negative priority for all other DCIDs. This command will set usable_regions to 1. If the " + "database has already recovered, this command does nothing.\n")); +} // namespace fdb_cli diff --git a/fdbcli/GetAuditStatusCommand.actor.cpp b/fdbcli/GetAuditStatusCommand.actor.cpp deleted file mode 100644 index e66c610c3e4..00000000000 --- a/fdbcli/GetAuditStatusCommand.actor.cpp +++ /dev/null @@ -1,262 +0,0 @@ -/* - * GetAuditStatusCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/Audit.h" -#include "fdbclient/AuditUtils.actor.h" -#include "fdbclient/IClientApi.h" -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" - -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future getAuditProgressByRange(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) { - state KeyRange rangeToRead = auditRange; - state Key rangeToReadBegin = auditRange.begin; - state int retryCount = 0; - state int64_t finishCount = 0; - while (rangeToReadBegin < auditRange.end) { - loop { - try { - rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end); - state std::vector auditStates = - wait(getAuditStateByRange(cx, auditType, auditId, rangeToRead)); - for (int i = 0; i < auditStates.size(); i++) { - AuditPhase phase = auditStates[i].getPhase(); - if (phase == AuditPhase::Invalid) { - fmt::println("( Ongoing ) {}", auditStates[i].range.toString()); - } else if (phase == AuditPhase::Error) { - fmt::println("( Error ) {}", auditStates[i].range.toString()); - ++finishCount; - } else { - ++finishCount; - continue; - } - } - rangeToReadBegin = auditStates.back().range.end; - break; - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) { - throw e; - } - if (retryCount > 30) { - fmt::println("Incomplete check"); - return Void(); - } - wait(delay(0.5)); - retryCount++; - } - } - } - fmt::println("Finished range count: {}", finishCount); - return Void(); -} - -ACTOR Future> getStorageServers(Database cx) { - state Transaction tr(cx); - loop { - tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr.setOption(FDBTransactionOptions::LOCK_AWARE); - try { - RangeResult serverList = wait(tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY)); - ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY); - std::vector servers; - servers.reserve(serverList.size()); - for (int i = 0; i < serverList.size(); i++) - servers.push_back(decodeServerListValue(serverList[i].value)); - return servers; - } catch (Error& e) { - wait(tr.onError(e)); - } - } -} - -ACTOR Future getAuditProgressByServer(Database cx, - AuditType auditType, - UID auditId, - KeyRange auditRange, - UID serverId) { - state KeyRange rangeToRead = auditRange; - state Key rangeToReadBegin = auditRange.begin; - state int retryCount = 0; - while (rangeToReadBegin < auditRange.end) { - loop { - try { - rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end); - state std::vector auditStates = - wait(getAuditStateByServer(cx, auditType, auditId, serverId, rangeToRead)); - for (int i = 0; i < auditStates.size(); i++) { - AuditPhase phase = auditStates[i].getPhase(); - if (phase == AuditPhase::Invalid) { - return AuditPhase::Running; - } else if (phase == AuditPhase::Error) { - return AuditPhase::Error; - } - } - rangeToReadBegin = auditStates.back().range.end; - break; - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) { - throw e; - } - if (retryCount > 30) { - return AuditPhase::Invalid; - } - wait(delay(0.5)); - retryCount++; - } - } - } - return AuditPhase::Complete; -} - -ACTOR Future getAuditProgress(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) { - if (auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica || - auditType == AuditType::ValidateLocationMetadata || auditType == AuditType::ValidateRestore) { - wait(getAuditProgressByRange(cx, auditType, auditId, auditRange)); - } else if (auditType == AuditType::ValidateStorageServerShard) { - state std::vector> fs; - state std::unordered_map res; - state std::vector interfs = wait(getStorageServers(cx)); - state int i = 0; - state int numCompleteServers = 0; - state int numOngoingServers = 0; - state int numErrorServers = 0; - state int numTSSes = 0; - for (; i < interfs.size(); i++) { - if (interfs[i].isTss()) { - numTSSes++; - continue; // SSShard audit does not test TSS - } - AuditPhase serverPhase = wait(getAuditProgressByServer(cx, auditType, auditId, allKeys, interfs[i].id())); - if (serverPhase == AuditPhase::Running) { - numOngoingServers++; - } else if (serverPhase == AuditPhase::Complete) { - numCompleteServers++; - } else if (serverPhase == AuditPhase::Error) { - numErrorServers++; - } else if (serverPhase == AuditPhase::Invalid) { - fmt::println("SS {} partial progress fetched", interfs[i].id().toString()); - } - } - fmt::println("CompleteServers: {}", numCompleteServers); - fmt::println("OngoingServers: {}", numOngoingServers); - fmt::println("ErrorServers: {}", numErrorServers); - fmt::println("IgnoredTSSes: {}", numTSSes); - } else { - fmt::println("AuditType not implemented"); - } - return Void(); -} - -ACTOR Future getAuditStatusCommandActor(Database cx, std::vector tokens) { - if (tokens.size() < 2 || tokens.size() > 5) { - printUsage(tokens[0]); - return false; - } - - AuditType type = AuditType::Invalid; - if (tokencmp(tokens[1], "ha")) { - type = AuditType::ValidateHA; - } else if (tokencmp(tokens[1], "replica")) { - type = AuditType::ValidateReplica; - } else if (tokencmp(tokens[1], "locationmetadata")) { - type = AuditType::ValidateLocationMetadata; - } else if (tokencmp(tokens[1], "ssshard")) { - type = AuditType::ValidateStorageServerShard; - } else if (tokencmp(tokens[1], "validate_restore")) { - type = AuditType::ValidateRestore; - } else { - printUsage(tokens[0]); - return false; - } - - if (tokencmp(tokens[2], "id")) { - if (tokens.size() != 4) { - printUsage(tokens[0]); - return false; - } - const UID id = UID::fromString(tokens[3].toString()); - AuditStorageState res = wait(getAuditState(cx, type, id)); - fmt::println("Audit result is:\n{}", res.toString()); - } else if (tokencmp(tokens[2], "progress")) { - if (tokens.size() != 4) { - printUsage(tokens[0]); - return false; - } - const UID id = UID::fromString(tokens[3].toString()); - state AuditStorageState res = wait(getAuditState(cx, type, id)); - if (res.getPhase() == AuditPhase::Running) { - wait(getAuditProgress(cx, res.getType(), res.id, res.range)); - } else { - fmt::println("Already complete"); - } - } else if (tokencmp(tokens[2], "recent")) { - int count = CLIENT_KNOBS->TOO_MANY; - if (tokens.size() == 4) { - count = std::stoi(tokens[3].toString()); - } - std::vector res = wait(getAuditStates(cx, type, /*newFirst=*/true, count)); - for (const auto& it : res) { - fmt::println("Audit result is:\n{}", it.toString()); - } - } else if (tokencmp(tokens[2], "phase")) { - AuditPhase phase = stringToAuditPhase(tokens[3].toString()); - if (phase == AuditPhase::Invalid) { - printUsage(tokens[0]); - return false; - } - int count = CLIENT_KNOBS->TOO_MANY; - if (tokens.size() == 5) { - count = std::stoi(tokens[4].toString()); - } - std::vector res = wait(getAuditStates(cx, type, /*newFirst=*/true, count, phase)); - for (const auto& it : res) { - fmt::println("Audit result is:\n{}", it.toString()); - } - } else { - printUsage(tokens[0]); - return false; - } - - return true; -} - -CommandFactory getAuditStatusFactory( - "get_audit_status", - CommandHelp( - "get_audit_status [ha|replica|locationmetadata|ssshard|validate_restore] [id|recent|phase|progress] [ARGs]", - "Retrieve audit storage status", - "To fetch audit status via ID: `get_audit_status [Type] id [ID]'\n" - "To fetch status of most recent audit: `get_audit_status [Type] recent [Count]'\n" - "To fetch status of audits in a specific phase: `get_audit_status [Type] phase " - "[running|complete|failed|error] count'\n" - "To fetch audit progress via ID: `get_audit_status [Type] progress [ID]'\n" - "Supported types include: 'ha', `replica`, `locationmetadata`, `ssshard`, `validate_restore`. \n" - "If specified, `Count' is how many rows to audit.\n" - "If not specified, check all rows in audit.\n" - "Phase can be `Invalid=0', `Running=1', `Complete=2', `Error=3', or `Failed=4'.\n" - "See also `audit_storage' command.")); -} // namespace fdb_cli diff --git a/fdbcli/GetAuditStatusCommand.cpp b/fdbcli/GetAuditStatusCommand.cpp new file mode 100644 index 00000000000..0a9cf1d5983 --- /dev/null +++ b/fdbcli/GetAuditStatusCommand.cpp @@ -0,0 +1,265 @@ +/* + * GetAuditStatusCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "fdbcli/fdbcli.h" +#include "fdbclient/Audit.h" +#include "fdbclient/AuditUtils.h" +#include "fdbclient/IClientApi.h" +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace fdb_cli { + +Future getAuditProgressByRange(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) { + KeyRange rangeToRead = auditRange; + Key rangeToReadBegin = auditRange.begin; + int retryCount = 0; + int64_t finishCount = 0; + while (rangeToReadBegin < auditRange.end) { + while (true) { + Error err; + try { + rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end); + std::vector auditStates = + co_await getAuditStateByRange(cx, auditType, auditId, rangeToRead); + for (int i = 0; i < auditStates.size(); i++) { + AuditPhase phase = auditStates[i].getPhase(); + if (phase == AuditPhase::Invalid) { + fmt::println("( Ongoing ) {}", auditStates[i].range.toString()); + } else if (phase == AuditPhase::Error) { + fmt::println("( Error ) {}", auditStates[i].range.toString()); + ++finishCount; + } else { + ++finishCount; + continue; + } + } + rangeToReadBegin = auditStates.back().range.end; + break; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_actor_cancelled) { + throw err; + } + if (retryCount > 30) { + fmt::println("Incomplete check"); + co_return; + } + co_await delay(0.5); + retryCount++; + } + } + fmt::println("Finished range count: {}", finishCount); +} + +AsyncResult> getStorageServers(Database cx) { + Transaction tr(cx); + while (true) { + tr.setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr.setOption(FDBTransactionOptions::LOCK_AWARE); + Error err; + try { + RangeResult serverList = co_await tr.getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY); + ASSERT(!serverList.more && serverList.size() < CLIENT_KNOBS->TOO_MANY); + std::vector servers; + servers.reserve(serverList.size()); + for (int i = 0; i < serverList.size(); i++) + servers.push_back(decodeServerListValue(serverList[i].value)); + co_return servers; + } catch (Error& e) { + err = e; + } + co_await tr.onError(err); + } +} + +Future getAuditProgressByServer(Database cx, + AuditType auditType, + UID auditId, + KeyRange auditRange, + UID serverId) { + KeyRange rangeToRead = auditRange; + Key rangeToReadBegin = auditRange.begin; + int retryCount = 0; + while (rangeToReadBegin < auditRange.end) { + while (true) { + Error err; + try { + rangeToRead = KeyRangeRef(rangeToReadBegin, auditRange.end); + std::vector auditStates = + co_await getAuditStateByServer(cx, auditType, auditId, serverId, rangeToRead); + for (int i = 0; i < auditStates.size(); i++) { + AuditPhase phase = auditStates[i].getPhase(); + if (phase == AuditPhase::Invalid) { + co_return AuditPhase::Running; + } else if (phase == AuditPhase::Error) { + co_return AuditPhase::Error; + } + } + rangeToReadBegin = auditStates.back().range.end; + break; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_actor_cancelled) { + throw err; + } + if (retryCount > 30) { + co_return AuditPhase::Invalid; + } + co_await delay(0.5); + retryCount++; + } + } + co_return AuditPhase::Complete; +} + +Future getAuditProgress(Database cx, AuditType auditType, UID auditId, KeyRange auditRange) { + if (auditType == AuditType::ValidateHA || auditType == AuditType::ValidateReplica || + auditType == AuditType::ValidateLocationMetadata || auditType == AuditType::ValidateRestore) { + co_await getAuditProgressByRange(cx, auditType, auditId, auditRange); + } else if (auditType == AuditType::ValidateStorageServerShard) { + std::vector> fs; + std::unordered_map res; + std::vector interfs = co_await getStorageServers(cx); + int i = 0; + int numCompleteServers = 0; + int numOngoingServers = 0; + int numErrorServers = 0; + int numTSSes = 0; + for (; i < interfs.size(); i++) { + if (interfs[i].isTss()) { + numTSSes++; + continue; // SSShard audit does not test TSS + } + AuditPhase serverPhase = + co_await getAuditProgressByServer(cx, auditType, auditId, allKeys, interfs[i].id()); + if (serverPhase == AuditPhase::Running) { + numOngoingServers++; + } else if (serverPhase == AuditPhase::Complete) { + numCompleteServers++; + } else if (serverPhase == AuditPhase::Error) { + numErrorServers++; + } else if (serverPhase == AuditPhase::Invalid) { + fmt::println("SS {} partial progress fetched", interfs[i].id().toString()); + } + } + fmt::println("CompleteServers: {}", numCompleteServers); + fmt::println("OngoingServers: {}", numOngoingServers); + fmt::println("ErrorServers: {}", numErrorServers); + fmt::println("IgnoredTSSes: {}", numTSSes); + } else { + fmt::println("AuditType not implemented"); + } +} + +Future getAuditStatusCommandActor(Database cx, std::vector tokens) { + if (tokens.size() < 2 || tokens.size() > 5) { + printUsage(tokens[0]); + co_return false; + } + + AuditType type = AuditType::Invalid; + if (tokencmp(tokens[1], "ha")) { + type = AuditType::ValidateHA; + } else if (tokencmp(tokens[1], "replica")) { + type = AuditType::ValidateReplica; + } else if (tokencmp(tokens[1], "locationmetadata")) { + type = AuditType::ValidateLocationMetadata; + } else if (tokencmp(tokens[1], "ssshard")) { + type = AuditType::ValidateStorageServerShard; + } else if (tokencmp(tokens[1], "validate_restore")) { + type = AuditType::ValidateRestore; + } else { + printUsage(tokens[0]); + co_return false; + } + + if (tokencmp(tokens[2], "id")) { + if (tokens.size() != 4) { + printUsage(tokens[0]); + co_return false; + } + const UID id = UID::fromString(tokens[3].toString()); + AuditStorageState res = co_await getAuditState(cx, type, id); + fmt::println("Audit result is:\n{}", res.toString()); + } else if (tokencmp(tokens[2], "progress")) { + if (tokens.size() != 4) { + printUsage(tokens[0]); + co_return false; + } + const UID id = UID::fromString(tokens[3].toString()); + AuditStorageState res = co_await getAuditState(cx, type, id); + if (res.getPhase() == AuditPhase::Running) { + co_await getAuditProgress(cx, res.getType(), res.id, res.range); + } else { + fmt::println("Already complete"); + } + } else if (tokencmp(tokens[2], "recent")) { + int count = CLIENT_KNOBS->TOO_MANY; + if (tokens.size() == 4) { + count = std::stoi(tokens[3].toString()); + } + std::vector res = co_await getAuditStates(cx, type, /*newFirst=*/true, count); + for (const auto& it : res) { + fmt::println("Audit result is:\n{}", it.toString()); + } + } else if (tokencmp(tokens[2], "phase")) { + AuditPhase phase = stringToAuditPhase(tokens[3].toString()); + if (phase == AuditPhase::Invalid) { + printUsage(tokens[0]); + co_return false; + } + int count = CLIENT_KNOBS->TOO_MANY; + if (tokens.size() == 5) { + count = std::stoi(tokens[4].toString()); + } + std::vector res = co_await getAuditStates(cx, type, /*newFirst=*/true, count, phase); + for (const auto& it : res) { + fmt::println("Audit result is:\n{}", it.toString()); + } + } else { + printUsage(tokens[0]); + co_return false; + } + + co_return true; +} + +CommandFactory getAuditStatusFactory( + "get_audit_status", + CommandHelp( + "get_audit_status [ha|replica|locationmetadata|ssshard|validate_restore] [id|recent|phase|progress] [ARGs]", + "Retrieve audit storage status", + "To fetch audit status via ID: `get_audit_status [Type] id [ID]'\n" + "To fetch status of most recent audit: `get_audit_status [Type] recent [Count]'\n" + "To fetch status of audits in a specific phase: `get_audit_status [Type] phase " + "[running|complete|failed|error] count'\n" + "To fetch audit progress via ID: `get_audit_status [Type] progress [ID]'\n" + "Supported types include: 'ha', `replica`, `locationmetadata`, `ssshard`, `validate_restore`. \n" + "If specified, `Count' is how many rows to audit.\n" + "If not specified, check all rows in audit.\n" + "Phase can be `Invalid=0', `Running=1', `Complete=2', `Error=3', or `Failed=4'.\n" + "See also `audit_storage' command.")); +} // namespace fdb_cli diff --git a/fdbcli/HotRangeCommand.actor.cpp b/fdbcli/HotRangeCommand.actor.cpp deleted file mode 100644 index b3d6788fffc..00000000000 --- a/fdbcli/HotRangeCommand.actor.cpp +++ /dev/null @@ -1,128 +0,0 @@ -/* - * HotRangeCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/StorageServerInterface.h" - -#include "fdbclient/json_spirit/json_spirit_value.h" -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/NetworkAddress.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -ReadHotSubRangeRequest::SplitType parseSplitType(const std::string& typeStr) { - auto type = ReadHotSubRangeRequest::SplitType::BYTES; - if (typeStr == "bytes") { - type = ReadHotSubRangeRequest::BYTES; - } else if (typeStr == "readBytes") { - type = ReadHotSubRangeRequest::READ_BYTES; - } else if (typeStr == "readOps") { - type = ReadHotSubRangeRequest::READ_OPS; - } else { - fmt::print("Error: {} is not a valid split type. Will use bytes as the default split type\n", typeStr); - } - return type; -} - -} // namespace - -namespace fdb_cli { - -ACTOR Future hotRangeCommandActor(Database localdb, - Reference db, - std::vector tokens, - std::map* storage_interface) { - - if (tokens.size() == 1) { - // initialize storage interfaces - storage_interface->clear(); - wait(getStorageServerInterfaces(db, storage_interface)); - fmt::print("\nThe following {} storage servers can be queried:\n", storage_interface->size()); - for (const auto& [addr, _] : *storage_interface) { - fmt::print("{}\n", addr); - } - fmt::print("\n"); - } else if (tokens.size() == 6) { - if (storage_interface->size() == 0) { - fprintf(stderr, "ERROR: no storage processes to query. You must run the `hotrange’ command first.\n"); - return false; - } - state Key address = tokens[1]; - // At present we only support one process(IP:Port) at a time - if (!storage_interface->count(address.toString())) { - fprintf(stderr, "ERROR: storage process `%s' not recognized.\n", printable(address).c_str()); - return false; - } - state int splitCount; - try { - splitCount = boost::lexical_cast(tokens[5].toString()); - } catch (...) { - fmt::print("Error: splitCount value: '{}', cannot be parsed to an Integer\n", tokens[5].toString()); - return false; - } - ReadHotSubRangeRequest::SplitType splitType = parseSplitType(tokens[2].toString()); - KeyRangeRef range(tokens[3], tokens[4]); - // TODO: refactor this to support multiversion fdbcli in the future - Standalone> metrics = - wait(localdb->getHotRangeMetrics((*storage_interface)[address.toString()], range, splitType, splitCount)); - // next parse the result and form a json array for each object - json_spirit::mArray resultArray; - for (const auto& metric : metrics) { - json_spirit::mObject metricObj; - metricObj["begin"] = metric.keys.begin.toString(); - metricObj["end"] = metric.keys.end.toString(); - metricObj["readBytesPerSec"] = metric.readBandwidthSec; - metricObj["readOpsPerSec"] = metric.readOpsSec; - metricObj["bytes"] = metric.bytes; - resultArray.push_back(metricObj); - } - // print out the json array - const std::string result = - json_spirit::write_string(json_spirit::mValue(resultArray), json_spirit::pretty_print); - fmt::print("\n{}\n", result); - } else { - printUsage(tokens[0]); - return false; - } - - return true; -} - -CommandFactory hotRangeFactory( - "hotrange", - CommandHelp( - "hotrange ", - "Fetch read metrics from a given storage server to detect hot range", - "If no arguments are specified, populates the list of storage processes that can be queried. " - " specify the range you are interested in, " - " is the metric used to divide ranges, " - "splitCount is the number of returned ranges divided by the given metric. " - "The command will return an array of json object for each range with their metrics." - "Notice: the three metrics are sampled by a different way, so their values are not perfectly matched.\n")); - -} // namespace fdb_cli diff --git a/fdbcli/HotRangeCommand.cpp b/fdbcli/HotRangeCommand.cpp new file mode 100644 index 00000000000..2913ced41e8 --- /dev/null +++ b/fdbcli/HotRangeCommand.cpp @@ -0,0 +1,126 @@ +/* + * HotRangeCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/StorageServerInterface.h" + +#include "fdbclient/json_spirit/json_spirit_value.h" +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/NetworkAddress.h" +#include "flow/ThreadHelper.actor.h" +namespace { + +ReadHotSubRangeRequest::SplitType parseSplitType(const std::string& typeStr) { + auto type = ReadHotSubRangeRequest::SplitType::BYTES; + if (typeStr == "bytes") { + type = ReadHotSubRangeRequest::BYTES; + } else if (typeStr == "readBytes") { + type = ReadHotSubRangeRequest::READ_BYTES; + } else if (typeStr == "readOps") { + type = ReadHotSubRangeRequest::READ_OPS; + } else { + fmt::print("Error: {} is not a valid split type. Will use bytes as the default split type\n", typeStr); + } + return type; +} + +} // namespace + +namespace fdb_cli { + +Future hotRangeCommandActor(Database localdb, + Reference db, + std::vector const& tokens, + std::map* const& storage_interface) { + + if (tokens.size() == 1) { + // initialize storage interfaces + storage_interface->clear(); + co_await getStorageServerInterfaces(db, storage_interface); + fmt::print("\nThe following {} storage servers can be queried:\n", storage_interface->size()); + for (const auto& [addr, _] : *storage_interface) { + fmt::print("{}\n", addr); + } + fmt::print("\n"); + } else if (tokens.size() == 6) { + if (storage_interface->empty()) { + fprintf(stderr, "ERROR: no storage processes to query. You must run the `hotrange’ command first.\n"); + co_return false; + } + Key address = tokens[1]; + // At present we only support one process(IP:Port) at a time + if (!storage_interface->count(address.toString())) { + fprintf(stderr, "ERROR: storage process `%s' not recognized.\n", printable(address).c_str()); + co_return false; + } + int splitCount{ 0 }; + try { + splitCount = boost::lexical_cast(tokens[5].toString()); + } catch (...) { + fmt::print("Error: splitCount value: '{}', cannot be parsed to an Integer\n", tokens[5].toString()); + co_return false; + } + ReadHotSubRangeRequest::SplitType splitType = parseSplitType(tokens[2].toString()); + KeyRangeRef range(tokens[3], tokens[4]); + // TODO: refactor this to support multiversion fdbcli in the future + Standalone> metrics = co_await localdb->getHotRangeMetrics( + (*storage_interface)[address.toString()], range, splitType, splitCount); + // next parse the result and form a json array for each object + json_spirit::mArray resultArray; + for (const auto& metric : metrics) { + json_spirit::mObject metricObj; + metricObj["begin"] = metric.keys.begin.toString(); + metricObj["end"] = metric.keys.end.toString(); + metricObj["readBytesPerSec"] = metric.readBandwidthSec; + metricObj["readOpsPerSec"] = metric.readOpsSec; + metricObj["bytes"] = metric.bytes; + resultArray.push_back(metricObj); + } + // print out the json array + const std::string result = + json_spirit::write_string(json_spirit::mValue(resultArray), json_spirit::pretty_print); + fmt::print("\n{}\n", result); + } else { + printUsage(tokens[0]); + co_return false; + } + + co_return true; +} + +CommandFactory hotRangeFactory( + "hotrange", + CommandHelp( + "hotrange ", + "Fetch read metrics from a given storage server to detect hot range", + "If no arguments are specified, populates the list of storage processes that can be queried. " + " specify the range you are interested in, " + " is the metric used to divide ranges, " + "splitCount is the number of returned ranges divided by the given metric. " + "The command will return an array of json object for each range with their metrics." + "Notice: the three metrics are sampled by a different way, so their values are not perfectly matched.\n")); + +} // namespace fdb_cli diff --git a/fdbcli/IdempotencyIdsCommand.actor.cpp b/fdbcli/IdempotencyIdsCommand.actor.cpp deleted file mode 100644 index 56457027235..00000000000 --- a/fdbcli/IdempotencyIdsCommand.actor.cpp +++ /dev/null @@ -1,83 +0,0 @@ -/* - * IdempotencyIdsCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/IdempotencyId.actor.h" -#include "fdbclient/JsonBuilder.h" -#include "fdbclient/json_spirit/json_spirit_reader_template.h" -#include "flow/actorcompiler.h" // This must be the last include - -namespace { - -Optional parseAgeValue(StringRef token) { - try { - return std::stod(token.toString()); - } catch (...) { - return {}; - } -} - -} // namespace - -namespace fdb_cli { - -ACTOR Future idempotencyIdsCommandActor(Database db, std::vector tokens) { - if (tokens.size() < 2 || tokens.size() > 3) { - printUsage(tokens[0]); - return false; - } else { - auto const action = tokens[1]; - if (action == "status"_sr) { - if (tokens.size() != 2) { - printUsage(tokens[0]); - return false; - } - JsonBuilderObject status = wait(getIdmpKeyStatus(db)); - fmt::print("{}\n", status.getJson()); - return true; - } else if (action == "clear"_sr) { - if (tokens.size() != 3) { - printUsage(tokens[0]); - return false; - } - auto const age = parseAgeValue(tokens[2]); - if (!age.present()) { - printUsage(tokens[0]); - return false; - } - wait(cleanIdempotencyIds(db, age.get())); - fmt::print("Successfully cleared idempotency IDs.\n"); - return true; - } else { - printUsage(tokens[0]); - return false; - } - } -} - -CommandFactory idempotencyIdsCommandFactory( - "idempotencyids", - CommandHelp( - "idempotencyids [status | clear ]", - "View status of idempotency ids, or reclaim space used by idempotency ids older than the given age", - "View status of idempotency ids currently in the cluster, or reclaim space by clearing all the idempotency ids " - "older than min_age_seconds (which will expire all transaction versions older than this age)\n")); - -} // namespace fdb_cli diff --git a/fdbcli/IdempotencyIdsCommand.cpp b/fdbcli/IdempotencyIdsCommand.cpp new file mode 100644 index 00000000000..1984b769cbc --- /dev/null +++ b/fdbcli/IdempotencyIdsCommand.cpp @@ -0,0 +1,81 @@ +/* + * IdempotencyIdsCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" +#include "fdbclient/IdempotencyId.h" +#include "fdbclient/JsonBuilder.h" +#include "fdbclient/json_spirit/json_spirit_reader_template.h" +namespace { + +Optional parseAgeValue(StringRef token) { + try { + return std::stod(token.toString()); + } catch (...) { + return {}; + } +} + +} // namespace + +namespace fdb_cli { + +Future idempotencyIdsCommandActor(Database db, std::vector const& tokens) { + if (tokens.size() < 2 || tokens.size() > 3) { + printUsage(tokens[0]); + co_return false; + } else { + auto const action = tokens[1]; + if (action == "status"_sr) { + if (tokens.size() != 2) { + printUsage(tokens[0]); + co_return false; + } + JsonBuilderObject status = co_await getIdmpKeyStatus(db); + fmt::print("{}\n", status.getJson()); + co_return true; + } else if (action == "clear"_sr) { + if (tokens.size() != 3) { + printUsage(tokens[0]); + co_return false; + } + auto const age = parseAgeValue(tokens[2]); + if (!age.present()) { + printUsage(tokens[0]); + co_return false; + } + co_await cleanIdempotencyIds(db, age.get()); + fmt::print("Successfully cleared idempotency IDs.\n"); + co_return true; + } else { + printUsage(tokens[0]); + co_return false; + } + } +} + +CommandFactory idempotencyIdsCommandFactory( + "idempotencyids", + CommandHelp( + "idempotencyids [status | clear ]", + "View status of idempotency ids, or reclaim space used by idempotency ids older than the given age", + "View status of idempotency ids currently in the cluster, or reclaim space by clearing all the idempotency ids " + "older than min_age_seconds (which will expire all transaction versions older than this age)\n")); + +} // namespace fdb_cli diff --git a/fdbcli/IncludeCommand.actor.cpp b/fdbcli/IncludeCommand.actor.cpp deleted file mode 100644 index 40a42e435e6..00000000000 --- a/fdbcli/IncludeCommand.actor.cpp +++ /dev/null @@ -1,175 +0,0 @@ -/* - * IncludeCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -// Remove the given localities from the exclusion list. -// include localities by clearing the keys. -ACTOR Future includeLocalities(Reference db, - std::vector localities, - bool failed, - bool includeAll) { - state std::string versionKey = deterministicRandom()->randomUniqueID().toString(); - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - - if (includeAll) { - if (failed) { - tr->clear(fdb_cli::failedLocalitySpecialKeyRange); - } else { - tr->clear(fdb_cli::excludedLocalitySpecialKeyRange); - } - } else { - for (const auto& l : localities) { - Key locality = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l) - : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l); - tr->clear(locality); - } - } - wait(safeThreadFutureToFuture(tr->commit())); - return Void(); - } catch (Error& e) { - TraceEvent("IncludeLocalitiesError").errorUnsuppressed(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future includeServers(Reference db, std::vector servers, bool failed) { - state std::string versionKey = deterministicRandom()->randomUniqueID().toString(); - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - for (auto& s : servers) { - // include all, just clear the whole key range - if (!s.isValid()) { - if (failed) { - tr->clear(fdb_cli::failedServersSpecialKeyRange); - } else { - tr->clear(fdb_cli::excludedServersSpecialKeyRange); - } - } else { - Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString()) - : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString()); - tr->clear(addr); - // Eliminate both any ip-level exclusion (1.2.3.4) and any - // port-level exclusions (1.2.3.4:5) - // The range ['IP', 'IP;'] was originally deleted. ';' is - // char(':' + 1). This does not work, as other for all - // x between 0 and 9, 'IPx' will also be in this range. - // - // This is why we now make two clears: first only of the ip - // address, the second will delete all ports. - if (s.isWholeMachine()) - tr->clear(KeyRangeRef(addr.withSuffix(":"_sr), addr.withSuffix(";"_sr))); - } - } - wait(safeThreadFutureToFuture(tr->commit())); - return Void(); - } catch (Error& e) { - TraceEvent("IncludeServersError").errorUnsuppressed(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -// Includes the servers that could be IP addresses or localities back to the cluster. -ACTOR Future include(Reference db, std::vector tokens) { - state std::vector addresses; - state std::vector localities; - state bool failed = false; - state bool all = false; - for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) { - if (*t == "all"_sr) { - all = true; - } else if (*t == "failed"_sr) { - failed = true; - } else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && t->toString().find(':') != std::string::npos) { - // if the token starts with 'locality_' prefix. - localities.push_back(t->toString()); - } else { - auto a = AddressExclusion::parse(*t); - if (!a.isValid()) { - fprintf(stderr, - "ERROR: '%s' is neither a valid network endpoint address nor a locality\n", - t->toString().c_str()); - if (t->toString().find(":tls") != std::string::npos) - printf(" Do not include the `:tls' suffix when naming a process\n"); - return false; - } - addresses.push_back(a); - } - } - if (all) { - std::vector includeAll; - includeAll.push_back(AddressExclusion()); - wait(includeServers(db, includeAll, failed)); - wait(includeLocalities(db, localities, failed, all)); - } else { - if (!addresses.empty()) { - wait(includeServers(db, addresses, failed)); - } - if (!localities.empty()) { - // include the servers that belong to given localities. - wait(includeLocalities(db, localities, failed, all)); - } - } - return true; -}; - -} // namespace - -namespace fdb_cli { - -ACTOR Future includeCommandActor(Reference db, std::vector tokens) { - if (tokens.size() < 2) { - printUsage(tokens[0]); - return false; - } else { - bool result = wait(include(db, tokens)); - return result; - } -} - -CommandFactory includeFactory( - "include", - CommandHelp( - "include all|[] [locality_dcid:] [locality_zoneid:] " - "[locality_machineid:] [locality_processid:] or any locality data", - "permit previously-excluded servers and localities to rejoin the database", - "If `all' is specified, the excluded servers and localities list is cleared.\n\nFor each IP address or IP:port " - "pair in or any LocalityData (like dcid, zoneid, machineid, processid), removes any " - "matching exclusions from the excluded servers and localities list. " - "(A specified IP will match all IP:* exclusion entries)")); -} // namespace fdb_cli diff --git a/fdbcli/IncludeCommand.cpp b/fdbcli/IncludeCommand.cpp new file mode 100644 index 00000000000..2691915bc45 --- /dev/null +++ b/fdbcli/IncludeCommand.cpp @@ -0,0 +1,177 @@ +/* + * IncludeCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +// Remove the given localities from the exclusion list. +// include localities by clearing the keys. +Future includeLocalities(Reference db, + std::vector localities, + bool failed, + bool includeAll) { + std::string versionKey = deterministicRandom()->randomUniqueID().toString(); + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + if (includeAll) { + if (failed) { + tr->clear(fdb_cli::failedLocalitySpecialKeyRange); + } else { + tr->clear(fdb_cli::excludedLocalitySpecialKeyRange); + } + } else { + for (const auto& l : localities) { + Key locality = failed ? fdb_cli::failedLocalitySpecialKeyRange.begin.withSuffix(l) + : fdb_cli::excludedLocalitySpecialKeyRange.begin.withSuffix(l); + tr->clear(locality); + } + } + co_await safeThreadFutureToFuture(tr->commit()); + co_return; + } catch (Error& e) { + err = e; + } + TraceEvent("IncludeLocalitiesError").errorUnsuppressed(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future includeServers(Reference db, std::vector servers, bool failed) { + std::string versionKey = deterministicRandom()->randomUniqueID().toString(); + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + for (auto& s : servers) { + // include all, just clear the whole key range + if (!s.isValid()) { + if (failed) { + tr->clear(fdb_cli::failedServersSpecialKeyRange); + } else { + tr->clear(fdb_cli::excludedServersSpecialKeyRange); + } + } else { + Key addr = failed ? fdb_cli::failedServersSpecialKeyRange.begin.withSuffix(s.toString()) + : fdb_cli::excludedServersSpecialKeyRange.begin.withSuffix(s.toString()); + tr->clear(addr); + // Eliminate both any ip-level exclusion (1.2.3.4) and any + // port-level exclusions (1.2.3.4:5) + // The range ['IP', 'IP;'] was originally deleted. ';' is + // char(':' + 1). This does not work, as other for all + // x between 0 and 9, 'IPx' will also be in this range. + // + // This is why we now make two clears: first only of the ip + // address, the second will delete all ports. + if (s.isWholeMachine()) + tr->clear(KeyRangeRef(addr.withSuffix(":"_sr), addr.withSuffix(";"_sr))); + } + } + co_await safeThreadFutureToFuture(tr->commit()); + co_return; + } catch (Error& e) { + err = e; + } + TraceEvent("IncludeServersError").errorUnsuppressed(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +// Includes the servers that could be IP addresses or localities back to the cluster. +Future include(Reference db, std::vector tokens) { + std::vector addresses; + std::vector localities; + bool failed = false; + bool all = false; + for (auto t = tokens.begin() + 1; t != tokens.end(); ++t) { + if (*t == "all"_sr) { + all = true; + } else if (*t == "failed"_sr) { + failed = true; + } else if (t->startsWith(LocalityData::ExcludeLocalityPrefix) && t->toString().find(':') != std::string::npos) { + // if the token starts with 'locality_' prefix. + localities.push_back(t->toString()); + } else { + auto a = AddressExclusion::parse(*t); + if (!a.isValid()) { + fprintf(stderr, + "ERROR: '%s' is neither a valid network endpoint address nor a locality\n", + t->toString().c_str()); + if (t->toString().find(":tls") != std::string::npos) + printf(" Do not include the `:tls' suffix when naming a process\n"); + co_return false; + } + addresses.push_back(a); + } + } + if (all) { + std::vector includeAll; + includeAll.push_back(AddressExclusion()); + co_await includeServers(db, includeAll, failed); + co_await includeLocalities(db, localities, failed, all); + } else { + if (!addresses.empty()) { + co_await includeServers(db, addresses, failed); + } + if (!localities.empty()) { + // include the servers that belong to given localities. + co_await includeLocalities(db, localities, failed, all); + } + } + co_return true; +}; + +} // namespace + +namespace fdb_cli { + +Future includeCommandActor(Reference db, std::vector tokens) { + if (tokens.size() < 2) { + printUsage(tokens[0]); + co_return false; + } else { + bool result = co_await include(db, tokens); + co_return result; + } +} + +CommandFactory includeFactory( + "include", + CommandHelp( + "include all|[] [locality_dcid:] [locality_zoneid:] " + "[locality_machineid:] [locality_processid:] or any locality data", + "permit previously-excluded servers and localities to rejoin the database", + "If `all' is specified, the excluded servers and localities list is cleared.\n\nFor each IP address or IP:port " + "pair in or any LocalityData (like dcid, zoneid, machineid, processid), removes any " + "matching exclusions from the excluded servers and localities list. " + "(A specified IP will match all IP:* exclusion entries)")); +} // namespace fdb_cli diff --git a/fdbcli/KillCommand.actor.cpp b/fdbcli/KillCommand.actor.cpp deleted file mode 100644 index c520cd15c34..00000000000 --- a/fdbcli/KillCommand.actor.cpp +++ /dev/null @@ -1,135 +0,0 @@ -/* - * KillCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/algorithm/string.hpp" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/CodeProbe.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future killCommandActor(Reference db, - Reference tr, - std::vector tokens, - std::map>* address_interface) { - ASSERT(tokens.size() >= 1); - state bool result = true; - state std::string addressesStr; - if (tokens.size() == 1) { - // initialize worker interfaces - address_interface->clear(); - wait(getWorkerInterfaces(tr, address_interface, true)); - } - if (tokens.size() == 1 || tokencmp(tokens[1], "list")) { - if (address_interface->size() == 0) { - printf("\nNo addresses can be killed.\n"); - } else if (address_interface->size() == 1) { - printf("\nThe following address can be killed:\n"); - } else { - printf("\nThe following %zu addresses can be killed:\n", address_interface->size()); - } - for (auto it : *address_interface) { - printf("%s\n", printable(it.first).c_str()); - } - printf("\n"); - } else if (tokencmp(tokens[1], "all")) { - if (address_interface->size() == 0) { - result = false; - fprintf(stderr, - "ERROR: no processes to kill. You must run the `kill’ command before " - "running `kill all’.\n"); - } else { - std::vector addressesVec; - for (const auto& [address, _] : *address_interface) { - addressesVec.push_back(address.toString()); - } - addressesStr = boost::algorithm::join(addressesVec, ","); - // make sure we only call the interface once to send requests in parallel - int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0))); - if (!killRequestsSent) { - result = false; - fprintf(stderr, - "ERROR: failed to send requests to all processes, please run the `kill’ command again to fetch " - "latest addresses.\n"); - } else { - printf("Attempted to kill %zu processes\n", address_interface->size()); - } - } - } else { - state int i; - for (i = 1; i < tokens.size(); i++) { - if (!address_interface->count(tokens[i])) { - fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str()); - result = false; - break; - } - } - - if (result) { - std::vector addressesVec; - for (i = 1; i < tokens.size(); i++) { - addressesVec.push_back(tokens[i].toString()); - } - addressesStr = boost::algorithm::join(addressesVec, ","); - int64_t killRequestsSent = wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0))); - if (!killRequestsSent) { - result = false; - fprintf(stderr, - "ERROR: failed to send requests to kill processes `%s', please run the `kill’ command again to " - "fetch latest addresses.\n", - addressesStr.c_str()); - } else { - // delay in case the network queue is not flush before the client exits - wait(delay(3.0)); - printf("Attempted to kill %zu processes\n", tokens.size() - 1); - } - } - } - return result; -} - -void killGenerator(const char* text, - const char* line, - std::vector& lc, - std::vector const& tokens) { - const char* opts[] = { "all", "list", nullptr }; - arrayGenerator(text, line, opts, lc); -} - -CommandFactory killFactory( - "kill", - CommandHelp( - "kill all|list|", - "attempts to kill one or more processes in the cluster", - "If no addresses are specified, populates the list of processes which can be killed. Processes cannot be " - "killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known " - "processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is " - "unresponsive.\n\nFor each IP:port pair in
, attempt to kill the specified process."), - &killGenerator); -} // namespace fdb_cli diff --git a/fdbcli/KillCommand.cpp b/fdbcli/KillCommand.cpp new file mode 100644 index 00000000000..baa12ec976c --- /dev/null +++ b/fdbcli/KillCommand.cpp @@ -0,0 +1,134 @@ +/* + * KillCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/algorithm/string.hpp" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/CodeProbe.h" + +namespace fdb_cli { + +Future killCommandActor(Reference db, + Reference tr, + std::vector tokens, + std::map>* address_interface) { + ASSERT(!tokens.empty()); + bool result = true; + std::string addressesStr; + if (tokens.size() == 1) { + // initialize worker interfaces + address_interface->clear(); + co_await getWorkerInterfaces(tr, address_interface, true); + } + if (tokens.size() == 1 || tokencmp(tokens[1], "list")) { + if (address_interface->empty()) { + printf("\nNo addresses can be killed.\n"); + } else if (address_interface->size() == 1) { + printf("\nThe following address can be killed:\n"); + } else { + printf("\nThe following %zu addresses can be killed:\n", address_interface->size()); + } + for (const auto& it : *address_interface) { + printf("%s\n", printable(it.first).c_str()); + } + printf("\n"); + } else if (tokencmp(tokens[1], "all")) { + if (address_interface->empty()) { + result = false; + fprintf(stderr, + "ERROR: no processes to kill. You must run the `kill’ command before " + "running `kill all’.\n"); + } else { + std::vector addressesVec; + for (const auto& [address, _] : *address_interface) { + addressesVec.push_back(address.toString()); + } + addressesStr = boost::algorithm::join(addressesVec, ","); + // make sure we only call the interface once to send requests in parallel + int64_t killRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)); + if (!killRequestsSent) { + result = false; + fprintf(stderr, + "ERROR: failed to send requests to all processes, please run the `kill’ command again to fetch " + "latest addresses.\n"); + } else { + printf("Attempted to kill %zu processes\n", address_interface->size()); + } + } + } else { + int i{ 0 }; + for (i = 1; i < tokens.size(); i++) { + if (!address_interface->count(tokens[i])) { + fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str()); + result = false; + break; + } + } + + if (result) { + std::vector addressesVec; + for (i = 1; i < tokens.size(); i++) { + addressesVec.push_back(tokens[i].toString()); + } + addressesStr = boost::algorithm::join(addressesVec, ","); + int64_t killRequestsSent = co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, 0)); + if (!killRequestsSent) { + result = false; + fprintf(stderr, + "ERROR: failed to send requests to kill processes `%s', please run the `kill’ command again to " + "fetch latest addresses.\n", + addressesStr.c_str()); + } else { + // delay in case the network queue is not flush before the client exits + co_await delay(3.0); + printf("Attempted to kill %zu processes\n", tokens.size() - 1); + } + } + } + co_return result; +} + +void killGenerator(const char* text, + const char* line, + std::vector& lc, + std::vector const& tokens) { + const char* opts[] = { "all", "list", nullptr }; + arrayGenerator(text, line, opts, lc); +} + +CommandFactory killFactory( + "kill", + CommandHelp( + "kill all|list|", + "attempts to kill one or more processes in the cluster", + "If no addresses are specified, populates the list of processes which can be killed. Processes cannot be " + "killed before this list has been populated.\n\nIf `all' is specified, attempts to kill all known " + "processes.\n\nIf `list' is specified, displays all known processes. This is only useful when the database is " + "unresponsive.\n\nFor each IP:port pair in
, attempt to kill the specified process."), + &killGenerator); +} // namespace fdb_cli diff --git a/fdbcli/LocationMetadataCommand.actor.cpp b/fdbcli/LocationMetadataCommand.actor.cpp deleted file mode 100644 index c1923242b82..00000000000 --- a/fdbcli/LocationMetadataCommand.actor.cpp +++ /dev/null @@ -1,296 +0,0 @@ -/* - * LocationMetadataCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/Audit.h" -#include "fdbclient/AuditUtils.actor.h" -#include "fdbclient/IClientApi.h" -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" - -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { -ACTOR Future describeServers(Reference tr, std::vector ids) { - std::vector>> serverListEntries; - for (const UID& id : ids) { - serverListEntries.push_back(tr->get(serverListKeyFor(id))); - } - std::vector> serverListValues = wait(getAll(serverListEntries)); - std::string res; - for (auto& v : serverListValues) { - StorageServerInterface ssi = decodeServerListValue(v.get()); - if (!res.empty()) { - res += ", "; - } - res += fmt::format("ServerID: {}, Addr: {}", ssi.uniqueID.toString(), ssi.stableAddress().toString()); - } - - return res; -} - -ACTOR Future printKeyServersEntry(Reference tr, - RangeResultRef UIDtoTagMap, - Value entry, - KeyRangeRef range) { - state std::vector src; - state std::vector dest; - state UID srcId; - state UID destId; - decodeKeyServersValue(UIDtoTagMap, entry, src, dest, srcId, destId); - state std::string srcDesc = wait(describeServers(tr, src)); - std::string destDesc = wait(describeServers(tr, dest)); - printf("Range: %s, ShardID: %s, Src Servers: %s, Dest Servers: %s\n", - Traceable::toString(range).c_str(), - srcId.toString().c_str(), - srcDesc.c_str(), - destDesc.c_str()); - return Void(); -} - -ACTOR Future printRandomShards(Database cx, int n, bool physicalShard) { - state Key begin = allKeys.begin; - state int numShards = 0; - - while (begin < allKeys.end && numShards < n) { - // RYW to optimize re-reading the same key ranges - state Reference tr = makeReference(cx); - - loop { - try { - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - - state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY)); - ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY); - - state KeyRangeRef currentKeys(begin, allKeys.end); - state RangeResult shards = - wait(krmGetRanges(tr, keyServersPrefix, currentKeys, n, CLIENT_KNOBS->TOO_MANY)); - - state int i = 0; - for (; i < shards.size() - 1 && numShards < n; ++i) { - KeyRangeRef currentRange(shards[i].key, shards[i + 1].key); - std::vector src; - std::vector dest; - UID srcId; - UID destId; - decodeKeyServersValue(UIDtoTagMap, shards[i].value, src, dest, srcId, destId); - if (physicalShard == (srcId != anonymousShardId)) { - wait(printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange)); - ++numShards; - } - } - - begin = shards.back().key; - break; - } catch (Error& e) { - wait(tr->onError(e)); - } - } - } - - printf("Found %d %s shards\n", numShards, physicalShard ? "Physical" : "Non-physical"); - - return Void(); -} - -ACTOR Future printPhysicalShardCount(Database cx) { - state Key begin = allKeys.begin; - state int numShards = 0; - state int numPhysicalShards = 0; - - while (begin < allKeys.end) { - // RYW to optimize re-reading the same key ranges - state Reference tr = makeReference(cx); - - loop { - try { - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - - state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY)); - ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY); - - KeyRangeRef currentKeys(begin, allKeys.end); - RangeResult shards = wait( - krmGetRanges(tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY)); - - for (int i = 0; i < shards.size() - 1; ++i) { - std::vector src; - std::vector dest; - UID srcId; - UID destId; - decodeKeyServersValue(UIDtoTagMap, shards[i].value, src, dest, srcId, destId); - if (srcId != anonymousShardId) { - ++numPhysicalShards; - } - } - - begin = shards.back().key; - numShards += shards.size() - 1; - break; - } catch (Error& e) { - wait(tr->onError(e)); - } - } - } - - printf("Total number of shards: %d, number of physical shards: %d\n", numShards, numPhysicalShards); - - return Void(); -} - -ACTOR Future printServerShards(Database cx, UID serverId) { - state Key begin = allKeys.begin; - - while (begin < allKeys.end) { - // RYW to optimize re-reading the same key ranges - state Reference tr = makeReference(cx); - - loop { - try { - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - - RangeResult serverShards = - wait(krmGetRanges(tr, serverKeysPrefixFor(serverId), KeyRangeRef(begin, allKeys.end))); - - for (int i = 0; i < serverShards.size() - 1; ++i) { - KeyRangeRef currentRange(serverShards[i].key, serverShards[i + 1].key); - UID shardId; - bool assigned, emptyRange; - DataMoveType dataMoveType = DataMoveType::LOGICAL; - DataMovementReason dataMoveReason = DataMovementReason::INVALID; - decodeServerKeysValue( - serverShards[i].value, assigned, emptyRange, dataMoveType, shardId, dataMoveReason); - printf("Range: %s, ShardID: %s, Assigned: %s\n", - Traceable::toString(currentRange).c_str(), - shardId.toString().c_str(), - assigned ? "true" : "false"); - } - - begin = serverShards.back().key; - break; - } catch (Error& e) { - wait(tr->onError(e)); - } - } - } - - return Void(); -} - -ACTOR Future resolveRange(Database cx, KeyRange range) { - state Key begin = range.begin; - - while (begin < range.end) { - // RYW to optimize re-reading the same key ranges - state Reference tr = makeReference(cx); - - loop { - try { - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - - state RangeResult UIDtoTagMap = wait(tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY)); - ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY); - - state KeyRangeRef currentKeys(begin, range.end); - state RangeResult shards = wait( - krmGetRanges(tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY)); - - state int i = 0; - for (; i < shards.size() - 1; ++i) { - state KeyRangeRef currentRange(shards[i].key, shards[i + 1].key); - wait(printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange)); - } - - begin = shards.back().key; - break; - } catch (Error& e) { - wait(tr->onError(e)); - } - } - } - - return Void(); -} - -} // namespace - -namespace fdb_cli { - -ACTOR Future locationMetadataCommandActor(Database cx, std::vector tokens) { - if (tokens.size() < 2 || tokens.size() > 4) { - printUsage(tokens[0]); - return false; - } - - if (tokens.size() == 2) { - if (!tokencmp(tokens[1], "physicalshards")) { - printUsage(tokens[0]); - return false; - } - wait(printPhysicalShardCount(cx)); - } else if (tokencmp(tokens[1], "resolve")) { - if (tokens.size() == 3) { - wait(resolveRange(cx, singleKeyRange(tokens[2]))); - } else { - wait(resolveRange(cx, KeyRangeRef(tokens[2], tokens[3]))); - } - } else if (tokencmp(tokens[1], "servershards")) { - if (tokens.size() != 3) { - printUsage(tokens[0]); - return false; - } - wait(printServerShards(cx, UID::fromString(tokens[2].toString()))); - } else if (tokencmp(tokens[1], "listshards")) { - if (tokens.size() == 4 && !tokencmp(tokens[3], "physical")) { - printUsage(tokens[0]); - return false; - } - const bool physical = tokens.size() == 4; - wait(printRandomShards(cx, std::stoi(tokens[2].toString()), physical)); - } else { - printUsage(tokens[0]); - return false; - } - - return true; -} - -CommandFactory locationMetadataFactory( - "location_metadata", - CommandHelp( - "location_metadata [physicalshards|resolve|servershards|listshards] [||] [|physical]", - "Check location metadata", - "To check number of physical shards: `location_metadata physicalshards'\n" - "To check the location of a key: `location_metadata resolve '\n" - "To check the location of a keyrange: `location_metadata resolve '\n" - "To check shard assignments of a storage server: `location_metadata servershards '\n" - "To list random physical shards: `location_metadata listshards physical'\n" - "To list random non-physical shards: `location_metadata listshards '\n")); -} // namespace fdb_cli diff --git a/fdbcli/LocationMetadataCommand.cpp b/fdbcli/LocationMetadataCommand.cpp new file mode 100644 index 00000000000..94dfae46a1f --- /dev/null +++ b/fdbcli/LocationMetadataCommand.cpp @@ -0,0 +1,293 @@ +/* + * LocationMetadataCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" +#include "fdbclient/Audit.h" +#include "fdbclient/AuditUtils.h" +#include "fdbclient/IClientApi.h" +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { +Future describeServers(Reference tr, std::vector ids) { + std::vector>> serverListEntries; + for (const UID& id : ids) { + serverListEntries.push_back(tr->get(serverListKeyFor(id))); + } + std::vector> serverListValues = co_await getAll(serverListEntries); + std::string res; + for (auto& v : serverListValues) { + StorageServerInterface ssi = decodeServerListValue(v.get()); + if (!res.empty()) { + res += ", "; + } + res += fmt::format("ServerID: {}, Addr: {}", ssi.uniqueID.toString(), ssi.stableAddress().toString()); + } + + co_return res; +} + +Future printKeyServersEntry(Reference tr, + RangeResultRef UIDtoTagMap, + Value entry, + KeyRangeRef range) { + std::vector src; + std::vector dest; + UID srcId; + UID destId; + decodeKeyServersValue(UIDtoTagMap, entry, src, dest, srcId, destId); + std::string srcDesc = co_await describeServers(tr, src); + std::string destDesc = co_await describeServers(tr, dest); + printf("Range: %s, ShardID: %s, Src Servers: %s, Dest Servers: %s\n", + Traceable::toString(range).c_str(), + srcId.toString().c_str(), + srcDesc.c_str(), + destDesc.c_str()); +} + +Future printRandomShards(Database cx, int n, bool physicalShard) { + Key begin = allKeys.begin; + int numShards = 0; + + while (begin < allKeys.end && numShards < n) { + // RYW to optimize re-reading the same key ranges + auto tr = makeReference(cx); + + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY); + ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY); + + KeyRangeRef currentKeys(begin, allKeys.end); + RangeResult shards = + co_await krmGetRanges(tr, keyServersPrefix, currentKeys, n, CLIENT_KNOBS->TOO_MANY); + + int i = 0; + for (; i < shards.size() - 1 && numShards < n; ++i) { + KeyRangeRef currentRange(shards[i].key, shards[i + 1].key); + std::vector src; + std::vector dest; + UID srcId; + UID destId; + decodeKeyServersValue(UIDtoTagMap, shards[i].value, src, dest, srcId, destId); + if (physicalShard == (srcId != anonymousShardId)) { + co_await printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange); + ++numShards; + } + } + + begin = shards.back().key; + break; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } + } + + printf("Found %d %s shards\n", numShards, physicalShard ? "Physical" : "Non-physical"); +} + +Future printPhysicalShardCount(Database cx) { + Key begin = allKeys.begin; + int numShards = 0; + int numPhysicalShards = 0; + + while (begin < allKeys.end) { + // RYW to optimize re-reading the same key ranges + auto tr = makeReference(cx); + + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY); + ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY); + + KeyRangeRef currentKeys(begin, allKeys.end); + RangeResult shards = co_await krmGetRanges( + tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY); + + for (int i = 0; i < shards.size() - 1; ++i) { + std::vector src; + std::vector dest; + UID srcId; + UID destId; + decodeKeyServersValue(UIDtoTagMap, shards[i].value, src, dest, srcId, destId); + if (srcId != anonymousShardId) { + ++numPhysicalShards; + } + } + + begin = shards.back().key; + numShards += shards.size() - 1; + break; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } + } + + printf("Total number of shards: %d, number of physical shards: %d\n", numShards, numPhysicalShards); +} + +Future printServerShards(Database cx, UID serverId) { + Key begin = allKeys.begin; + + while (begin < allKeys.end) { + // RYW to optimize re-reading the same key ranges + auto tr = makeReference(cx); + + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + RangeResult serverShards = + co_await krmGetRanges(tr, serverKeysPrefixFor(serverId), KeyRangeRef(begin, allKeys.end)); + + for (int i = 0; i < serverShards.size() - 1; ++i) { + KeyRangeRef currentRange(serverShards[i].key, serverShards[i + 1].key); + UID shardId; + bool assigned, emptyRange; + DataMoveType dataMoveType = DataMoveType::LOGICAL; + DataMovementReason dataMoveReason = DataMovementReason::INVALID; + decodeServerKeysValue( + serverShards[i].value, assigned, emptyRange, dataMoveType, shardId, dataMoveReason); + printf("Range: %s, ShardID: %s, Assigned: %s\n", + Traceable::toString(currentRange).c_str(), + shardId.toString().c_str(), + assigned ? "true" : "false"); + } + + begin = serverShards.back().key; + break; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } + } +} + +Future resolveRange(Database cx, KeyRange range) { + Key begin = range.begin; + + while (begin < range.end) { + // RYW to optimize re-reading the same key ranges + auto tr = makeReference(cx); + + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + + RangeResult UIDtoTagMap = co_await tr->getRange(serverTagKeys, CLIENT_KNOBS->TOO_MANY); + ASSERT(!UIDtoTagMap.more && UIDtoTagMap.size() < CLIENT_KNOBS->TOO_MANY); + + KeyRangeRef currentKeys(begin, range.end); + RangeResult shards = co_await krmGetRanges( + tr, keyServersPrefix, currentKeys, CLIENT_KNOBS->TOO_MANY, CLIENT_KNOBS->TOO_MANY); + + int i = 0; + for (; i < shards.size() - 1; ++i) { + KeyRangeRef currentRange(shards[i].key, shards[i + 1].key); + co_await printKeyServersEntry(tr, UIDtoTagMap, shards[i].value, currentRange); + } + + begin = shards.back().key; + break; + } catch (Error& e) { + err = e; + } + co_await tr->onError(err); + } + } +} + +} // namespace + +namespace fdb_cli { + +Future locationMetadataCommandActor(Database cx, std::vector tokens) { + if (tokens.size() < 2 || tokens.size() > 4) { + printUsage(tokens[0]); + co_return false; + } + + if (tokens.size() == 2) { + if (!tokencmp(tokens[1], "physicalshards")) { + printUsage(tokens[0]); + co_return false; + } + co_await printPhysicalShardCount(cx); + } else if (tokencmp(tokens[1], "resolve")) { + if (tokens.size() == 3) { + co_await resolveRange(cx, singleKeyRange(tokens[2])); + } else { + co_await resolveRange(cx, KeyRangeRef(tokens[2], tokens[3])); + } + } else if (tokencmp(tokens[1], "servershards")) { + if (tokens.size() != 3) { + printUsage(tokens[0]); + co_return false; + } + co_await printServerShards(cx, UID::fromString(tokens[2].toString())); + } else if (tokencmp(tokens[1], "listshards")) { + if (tokens.size() == 4 && !tokencmp(tokens[3], "physical")) { + printUsage(tokens[0]); + co_return false; + } + const bool physical = tokens.size() == 4; + co_await printRandomShards(cx, std::stoi(tokens[2].toString()), physical); + } else { + printUsage(tokens[0]); + co_return false; + } + + co_return true; +} + +CommandFactory locationMetadataFactory( + "location_metadata", + CommandHelp( + "location_metadata [physicalshards|resolve|servershards|listshards] [||] [|physical]", + "Check location metadata", + "To check number of physical shards: `location_metadata physicalshards'\n" + "To check the location of a key: `location_metadata resolve '\n" + "To check the location of a keyrange: `location_metadata resolve '\n" + "To check shard assignments of a storage server: `location_metadata servershards '\n" + "To list random physical shards: `location_metadata listshards physical'\n" + "To list random non-physical shards: `location_metadata listshards '\n")); +} // namespace fdb_cli diff --git a/fdbcli/LockCommand.actor.cpp b/fdbcli/LockCommand.actor.cpp deleted file mode 100644 index 1ed959ae183..00000000000 --- a/fdbcli/LockCommand.actor.cpp +++ /dev/null @@ -1,120 +0,0 @@ -/* - * LockCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" -#include "fdbclient/Schemas.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -ACTOR Future lockDatabase(Reference db, UID id) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - tr->set(fdb_cli::lockSpecialKey, id.toString()); - wait(safeThreadFutureToFuture(tr->commit())); - printf("Database locked.\n"); - return true; - } catch (Error& e) { - state Error err(e); - if (e.code() == error_code_database_locked) - throw e; - else if (e.code() == error_code_special_keys_api_failure) { - std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); - fprintf(stderr, "%s\n", errorMsgStr.c_str()); - return false; - } - wait(safeThreadFutureToFuture(tr->onError(err))); - } - } -} - -} // namespace - -namespace fdb_cli { - -const KeyRef lockSpecialKey = "\xff\xff/management/db_locked"_sr; - -ACTOR Future lockCommandActor(Reference db, std::vector tokens) { - if (tokens.size() != 1) { - printUsage(tokens[0]); - return false; - } else { - state UID lockUID = deterministicRandom()->randomUniqueID(); - printf("Locking database with lockUID: %s\n", lockUID.toString().c_str()); - bool result = wait((lockDatabase(db, lockUID))); - return result; - } -} - -ACTOR Future unlockDatabaseActor(Reference db, UID uid) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - state ThreadFuture> valF = tr->get(fdb_cli::lockSpecialKey); - Optional val = wait(safeThreadFutureToFuture(valF)); - - if (!val.present()) - return true; - - if (val.present() && UID::fromString(val.get().toString()) != uid) { - printf("Unable to unlock database. Make sure to unlock with the correct lock UID.\n"); - return false; - } - - tr->clear(fdb_cli::lockSpecialKey); - wait(safeThreadFutureToFuture(tr->commit())); - printf("Database unlocked.\n"); - return true; - } catch (Error& e) { - state Error err(e); - if (e.code() == error_code_special_keys_api_failure) { - std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); - fprintf(stderr, "%s\n", errorMsgStr.c_str()); - return false; - } - wait(safeThreadFutureToFuture(tr->onError(err))); - } - } -} - -CommandFactory lockFactory( - "lock", - CommandHelp("lock", - "lock the database with a randomly generated lockUID", - "Randomly generates a lockUID, prints this lockUID, and then uses the lockUID to lock the database.")); - -CommandFactory unlockFactory( - "unlock", - CommandHelp("unlock ", - "unlock the database with the provided lockUID", - "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the " - "user will be asked to enter a passphrase to confirm their intent.")); -} // namespace fdb_cli diff --git a/fdbcli/LockCommand.cpp b/fdbcli/LockCommand.cpp new file mode 100644 index 00000000000..5570e7be763 --- /dev/null +++ b/fdbcli/LockCommand.cpp @@ -0,0 +1,120 @@ +/* + * LockCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/Schemas.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +namespace { + +Future lockDatabase(Reference db, UID id) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + tr->set(fdb_cli::lockSpecialKey, id.toString()); + co_await safeThreadFutureToFuture(tr->commit()); + printf("Database locked.\n"); + co_return true; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_database_locked) + throw err; + if (err.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr); + fprintf(stderr, "%s\n", errorMsgStr.c_str()); + co_return false; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +} // namespace + +namespace fdb_cli { + +const KeyRef lockSpecialKey = "\xff\xff/management/db_locked"_sr; + +Future lockCommandActor(Reference db, std::vector const& tokens) { + if (tokens.size() != 1) { + printUsage(tokens[0]); + co_return false; + } else { + UID lockUID = deterministicRandom()->randomUniqueID(); + printf("Locking database with lockUID: %s\n", lockUID.toString().c_str()); + bool result = co_await lockDatabase(db, lockUID); + co_return result; + } +} + +Future unlockDatabaseActor(Reference db, UID uid) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + ThreadFuture> valF = tr->get(fdb_cli::lockSpecialKey); + Optional val = co_await safeThreadFutureToFuture(valF); + + if (!val.present()) + co_return true; + + if (val.present() && UID::fromString(val.get().toString()) != uid) { + printf("Unable to unlock database. Make sure to unlock with the correct lock UID.\n"); + co_return false; + } + + tr->clear(fdb_cli::lockSpecialKey); + co_await safeThreadFutureToFuture(tr->commit()); + printf("Database unlocked.\n"); + co_return true; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr); + fprintf(stderr, "%s\n", errorMsgStr.c_str()); + co_return false; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +CommandFactory lockFactory( + "lock", + CommandHelp("lock", + "lock the database with a randomly generated lockUID", + "Randomly generates a lockUID, prints this lockUID, and then uses the lockUID to lock the database.")); + +CommandFactory unlockFactory( + "unlock", + CommandHelp("unlock ", + "unlock the database with the provided lockUID", + "Unlocks the database with the provided lockUID. This is a potentially dangerous operation, so the " + "user will be asked to enter a passphrase to confirm their intent.")); +} // namespace fdb_cli diff --git a/fdbcli/MaintenanceCommand.actor.cpp b/fdbcli/MaintenanceCommand.actor.cpp deleted file mode 100644 index ca4da82a803..00000000000 --- a/fdbcli/MaintenanceCommand.actor.cpp +++ /dev/null @@ -1,172 +0,0 @@ -/* - * MaintenanceCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "boost/lexical_cast.hpp" -#include "fmt/format.h" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBTypes.h" -#include "fdbclient/IClientApi.h" - -#include "fdbclient/Knobs.h" -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -// print zoneId under maintenance, only one is possible at the same time -ACTOR Future printHealthyZone(Reference db) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - // We need to keep the future as the returned standalone is not guaranteed to manage its memory when - // using an external client, but the ThreadFuture holds a reference to the memory - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - RangeResult res = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(res.size() <= 1); - if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { - printf("Data distribution has been disabled for all storage server failures in this cluster and thus " - "maintenance mode is not active.\n"); - } else if (!res.size() || boost::lexical_cast(res[0].value.toString()) <= 0) { - printf("No ongoing maintenance.\n"); - } else { - std::string zoneId = res[0].key.removePrefix(fdb_cli::maintenanceSpecialKeyRange.begin).toString(); - int64_t seconds = static_cast(boost::lexical_cast(res[0].value.toString())); - fmt::print("Maintenance for zone {0} will continue for {1} seconds.\n", zoneId, seconds); - } - return Void(); - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -} // namespace - -namespace fdb_cli { - -const KeyRangeRef maintenanceSpecialKeyRange = - KeyRangeRef("\xff\xff/management/maintenance/"_sr, "\xff\xff/management/maintenance0"_sr); -// The special key, if present, means data distribution is disabled for storage failures; -const KeyRef ignoreSSFailureSpecialKey = "\xff\xff/management/maintenance/IgnoreSSFailures"_sr; - -// add a zone to maintenance and specify the maintenance duration -ACTOR Future setHealthyZone(Reference db, StringRef zoneId, double seconds, bool printWarning) { - state Reference tr = db->createTransaction(); - TraceEvent("SetHealthyZone").detail("Zone", zoneId).detail("DurationSeconds", seconds); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - // hold the returned standalone object's memory - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - RangeResult res = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(res.size() <= 1); - if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { - if (printWarning) { - fprintf(stderr, - "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage " - "server failures. Use 'datadistribution on' to reenable data distribution.\n"); - } - return false; - } - tr->set(fdb_cli::maintenanceSpecialKeyRange.begin.withSuffix(zoneId), - boost::lexical_cast(seconds)); - wait(safeThreadFutureToFuture(tr->commit())); - return true; - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -// clear ongoing maintenance, let clearSSFailureZoneString = true to enable data distribution for storage -ACTOR Future clearHealthyZone(Reference db, bool printWarning, bool clearSSFailureZoneString) { - state Reference tr = db->createTransaction(); - TraceEvent("ClearHealthyZone").detail("ClearSSFailureZoneString", clearSSFailureZoneString); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - // hold the returned standalone object's memory - state ThreadFuture resultFuture = - tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - RangeResult res = wait(safeThreadFutureToFuture(resultFuture)); - ASSERT(res.size() <= 1); - if (!clearSSFailureZoneString && res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { - if (printWarning) { - fprintf(stderr, - "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage " - "server failures. Use 'datadistribution on' to reenable data distribution.\n"); - } - return false; - } - - tr->clear(fdb_cli::maintenanceSpecialKeyRange); - wait(safeThreadFutureToFuture(tr->commit())); - return true; - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future maintenanceCommandActor(Reference db, std::vector tokens) { - state bool result = true; - if (tokens.size() == 1) { - wait(printHealthyZone(db)); - } else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) { - bool clearResult = wait(clearHealthyZone(db, true)); - result = clearResult; - } else if (tokens.size() == 4 && tokencmp(tokens[1], "on")) { - double seconds; - int n = 0; - auto secondsStr = tokens[3].toString(); - if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) { - printUsage(tokens[0]); - result = false; - } else { - bool setResult = wait(setHealthyZone(db, tokens[2], seconds, true)); - result = setResult; - } - } else { - printUsage(tokens[0]); - result = false; - } - return result; -} - -CommandFactory maintenanceFactory( - "maintenance", - CommandHelp( - "maintenance [on|off] [ZONEID] [SECONDS]", - "mark a zone for maintenance", - "Calling this command with `on' prevents data distribution from moving data away from the processes with the " - "specified ZONEID. Data distribution will automatically be turned back on for ZONEID after the specified " - "SECONDS have elapsed, or after a storage server with a different ZONEID fails. Only one ZONEID can be marked " - "for maintenance. Calling this command with no arguments will display any ongoing maintenance. Calling this " - "command with `off' will disable maintenance.\n")); -} // namespace fdb_cli diff --git a/fdbcli/MaintenanceCommand.cpp b/fdbcli/MaintenanceCommand.cpp new file mode 100644 index 00000000000..bcd7cf99e93 --- /dev/null +++ b/fdbcli/MaintenanceCommand.cpp @@ -0,0 +1,177 @@ +/* + * MaintenanceCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "boost/lexical_cast.hpp" +#include "fmt/format.h" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBTypes.h" +#include "fdbclient/IClientApi.h" + +#include "fdbclient/Knobs.h" +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +// print zoneId under maintenance, only one is possible at the same time +Future printHealthyZone(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + // We need to keep the future as the returned standalone is not guaranteed to manage its memory when + // using an external client, but the ThreadFuture holds a reference to the memory + ThreadFuture resultFuture = + tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult res = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(res.size() <= 1); + if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { + printf("Data distribution has been disabled for all storage server failures in this cluster and thus " + "maintenance mode is not active.\n"); + } else if (res.empty() || boost::lexical_cast(res[0].value.toString()) <= 0) { + printf("No ongoing maintenance.\n"); + } else { + std::string zoneId = res[0].key.removePrefix(fdb_cli::maintenanceSpecialKeyRange.begin).toString(); + int64_t seconds = static_cast(boost::lexical_cast(res[0].value.toString())); + fmt::print("Maintenance for zone {0} will continue for {1} seconds.\n", zoneId, seconds); + } + co_return; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +} // namespace + +namespace fdb_cli { + +const KeyRangeRef maintenanceSpecialKeyRange = + KeyRangeRef("\xff\xff/management/maintenance/"_sr, "\xff\xff/management/maintenance0"_sr); +// The special key, if present, means data distribution is disabled for storage failures; +const KeyRef ignoreSSFailureSpecialKey = "\xff\xff/management/maintenance/IgnoreSSFailures"_sr; + +// add a zone to maintenance and specify the maintenance duration +Future setHealthyZone(Reference db, StringRef zoneId, double seconds, bool printWarning) { + Reference tr = db->createTransaction(); + TraceEvent("SetHealthyZone").detail("Zone", zoneId).detail("DurationSeconds", seconds); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + // hold the returned standalone object's memory + ThreadFuture resultFuture = + tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult res = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(res.size() <= 1); + if (res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { + if (printWarning) { + fprintf(stderr, + "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage " + "server failures. Use 'datadistribution on' to reenable data distribution.\n"); + } + co_return false; + } + tr->set(fdb_cli::maintenanceSpecialKeyRange.begin.withSuffix(zoneId), + boost::lexical_cast(seconds)); + co_await safeThreadFutureToFuture(tr->commit()); + co_return true; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +// clear ongoing maintenance, let clearSSFailureZoneString = true to enable data distribution for storage +Future clearHealthyZone(Reference db, bool printWarning, bool clearSSFailureZoneString) { + Reference tr = db->createTransaction(); + TraceEvent("ClearHealthyZone").detail("ClearSSFailureZoneString", clearSSFailureZoneString); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + // hold the returned standalone object's memory + ThreadFuture resultFuture = + tr->getRange(fdb_cli::maintenanceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + RangeResult res = co_await safeThreadFutureToFuture(resultFuture); + ASSERT(res.size() <= 1); + if (!clearSSFailureZoneString && res.size() == 1 && res[0].key == fdb_cli::ignoreSSFailureSpecialKey) { + if (printWarning) { + fprintf(stderr, + "ERROR: Maintenance mode cannot be used while data distribution is disabled for storage " + "server failures. Use 'datadistribution on' to reenable data distribution.\n"); + } + co_return false; + } + + tr->clear(fdb_cli::maintenanceSpecialKeyRange); + co_await safeThreadFutureToFuture(tr->commit()); + co_return true; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future maintenanceCommandActor(Reference db, std::vector tokens) { + bool result = true; + if (tokens.size() == 1) { + co_await printHealthyZone(db); + } else if (tokens.size() == 2 && tokencmp(tokens[1], "off")) { + bool clearResult = co_await clearHealthyZone(db, true); + result = clearResult; + } else if (tokens.size() == 4 && tokencmp(tokens[1], "on")) { + double seconds; + int n = 0; + auto secondsStr = tokens[3].toString(); + if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) { + printUsage(tokens[0]); + result = false; + } else { + bool setResult = co_await setHealthyZone(db, tokens[2], seconds, true); + result = setResult; + } + } else { + printUsage(tokens[0]); + result = false; + } + co_return result; +} + +CommandFactory maintenanceFactory( + "maintenance", + CommandHelp( + "maintenance [on|off] [ZONEID] [SECONDS]", + "mark a zone for maintenance", + "Calling this command with `on' prevents data distribution from moving data away from the processes with the " + "specified ZONEID. Data distribution will automatically be turned back on for ZONEID after the specified " + "SECONDS have elapsed, or after a storage server with a different ZONEID fails. Only one ZONEID can be marked " + "for maintenance. Calling this command with no arguments will display any ongoing maintenance. Calling this " + "command with `off' will disable maintenance.\n")); +} // namespace fdb_cli diff --git a/fdbcli/ProfileCommand.actor.cpp b/fdbcli/ProfileCommand.actor.cpp deleted file mode 100644 index a997e7f1a39..00000000000 --- a/fdbcli/ProfileCommand.actor.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/* - * ProfileCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/lexical_cast.hpp" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/GlobalConfig.actor.h" -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" -#include "fdbclient/Tuple.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future profileCommandActor(Database db, - Reference tr, - std::vector tokens, - bool intrans) { - state bool result = true; - if (tokens.size() == 1) { - printUsage(tokens[0]); - result = false; - } else if (tokencmp(tokens[1], "client")) { - if (tokens.size() == 2) { - fprintf(stderr, "ERROR: Usage: profile client \n"); - return false; - } - wait(db->globalConfig->onInitialized()); - if (tokencmp(tokens[2], "get")) { - if (tokens.size() != 3) { - fprintf(stderr, "ERROR: Additional arguments to `get` are not supported.\n"); - return false; - } - std::string sampleRateStr = "default"; - std::string sizeLimitStr = "default"; - const double sampleRateDbl = - db->globalConfig->get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); - if (!std::isinf(sampleRateDbl)) { - sampleRateStr = std::to_string(sampleRateDbl); - } - const int64_t sizeLimit = db->globalConfig->get(fdbClientInfoTxnSizeLimit, -1); - if (sizeLimit != -1) { - sizeLimitStr = boost::lexical_cast(sizeLimit); - } - printf("Client profiling rate is set to %s and size limit is set to %s.\n", - sampleRateStr.c_str(), - sizeLimitStr.c_str()); - } else if (tokencmp(tokens[2], "set")) { - if (tokens.size() != 5) { - fprintf(stderr, "ERROR: Usage: profile client set \n"); - return false; - } - double sampleRate; - if (tokencmp(tokens[3], "default")) { - sampleRate = std::numeric_limits::infinity(); - } else { - char* end; - sampleRate = std::strtod((const char*)tokens[3].begin(), &end); - if (!std::isspace(*end)) { - fprintf(stderr, "ERROR: %s failed to parse.\n", printable(tokens[3]).c_str()); - return false; - } - } - int64_t sizeLimit; - if (tokencmp(tokens[4], "default")) { - sizeLimit = -1; - } else { - Optional parsed = parse_with_suffix(tokens[4].toString()); - if (parsed.present()) { - sizeLimit = parsed.get(); - } else { - fprintf(stderr, "ERROR: `%s` failed to parse.\n", printable(tokens[4]).c_str()); - return false; - } - } - - Tuple rate = Tuple::makeTuple(sampleRate); - Tuple size = Tuple::makeTuple(sizeLimit); - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack()); - tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack()); - if (!intrans) { - wait(safeThreadFutureToFuture(tr->commit())); - } - } else { - fprintf(stderr, "ERROR: Unknown action: %s\n", printable(tokens[2]).c_str()); - result = false; - } - } else if (tokencmp(tokens[1], "list")) { - if (tokens.size() != 2) { - fprintf(stderr, "ERROR: Usage: profile list\n"); - return false; - } - // Hold the reference to the standalone's memory - state ThreadFuture kvsFuture = tr->getRange( - KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY); - RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture)); - ASSERT(!kvs.more); - for (const auto& pair : kvs) { - auto ip_port = (pair.key.endsWith(":tls"_sr) ? pair.key.removeSuffix(":tls"_sr) : pair.key) - .removePrefix("\xff\xff/worker_interfaces/"_sr); - printf("%s\n", printable(ip_port).c_str()); - } - } else { - fprintf(stderr, "ERROR: Unknown type: %s\n", printable(tokens[1]).c_str()); - result = false; - } - return result; -} - -CommandFactory profileFactory("profile", - CommandHelp("profile ", - "namespace for all the profiling-related commands.", - "Different types support different actions. Run `profile` to get a list of " - "types, and iteratively explore the help.\n")); -} // namespace fdb_cli diff --git a/fdbcli/ProfileCommand.cpp b/fdbcli/ProfileCommand.cpp new file mode 100644 index 00000000000..a08392701ad --- /dev/null +++ b/fdbcli/ProfileCommand.cpp @@ -0,0 +1,135 @@ +/* + * ProfileCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/lexical_cast.hpp" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/GlobalConfig.h" +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/Tuple.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace fdb_cli { + +Future profileCommandActor(Database db, Reference tr, std::vector tokens, bool intrans) { + bool result = true; + if (tokens.size() == 1) { + printUsage(tokens[0]); + result = false; + } else if (tokencmp(tokens[1], "client")) { + if (tokens.size() == 2) { + fprintf(stderr, "ERROR: Usage: profile client \n"); + co_return false; + } + co_await db->globalConfig->onInitialized(); + if (tokencmp(tokens[2], "get")) { + if (tokens.size() != 3) { + fprintf(stderr, "ERROR: Additional arguments to `get` are not supported.\n"); + co_return false; + } + std::string sampleRateStr = "default"; + std::string sizeLimitStr = "default"; + const double sampleRateDbl = + db->globalConfig->get(fdbClientInfoTxnSampleRate, std::numeric_limits::infinity()); + if (!std::isinf(sampleRateDbl)) { + sampleRateStr = std::to_string(sampleRateDbl); + } + const int64_t sizeLimit = db->globalConfig->get(fdbClientInfoTxnSizeLimit, -1); + if (sizeLimit != -1) { + sizeLimitStr = boost::lexical_cast(sizeLimit); + } + printf("Client profiling rate is set to %s and size limit is set to %s.\n", + sampleRateStr.c_str(), + sizeLimitStr.c_str()); + } else if (tokencmp(tokens[2], "set")) { + if (tokens.size() != 5) { + fprintf(stderr, "ERROR: Usage: profile client set \n"); + co_return false; + } + double sampleRate; + if (tokencmp(tokens[3], "default")) { + sampleRate = std::numeric_limits::infinity(); + } else { + char* end; + sampleRate = std::strtod((const char*)tokens[3].begin(), &end); + if (!std::isspace(*end)) { + fprintf(stderr, "ERROR: %s failed to parse.\n", printable(tokens[3]).c_str()); + co_return false; + } + } + int64_t sizeLimit; + if (tokencmp(tokens[4], "default")) { + sizeLimit = -1; + } else { + Optional parsed = parse_with_suffix(tokens[4].toString()); + if (parsed.present()) { + sizeLimit = parsed.get(); + } else { + fprintf(stderr, "ERROR: `%s` failed to parse.\n", printable(tokens[4]).c_str()); + co_return false; + } + } + + Tuple rate = Tuple::makeTuple(sampleRate); + Tuple size = Tuple::makeTuple(sizeLimit); + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSampleRate), rate.pack()); + tr->set(GlobalConfig::prefixedKey(fdbClientInfoTxnSizeLimit), size.pack()); + if (!intrans) { + co_await safeThreadFutureToFuture(tr->commit()); + } + } else { + fprintf(stderr, "ERROR: Unknown action: %s\n", printable(tokens[2]).c_str()); + result = false; + } + } else if (tokencmp(tokens[1], "list")) { + if (tokens.size() != 2) { + fprintf(stderr, "ERROR: Usage: profile list\n"); + co_return false; + } + // Hold the reference to the standalone's memory + ThreadFuture kvsFuture = tr->getRange( + KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY); + RangeResult kvs = co_await safeThreadFutureToFuture(kvsFuture); + ASSERT(!kvs.more); + for (const auto& pair : kvs) { + auto ip_port = (pair.key.endsWith(":tls"_sr) ? pair.key.removeSuffix(":tls"_sr) : pair.key) + .removePrefix("\xff\xff/worker_interfaces/"_sr); + printf("%s\n", printable(ip_port).c_str()); + } + } else { + fprintf(stderr, "ERROR: Unknown type: %s\n", printable(tokens[1]).c_str()); + result = false; + } + co_return result; +} + +CommandFactory profileFactory("profile", + CommandHelp("profile ", + "namespace for all the profiling-related commands.", + "Different types support different actions. Run `profile` to get a list of " + "types, and iteratively explore the help.\n")); +} // namespace fdb_cli diff --git a/fdbcli/RangeConfigCommand.actor.cpp b/fdbcli/RangeConfigCommand.actor.cpp deleted file mode 100644 index bf83ddecfa8..00000000000 --- a/fdbcli/RangeConfigCommand.actor.cpp +++ /dev/null @@ -1,167 +0,0 @@ -/* - * RangeConfigCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include - -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/NativeAPI.actor.h" -#include "fdbclient/DataDistributionConfig.actor.h" - -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future rangeConfigCommandActor(Database cx, std::vector tokens) { - state std::function fail = [&](std::string msg) { - if (!msg.empty()) { - fmt::print(stderr, "ERROR: {}\n", msg); - } - printUsage(tokens[0]); - return false; - }; - - state std::list args(tokens.begin() + 1, tokens.end()); - state std::function nextArg = [&]() { - ASSERT(!args.empty()); - auto s = args.front(); - args.pop_front(); - return s; - }; - state std::function nextArgInt = [&]() { - if (args.empty()) { - throw boost::bad_lexical_cast(); - } - return boost::lexical_cast(nextArg().toString()); - }; - - if (args.size() < 1) { - return fail("No subcommand given."); - } - - state StringRef cmd = nextArg(); - - if (cmd == "show"_sr) { - state bool includeDefault = false; - - while (!args.empty()) { - auto arg = nextArg(); - if (arg == "includeDefault"_sr) { - includeDefault = true; - } else { - return fail(fmt::format("Unknown argument: '{}'", arg.printable())); - } - } - - Reference config = - wait(DDConfiguration().userRangeConfig().getSnapshot( - SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end)); - fmt::print( - "{}\n", - json_spirit::write_string(DDConfiguration::toJSON(*config, includeDefault), json_spirit::pretty_print)); - - } else if (cmd == "update"_sr || cmd == "set"_sr) { - if (args.size() < 3) { - return fail("Begin, end, and at least one configuration option are required."); - } - - state KeyRef begin = nextArg(); - state KeyRef end = nextArg(); - if (end <= begin) { - return fail("Range end must be > range begin."); - } - - state DDRangeConfig rangeConfig; - - while (!args.empty()) { - state StringRef option = nextArg(); - - try { - if (option == "replication"_sr) { - rangeConfig.replicationFactor = nextArgInt(); - } else if (option == "teamID"_sr) { - rangeConfig.teamID = nextArgInt(); - } else if (option == "default"_sr) { - rangeConfig = DDRangeConfig(); - } else { - return fail(fmt::format("Unknown range option: '{}'", option.printable())); - } - } catch (...) { - return fail( - fmt::format("Required argument for range option '{}' missing or invalid.", option.toString())); - } - - wait(DDConfiguration().userRangeConfig().updateRange( - SystemDBWriteLockedNow(cx.getReference()), begin, end, rangeConfig, cmd == "set"_sr)); - } - } else { - return fail(fmt::format("Unknown command: '{}'", cmd.printable())); - } - - return true; -} - -std::vector rangeConfigGenerator(std::vector const& tokens, bool inArgument) { - if (tokens.size() == 1) { - return { "", "[ARGS]" }; - } - auto cmd = tokens[1]; - if (cmd == "show"_sr && tokens.size() == 2) { - return { "[includeDefault]" }; - } else if (cmd == "set"_sr || cmd == "update"_sr) { - static std::vector opts = { - "", "", "[default]", "[replication ]", "[teamID ]" - }; - // Subtract the two known tokens, command and subcommand, and then possibly two more tokens for begin and end if - // present - return std::vector(opts.begin() + std::min(tokens.size(), 4) - 2, opts.end()); - } else { - return {}; - } -} - -CommandFactory rangeConfigFactory( - "rangeconfig", - CommandHelp( - "rangeconfig [includeDefault] | [default] [replication ] [teamID " - "]", - "Show or change the per-keyrange configuration options.", - "The `show' command will print the range configuration in JSON. By default, ranges with no configured " - "options are not shown, these are called 'default ranges' and can be shown with the `includeDefault' flag.\n\n" - "The `update' command will apply the given options to the given key range. Any option not explicitly given " - "will remain at its present setting for the range in the configuration.\n\n" - "The `set' command will change the given key range to be exactly given configuration, meaning that any option " - "not explicitly given will be changed to unset for the range.\n\n" - "Range Options:\n" - " default - Resets the configuration to apply to have no options set. This can be used with `set' to " - "explicitly clear all configured options for a range.\n\n" - " replication - Set replication factor for the range. Ranges set to a replication factor lower than the " - "cluster's configured replication level will be treated as the same as the cluster's replication level.\n\n" - " teamID - This provides a way to indicate that shards should be on different teams. Ranges with " - "different teamID settings should be assigned to different storage teams. Shards with the same team ID can be " - "assigned to the same storage team, but nothing enforces this.\n\n" - "Note that key range configuration options do not alter the shard map directly, rather they are hints which " - "DataDistribution should honor.\n\n" - "For help with escaping BEGINKEY or ENDKEY, see `help escaping'.\n"), - nullptr, - &rangeConfigGenerator); -} // namespace fdb_cli diff --git a/fdbcli/RangeConfigCommand.cpp b/fdbcli/RangeConfigCommand.cpp new file mode 100644 index 00000000000..f8b4e0e3d93 --- /dev/null +++ b/fdbcli/RangeConfigCommand.cpp @@ -0,0 +1,165 @@ +/* + * RangeConfigCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include "fdbcli/fdbcli.h" +#include "fdbclient/NativeAPI.actor.h" +#include "fdbclient/DataDistributionConfig.h" + +namespace fdb_cli { + +Future rangeConfigCommandActor(Database cx, std::vector tokens) { + std::function fail = [&](std::string msg) { + if (!msg.empty()) { + fmt::print(stderr, "ERROR: {}\n", msg); + } + printUsage(tokens[0]); + return false; + }; + + std::list args(tokens.begin() + 1, tokens.end()); + std::function nextArg = [&]() { + ASSERT(!args.empty()); + auto s = args.front(); + args.pop_front(); + return s; + }; + std::function nextArgInt = [&]() { + if (args.empty()) { + throw boost::bad_lexical_cast(); + } + return boost::lexical_cast(nextArg().toString()); + }; + + if (args.empty()) { + co_return fail("No subcommand given."); + } + + StringRef cmd = nextArg(); + + if (cmd == "show"_sr) { + bool includeDefault = false; + + while (!args.empty()) { + auto arg = nextArg(); + if (arg == "includeDefault"_sr) { + includeDefault = true; + } else { + co_return fail(fmt::format("Unknown argument: '{}'", arg.printable())); + } + } + + Reference config = + co_await DDConfiguration().userRangeConfig().getSnapshot( + SystemDBWriteLockedNow(cx.getReference()), allKeys.begin, allKeys.end); + fmt::print( + "{}\n", + json_spirit::write_string(DDConfiguration::toJSON(*config, includeDefault), json_spirit::pretty_print)); + + } else if (cmd == "update"_sr || cmd == "set"_sr) { + if (args.size() < 3) { + co_return fail("Begin, end, and at least one configuration option are required."); + } + + KeyRef begin = nextArg(); + KeyRef end = nextArg(); + if (end <= begin) { + co_return fail("Range end must be > range begin."); + } + + DDRangeConfig rangeConfig; + + while (!args.empty()) { + StringRef option = nextArg(); + + try { + if (option == "replication"_sr) { + rangeConfig.replicationFactor = nextArgInt(); + } else if (option == "teamID"_sr) { + rangeConfig.teamID = nextArgInt(); + } else if (option == "default"_sr) { + rangeConfig = DDRangeConfig(); + } else { + co_return fail(fmt::format("Unknown range option: '{}'", option.printable())); + } + } catch (...) { + co_return fail( + fmt::format("Required argument for range option '{}' missing or invalid.", option.toString())); + } + + co_await DDConfiguration().userRangeConfig().updateRange( + SystemDBWriteLockedNow(cx.getReference()), begin, end, rangeConfig, cmd == "set"_sr); + } + } else { + co_return fail(fmt::format("Unknown command: '{}'", cmd.printable())); + } + + co_return true; +} + +std::vector rangeConfigGenerator(std::vector const& tokens, bool inArgument) { + if (tokens.size() == 1) { + return { "", "[ARGS]" }; + } + auto cmd = tokens[1]; + if (cmd == "show"_sr && tokens.size() == 2) { + return { "[includeDefault]" }; + } else if (cmd == "set"_sr || cmd == "update"_sr) { + static std::vector opts = { + "", "", "[default]", "[replication ]", "[teamID ]" + }; + // Subtract the two known tokens, command and subcommand, and then possibly two more tokens for begin and end if + // present + return std::vector(opts.begin() + std::min(tokens.size(), 4) - 2, opts.end()); + } else { + return {}; + } +} + +CommandFactory rangeConfigFactory( + "rangeconfig", + CommandHelp( + "rangeconfig [includeDefault] | [default] [replication ] [teamID " + "]", + "Show or change the per-keyrange configuration options.", + "The `show' command will print the range configuration in JSON. By default, ranges with no configured " + "options are not shown, these are called 'default ranges' and can be shown with the `includeDefault' flag.\n\n" + "The `update' command will apply the given options to the given key range. Any option not explicitly given " + "will remain at its present setting for the range in the configuration.\n\n" + "The `set' command will change the given key range to be exactly given configuration, meaning that any option " + "not explicitly given will be changed to unset for the range.\n\n" + "Range Options:\n" + " default - Resets the configuration to apply to have no options set. This can be used with `set' to " + "explicitly clear all configured options for a range.\n\n" + " replication - Set replication factor for the range. Ranges set to a replication factor lower than the " + "cluster's configured replication level will be treated as the same as the cluster's replication level.\n\n" + " teamID - This provides a way to indicate that shards should be on different teams. Ranges with " + "different teamID settings should be assigned to different storage teams. Shards with the same team ID can be " + "assigned to the same storage team, but nothing enforces this.\n\n" + "Note that key range configuration options do not alter the shard map directly, rather they are hints which " + "DataDistribution should honor.\n\n" + "For help with escaping BEGINKEY or ENDKEY, see `help escaping'.\n"), + nullptr, + &rangeConfigGenerator); +} // namespace fdb_cli diff --git a/fdbcli/SetClassCommand.actor.cpp b/fdbcli/SetClassCommand.actor.cpp deleted file mode 100644 index 80f8e4a2333..00000000000 --- a/fdbcli/SetClassCommand.actor.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/* - * SetClassCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fmt/format.h" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -ACTOR Future printProcessClass(Reference db) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - // Hold the reference to the memory - state ThreadFuture classTypeFuture = - tr->getRange(fdb_cli::processClassTypeSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - state ThreadFuture classSourceFuture = - tr->getRange(fdb_cli::processClassSourceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); - wait(success(safeThreadFutureToFuture(classSourceFuture)) && - success(safeThreadFutureToFuture(classTypeFuture))); - RangeResult processTypeList = classTypeFuture.get(); - RangeResult processSourceList = classSourceFuture.get(); - ASSERT(processSourceList.size() == processTypeList.size()); - if (!processTypeList.size()) - printf("No processes are registered in the database.\n"); - fmt::print("There are currently {} processes in the database:\n", processTypeList.size()); - for (int index = 0; index < processTypeList.size(); index++) { - std::string address = - processTypeList[index].key.removePrefix(fdb_cli::processClassTypeSpecialKeyRange.begin).toString(); - // check the addresses are the same in each list - std::string addressFromSourceList = - processSourceList[index] - .key.removePrefix(fdb_cli::processClassSourceSpecialKeyRange.begin) - .toString(); - ASSERT(address == addressFromSourceList); - printf(" %s: %s (%s)\n", - address.c_str(), - processTypeList[index].value.toString().c_str(), - processSourceList[index].value.toString().c_str()); - } - return Void(); - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -}; - -ACTOR Future setProcessClass(Reference db, KeyRef network_address, KeyRef class_type) { - state Reference tr = db->createTransaction(); - loop { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - try { - state ThreadFuture> result = - tr->get(network_address.withPrefix(fdb_cli::processClassTypeSpecialKeyRange.begin)); - Optional val = wait(safeThreadFutureToFuture(result)); - if (!val.present()) { - printf("No matching addresses found\n"); - return false; - } - tr->set(network_address.withPrefix(fdb_cli::processClassTypeSpecialKeyRange.begin), class_type); - wait(safeThreadFutureToFuture(tr->commit())); - return true; - } catch (Error& e) { - state Error err(e); - if (e.code() == error_code_special_keys_api_failure) { - std::string errorMsgStr = wait(fdb_cli::getSpecialKeysFailureErrorMessage(tr)); - // error message already has \n at the end - fprintf(stderr, "%s", errorMsgStr.c_str()); - return false; - } - wait(safeThreadFutureToFuture(tr->onError(err))); - } - } -} - -} // namespace - -namespace fdb_cli { - -const KeyRangeRef processClassSourceSpecialKeyRange = - KeyRangeRef("\xff\xff/configuration/process/class_source/"_sr, "\xff\xff/configuration/process/class_source0"_sr); - -const KeyRangeRef processClassTypeSpecialKeyRange = - KeyRangeRef("\xff\xff/configuration/process/class_type/"_sr, "\xff\xff/configuration/process/class_type0"_sr); - -ACTOR Future setClassCommandActor(Reference db, std::vector tokens) { - if (tokens.size() != 3 && tokens.size() != 1) { - printUsage(tokens[0]); - return false; - } else if (tokens.size() == 1) { - wait(printProcessClass(db)); - } else { - bool successful = wait(setProcessClass(db, tokens[1], tokens[2])); - return successful; - } - return true; -} - -CommandFactory setClassFactory( - "setclass", - CommandHelp("setclass [
]", - "change the class of a process", - "If no address and class are specified, lists the classes of all servers.\n\nSetting the class to " - "`default' resets the process class to the class specified on the command line. The available " - "classes are `unset', `storage', `transaction', `resolution', `commit_proxy', `grv_proxy', " - "`master', `test', " - "`stateless', `log', `router', `cluster_controller', `fast_restore', `data_distributor', " - "`coordinator', `ratekeeper', `backup', and `default'.")); - -} // namespace fdb_cli diff --git a/fdbcli/SetClassCommand.cpp b/fdbcli/SetClassCommand.cpp new file mode 100644 index 00000000000..d5155d3ab75 --- /dev/null +++ b/fdbcli/SetClassCommand.cpp @@ -0,0 +1,139 @@ +/* + * SetClassCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fmt/format.h" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +Future printProcessClass(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + // Hold the reference to the memory + ThreadFuture classTypeFuture = + tr->getRange(fdb_cli::processClassTypeSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + ThreadFuture classSourceFuture = + tr->getRange(fdb_cli::processClassSourceSpecialKeyRange, CLIENT_KNOBS->TOO_MANY); + co_await (success(safeThreadFutureToFuture(classSourceFuture)) && + success(safeThreadFutureToFuture(classTypeFuture))); + RangeResult processTypeList = classTypeFuture.get(); + RangeResult processSourceList = classSourceFuture.get(); + ASSERT(processSourceList.size() == processTypeList.size()); + if (processTypeList.empty()) + printf("No processes are registered in the database.\n"); + fmt::print("There are currently {} processes in the database:\n", processTypeList.size()); + for (int index = 0; index < processTypeList.size(); index++) { + std::string address = + processTypeList[index].key.removePrefix(fdb_cli::processClassTypeSpecialKeyRange.begin).toString(); + // check the addresses are the same in each list + std::string addressFromSourceList = + processSourceList[index] + .key.removePrefix(fdb_cli::processClassSourceSpecialKeyRange.begin) + .toString(); + ASSERT(address == addressFromSourceList); + printf(" %s: %s (%s)\n", + address.c_str(), + processTypeList[index].value.toString().c_str(), + processSourceList[index].value.toString().c_str()); + } + co_return; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +}; + +Future setProcessClass(Reference db, KeyRef network_address, KeyRef class_type) { + Reference tr = db->createTransaction(); + while (true) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Error err; + try { + ThreadFuture> result = + tr->get(network_address.withPrefix(fdb_cli::processClassTypeSpecialKeyRange.begin)); + Optional val = co_await safeThreadFutureToFuture(result); + if (!val.present()) { + printf("No matching addresses found\n"); + co_return false; + } + tr->set(network_address.withPrefix(fdb_cli::processClassTypeSpecialKeyRange.begin), class_type); + co_await safeThreadFutureToFuture(tr->commit()); + co_return true; + } catch (Error& e) { + err = e; + } + if (err.code() == error_code_special_keys_api_failure) { + std::string errorMsgStr = co_await fdb_cli::getSpecialKeysFailureErrorMessage(tr); + // error message already has \n at the end + fprintf(stderr, "%s", errorMsgStr.c_str()); + co_return false; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +} // namespace + +namespace fdb_cli { + +const KeyRangeRef processClassSourceSpecialKeyRange = + KeyRangeRef("\xff\xff/configuration/process/class_source/"_sr, "\xff\xff/configuration/process/class_source0"_sr); + +const KeyRangeRef processClassTypeSpecialKeyRange = + KeyRangeRef("\xff\xff/configuration/process/class_type/"_sr, "\xff\xff/configuration/process/class_type0"_sr); + +Future setClassCommandActor(Reference db, std::vector tokens) { + if (tokens.size() != 3 && tokens.size() != 1) { + printUsage(tokens[0]); + co_return false; + } else if (tokens.size() == 1) { + co_await printProcessClass(db); + } else { + bool successful = co_await setProcessClass(db, tokens[1], tokens[2]); + co_return successful; + } + co_return true; +} + +CommandFactory setClassFactory( + "setclass", + CommandHelp("setclass [
]", + "change the class of a process", + "If no address and class are specified, lists the classes of all servers.\n\nSetting the class to " + "`default' resets the process class to the class specified on the command line. The available " + "classes are `unset', `storage', `transaction', `resolution', `commit_proxy', `grv_proxy', " + "`master', `test', " + "`stateless', `log', `router', `cluster_controller', `data_distributor', " + "`coordinator', `ratekeeper', `backup', and `default'.")); + +} // namespace fdb_cli diff --git a/fdbcli/SnapshotCommand.actor.cpp b/fdbcli/SnapshotCommand.actor.cpp deleted file mode 100644 index 844ae88d588..00000000000 --- a/fdbcli/SnapshotCommand.actor.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * SnapshotCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/IClientApi.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future snapshotCommandActor(Reference db, std::vector tokens) { - state bool result = true; - if (tokens.size() < 2) { - printUsage(tokens[0]); - result = false; - } else { - Standalone snap_cmd; - state Key uid(deterministicRandom()->randomUniqueID().toString()); - for (int i = 1; i < tokens.size(); i++) { - snap_cmd = snap_cmd.withSuffix(tokens[i]); - if (i != tokens.size() - 1) { - snap_cmd = snap_cmd.withSuffix(" "_sr); - } - } - try { - wait(safeThreadFutureToFuture(db->createSnapshot(uid, snap_cmd))); - printf("Snapshot command succeeded with UID %s\n", uid.toString().c_str()); - } catch (Error& e) { - fprintf(stderr, - "Snapshot command failed %d (%s)." - " Please cleanup any instance level snapshots created with UID %s.\n", - e.code(), - e.what(), - uid.toString().c_str()); - result = false; - } - } - return result; -} - -// hidden commands, no help text for now -CommandFactory snapshotFactory("snapshot"); -} // namespace fdb_cli diff --git a/fdbcli/SnapshotCommand.cpp b/fdbcli/SnapshotCommand.cpp new file mode 100644 index 00000000000..d02a25c52c3 --- /dev/null +++ b/fdbcli/SnapshotCommand.cpp @@ -0,0 +1,62 @@ +/* + * SnapshotCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/IClientApi.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +namespace fdb_cli { + +Future snapshotCommandActor(Reference db, std::vector const& tokens) { + bool result = true; + if (tokens.size() < 2) { + printUsage(tokens[0]); + result = false; + } else { + Standalone snap_cmd; + Key uid(deterministicRandom()->randomUniqueID().toString()); + for (int i = 1; i < tokens.size(); i++) { + snap_cmd = snap_cmd.withSuffix(tokens[i]); + if (i != tokens.size() - 1) { + snap_cmd = snap_cmd.withSuffix(" "_sr); + } + } + try { + co_await safeThreadFutureToFuture(db->createSnapshot(uid, snap_cmd)); + printf("Snapshot command succeeded with UID %s\n", uid.toString().c_str()); + } catch (Error& e) { + fprintf(stderr, + "Snapshot command failed %d (%s)." + " Please cleanup any instance level snapshots created with UID %s.\n", + e.code(), + e.what(), + uid.toString().c_str()); + result = false; + } + } + co_return result; +} + +// hidden commands, no help text for now +CommandFactory snapshotFactory("snapshot"); +} // namespace fdb_cli diff --git a/fdbcli/StatusCommand.actor.cpp b/fdbcli/StatusCommand.actor.cpp deleted file mode 100644 index 0a144d33fc1..00000000000 --- a/fdbcli/StatusCommand.actor.cpp +++ /dev/null @@ -1,1291 +0,0 @@ -/* - * StatusCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" -#include "fmt/chrono.h" -#include "fmt/core.h" -#include "fmt/format.h" -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" -#include "fdbclient/StatusClient.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -std::string getCoordinatorsInfoString(StatusObjectReader statusObj) { - std::string outputString; - try { - StatusArray coordinatorsArr = statusObj["client.coordinators.coordinators"].get_array(); - for (StatusObjectReader coor : coordinatorsArr) - outputString += format("\n %s (%s)", - coor["address"].get_str().c_str(), - coor["reachable"].get_bool() ? "reachable" : "unreachable"); - } catch (std::runtime_error&) { - outputString = "\n Unable to retrieve list of coordination servers"; - } - - return outputString; -} - -std::string lineWrap(const char* text, int col) { - const char* iter = text; - const char* start = text; - const char* space = nullptr; - std::string out = ""; - do { - iter++; - if (*iter == '\n' || *iter == ' ' || *iter == '\0') - space = iter; - if (*iter == '\n' || *iter == '\0' || (iter - start == col)) { - if (!space) - space = iter; - out += format("%.*s\n", (int)(space - start), start); - start = space; - if (*start == ' ' /* || *start == '\n'*/) - start++; - space = nullptr; - } - } while (*iter); - return out; -} - -std::pair getNumOfNonExcludedProcessAndZones(StatusObjectReader statusObjCluster) { - StatusObjectReader processesMap; - std::set zones; - int numOfNonExcludedProcesses = 0; - if (statusObjCluster.get("processes", processesMap)) { - for (auto proc : processesMap.obj()) { - StatusObjectReader process(proc.second); - if (process.has("excluded") && process.last().get_bool()) - continue; - numOfNonExcludedProcesses++; - std::string zoneId; - if (process.get("locality.zoneid", zoneId)) { - zones.insert(zoneId); - } - } - } - return { numOfNonExcludedProcesses, zones.size() }; -} - -int getNumofNonExcludedMachines(StatusObjectReader statusObjCluster) { - StatusObjectReader machineMap; - int numOfNonExcludedMachines = 0; - if (statusObjCluster.get("machines", machineMap)) { - for (auto mach : machineMap.obj()) { - StatusObjectReader machine(mach.second); - if (machine.has("excluded") && !machine.last().get_bool()) - numOfNonExcludedMachines++; - } - } - return numOfNonExcludedMachines; -} - -std::string getDateInfoString(StatusObjectReader statusObj, std::string key) { - time_t curTime; - if (!statusObj.has(key)) { - return ""; - } - curTime = statusObj.last().get_int64(); - char buffer[128]; - struct tm* timeinfo; - timeinfo = localtime(&curTime); - strftime(buffer, 128, "%m/%d/%y %H:%M:%S", timeinfo); - return std::string(buffer); -} - -std::string getProcessAddressByServerID(StatusObjectReader processesMap, std::string serverID) { - if (serverID == "") - return "unknown"; - - for (auto proc : processesMap.obj()) { - try { - StatusArray rolesArray = proc.second.get_obj()["roles"].get_array(); - for (StatusObjectReader role : rolesArray) { - if (role["id"].get_str().find(serverID) == 0) { - // If this next line throws, then we found the serverID but the role has no address, so the role is - // skipped. - return proc.second.get_obj()["address"].get_str(); - } - } - } catch (std::exception&) { - // If an entry in the process map is badly formed then something will throw. Since we are - // looking for a positive match, just ignore any read exceptions and move on to the next proc - } - } - return "unknown"; -} - -std::string getWorkloadRates(StatusObjectReader statusObj, - bool unknown, - std::string first, - std::string second, - bool transactionSection = false) { - // Re-point statusObj at either the transactions sub-doc or the operations sub-doc depending on transactionSection - // flag - if (transactionSection) { - if (!statusObj.get("transactions", statusObj)) - return "unknown"; - } else { - if (!statusObj.get("operations", statusObj)) - return "unknown"; - } - - std::string path = first + "." + second; - double value; - if (!unknown && statusObj.get(path, value)) { - return format("%d Hz", (int)round(value)); - } - return "unknown"; -} - -void getBackupDRTags(StatusObjectReader& statusObjCluster, - const char* context, - std::map& tagMap) { - std::string path = format("layers.%s.tags", context); - StatusObjectReader tags; - if (statusObjCluster.tryGet(path, tags)) { - for (auto itr : tags.obj()) { - JSONDoc tag(itr.second); - bool running = false; - tag.tryGet("running_backup", running); - if (running) { - std::string uid; - if (tag.tryGet("mutation_stream_id", uid)) { - tagMap[itr.first] = uid; - } else { - tagMap[itr.first] = ""; - } - } - } - } -} - -std::string logBackupDR(const char* context, std::map const& tagMap) { - std::string outputString = ""; - if (tagMap.size() > 0) { - outputString += format("\n\n%s:", context); - for (auto itr : tagMap) { - outputString += format("\n %-22s", itr.first.c_str()); - if (itr.second.size() > 0) { - outputString += format(" - %s", itr.second.c_str()); - } - } - } - - return outputString; -} - -} // namespace - -namespace fdb_cli { - -std::string toBytesString(double bytes) { - if (bytes >= 1e12) { - return format("%.3f TB", (bytes / 1e12)); - } else if (bytes >= 1e9) { - return format("%.3f GB", (bytes / 1e9)); - } else { - // no decimal points for MB - return format("%d MB", (int)round(bytes / 1e6)); - } -} - -void printStatus(StatusObjectReader statusObj, - StatusClient::StatusLevel level, - bool displayDatabaseAvailable, - bool hideErrorMessages) { - if (FlowTransport::transport().incompatibleOutgoingConnectionsPresent()) { - fprintf( - stderr, - "WARNING: One or more of the processes in the cluster is incompatible with this version of fdbcli.\n\n"); - } - - try { - bool printedCoordinators = false; - - // status or status details - if (level == StatusClient::NORMAL || level == StatusClient::DETAILED) { - - StatusObjectReader statusObjClient; - statusObj.get("client", statusObjClient); - - // The way the output string is assembled is to add new line character before addition to the string rather - // than after - std::string outputString = ""; - std::string clusterFilePath; - if (statusObjClient.get("cluster_file.path", clusterFilePath)) - outputString = format("Using cluster file `%s'.\n", clusterFilePath.c_str()); - else - outputString = "Using unknown cluster file.\n"; - - StatusObjectReader statusObjCoordinators; - StatusArray coordinatorsArr; - - if (statusObjClient.get("coordinators", statusObjCoordinators)) { - // Look for a second "coordinators", under the first one. - if (statusObjCoordinators.has("coordinators")) - coordinatorsArr = statusObjCoordinators.last().get_array(); - } - - // Check if any coordination servers are unreachable - bool quorum_reachable; - if (statusObjCoordinators.get("quorum_reachable", quorum_reachable) && !quorum_reachable) { - outputString += "\nCould not communicate with a quorum of coordination servers:"; - outputString += getCoordinatorsInfoString(statusObj); - - printf("%s\n", outputString.c_str()); - return; - } else { - for (StatusObjectReader coor : coordinatorsArr) { - bool reachable; - if (coor.get("reachable", reachable) && !reachable) { - outputString += "\nCould not communicate with all of the coordination servers." - "\n The database will remain operational as long as we" - "\n can connect to a quorum of servers, however the fault" - "\n tolerance of the system is reduced as long as the" - "\n servers remain disconnected.\n"; - outputString += getCoordinatorsInfoString(statusObj); - outputString += "\n"; - printedCoordinators = true; - break; - } - } - } - - // print any client messages - if (statusObjClient.has("messages")) { - for (StatusObjectReader message : statusObjClient.last().get_array()) { - std::string desc; - if (message.get("description", desc)) - outputString += "\n" + lineWrap(desc.c_str(), 80); - } - } - - bool fatalRecoveryState = false; - StatusObjectReader statusObjCluster; - try { - if (statusObj.get("cluster", statusObjCluster)) { - - StatusObjectReader recoveryState; - if (statusObjCluster.get("recovery_state", recoveryState)) { - std::string name; - std::string description; - if (recoveryState.get("name", name) && recoveryState.get("description", description) && - name != "accepting_commits" && name != "all_logs_recruited" && - name != "storage_recovered" && name != "fully_recovered") { - fatalRecoveryState = true; - - if (name == "recruiting_transaction_servers") { - description += - format("\nNeed at least %d log servers across unique zones, %d commit proxies, " - "%d GRV proxies and %d resolvers.", - recoveryState["required_logs"].get_int(), - recoveryState["required_commit_proxies"].get_int(), - recoveryState["required_grv_proxies"].get_int(), - recoveryState["required_resolvers"].get_int()); - if (statusObjCluster.has("machines") && statusObjCluster.has("processes")) { - auto numOfNonExcludedProcessesAndZones = - getNumOfNonExcludedProcessAndZones(statusObjCluster); - description += - format("\nHave %d non-excluded processes on %d machines across %d zones.", - numOfNonExcludedProcessesAndZones.first, - getNumofNonExcludedMachines(statusObjCluster), - numOfNonExcludedProcessesAndZones.second); - } - } else if (name == "locking_old_transaction_servers" && - recoveryState["missing_logs"].get_str().size()) { - description += format("\nNeed one or more of the following log servers: %s", - recoveryState["missing_logs"].get_str().c_str()); - } - description = lineWrap(description.c_str(), 80); - if (!printedCoordinators && - (name == "reading_coordinated_state" || name == "locking_coordinated_state" || - name == "configuration_never_created" || name == "writing_coordinated_state")) { - description += getCoordinatorsInfoString(statusObj); - description += "\n"; - printedCoordinators = true; - } - - outputString += "\n" + description; - } - } - } - } catch (std::runtime_error&) { - } - - // Check if cluster controllable is reachable - try { - // print any cluster messages - if (statusObjCluster.has("messages") && statusObjCluster.last().get_array().size()) { - - // any messages we don't want to display - std::set skipMsgs = { "unreachable_process", "" }; - if (fatalRecoveryState) { - skipMsgs.insert("status_incomplete"); - skipMsgs.insert("unreadable_configuration"); - skipMsgs.insert("immediate_priority_transaction_start_probe_timeout"); - skipMsgs.insert("batch_priority_transaction_start_probe_timeout"); - skipMsgs.insert("transaction_start_probe_timeout"); - skipMsgs.insert("read_probe_timeout"); - skipMsgs.insert("commit_probe_timeout"); - } - - for (StatusObjectReader msgObj : statusObjCluster.last().get_array()) { - std::string messageName; - if (!msgObj.get("name", messageName)) { - continue; - } - if (skipMsgs.count(messageName)) { - continue; - } else if (messageName == "client_issues") { - if (msgObj.has("issues")) { - for (StatusObjectReader issue : msgObj["issues"].get_array()) { - std::string issueName; - if (!issue.get("name", issueName)) { - continue; - } - - std::string description; - if (!issue.get("description", description)) { - description = issueName; - } - - std::string countStr; - StatusArray addresses; - if (!issue.has("addresses")) { - countStr = "Some client(s)"; - } else { - addresses = issue["addresses"].get_array(); - countStr = format("%d client(s)", addresses.size()); - } - outputString += - format("\n%s reported: %s\n", countStr.c_str(), description.c_str()); - - if (level == StatusClient::StatusLevel::DETAILED) { - for (int i = 0; i < addresses.size() && i < 4; ++i) { - outputString += format(" %s\n", addresses[i].get_str().c_str()); - } - if (addresses.size() > 4) { - outputString += " ...\n"; - } - } - } - } - } else { - if (msgObj.has("description")) - outputString += "\n" + lineWrap(msgObj.last().get_str().c_str(), 80); - } - } - } - } catch (std::runtime_error&) { - } - - if (fatalRecoveryState) { - printf("%s", outputString.c_str()); - return; - } - - StatusObjectReader statusObjConfig; - StatusArray excludedServersArr; - Optional activePrimaryDC; - - if (statusObjCluster.has("active_primary_dc")) { - activePrimaryDC = statusObjCluster["active_primary_dc"].get_str(); - } - if (statusObjCluster.get("configuration", statusObjConfig)) { - if (statusObjConfig.has("excluded_servers")) - excludedServersArr = statusObjConfig.last().get_array(); - } - - // If there is a configuration message then there is no configuration information to display - outputString += "\nConfiguration:"; - std::string outputStringCache = outputString; - bool isOldMemory = false; - try { - // Configuration section - // FIXME: Should we suppress this if there are cluster messages implying that the database has no - // configuration? - - outputString += "\n Redundancy mode - "; - std::string strVal; - - if (statusObjConfig.get("redundancy_mode", strVal)) { - outputString += strVal; - } else - outputString += "unknown"; - - outputString += "\n Storage engine - "; - if (statusObjConfig.get("storage_engine", strVal)) { - if (strVal == "memory-1") { - isOldMemory = true; - } - outputString += strVal; - } else - outputString += "unknown"; - - outputString += - "\n Log engine - " + (statusObjConfig.get("log_engine", strVal) ? strVal : "unknown"); - - int intVal = 0; - - outputString += "\n Coordinators - "; - if (statusObjConfig.get("coordinators_count", intVal)) { - outputString += std::to_string(intVal); - } else - outputString += "unknown"; - - if (excludedServersArr.size()) { - outputString += format("\n Exclusions - %d (type `exclude' for details)", - excludedServersArr.size()); - } - - if (statusObjConfig.get("commit_proxies", intVal)) - outputString += format("\n Desired Commit Proxies - %d", intVal); - - if (statusObjConfig.get("grv_proxies", intVal)) - outputString += format("\n Desired GRV Proxies - %d", intVal); - - if (statusObjConfig.get("resolvers", intVal)) - outputString += format("\n Desired Resolvers - %d", intVal); - - if (statusObjConfig.get("logs", intVal)) - outputString += format("\n Desired Logs - %d", intVal); - - if (statusObjConfig.get("remote_logs", intVal)) - outputString += format("\n Desired Remote Logs - %d", intVal); - - if (statusObjConfig.get("log_routers", intVal)) - outputString += format("\n Desired Log Routers - %d", intVal); - - if (statusObjConfig.get("tss_count", intVal) && intVal > 0) { - int activeTss = 0; - if (statusObjCluster.has("active_tss_count")) { - statusObjCluster.get("active_tss_count", activeTss); - } - outputString += format("\n TSS - %d/%d", activeTss, intVal); - - if (statusObjConfig.get("tss_storage_engine", strVal)) - outputString += format("\n TSS Storage Engine - %s", strVal.c_str()); - } - - outputString += "\n Usable Regions - "; - if (statusObjConfig.get("usable_regions", intVal)) { - outputString += std::to_string(intVal); - } else { - outputString += "unknown"; - } - - StatusArray regions; - if (statusObjConfig.has("regions")) { - outputString += "\n Regions: "; - regions = statusObjConfig["regions"].get_array(); - for (StatusObjectReader region : regions) { - bool isPrimary = false; - std::vector regionSatelliteDCs; - std::string regionDC; - for (StatusObjectReader dc : region["datacenters"].get_array()) { - if (!dc.has("satellite")) { - regionDC = dc["id"].get_str(); - if (activePrimaryDC.present() && dc["id"].get_str() == activePrimaryDC.get()) { - isPrimary = true; - } - } else if (dc["satellite"].get_int() == 1) { - regionSatelliteDCs.push_back(dc["id"].get_str()); - } - } - if (activePrimaryDC.present()) { - if (isPrimary) { - outputString += "\n Primary -"; - } else { - outputString += "\n Remote -"; - } - } else { - outputString += "\n Region -"; - } - outputString += format("\n Datacenter - %s", regionDC.c_str()); - if (regionSatelliteDCs.size() > 0) { - outputString += "\n Satellite datacenters - "; - for (int i = 0; i < regionSatelliteDCs.size(); i++) { - if (i != regionSatelliteDCs.size() - 1) { - outputString += format("%s, ", regionSatelliteDCs[i].c_str()); - } else { - outputString += format("%s", regionSatelliteDCs[i].c_str()); - } - } - } - isPrimary = false; - if (region.get("satellite_redundancy_mode", strVal)) { - outputString += format("\n Satellite Redundancy Mode - %s", strVal.c_str()); - } - if (region.get("satellite_anti_quorum", intVal)) { - outputString += format("\n Satellite Anti Quorum - %d", intVal); - } - if (region.get("satellite_logs", intVal)) { - outputString += format("\n Satellite Logs - %d", intVal); - } - if (region.get("satellite_log_policy", strVal)) { - outputString += format("\n Satellite Log Policy - %s", strVal.c_str()); - } - if (region.get("satellite_log_replicas", intVal)) { - outputString += format("\n Satellite Log Replicas - %d", intVal); - } - if (region.get("satellite_usable_dcs", intVal)) { - outputString += format("\n Satellite Usable DCs - %d", intVal); - } - } - } - } catch (std::runtime_error&) { - outputString = outputStringCache; - outputString += "\n Unable to retrieve configuration status"; - } - - // Cluster section - outputString += "\n\nCluster:"; - StatusObjectReader processesMap; - StatusObjectReader machinesMap; - - outputStringCache = outputString; - - bool machinesAreZones = true; - std::map zones; - try { - outputString += "\n FoundationDB processes - "; - if (statusObjCluster.get("processes", processesMap)) { - - outputString += format("%d", processesMap.obj().size()); - - int errors = 0; - int processExclusions = 0; - for (auto p : processesMap.obj()) { - StatusObjectReader process(p.second); - bool excluded = process.has("excluded") && process.last().get_bool(); - if (excluded) { - processExclusions++; - } - if (process.has("messages") && process.last().get_array().size()) { - errors++; - } - - std::string zoneId; - if (process.get("locality.zoneid", zoneId)) { - std::string machineId; - if (!process.get("locality.machineid", machineId) || machineId != zoneId) { - machinesAreZones = false; - } - int& nonExcluded = zones[zoneId]; - if (!excluded) { - nonExcluded = 1; - } - } - } - - if (errors > 0 || processExclusions) { - outputString += format(" (less %d excluded; %d with errors)", processExclusions, errors); - } - - } else - outputString += "unknown"; - - if (zones.size() > 0) { - outputString += format("\n Zones - %d", zones.size()); - int zoneExclusions = 0; - for (auto itr : zones) { - if (itr.second == 0) { - ++zoneExclusions; - } - } - if (zoneExclusions > 0) { - outputString += format(" (less %d excluded)", zoneExclusions); - } - } else { - outputString += "\n Zones - unknown"; - } - - outputString += "\n Machines - "; - if (statusObjCluster.get("machines", machinesMap)) { - outputString += format("%d", machinesMap.obj().size()); - - int machineExclusions = 0; - for (auto mach : machinesMap.obj()) { - StatusObjectReader machine(mach.second); - if (machine.has("excluded") && machine.last().get_bool()) - machineExclusions++; - } - - if (machineExclusions) { - outputString += format(" (less %d excluded)", machineExclusions); - } - - int64_t minMemoryAvailable = std::numeric_limits::max(); - for (auto proc : processesMap.obj()) { - StatusObjectReader process(proc.second); - int64_t availBytes; - if (process.get("memory.available_bytes", availBytes)) { - minMemoryAvailable = std::min(minMemoryAvailable, availBytes); - } - } - - if (minMemoryAvailable < std::numeric_limits::max()) { - double worstServerGb = minMemoryAvailable / (1024.0 * 1024 * 1024); - outputString += "\n Memory availability - "; - outputString += format("%.1f GB per process on machine with least available", worstServerGb); - outputString += minMemoryAvailable < 4294967296 - ? "\n >>>>> (WARNING: 4.0 GB recommended) <<<<<" - : ""; - } - - double retransCount = 0; - for (auto mach : machinesMap.obj()) { - StatusObjectReader machine(mach.second); - double hz; - if (machine.get("network.tcp_segments_retransmitted.hz", hz)) - retransCount += hz; - } - - if (retransCount > 0) { - outputString += format("\n Retransmissions rate - %d Hz", (int)round(retransCount)); - } - } else - outputString += "\n Machines - unknown"; - - StatusObjectReader faultTolerance; - if (statusObjCluster.get("fault_tolerance", faultTolerance)) { - int availLoss, dataLoss; - - if (faultTolerance.get("max_zone_failures_without_losing_availability", availLoss) && - faultTolerance.get("max_zone_failures_without_losing_data", dataLoss)) { - - outputString += "\n Fault Tolerance - "; - - int minLoss = std::min(availLoss, dataLoss); - const char* faultDomain = machinesAreZones ? "machine" : "zone"; - outputString += format("%d %ss", minLoss, faultDomain); - - if (dataLoss > availLoss) { - outputString += format(" (%d without data loss)", dataLoss); - } - - if (dataLoss == -1) { - ASSERT_WE_THINK(availLoss == -1); - StatusObjectReader logs; - std::string epochData; - int32_t minFaultTolerance = 0; - if (statusObjCluster.has("logs")) { - for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) { - bool possiblyLosingData; - if (logEpoch.get("possibly_losing_data", possiblyLosingData) && - !possiblyLosingData) { - continue; - } - // Current epoch doesn't have an end version. - int64_t epoch, beginVersion, endVersion = invalidVersion; - bool current; - int32_t faultTolerance; - logEpoch.get("epoch", epoch); - logEpoch.get("begin_version", beginVersion); - logEpoch.get("end_version", endVersion); - logEpoch.get("current", current); - logEpoch.get("remote_log_fault_tolerance", faultTolerance); - minFaultTolerance = std::min(minFaultTolerance, faultTolerance); - std::string missing_log_interfaces; - if (logEpoch.has("log_interfaces")) { - for (StatusObjectReader logInterface : logEpoch.last().get_array()) { - bool healthy; - std::string address, id; - if (logInterface.get("healthy", healthy) && !healthy) { - logInterface.get("id", id); - logInterface.get("address", address); - missing_log_interfaces += - format("%s,%s ", - id.c_str(), - address.empty() ? "unknown" : address.c_str()); - } - } - } - epochData += format( - " %s log epoch: %lld begin: %lld end: %s, missing " - "log interfaces(id,address): %s\n", - current ? "Current" : "Old", - epoch, - beginVersion, - endVersion == invalidVersion ? "(unknown)" : format("%lld", endVersion).c_str(), - missing_log_interfaces.c_str()); - } - } - outputString += format("\n\n "); - if (minFaultTolerance < 0) { - outputString += format("Warning: the database may have data loss and availability loss. "); - } - outputString += format( - "Please restart " - "following tlog interfaces, otherwise storage servers may never be able to catch " - "up.\n") + epochData; - } - } - } - - std::string serverTime = getDateInfoString(statusObjCluster, "cluster_controller_timestamp"); - if (serverTime != "") { - outputString += "\n Server time - " + serverTime; - } - } catch (std::runtime_error&) { - outputString = outputStringCache; - outputString += "\n Unable to retrieve cluster status"; - } - - StatusObjectReader statusObjData; - statusObjCluster.get("data", statusObjData); - - // Data section - outputString += "\n\nData:"; - outputStringCache = outputString; - try { - outputString += "\n Replication health - "; - - StatusObjectReader statusObjDataState; - statusObjData.get("state", statusObjDataState); - - std::string dataState; - statusObjDataState.get("name", dataState); - - std::string description = ""; - statusObjDataState.get("description", description); - - bool healthy; - if (statusObjDataState.get("healthy", healthy) && healthy) { - outputString += "Healthy" + (description != "" ? " (" + description + ")" : ""); - } else if (dataState == "missing_data") { - outputString += "UNHEALTHY" + (description != "" ? ": " + description : ""); - } else if (dataState == "healing") { - outputString += "HEALING" + (description != "" ? ": " + description : ""); - } else if (description != "") { - outputString += description; - } else { - outputString += "unknown"; - } - - if (statusObjData.has("moving_data")) { - StatusObjectReader movingData = statusObjData.last(); - double dataInQueue, dataInFlight; - if (movingData.get("in_queue_bytes", dataInQueue) && - movingData.get("in_flight_bytes", dataInFlight)) - outputString += format("\n Moving data - %.3f GB", - ((double)dataInQueue + (double)dataInFlight) / 1e9); - } else if (dataState == "initializing") { - outputString += "\n Moving data - unknown (initializing)"; - } else { - outputString += "\n Moving data - unknown"; - } - - outputString += "\n Sum of key-value sizes - "; - - if (statusObjData.has("total_kv_size_bytes")) { - double totalDBBytes = statusObjData.last().get_int64(); - outputString += toBytesString(totalDBBytes); - } else { - outputString += "unknown"; - } - - outputString += "\n System keyspace sizes - "; - - if (statusObjData.has("system_kv_size_bytes")) { - double systemBytes = statusObjData.last().get_int64(); - outputString += toBytesString(systemBytes); - } else { - outputString += "unknown"; - } - - outputString += "\n Disk space used - "; - - if (statusObjData.has("total_disk_used_bytes")) { - double totalDiskUsed = statusObjData.last().get_int64(); - outputString += toBytesString(totalDiskUsed); - } else - outputString += "unknown"; - - } catch (std::runtime_error&) { - outputString = outputStringCache; - outputString += "\n Unable to retrieve data status"; - } - // Storage Wiggle section - StatusObjectReader storageWigglerObj; - std::string storageWigglerString; - try { - if (statusObjCluster.get("storage_wiggler", storageWigglerObj)) { - int size = 0; - if (storageWigglerObj.has("wiggle_server_addresses")) { - storageWigglerString += "\n Wiggle server addresses-"; - for (auto& v : storageWigglerObj.obj().at("wiggle_server_addresses").get_array()) { - storageWigglerString += " " + v.get_str(); - size += 1; - } - } - storageWigglerString += "\n Wiggle server count - " + std::to_string(size); - } - } catch (std::runtime_error&) { - storageWigglerString += "\n Unable to retrieve storage wiggler status"; - } - if (storageWigglerString.size()) { - outputString += "\n\nStorage wiggle:"; - outputString += storageWigglerString; - } - - // Operating space section - outputString += "\n\nOperating space:"; - std::string operatingSpaceString = ""; - try { - int64_t val; - if (statusObjData.get("least_operating_space_bytes_storage_server", val)) - operatingSpaceString += format("\n Storage server - %.1f GB free on most full server", - std::max(val / 1e9, 0.0)); - - if (statusObjData.get("least_operating_space_bytes_log_server", val)) - operatingSpaceString += format("\n Log server - %.1f GB free on most full server", - std::max(val / 1e9, 0.0)); - - } catch (std::runtime_error&) { - operatingSpaceString = ""; - } - - if (operatingSpaceString.empty()) { - operatingSpaceString += "\n Unable to retrieve operating space status"; - } - outputString += operatingSpaceString; - - // Workload section - outputString += "\n\nWorkload:"; - outputStringCache = outputString; - bool foundLogAndStorage = false; - try { - // Determine which rates are unknown - StatusObjectReader statusObjWorkload; - statusObjCluster.get("workload", statusObjWorkload); - - std::string performanceLimited = ""; - bool unknownMCT = false; - bool unknownRP = false; - - // Print performance limit details if known. - try { - StatusObjectReader limit = statusObjCluster["qos.performance_limited_by"]; - std::string name = limit["name"].get_str(); - if (name != "workload") { - std::string desc = limit["description"].get_str(); - std::string serverID; - limit.get("reason_server_id", serverID); - std::string procAddr = getProcessAddressByServerID(processesMap, serverID); - performanceLimited = format("\n Performance limited by %s: %s", - (procAddr == "unknown") - ? ("server" + (serverID == "" ? "" : (" " + serverID))).c_str() - : "process", - desc.c_str()); - if (procAddr != "unknown") - performanceLimited += format("\n Most limiting process: %s", procAddr.c_str()); - } - } catch (std::exception&) { - // If anything here throws (such as for an incompatible type) ignore it. - } - - // display the known rates - outputString += "\n Read rate - "; - outputString += getWorkloadRates(statusObjWorkload, unknownRP, "reads", "hz"); - - outputString += "\n Write rate - "; - outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "writes", "hz"); - - outputString += "\n Transactions started - "; - outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "started", "hz", true); - - outputString += "\n Transactions committed - "; - outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "committed", "hz", true); - - outputString += "\n Conflict rate - "; - outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "conflicted", "hz", true); - - outputString += unknownRP ? "" : performanceLimited; - - // display any process messages - // FIXME: Above comment is not what this code block does, it actually just looks for a specific message - // in the process map, *by description*, and adds process addresses that have it to a vector. Either - // change the comment or the code. - std::vector messagesAddrs; - for (auto proc : processesMap.obj()) { - StatusObjectReader process(proc.second); - if (process.has("roles")) { - StatusArray rolesArray = proc.second.get_obj()["roles"].get_array(); - bool storageRole = false; - bool logRole = false; - for (StatusObjectReader role : rolesArray) { - if (role["role"].get_str() == "storage") { - storageRole = true; - } else if (role["role"].get_str() == "log") { - logRole = true; - } - } - if (storageRole && logRole) { - foundLogAndStorage = true; - } - } - if (process.has("messages")) { - StatusArray processMessagesArr = process.last().get_array(); - if (processMessagesArr.size()) { - for (StatusObjectReader msg : processMessagesArr) { - std::string desc; - std::string addr; - if (msg.get("description", desc) && desc == "Unable to update cluster file." && - process.get("address", addr)) { - messagesAddrs.push_back(addr); - } - } - } - } - } - if (messagesAddrs.size()) { - outputString += format("\n\n%d FoundationDB processes reported unable to update cluster file:", - messagesAddrs.size()); - for (auto msg : messagesAddrs) { - outputString += "\n " + msg; - } - } - } catch (std::runtime_error&) { - outputString = outputStringCache; - outputString += "\n Unable to retrieve workload status"; - } - - // Backup and DR section - outputString += "\n\nBackup and DR:"; - - std::map backupTags; - getBackupDRTags(statusObjCluster, "backup", backupTags); - - std::map drPrimaryTags; - getBackupDRTags(statusObjCluster, "dr_backup", drPrimaryTags); - - std::map drSecondaryTags; - getBackupDRTags(statusObjCluster, "dr_backup_dest", drSecondaryTags); - - outputString += format("\n Running backups - %d", backupTags.size()); - outputString += format("\n Running DRs - "); - - if (drPrimaryTags.size() == 0 && drSecondaryTags.size() == 0) { - outputString += format("%d", 0); - } else { - if (drPrimaryTags.size() > 0) { - outputString += format("%d as primary", drPrimaryTags.size()); - if (drSecondaryTags.size() > 0) { - outputString += ", "; - } - } - if (drSecondaryTags.size() > 0) { - outputString += format("%d as secondary", drSecondaryTags.size()); - } - } - - // status details - if (level == StatusClient::DETAILED) { - outputString += logBackupDR("Running backup tags", backupTags); - outputString += logBackupDR("Running DR tags (as primary)", drPrimaryTags); - outputString += logBackupDR("Running DR tags (as secondary)", drSecondaryTags); - - outputString += "\n\nProcess performance details:"; - outputStringCache = outputString; - try { - // constructs process performance details output - std::map workerDetails; - for (auto proc : processesMap.obj()) { - StatusObjectReader procObj(proc.second); - std::string address; - procObj.get("address", address); - - std::string line; - - NetworkAddress parsedAddress; - try { - parsedAddress = NetworkAddress::parse(address); - } catch (Error&) { - // Groups all invalid IP address/port pair in the end of this detail group. - line = format(" %-22s (invalid IP address or port)", address.c_str()); - IPAddress::IPAddressStore maxIp; - for (int i = 0; i < maxIp.size(); ++i) { - maxIp[i] = std::numeric_limits::type>::max(); - } - std::string& lastline = - workerDetails[NetworkAddress(IPAddress(maxIp), std::numeric_limits::max())]; - if (!lastline.empty()) - lastline.append("\n"); - lastline += line; - continue; - } - - try { - double tx = -1, rx = -1, mCPUUtil = -1; - int64_t processRSS; - - // Get the machine for this process - // StatusObjectReader mach = machinesMap[procObj["machine_id"].get_str()]; - StatusObjectReader mach; - if (machinesMap.get(procObj["machine_id"].get_str(), mach, false)) { - StatusObjectReader machCPU; - if (mach.get("cpu", machCPU)) { - - machCPU.get("logical_core_utilization", mCPUUtil); - - StatusObjectReader network; - if (mach.get("network", network)) { - network.get("megabits_sent.hz", tx); - network.get("megabits_received.hz", rx); - } - } - } - - procObj.get("memory.rss_bytes", processRSS); - - StatusObjectReader procCPUObj; - procObj.get("cpu", procCPUObj); - - line = format(" %-22s (", address.c_str()); - - double usageCores; - if (procCPUObj.get("usage_cores", usageCores)) - line += format("%3.0f%% cpu;", usageCores * 100); - - line += mCPUUtil != -1 ? format("%3.0f%% machine;", mCPUUtil * 100) : ""; - line += std::min(tx, rx) != -1 ? format("%6.3f Gbps;", std::max(tx, rx) / 1000.0) : ""; - - double diskBusy; - if (procObj.get("disk.busy", diskBusy)) - line += format("%3.0f%% disk IO;", 100.0 * diskBusy); - - line += processRSS != -1 ? format("%4.1f GB", processRSS / (1024.0 * 1024 * 1024)) : ""; - - double availableBytes; - if (procObj.get("memory.available_bytes", availableBytes)) - line += format(" / %3.1f GB RAM )", availableBytes / (1024.0 * 1024 * 1024)); - else - line += " )"; - - if (procObj.has("messages")) { - for (StatusObjectReader message : procObj.last().get_array()) { - std::string desc; - if (message.get("description", desc)) { - if (message.has("type")) { - line += "\n Last logged error: " + desc; - } else { - line += "\n " + desc; - } - } - } - } - - workerDetails[parsedAddress] = line; - } - - catch (std::runtime_error&) { - std::string noMetrics = format(" %-22s (no metrics available)", address.c_str()); - workerDetails[parsedAddress] = noMetrics; - } - } - for (auto w : workerDetails) - outputString += "\n" + format("%s", w.second.c_str()); - } catch (std::runtime_error&) { - outputString = outputStringCache; - outputString += "\n Unable to retrieve process performance details"; - } - - if (!printedCoordinators) { - printedCoordinators = true; - outputString += "\n\nCoordination servers:"; - outputString += getCoordinatorsInfoString(statusObj); - } - } - - // client time - std::string clientTime = getDateInfoString(statusObjClient, "timestamp"); - if (clientTime != "") { - outputString += "\n\nClient time: " + clientTime; - } - - if (processesMap.obj().size() > 1 && isOldMemory) { - outputString += "\n\nWARNING: type `configure memory' to switch to a safer method of persisting data " - "on the transaction logs."; - } - if (processesMap.obj().size() > 9 && foundLogAndStorage) { - outputString += - "\n\nWARNING: A single process is both a transaction log and a storage server.\n For best " - "performance use dedicated disks for the transaction logs by setting process classes."; - } - - if (statusObjCluster.has("data_distribution_disabled")) { - outputString += "\n\nWARNING: Data distribution is off."; - } else { - if (statusObjCluster.has("data_distribution_disabled_for_ss_failures")) { - outputString += "\n\nWARNING: Data distribution is currently turned on but disabled for all " - "storage server failures."; - } - if (statusObjCluster.has("data_distribution_disabled_for_rebalance")) { - outputString += "\n\nWARNING: Data distribution is currently turned on but one or both of shard " - "size and read-load based balancing are disabled."; - // data_distribution_disabled_hex - if (statusObjCluster.has("data_distribution_disabled_hex")) { - outputString += " Ignore code: " + statusObjCluster["data_distribution_disabled_hex"].get_str(); - } - } - } - - printf("%s\n", outputString.c_str()); - } - - // status minimal - else if (level == StatusClient::MINIMAL) { - // Checking for field existence is not necessary here because if a field is missing there is no additional - // information that we would be able to display if we continued execution. Instead, any missing fields will - // throw and the catch will display the proper message. - try { - // If any of these throw, can't get status because the result makes no sense. - StatusObjectReader statusObjClient = statusObj["client"].get_obj(); - StatusObjectReader statusObjClientDatabaseStatus = statusObjClient["database_status"].get_obj(); - - bool available = statusObjClientDatabaseStatus["available"].get_bool(); - - // Database unavailable - if (!available) { - printf("%s", "The database is unavailable; type `status' for more information.\n"); - } else { - try { - bool healthy = statusObjClientDatabaseStatus["healthy"].get_bool(); - - // Database available without issues - if (healthy) { - if (displayDatabaseAvailable) { - printf("The database is available.\n"); - } - } else { // Database running but with issues - printf("The database is available, but has issues (type 'status' for more information).\n"); - } - } catch (std::runtime_error&) { - printf("The database is available, but has issues (type 'status' for more information).\n"); - } - } - - bool upToDate; - if (!statusObjClient.get("cluster_file.up_to_date", upToDate) || !upToDate) { - fprintf(stderr, - "WARNING: The cluster file is not up to date. Type 'status' for more information.\n"); - } - } catch (std::runtime_error&) { - printf("Unable to determine database state, type 'status' for more information.\n"); - } - - } - - // status JSON - else if (level == StatusClient::JSON) { - printf("%s\n", - json_spirit::write_string(json_spirit::mValue(statusObj.obj()), - json_spirit::Output_options::pretty_print) - .c_str()); - } - } catch (Error&) { - if (hideErrorMessages) - return; - if (level == StatusClient::MINIMAL) { - printf("Unable to determine database state, type 'status' for more information.\n"); - } else if (level == StatusClient::JSON) { - printf("Could not retrieve status json.\n\n"); - } else { - printf("Could not retrieve status, type 'status json' for more information.\n"); - } - } -} - -// "db" is the handler to the multiversion database -// localDb is the native Database object -// localDb is rarely needed except the "db" has not established a connection to the cluster where the operation will -// return Never as we expect status command to always return, we use "localDb" to return the default result -ACTOR Future statusCommandActor(Reference db, - Database localDb, - std::vector tokens, - bool isExecMode) { - - state StatusClient::StatusLevel level; - if (tokens.size() == 1) - level = StatusClient::NORMAL; - else if (tokens.size() == 2 && tokencmp(tokens[1], "details")) - level = StatusClient::DETAILED; - else if (tokens.size() == 2 && tokencmp(tokens[1], "minimal")) - level = StatusClient::MINIMAL; - else if (tokens.size() == 2 && tokencmp(tokens[1], "json")) - level = StatusClient::JSON; - else { - printUsage(tokens[0]); - return false; - } - - state StatusObject s; - state Reference tr = db->createTransaction(); - if (!tr->isValid()) { - StatusObject _s = wait(StatusClient::statusFetcher(localDb)); - s = _s; - } else { - state ThreadFuture> statusValueF = tr->get("\xff\xff/status/json"_sr); - Optional statusValue = wait(safeThreadFutureToFuture(statusValueF)); - if (!statusValue.present()) { - fprintf(stderr, "ERROR: Failed to get status json from the cluster\n"); - } - json_spirit::mValue mv; - json_spirit::read_string(statusValue.get().toString(), mv); - s = StatusObject(mv.get_obj()); - } - - if (!isExecMode) - printf("\n"); - printStatus(s, level); - if (!isExecMode) - printf("\n"); - return true; -} - -void statusGenerator(const char* text, - const char* line, - std::vector& lc, - std::vector const& tokens) { - if (tokens.size() == 1) { - const char* opts[] = { "minimal", "details", "json", nullptr }; - arrayGenerator(text, line, opts, lc); - } -} - -CommandFactory statusFactory( - "status", - CommandHelp("status [minimal|details|json]", - "get the status of a FoundationDB cluster", - "If the cluster is down, this command will print a diagnostic which may be useful in figuring out " - "what is wrong. If the cluster is running, this command will print cluster " - "statistics.\n\nSpecifying `minimal' will provide a minimal description of the status of your " - "database.\n\nSpecifying `details' will provide load information for individual " - "workers.\n\nSpecifying `json' will provide status information in a machine readable JSON format."), - &statusGenerator); -} // namespace fdb_cli diff --git a/fdbcli/StatusCommand.cpp b/fdbcli/StatusCommand.cpp new file mode 100644 index 00000000000..a709ef9e327 --- /dev/null +++ b/fdbcli/StatusCommand.cpp @@ -0,0 +1,1309 @@ +/* + * StatusCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" +#include "fmt/chrono.h" +#include "fmt/core.h" +#include "fmt/format.h" +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/StatusClient.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +std::string getCoordinatorsInfoString(StatusObjectReader statusObj) { + std::string outputString; + try { + StatusArray coordinatorsArr = statusObj["client.coordinators.coordinators"].get_array(); + for (StatusObjectReader coor : coordinatorsArr) + outputString += format("\n %s (%s)", + coor["address"].get_str().c_str(), + coor["reachable"].get_bool() ? "reachable" : "unreachable"); + } catch (std::runtime_error&) { + outputString = "\n Unable to retrieve list of coordination servers"; + } + + return outputString; +} + +std::string lineWrap(const char* text, int col) { + const char* iter = text; + const char* start = text; + const char* space = nullptr; + std::string out = ""; + do { + iter++; + if (*iter == '\n' || *iter == ' ' || *iter == '\0') + space = iter; + if (*iter == '\n' || *iter == '\0' || (iter - start == col)) { + if (!space) + space = iter; + out += format("%.*s\n", (int)(space - start), start); + start = space; + if (*start == ' ' /* || *start == '\n'*/) + start++; + space = nullptr; + } + } while (*iter); + return out; +} + +std::pair getNumOfNonExcludedProcessAndZones(StatusObjectReader statusObjCluster) { + StatusObjectReader processesMap; + std::set zones; + int numOfNonExcludedProcesses = 0; + if (statusObjCluster.get("processes", processesMap)) { + for (auto proc : processesMap.obj()) { + StatusObjectReader process(proc.second); + if (process.has("excluded") && process.last().get_bool()) + continue; + numOfNonExcludedProcesses++; + std::string zoneId; + if (process.get("locality.zoneid", zoneId)) { + zones.insert(zoneId); + } + } + } + return { numOfNonExcludedProcesses, zones.size() }; +} + +int getNumofNonExcludedMachines(StatusObjectReader statusObjCluster) { + StatusObjectReader machineMap; + int numOfNonExcludedMachines = 0; + if (statusObjCluster.get("machines", machineMap)) { + for (auto mach : machineMap.obj()) { + StatusObjectReader machine(mach.second); + if (machine.has("excluded") && !machine.last().get_bool()) + numOfNonExcludedMachines++; + } + } + return numOfNonExcludedMachines; +} + +bool logEpochsMayBeLosingData(StatusObjectReader statusObjCluster) { + if (!statusObjCluster.has("logs")) { + return true; + } + + bool sawLogEpoch = false; + + for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) { + sawLogEpoch = true; + bool possiblyLosingData = true; + int satFaultTolerance = -1; + + if (!logEpoch.get("possibly_losing_data", possiblyLosingData) || + (possiblyLosingData && + (!logEpoch.get("satellite_log_fault_tolerance", satFaultTolerance) || satFaultTolerance < 0))) { + return true; + } + } + + return !sawLogEpoch; +} + +std::string getDateInfoString(StatusObjectReader statusObj, std::string key) { + time_t curTime; + if (!statusObj.has(key)) { + return ""; + } + curTime = statusObj.last().get_int64(); + char buffer[128]; + struct tm* timeinfo; + timeinfo = localtime(&curTime); + strftime(buffer, 128, "%m/%d/%y %H:%M:%S", timeinfo); + return std::string(buffer); +} + +std::string getProcessAddressByServerID(StatusObjectReader processesMap, std::string serverID) { + if (serverID.empty()) + return "unknown"; + + for (auto proc : processesMap.obj()) { + try { + StatusArray rolesArray = proc.second.get_obj()["roles"].get_array(); + for (StatusObjectReader role : rolesArray) { + if (role["id"].get_str().find(serverID) == 0) { + // If this next line throws, then we found the serverID but the role has no address, so the role is + // skipped. + return proc.second.get_obj()["address"].get_str(); + } + } + } catch (std::exception&) { + // If an entry in the process map is badly formed then something will throw. Since we are + // looking for a positive match, just ignore any read exceptions and move on to the next proc + } + } + return "unknown"; +} + +std::string getWorkloadRates(StatusObjectReader statusObj, + bool unknown, + std::string first, + std::string second, + bool transactionSection = false) { + // Re-point statusObj at either the transactions sub-doc or the operations sub-doc depending on transactionSection + // flag + if (transactionSection) { + if (!statusObj.get("transactions", statusObj)) + return "unknown"; + } else { + if (!statusObj.get("operations", statusObj)) + return "unknown"; + } + + std::string path = first + "." + second; + double value; + if (!unknown && statusObj.get(path, value)) { + return format("%d Hz", (int)round(value)); + } + return "unknown"; +} + +void getBackupDRTags(StatusObjectReader& statusObjCluster, + const char* context, + std::map& tagMap) { + std::string path = format("layers.%s.tags", context); + StatusObjectReader tags; + if (statusObjCluster.tryGet(path, tags)) { + for (auto itr : tags.obj()) { + JSONDoc tag(itr.second); + bool running = false; + tag.tryGet("running_backup", running); + if (running) { + std::string uid; + if (tag.tryGet("mutation_stream_id", uid)) { + tagMap[itr.first] = uid; + } else { + tagMap[itr.first] = ""; + } + } + } + } +} + +std::string logBackupDR(const char* context, std::map const& tagMap) { + std::string outputString = ""; + if (!tagMap.empty()) { + outputString += format("\n\n%s:", context); + for (const auto& itr : tagMap) { + outputString += format("\n %-22s", itr.first.c_str()); + if (!itr.second.empty()) { + outputString += format(" - %s", itr.second.c_str()); + } + } + } + + return outputString; +} + +} // namespace + +namespace fdb_cli { + +std::string toBytesString(double bytes) { + if (bytes >= 1e12) { + return format("%.3f TB", (bytes / 1e12)); + } else if (bytes >= 1e9) { + return format("%.3f GB", (bytes / 1e9)); + } else { + // no decimal points for MB + return format("%d MB", (int)round(bytes / 1e6)); + } +} + +void printStatus(StatusObjectReader statusObj, + StatusClient::StatusLevel level, + bool displayDatabaseAvailable, + bool hideErrorMessages) { + if (FlowTransport::transport().incompatibleOutgoingConnectionsPresent()) { + fprintf( + stderr, + "WARNING: One or more of the processes in the cluster is incompatible with this version of fdbcli.\n\n"); + } + + try { + bool printedCoordinators = false; + + // status or status details + if (level == StatusClient::NORMAL || level == StatusClient::DETAILED) { + + StatusObjectReader statusObjClient; + statusObj.get("client", statusObjClient); + + // The way the output string is assembled is to add new line character before addition to the string rather + // than after + std::string outputString = ""; + std::string clusterFilePath; + if (statusObjClient.get("cluster_file.path", clusterFilePath)) + outputString = format("Using cluster file `%s'.\n", clusterFilePath.c_str()); + else + outputString = "Using unknown cluster file.\n"; + + StatusObjectReader statusObjCoordinators; + StatusArray coordinatorsArr; + + if (statusObjClient.get("coordinators", statusObjCoordinators)) { + // Look for a second "coordinators", under the first one. + if (statusObjCoordinators.has("coordinators")) + coordinatorsArr = statusObjCoordinators.last().get_array(); + } + + // Check if any coordination servers are unreachable + bool quorum_reachable; + if (statusObjCoordinators.get("quorum_reachable", quorum_reachable) && !quorum_reachable) { + outputString += "\nCould not communicate with a quorum of coordination servers:"; + outputString += getCoordinatorsInfoString(statusObj); + + printf("%s\n", outputString.c_str()); + return; + } else { + for (StatusObjectReader coor : coordinatorsArr) { + bool reachable; + if (coor.get("reachable", reachable) && !reachable) { + outputString += "\nCould not communicate with all of the coordination servers." + "\n The database will remain operational as long as we" + "\n can connect to a quorum of servers, however the fault" + "\n tolerance of the system is reduced as long as the" + "\n servers remain disconnected.\n"; + outputString += getCoordinatorsInfoString(statusObj); + outputString += "\n"; + printedCoordinators = true; + break; + } + } + } + + // print any client messages + if (statusObjClient.has("messages")) { + for (StatusObjectReader message : statusObjClient.last().get_array()) { + std::string desc; + if (message.get("description", desc)) + outputString += "\n" + lineWrap(desc.c_str(), 80); + } + } + + bool fatalRecoveryState = false; + StatusObjectReader statusObjCluster; + try { + if (statusObj.get("cluster", statusObjCluster)) { + + StatusObjectReader recoveryState; + if (statusObjCluster.get("recovery_state", recoveryState)) { + std::string name; + std::string description; + if (recoveryState.get("name", name) && recoveryState.get("description", description) && + name != "accepting_commits" && name != "all_logs_recruited" && + name != "storage_recovered" && name != "fully_recovered") { + fatalRecoveryState = true; + + if (name == "recruiting_transaction_servers") { + description += + format("\nNeed at least %d log servers across unique zones, %d commit proxies, " + "%d GRV proxies and %d resolvers.", + recoveryState["required_logs"].get_int(), + recoveryState["required_commit_proxies"].get_int(), + recoveryState["required_grv_proxies"].get_int(), + recoveryState["required_resolvers"].get_int()); + if (statusObjCluster.has("machines") && statusObjCluster.has("processes")) { + auto numOfNonExcludedProcessesAndZones = + getNumOfNonExcludedProcessAndZones(statusObjCluster); + description += + format("\nHave %d non-excluded processes on %d machines across %d zones.", + numOfNonExcludedProcessesAndZones.first, + getNumofNonExcludedMachines(statusObjCluster), + numOfNonExcludedProcessesAndZones.second); + } + } else if (name == "locking_old_transaction_servers" && + !recoveryState["missing_logs"].get_str().empty()) { + description += format("\nNeed one or more of the following log servers: %s", + recoveryState["missing_logs"].get_str().c_str()); + } + description = lineWrap(description.c_str(), 80); + if (!printedCoordinators && + (name == "reading_coordinated_state" || name == "locking_coordinated_state" || + name == "configuration_never_created" || name == "writing_coordinated_state")) { + description += getCoordinatorsInfoString(statusObj); + description += "\n"; + printedCoordinators = true; + } + + outputString += "\n" + description; + } + } + } + } catch (std::runtime_error&) { + } + + // Check if cluster controllable is reachable + try { + // print any cluster messages + if (statusObjCluster.has("messages") && !statusObjCluster.last().get_array().empty()) { + + // any messages we don't want to display + std::set skipMsgs = { "unreachable_process", "" }; + if (fatalRecoveryState) { + skipMsgs.insert("status_incomplete"); + skipMsgs.insert("unreadable_configuration"); + skipMsgs.insert("immediate_priority_transaction_start_probe_timeout"); + skipMsgs.insert("batch_priority_transaction_start_probe_timeout"); + skipMsgs.insert("transaction_start_probe_timeout"); + skipMsgs.insert("read_probe_timeout"); + skipMsgs.insert("commit_probe_timeout"); + } + + for (StatusObjectReader msgObj : statusObjCluster.last().get_array()) { + std::string messageName; + if (!msgObj.get("name", messageName)) { + continue; + } + if (skipMsgs.contains(messageName)) { + continue; + } else if (messageName == "client_issues") { + if (msgObj.has("issues")) { + for (StatusObjectReader issue : msgObj["issues"].get_array()) { + std::string issueName; + if (!issue.get("name", issueName)) { + continue; + } + + std::string description; + if (!issue.get("description", description)) { + description = issueName; + } + + std::string countStr; + StatusArray addresses; + if (!issue.has("addresses")) { + countStr = "Some client(s)"; + } else { + addresses = issue["addresses"].get_array(); + countStr = format("%d client(s)", addresses.size()); + } + outputString += + format("\n%s reported: %s\n", countStr.c_str(), description.c_str()); + + if (level == StatusClient::StatusLevel::DETAILED) { + for (int i = 0; i < addresses.size() && i < 4; ++i) { + outputString += format(" %s\n", addresses[i].get_str().c_str()); + } + if (addresses.size() > 4) { + outputString += " ...\n"; + } + } + } + } + } else { + if (msgObj.has("description")) + outputString += "\n" + lineWrap(msgObj.last().get_str().c_str(), 80); + } + } + } + } catch (std::runtime_error&) { + } + + if (fatalRecoveryState) { + printf("%s", outputString.c_str()); + return; + } + + StatusObjectReader statusObjConfig; + StatusArray excludedServersArr; + Optional activePrimaryDC; + + if (statusObjCluster.has("active_primary_dc")) { + activePrimaryDC = statusObjCluster["active_primary_dc"].get_str(); + } + if (statusObjCluster.get("configuration", statusObjConfig)) { + if (statusObjConfig.has("excluded_servers")) + excludedServersArr = statusObjConfig.last().get_array(); + } + + // If there is a configuration message then there is no configuration information to display + outputString += "\nConfiguration:"; + std::string outputStringCache = outputString; + bool isOldMemory = false; + try { + // Configuration section + // FIXME: Should we suppress this if there are cluster messages implying that the database has no + // configuration? + + outputString += "\n Redundancy mode - "; + std::string strVal; + + if (statusObjConfig.get("redundancy_mode", strVal)) { + outputString += strVal; + } else + outputString += "unknown"; + + outputString += "\n Storage engine - "; + if (statusObjConfig.get("storage_engine", strVal)) { + if (strVal == "memory-1") { + isOldMemory = true; + } + outputString += strVal; + } else + outputString += "unknown"; + + outputString += + "\n Log engine - " + (statusObjConfig.get("log_engine", strVal) ? strVal : "unknown"); + + int intVal = 0; + + outputString += "\n Coordinators - "; + if (statusObjConfig.get("coordinators_count", intVal)) { + outputString += std::to_string(intVal); + } else + outputString += "unknown"; + + if (!excludedServersArr.empty()) { + outputString += format("\n Exclusions - %d (type `exclude' for details)", + excludedServersArr.size()); + } + + if (statusObjConfig.get("commit_proxies", intVal)) + outputString += format("\n Desired Commit Proxies - %d", intVal); + + if (statusObjConfig.get("grv_proxies", intVal)) + outputString += format("\n Desired GRV Proxies - %d", intVal); + + if (statusObjConfig.get("resolvers", intVal)) + outputString += format("\n Desired Resolvers - %d", intVal); + + if (statusObjConfig.get("logs", intVal)) + outputString += format("\n Desired Logs - %d", intVal); + + if (statusObjConfig.get("remote_logs", intVal)) + outputString += format("\n Desired Remote Logs - %d", intVal); + + if (statusObjConfig.get("log_routers", intVal)) + outputString += format("\n Desired Log Routers - %d", intVal); + + if (statusObjConfig.get("tss_count", intVal) && intVal > 0) { + int activeTss = 0; + if (statusObjCluster.has("active_tss_count")) { + statusObjCluster.get("active_tss_count", activeTss); + } + outputString += format("\n TSS - %d/%d", activeTss, intVal); + + if (statusObjConfig.get("tss_storage_engine", strVal)) + outputString += format("\n TSS Storage Engine - %s", strVal.c_str()); + } + + outputString += "\n Usable Regions - "; + if (statusObjConfig.get("usable_regions", intVal)) { + outputString += std::to_string(intVal); + } else { + outputString += "unknown"; + } + + StatusArray regions; + if (statusObjConfig.has("regions")) { + outputString += "\n Regions: "; + regions = statusObjConfig["regions"].get_array(); + for (StatusObjectReader region : regions) { + bool isPrimary = false; + std::vector regionSatelliteDCs; + std::string regionDC; + for (StatusObjectReader dc : region["datacenters"].get_array()) { + if (!dc.has("satellite")) { + regionDC = dc["id"].get_str(); + if (activePrimaryDC.present() && dc["id"].get_str() == activePrimaryDC.get()) { + isPrimary = true; + } + } else if (dc["satellite"].get_int() == 1) { + regionSatelliteDCs.push_back(dc["id"].get_str()); + } + } + if (activePrimaryDC.present()) { + if (isPrimary) { + outputString += "\n Primary -"; + } else { + outputString += "\n Remote -"; + } + } else { + outputString += "\n Region -"; + } + outputString += format("\n Datacenter - %s", regionDC.c_str()); + if (!regionSatelliteDCs.empty()) { + outputString += "\n Satellite datacenters - "; + for (int i = 0; i < regionSatelliteDCs.size(); i++) { + if (i != regionSatelliteDCs.size() - 1) { + outputString += format("%s, ", regionSatelliteDCs[i].c_str()); + } else { + outputString += format("%s", regionSatelliteDCs[i].c_str()); + } + } + } + isPrimary = false; + if (region.get("satellite_redundancy_mode", strVal)) { + outputString += format("\n Satellite Redundancy Mode - %s", strVal.c_str()); + } + if (region.get("satellite_anti_quorum", intVal)) { + outputString += format("\n Satellite Anti Quorum - %d", intVal); + } + if (region.get("satellite_logs", intVal)) { + outputString += format("\n Satellite Logs - %d", intVal); + } + if (region.get("satellite_log_policy", strVal)) { + outputString += format("\n Satellite Log Policy - %s", strVal.c_str()); + } + if (region.get("satellite_log_replicas", intVal)) { + outputString += format("\n Satellite Log Replicas - %d", intVal); + } + if (region.get("satellite_usable_dcs", intVal)) { + outputString += format("\n Satellite Usable DCs - %d", intVal); + } + } + } + } catch (std::runtime_error&) { + outputString = outputStringCache; + outputString += "\n Unable to retrieve configuration status"; + } + + // Cluster section + outputString += "\n\nCluster:"; + StatusObjectReader processesMap; + StatusObjectReader machinesMap; + + outputStringCache = outputString; + + bool machinesAreZones = true; + std::map zones; + try { + outputString += "\n FoundationDB processes - "; + if (statusObjCluster.get("processes", processesMap)) { + + outputString += format("%d", processesMap.obj().size()); + + int errors = 0; + int processExclusions = 0; + for (auto p : processesMap.obj()) { + StatusObjectReader process(p.second); + bool excluded = process.has("excluded") && process.last().get_bool(); + if (excluded) { + processExclusions++; + } + if (process.has("messages") && !process.last().get_array().empty()) { + errors++; + } + + std::string zoneId; + if (process.get("locality.zoneid", zoneId)) { + std::string machineId; + if (!process.get("locality.machineid", machineId) || machineId != zoneId) { + machinesAreZones = false; + } + int& nonExcluded = zones[zoneId]; + if (!excluded) { + nonExcluded = 1; + } + } + } + + if (errors > 0 || processExclusions) { + outputString += format(" (less %d excluded; %d with errors)", processExclusions, errors); + } + + } else + outputString += "unknown"; + + if (!zones.empty()) { + outputString += format("\n Zones - %d", zones.size()); + int zoneExclusions = 0; + for (const auto& itr : zones) { + if (itr.second == 0) { + ++zoneExclusions; + } + } + if (zoneExclusions > 0) { + outputString += format(" (less %d excluded)", zoneExclusions); + } + } else { + outputString += "\n Zones - unknown"; + } + + outputString += "\n Machines - "; + if (statusObjCluster.get("machines", machinesMap)) { + outputString += format("%d", machinesMap.obj().size()); + + int machineExclusions = 0; + for (auto mach : machinesMap.obj()) { + StatusObjectReader machine(mach.second); + if (machine.has("excluded") && machine.last().get_bool()) + machineExclusions++; + } + + if (machineExclusions) { + outputString += format(" (less %d excluded)", machineExclusions); + } + + int64_t minMemoryAvailable = std::numeric_limits::max(); + for (auto proc : processesMap.obj()) { + StatusObjectReader process(proc.second); + int64_t availBytes; + if (process.get("memory.available_bytes", availBytes)) { + minMemoryAvailable = std::min(minMemoryAvailable, availBytes); + } + } + + if (minMemoryAvailable < std::numeric_limits::max()) { + double worstServerGb = minMemoryAvailable / (1024.0 * 1024 * 1024); + outputString += "\n Memory availability - "; + outputString += format("%.1f GB per process on machine with least available", worstServerGb); + outputString += minMemoryAvailable < 4294967296 + ? "\n >>>>> (WARNING: 4.0 GB recommended) <<<<<" + : ""; + } + + double retransCount = 0; + for (auto mach : machinesMap.obj()) { + StatusObjectReader machine(mach.second); + double hz; + if (machine.get("network.tcp_segments_retransmitted.hz", hz)) + retransCount += hz; + } + + if (retransCount > 0) { + outputString += format("\n Retransmissions rate - %d Hz", (int)round(retransCount)); + } + } else + outputString += "\n Machines - unknown"; + + StatusObjectReader faultTolerance; + if (statusObjCluster.get("fault_tolerance", faultTolerance)) { + int availLoss, dataLoss; + + if (faultTolerance.get("max_zone_failures_without_losing_availability", availLoss) && + faultTolerance.get("max_zone_failures_without_losing_data", dataLoss)) { + + outputString += "\n Fault Tolerance - "; + + int minLoss = std::min(availLoss, dataLoss); + const char* faultDomain = machinesAreZones ? "machine" : "zone"; + outputString += format("%d %ss", minLoss, faultDomain); + + if (dataLoss > availLoss) { + outputString += format(" (%d without data loss)", dataLoss); + } + + if (dataLoss == -1) { + ASSERT_WE_THINK(availLoss == -1); + const bool possiblyLosingData = logEpochsMayBeLosingData(statusObjCluster); + if (possiblyLosingData) { + outputString += format( + "\n\n Warning: the database may have data loss and availability loss. Please " + "restart following tlog interfaces, otherwise storage servers may never be able " + "to catch up.\n"); + } else { + outputString += format( + "\n\n Warning: the database may have availability loss. The current log state " + "does not indicate data loss.\n"); + } + if (statusObjCluster.has("logs")) { + for (StatusObjectReader logEpoch : statusObjCluster.last().get_array()) { + bool logEpochPossiblyLosingData; + if (logEpoch.get("possibly_losing_data", logEpochPossiblyLosingData) && + !logEpochPossiblyLosingData) { + continue; + } + // Current epoch doesn't have an end version. + int64_t epoch, beginVersion, endVersion = invalidVersion; + bool current; + logEpoch.get("epoch", epoch); + logEpoch.get("begin_version", beginVersion); + logEpoch.get("end_version", endVersion); + logEpoch.get("current", current); + std::string missing_log_interfaces; + if (logEpoch.has("log_interfaces")) { + for (StatusObjectReader logInterface : logEpoch.last().get_array()) { + bool healthy; + std::string address, id; + if (logInterface.get("healthy", healthy) && !healthy) { + logInterface.get("id", id); + logInterface.get("address", address); + missing_log_interfaces += + format("%s,%s ", + id.c_str(), + address.empty() ? "unknown" : address.c_str()); + } + } + } + outputString += format( + " %s log epoch: %lld begin: %lld end: %s, missing " + "log interfaces(id,address): %s\n", + current ? "Current" : "Old", + epoch, + beginVersion, + endVersion == invalidVersion ? "(unknown)" : format("%lld", endVersion).c_str(), + missing_log_interfaces.c_str()); + } + } + } + } + } + + std::string serverTime = getDateInfoString(statusObjCluster, "cluster_controller_timestamp"); + if (!serverTime.empty()) { + outputString += "\n Server time - " + serverTime; + } + } catch (std::runtime_error&) { + outputString = outputStringCache; + outputString += "\n Unable to retrieve cluster status"; + } + + StatusObjectReader statusObjData; + statusObjCluster.get("data", statusObjData); + + // Data section + outputString += "\n\nData:"; + outputStringCache = outputString; + try { + outputString += "\n Replication health - "; + + StatusObjectReader statusObjDataState; + statusObjData.get("state", statusObjDataState); + + std::string dataState; + statusObjDataState.get("name", dataState); + + std::string description = ""; + statusObjDataState.get("description", description); + + bool healthy; + if (statusObjDataState.get("healthy", healthy) && healthy) { + outputString += "Healthy" + (!description.empty() ? " (" + description + ")" : ""); + } else if (dataState == "missing_data") { + outputString += "UNHEALTHY" + (!description.empty() ? ": " + description : ""); + } else if (dataState == "healing") { + outputString += "HEALING" + (!description.empty() ? ": " + description : ""); + } else if (!description.empty()) { + outputString += description; + } else { + outputString += "unknown"; + } + + if (statusObjData.has("moving_data")) { + StatusObjectReader movingData = statusObjData.last(); + double dataInQueue, dataInFlight; + if (movingData.get("in_queue_bytes", dataInQueue) && + movingData.get("in_flight_bytes", dataInFlight)) + outputString += format("\n Moving data - %.3f GB", + ((double)dataInQueue + (double)dataInFlight) / 1e9); + } else if (dataState == "initializing") { + outputString += "\n Moving data - unknown (initializing)"; + } else { + outputString += "\n Moving data - unknown"; + } + + outputString += "\n Sum of key-value sizes - "; + + if (statusObjData.has("total_kv_size_bytes")) { + double totalDBBytes = statusObjData.last().get_int64(); + outputString += toBytesString(totalDBBytes); + } else { + outputString += "unknown"; + } + + outputString += "\n System keyspace sizes - "; + + if (statusObjData.has("system_kv_size_bytes")) { + double systemBytes = statusObjData.last().get_int64(); + outputString += toBytesString(systemBytes); + } else { + outputString += "unknown"; + } + + outputString += "\n Disk space used - "; + + if (statusObjData.has("total_disk_used_bytes")) { + double totalDiskUsed = statusObjData.last().get_int64(); + outputString += toBytesString(totalDiskUsed); + } else + outputString += "unknown"; + + } catch (std::runtime_error&) { + outputString = outputStringCache; + outputString += "\n Unable to retrieve data status"; + } + // Storage Wiggle section + StatusObjectReader storageWigglerObj; + std::string storageWigglerString; + try { + if (statusObjCluster.get("storage_wiggler", storageWigglerObj)) { + int size = 0; + if (storageWigglerObj.has("wiggle_server_addresses")) { + storageWigglerString += "\n Wiggle server addresses-"; + for (auto& v : storageWigglerObj.obj().at("wiggle_server_addresses").get_array()) { + storageWigglerString += " " + v.get_str(); + size += 1; + } + } + storageWigglerString += "\n Wiggle server count - " + std::to_string(size); + } + } catch (std::runtime_error&) { + storageWigglerString += "\n Unable to retrieve storage wiggler status"; + } + if (!storageWigglerString.empty()) { + outputString += "\n\nStorage wiggle:"; + outputString += storageWigglerString; + } + + // Operating space section + outputString += "\n\nOperating space:"; + std::string operatingSpaceString = ""; + try { + int64_t val; + if (statusObjData.get("least_operating_space_bytes_storage_server", val)) + operatingSpaceString += format("\n Storage server - %.1f GB free on most full server", + std::max(val / 1e9, 0.0)); + + if (statusObjData.get("least_operating_space_bytes_log_server", val)) + operatingSpaceString += format("\n Log server - %.1f GB free on most full server", + std::max(val / 1e9, 0.0)); + + } catch (std::runtime_error&) { + operatingSpaceString = ""; + } + + if (operatingSpaceString.empty()) { + operatingSpaceString += "\n Unable to retrieve operating space status"; + } + outputString += operatingSpaceString; + + // Workload section + outputString += "\n\nWorkload:"; + outputStringCache = outputString; + bool foundLogAndStorage = false; + try { + // Determine which rates are unknown + StatusObjectReader statusObjWorkload; + statusObjCluster.get("workload", statusObjWorkload); + + std::string performanceLimited = ""; + bool unknownMCT = false; + bool unknownRP = false; + + // Print performance limit details if known. + try { + StatusObjectReader limit = statusObjCluster["qos.performance_limited_by"]; + std::string name = limit["name"].get_str(); + if (name != "workload") { + std::string desc = limit["description"].get_str(); + std::string serverID; + limit.get("reason_server_id", serverID); + std::string procAddr = getProcessAddressByServerID(processesMap, serverID); + performanceLimited = format( + "\n Performance limited by %s: %s", + (procAddr == "unknown") ? ("server" + (serverID.empty() ? "" : (" " + serverID))).c_str() + : "process", + desc.c_str()); + if (procAddr != "unknown") + performanceLimited += format("\n Most limiting process: %s", procAddr.c_str()); + } + } catch (std::exception&) { + // If anything here throws (such as for an incompatible type) ignore it. + } + + // display the known rates + outputString += "\n Read rate - "; + outputString += getWorkloadRates(statusObjWorkload, unknownRP, "reads", "hz"); + + outputString += "\n Write rate - "; + outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "writes", "hz"); + + outputString += "\n Transactions started - "; + outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "started", "hz", true); + + outputString += "\n Transactions committed - "; + outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "committed", "hz", true); + + outputString += "\n Conflict rate - "; + outputString += getWorkloadRates(statusObjWorkload, unknownMCT, "conflicted", "hz", true); + + outputString += unknownRP ? "" : performanceLimited; + + // display any process messages + // FIXME: Above comment is not what this code block does, it actually just looks for a specific message + // in the process map, *by description*, and adds process addresses that have it to a vector. Either + // change the comment or the code. + std::vector messagesAddrs; + for (auto proc : processesMap.obj()) { + StatusObjectReader process(proc.second); + if (process.has("roles")) { + StatusArray rolesArray = proc.second.get_obj()["roles"].get_array(); + bool storageRole = false; + bool logRole = false; + for (StatusObjectReader role : rolesArray) { + if (role["role"].get_str() == "storage") { + storageRole = true; + } else if (role["role"].get_str() == "log") { + logRole = true; + } + } + if (storageRole && logRole) { + foundLogAndStorage = true; + } + } + if (process.has("messages")) { + StatusArray processMessagesArr = process.last().get_array(); + if (!processMessagesArr.empty()) { + for (StatusObjectReader msg : processMessagesArr) { + std::string desc; + std::string addr; + if (msg.get("description", desc) && desc == "Unable to update cluster file." && + process.get("address", addr)) { + messagesAddrs.push_back(addr); + } + } + } + } + } + if (!messagesAddrs.empty()) { + outputString += format("\n\n%d FoundationDB processes reported unable to update cluster file:", + messagesAddrs.size()); + for (const auto& msg : messagesAddrs) { + outputString += "\n " + msg; + } + } + } catch (std::runtime_error&) { + outputString = outputStringCache; + outputString += "\n Unable to retrieve workload status"; + } + + // Backup and DR section + outputString += "\n\nBackup and DR:"; + + std::map backupTags; + getBackupDRTags(statusObjCluster, "backup", backupTags); + + std::map drPrimaryTags; + getBackupDRTags(statusObjCluster, "dr_backup", drPrimaryTags); + + std::map drSecondaryTags; + getBackupDRTags(statusObjCluster, "dr_backup_dest", drSecondaryTags); + + outputString += format("\n Running backups - %d", backupTags.size()); + outputString += format("\n Running DRs - "); + + if (drPrimaryTags.empty() && drSecondaryTags.empty()) { + outputString += format("%d", 0); + } else { + if (!drPrimaryTags.empty()) { + outputString += format("%d as primary", drPrimaryTags.size()); + if (!drSecondaryTags.empty()) { + outputString += ", "; + } + } + if (!drSecondaryTags.empty()) { + outputString += format("%d as secondary", drSecondaryTags.size()); + } + } + + // status details + if (level == StatusClient::DETAILED) { + outputString += logBackupDR("Running backup tags", backupTags); + outputString += logBackupDR("Running DR tags (as primary)", drPrimaryTags); + outputString += logBackupDR("Running DR tags (as secondary)", drSecondaryTags); + + outputString += "\n\nProcess performance details:"; + outputStringCache = outputString; + try { + // constructs process performance details output + std::map workerDetails; + for (auto proc : processesMap.obj()) { + StatusObjectReader procObj(proc.second); + std::string address; + procObj.get("address", address); + + std::string line; + + NetworkAddress parsedAddress; + try { + parsedAddress = NetworkAddress::parse(address); + } catch (Error&) { + // Groups all invalid IP address/port pair in the end of this detail group. + line = format(" %-22s (invalid IP address or port)", address.c_str()); + IPAddress::IPAddressStore maxIp; + for (int i = 0; i < maxIp.size(); ++i) { + maxIp[i] = std::numeric_limits::type>::max(); + } + std::string& lastline = + workerDetails[NetworkAddress(IPAddress(maxIp), std::numeric_limits::max())]; + if (!lastline.empty()) + lastline.append("\n"); + lastline += line; + continue; + } + + try { + double tx = -1, rx = -1, mCPUUtil = -1; + int64_t processRSS; + + // Get the machine for this process + // StatusObjectReader mach = machinesMap[procObj["machine_id"].get_str()]; + StatusObjectReader mach; + if (machinesMap.get(procObj["machine_id"].get_str(), mach, false)) { + StatusObjectReader machCPU; + if (mach.get("cpu", machCPU)) { + + machCPU.get("logical_core_utilization", mCPUUtil); + + StatusObjectReader network; + if (mach.get("network", network)) { + network.get("megabits_sent.hz", tx); + network.get("megabits_received.hz", rx); + } + } + } + + procObj.get("memory.rss_bytes", processRSS); + + StatusObjectReader procCPUObj; + procObj.get("cpu", procCPUObj); + + line = format(" %-22s (", address.c_str()); + + double usageCores; + if (procCPUObj.get("usage_cores", usageCores)) + line += format("%3.0f%% cpu;", usageCores * 100); + + line += mCPUUtil != -1 ? format("%3.0f%% machine;", mCPUUtil * 100) : ""; + line += std::min(tx, rx) != -1 ? format("%6.3f Gbps;", std::max(tx, rx) / 1000.0) : ""; + + double diskBusy; + if (procObj.get("disk.busy", diskBusy)) + line += format("%3.0f%% disk IO;", 100.0 * diskBusy); + + line += processRSS != -1 ? format("%4.1f GB", processRSS / (1024.0 * 1024 * 1024)) : ""; + + double availableBytes; + if (procObj.get("memory.available_bytes", availableBytes)) + line += format(" / %3.1f GB RAM )", availableBytes / (1024.0 * 1024 * 1024)); + else + line += " )"; + + if (procObj.has("messages")) { + for (StatusObjectReader message : procObj.last().get_array()) { + std::string desc; + if (message.get("description", desc)) { + if (message.has("type")) { + line += "\n Last logged error: " + desc; + } else { + line += "\n " + desc; + } + } + } + } + + workerDetails[parsedAddress] = line; + } + + catch (std::runtime_error&) { + std::string noMetrics = format(" %-22s (no metrics available)", address.c_str()); + workerDetails[parsedAddress] = noMetrics; + } + } + for (const auto& w : workerDetails) + outputString += "\n" + format("%s", w.second.c_str()); + } catch (std::runtime_error&) { + outputString = outputStringCache; + outputString += "\n Unable to retrieve process performance details"; + } + + if (!printedCoordinators) { + printedCoordinators = true; + outputString += "\n\nCoordination servers:"; + outputString += getCoordinatorsInfoString(statusObj); + } + } + + // client time + std::string clientTime = getDateInfoString(statusObjClient, "timestamp"); + if (!clientTime.empty()) { + outputString += "\n\nClient time: " + clientTime; + } + + if (processesMap.obj().size() > 1 && isOldMemory) { + outputString += "\n\nWARNING: type `configure memory' to switch to a safer method of persisting data " + "on the transaction logs."; + } + if (processesMap.obj().size() > 9 && foundLogAndStorage) { + outputString += + "\n\nWARNING: A single process is both a transaction log and a storage server.\n For best " + "performance use dedicated disks for the transaction logs by setting process classes."; + } + + if (statusObjCluster.has("data_distribution_disabled")) { + outputString += "\n\nWARNING: Data distribution is off."; + } else { + if (statusObjCluster.has("data_distribution_disabled_for_ss_failures")) { + outputString += "\n\nWARNING: Data distribution is currently turned on but disabled for all " + "storage server failures."; + } + if (statusObjCluster.has("data_distribution_disabled_for_rebalance")) { + outputString += "\n\nWARNING: Data distribution is currently turned on but one or both of shard " + "size and read-load based balancing are disabled."; + // data_distribution_disabled_hex + if (statusObjCluster.has("data_distribution_disabled_hex")) { + outputString += " Ignore code: " + statusObjCluster["data_distribution_disabled_hex"].get_str(); + } + } + } + + printf("%s\n", outputString.c_str()); + } + + // status minimal + else if (level == StatusClient::MINIMAL) { + // Checking for field existence is not necessary here because if a field is missing there is no additional + // information that we would be able to display if we continued execution. Instead, any missing fields will + // throw and the catch will display the proper message. + try { + // If any of these throw, can't get status because the result makes no sense. + StatusObjectReader statusObjClient = statusObj["client"].get_obj(); + StatusObjectReader statusObjClientDatabaseStatus = statusObjClient["database_status"].get_obj(); + + bool available = statusObjClientDatabaseStatus["available"].get_bool(); + + // Database unavailable + if (!available) { + printf("%s", "The database is unavailable; type `status' for more information.\n"); + } else { + try { + bool healthy = statusObjClientDatabaseStatus["healthy"].get_bool(); + + // Database available without issues + if (healthy) { + if (displayDatabaseAvailable) { + printf("The database is available.\n"); + } + } else { // Database running but with issues + printf("The database is available, but has issues (type 'status' for more information).\n"); + } + } catch (std::runtime_error&) { + printf("The database is available, but has issues (type 'status' for more information).\n"); + } + } + + bool upToDate; + if (!statusObjClient.get("cluster_file.up_to_date", upToDate) || !upToDate) { + fprintf(stderr, + "WARNING: The cluster file is not up to date. Type 'status' for more information.\n"); + } + } catch (std::runtime_error&) { + printf("Unable to determine database state, type 'status' for more information.\n"); + } + + } + + // status JSON + else if (level == StatusClient::JSON) { + printf("%s\n", + json_spirit::write_string(json_spirit::mValue(statusObj.obj()), + json_spirit::Output_options::pretty_print) + .c_str()); + } + } catch (Error&) { + if (hideErrorMessages) + return; + if (level == StatusClient::MINIMAL) { + printf("Unable to determine database state, type 'status' for more information.\n"); + } else if (level == StatusClient::JSON) { + printf("Could not retrieve status json.\n\n"); + } else { + printf("Could not retrieve status, type 'status json' for more information.\n"); + } + } +} + +// "db" is the handler to the multiversion database +// localDb is the native Database object +// localDb is rarely needed except the "db" has not established a connection to the cluster where the operation will +// return Never as we expect status command to always return, we use "localDb" to return the default result +Future statusCommandActor(Reference db, + Database localDb, + std::vector tokens, + bool isExecMode) { + + StatusClient::StatusLevel level; + if (tokens.size() == 1) + level = StatusClient::NORMAL; + else if (tokens.size() == 2 && tokencmp(tokens[1], "details")) + level = StatusClient::DETAILED; + else if (tokens.size() == 2 && tokencmp(tokens[1], "minimal")) + level = StatusClient::MINIMAL; + else if (tokens.size() == 2 && tokencmp(tokens[1], "json")) + level = StatusClient::JSON; + else { + printUsage(tokens[0]); + co_return false; + } + + StatusObject s; + Reference tr = db->createTransaction(); + if (!tr->isValid()) { + StatusObject _s = co_await StatusClient::statusFetcher(localDb); + s = _s; + } else { + ThreadFuture> statusValueF = tr->get("\xff\xff/status/json"_sr); + Optional statusValue = co_await safeThreadFutureToFuture(statusValueF); + if (!statusValue.present()) { + fprintf(stderr, "ERROR: Failed to get status json from the cluster\n"); + } + json_spirit::mValue mv; + json_spirit::read_string(statusValue.get().toString(), mv); + s = StatusObject(mv.get_obj()); + } + + if (!isExecMode) + printf("\n"); + printStatus(s, level); + if (!isExecMode) + printf("\n"); + co_return true; +} + +void statusGenerator(const char* text, + const char* line, + std::vector& lc, + std::vector const& tokens) { + if (tokens.size() == 1) { + const char* opts[] = { "minimal", "details", "json", nullptr }; + arrayGenerator(text, line, opts, lc); + } +} + +CommandFactory statusFactory( + "status", + CommandHelp("status [minimal|details|json]", + "get the status of a FoundationDB cluster", + "If the cluster is down, this command will print a diagnostic which may be useful in figuring out " + "what is wrong. If the cluster is running, this command will print cluster " + "statistics.\n\nSpecifying `minimal' will provide a minimal description of the status of your " + "database.\n\nSpecifying `details' will provide load information for individual " + "workers.\n\nSpecifying `json' will provide status information in a machine readable JSON format."), + &statusGenerator); +} // namespace fdb_cli diff --git a/fdbcli/SuspendCommand.actor.cpp b/fdbcli/SuspendCommand.actor.cpp deleted file mode 100644 index e4bc2a46534..00000000000 --- a/fdbcli/SuspendCommand.actor.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * SuspendCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/algorithm/string.hpp" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/Knobs.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future suspendCommandActor(Reference db, - Reference tr, - std::vector tokens, - std::map>* address_interface) { - ASSERT(tokens.size() >= 1); - state bool result = true; - state std::string addressesStr; - if (tokens.size() == 1) { - // initialize worker interfaces - address_interface->clear(); - wait(getWorkerInterfaces(tr, address_interface, true)); - if (address_interface->size() == 0) { - printf("\nNo addresses can be suspended.\n"); - } else if (address_interface->size() == 1) { - printf("\nThe following address can be suspended:\n"); - } else { - printf("\nThe following %zu addresses can be suspended:\n", address_interface->size()); - } - for (auto it : *address_interface) { - printf("%s\n", printable(it.first).c_str()); - } - printf("\n"); - } else if (tokens.size() == 2) { - printUsage(tokens[0]); - result = false; - } else { - for (int i = 2; i < tokens.size(); i++) { - if (!address_interface->count(tokens[i])) { - fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str()); - result = false; - break; - } - } - - if (result) { - state double seconds; - int n = 0; - state int i; - auto secondsStr = tokens[1].toString(); - if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) { - printUsage(tokens[0]); - result = false; - } else { - std::vector addressesVec; - for (i = 2; i < tokens.size(); i++) { - addressesVec.push_back(tokens[i].toString()); - } - addressesStr = boost::algorithm::join(addressesVec, ","); - int64_t suspendRequestSent = - wait(safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, static_cast(seconds)))); - if (!suspendRequestSent) { - result = false; - fprintf( - stderr, - "ERROR: failed to send requests to suspend processes `%s', please run the `suspend’ command " - "to fetch latest addresses.\n", - addressesStr.c_str()); - } else { - printf("Attempted to suspend %zu processes\n", tokens.size() - 2); - } - } - } - } - return result; -} - -CommandFactory suspendFactory( - "suspend", - CommandHelp( - "suspend ", - "attempts to suspend one or more processes in the cluster", - "If no parameters are specified, populates the list of processes which can be suspended. Processes cannot be " - "suspended before this list has been populated.\n\nFor each IP:port pair in , attempt to suspend " - "the processes for the specified SECONDS after which the process will die.")); -} // namespace fdb_cli diff --git a/fdbcli/SuspendCommand.cpp b/fdbcli/SuspendCommand.cpp new file mode 100644 index 00000000000..9176aaf48d1 --- /dev/null +++ b/fdbcli/SuspendCommand.cpp @@ -0,0 +1,108 @@ +/* + * SuspendCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/algorithm/string.hpp" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/Knobs.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +namespace fdb_cli { + +Future suspendCommandActor(Reference db, + Reference tr, + std::vector const& tokens, + std::map>* address_interface) { + ASSERT(!tokens.empty()); + bool result = true; + std::string addressesStr; + if (tokens.size() == 1) { + // initialize worker interfaces + address_interface->clear(); + co_await getWorkerInterfaces(tr, address_interface, true); + if (address_interface->empty()) { + printf("\nNo addresses can be suspended.\n"); + } else if (address_interface->size() == 1) { + printf("\nThe following address can be suspended:\n"); + } else { + printf("\nThe following %zu addresses can be suspended:\n", address_interface->size()); + } + for (const auto& it : *address_interface) { + printf("%s\n", printable(it.first).c_str()); + } + printf("\n"); + } else if (tokens.size() == 2) { + printUsage(tokens[0]); + result = false; + } else { + for (int i = 2; i < tokens.size(); i++) { + if (!address_interface->count(tokens[i])) { + fprintf(stderr, "ERROR: process `%s' not recognized.\n", printable(tokens[i]).c_str()); + result = false; + break; + } + } + + if (result) { + double seconds{ 0 }; + int n = 0; + int i{ 0 }; + auto secondsStr = tokens[1].toString(); + if (sscanf(secondsStr.c_str(), "%lf%n", &seconds, &n) != 1 || n != secondsStr.size()) { + printUsage(tokens[0]); + result = false; + } else { + std::vector addressesVec; + for (i = 2; i < tokens.size(); i++) { + addressesVec.push_back(tokens[i].toString()); + } + addressesStr = boost::algorithm::join(addressesVec, ","); + int64_t suspendRequestSent = + co_await safeThreadFutureToFuture(db->rebootWorker(addressesStr, false, static_cast(seconds))); + if (!suspendRequestSent) { + result = false; + fprintf( + stderr, + "ERROR: failed to send requests to suspend processes `%s', please run the `suspend’ command " + "to fetch latest addresses.\n", + addressesStr.c_str()); + } else { + printf("Attempted to suspend %zu processes\n", tokens.size() - 2); + } + } + } + } + co_return result; +} + +CommandFactory suspendFactory( + "suspend", + CommandHelp( + "suspend ", + "attempts to suspend one or more processes in the cluster", + "If no parameters are specified, populates the list of processes which can be suspended. Processes cannot be " + "suspended before this list has been populated.\n\nFor each IP:port pair in , attempt to suspend " + "the processes for the specified SECONDS after which the process will die.")); +} // namespace fdb_cli diff --git a/fdbcli/ThrottleCommand.actor.cpp b/fdbcli/ThrottleCommand.actor.cpp deleted file mode 100644 index 02a3cd40633..00000000000 --- a/fdbcli/ThrottleCommand.actor.cpp +++ /dev/null @@ -1,413 +0,0 @@ -/* - * ThrottleCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/IClientApi.h" -#include "fdbclient/TagThrottle.actor.h" -#include "fdbclient/Knobs.h" -#include "fdbclient/SystemData.h" -#include "fdbclient/CommitTransaction.h" - -#include "flow/Arena.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/genericactors.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -static constexpr int defaultThrottleListLimit = 100; - -ACTOR Future throttleCommandActor(Reference db, std::vector tokens) { - - if (tokens.size() == 1) { - printUsage(tokens[0]); - return false; - } else if (tokencmp(tokens[1], "list")) { - if (tokens.size() > 4) { - fmt::print("Usage: throttle list [throttled|recommended|all] [LIMIT]\n\n"); - fmt::print("Lists tags that are currently throttled.\n"); - fmt::print("The default LIMIT is {} tags.\n", defaultThrottleListLimit); - return false; - } - - state bool reportThrottled = true; - state bool reportRecommended = false; - if (tokens.size() >= 3) { - if (tokencmp(tokens[2], "recommended")) { - reportThrottled = false; - reportRecommended = true; - } else if (tokencmp(tokens[2], "all")) { - reportThrottled = true; - reportRecommended = true; - } else if (!tokencmp(tokens[2], "throttled")) { - printf("ERROR: failed to parse `%s'.\n", printable(tokens[2]).c_str()); - return false; - } - } - - state int throttleListLimit = defaultThrottleListLimit; - if (tokens.size() >= 4) { - char* end; - throttleListLimit = std::strtol((const char*)tokens[3].begin(), &end, 10); - if ((tokens.size() > 4 && !std::isspace(*end)) || (tokens.size() == 4 && *end != '\0')) { - fprintf(stderr, "ERROR: failed to parse limit `%s'.\n", printable(tokens[3]).c_str()); - return false; - } - } - - state std::vector tags; - if (reportThrottled && reportRecommended) { - wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit, ContainsRecommended::True))); - } else if (reportThrottled) { - wait(store(tags, ThrottleApi::getThrottledTags(db, throttleListLimit))); - } else if (reportRecommended) { - wait(store(tags, ThrottleApi::getRecommendedTags(db, throttleListLimit))); - } - - bool anyLogged = false; - for (auto itr = tags.begin(); itr != tags.end(); ++itr) { - if (itr->expirationTime > now()) { - if (!anyLogged) { - printf("Throttled tags:\n\n"); - printf(" Rate (txn/s) | Expiration (s) | Priority | Type | Reason |Tag\n"); - printf(" --------------+----------------+-----------+--------+------------+------\n"); - - anyLogged = true; - } - - std::string reasonStr = "unset"; - if (itr->reason == TagThrottledReason::MANUAL) { - reasonStr = "manual"; - } else if (itr->reason == TagThrottledReason::BUSY_WRITE) { - reasonStr = "busy write"; - } else if (itr->reason == TagThrottledReason::BUSY_READ) { - reasonStr = "busy read"; - } - - printf(" %12d | %13ds | %9s | %6s | %10s |%s\n", - (int)(itr->tpsRate), - std::min((int)(itr->expirationTime - now()), (int)(itr->initialDuration)), - transactionPriorityToString(itr->priority, false), - itr->throttleType == TagThrottleType::AUTO ? "auto" : "manual", - reasonStr.c_str(), - itr->tag.toString().c_str()); - } - } - - if (tags.size() == throttleListLimit) { - printf("\nThe tag limit `%d' was reached. Use the [LIMIT] argument to view additional tags.\n", - throttleListLimit); - printf("Usage: throttle list [LIMIT]\n"); - } - if (!anyLogged) { - printf("There are no %s tags\n", reportThrottled ? "throttled" : "recommended"); - } - } else if (tokencmp(tokens[1], "on")) { - if (tokens.size() < 4 || !tokencmp(tokens[2], "tag") || tokens.size() > 7) { - printf("Usage: throttle on tag [RATE] [DURATION] [PRIORITY]\n"); - printf("\n"); - printf("Enables throttling for transactions with the specified tag.\n"); - printf("An optional transactions per second rate can be specified (default 0).\n"); - printf("An optional duration can be specified, which must include a time suffix (s, m, h, " - "d) (default 1h).\n"); - printf("An optional priority can be specified. Choices are `default', `immediate', and " - "`batch' (default `default').\n"); - return false; - } - - double tpsRate = 0.0; - uint64_t duration = 3600; - TransactionPriority priority = TransactionPriority::DEFAULT; - - if (tokens.size() >= 5) { - char* end; - tpsRate = std::strtod((const char*)tokens[4].begin(), &end); - if ((tokens.size() > 5 && !std::isspace(*end)) || (tokens.size() == 5 && *end != '\0')) { - fprintf(stderr, "ERROR: failed to parse rate `%s'.\n", printable(tokens[4]).c_str()); - return false; - } - if (tpsRate < 0) { - fprintf(stderr, "ERROR: rate cannot be negative `%f'\n", tpsRate); - return false; - } - } - if (tokens.size() == 6) { - Optional parsedDuration = parseDuration(tokens[5].toString()); - if (!parsedDuration.present()) { - fprintf(stderr, "ERROR: failed to parse duration `%s'.\n", printable(tokens[5]).c_str()); - return false; - } - duration = parsedDuration.get(); - - if (duration == 0) { - fprintf(stderr, "ERROR: throttle duration cannot be 0\n"); - return false; - } - } - if (tokens.size() == 7) { - if (tokens[6] == "default"_sr) { - priority = TransactionPriority::DEFAULT; - } else if (tokens[6] == "immediate"_sr) { - priority = TransactionPriority::IMMEDIATE; - } else if (tokens[6] == "batch"_sr) { - priority = TransactionPriority::BATCH; - } else { - fprintf(stderr, - "ERROR: unrecognized priority `%s'. Must be one of `default',\n `immediate', " - "or `batch'.\n", - tokens[6].toString().c_str()); - return false; - } - } - - TagSet tagSet; - tagSet.addTag(tokens[3]); - - wait(ThrottleApi::throttleTags(db, tagSet, tpsRate, duration, TagThrottleType::MANUAL, priority)); - printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str()); - } else if (tokencmp(tokens[1], "off")) { - int nextIndex = 2; - state TagSet tagSet; - bool throttleTypeSpecified = false; - bool is_error = false; - Optional throttleType = TagThrottleType::MANUAL; - Optional priority; - - if (tokens.size() == 2) { - is_error = true; - } - - while (nextIndex < tokens.size() && !is_error) { - if (tokencmp(tokens[nextIndex], "all")) { - if (throttleTypeSpecified) { - is_error = true; - continue; - } - throttleTypeSpecified = true; - throttleType = Optional(); - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "auto")) { - if (throttleTypeSpecified) { - is_error = true; - continue; - } - throttleTypeSpecified = true; - throttleType = TagThrottleType::AUTO; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "manual")) { - if (throttleTypeSpecified) { - is_error = true; - continue; - } - throttleTypeSpecified = true; - throttleType = TagThrottleType::MANUAL; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "default")) { - if (priority.present()) { - is_error = true; - continue; - } - priority = TransactionPriority::DEFAULT; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "immediate")) { - if (priority.present()) { - is_error = true; - continue; - } - priority = TransactionPriority::IMMEDIATE; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "batch")) { - if (priority.present()) { - is_error = true; - continue; - } - priority = TransactionPriority::BATCH; - ++nextIndex; - } else if (tokencmp(tokens[nextIndex], "tag")) { - if (tagSet.size() > 0 || nextIndex == tokens.size() - 1) { - is_error = true; - continue; - } - tagSet.addTag(tokens[nextIndex + 1]); - nextIndex += 2; - } else { - is_error = true; - } - } - - if (!is_error) { - state const char* throttleTypeString = - !throttleType.present() ? "" : (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually "); - state std::string priorityString = - priority.present() ? format(" at %s priority", transactionPriorityToString(priority.get(), false)) : ""; - - if (tagSet.size() > 0) { - bool success = wait(ThrottleApi::unthrottleTags(db, tagSet, throttleType, priority)); - if (success) { - fmt::print("Unthrottled {0}{1}\n", tagSet.toString(), priorityString); - } else { - fmt::print("{0} was not {1}throttled{2}\n", - tagSet.toString(Capitalize::True), - throttleTypeString, - priorityString); - } - } else { - bool unthrottled = wait(ThrottleApi::unthrottleAll(db, throttleType, priority)); - if (unthrottled) { - printf("Unthrottled all %sthrottled tags%s\n", throttleTypeString, priorityString.c_str()); - } else { - printf("There were no tags being %sthrottled%s\n", throttleTypeString, priorityString.c_str()); - } - } - } else { - printf("Usage: throttle off [all|auto|manual] [tag ] [PRIORITY]\n"); - printf("\n"); - printf("Disables throttling for throttles matching the specified filters. At least one " - "filter must be used.\n\n"); - printf("An optional qualifier `all', `auto', or `manual' can be used to specify the type " - "of throttle\n"); - printf("affected. `all' targets all throttles, `auto' targets those created by the " - "cluster, and\n"); - printf("`manual' targets those created manually (default `manual').\n\n"); - printf("The `tag' filter can be use to turn off only a specific tag.\n\n"); - printf("The priority filter can be used to turn off only throttles at specific priorities. " - "Choices are\n"); - printf("`default', `immediate', or `batch'. By default, all priorities are targeted.\n"); - } - } else if (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) { - if (tokens.size() != 3 || !tokencmp(tokens[2], "auto")) { - printf("Usage: throttle auto\n"); - printf("\n"); - printf("Enables or disable automatic tag throttling.\n"); - return false; - } - state bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable"); - wait(ThrottleApi::enableAuto(db, autoTagThrottlingEnabled)); - printf("Automatic tag throttling has been %s\n", autoTagThrottlingEnabled ? "enabled" : "disabled"); - } else { - printUsage(tokens[0]); - return false; - } - - return true; -} - -void throttleGenerator(const char* text, - const char* line, - std::vector& lc, - std::vector const& tokens) { - if (tokens.size() == 1) { - const char* opts[] = { "on tag", "off", "enable auto", "disable auto", "list", nullptr }; - arrayGenerator(text, line, opts, lc); - } else if (tokens.size() >= 2 && tokencmp(tokens[1], "on")) { - if (tokens.size() == 2) { - const char* opts[] = { "tag", nullptr }; - arrayGenerator(text, line, opts, lc); - } else if (tokens.size() == 6) { - const char* opts[] = { "default", "immediate", "batch", nullptr }; - arrayGenerator(text, line, opts, lc); - } - } else if (tokens.size() >= 2 && tokencmp(tokens[1], "off") && !tokencmp(tokens[tokens.size() - 1], "tag")) { - const char* opts[] = { "all", "auto", "manual", "tag", "default", "immediate", "batch", nullptr }; - arrayGenerator(text, line, opts, lc); - } else if (tokens.size() == 2 && (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable"))) { - const char* opts[] = { "auto", nullptr }; - arrayGenerator(text, line, opts, lc); - } else if (tokens.size() >= 2 && tokencmp(tokens[1], "list")) { - if (tokens.size() == 2) { - const char* opts[] = { "throttled", "recommended", "all", nullptr }; - arrayGenerator(text, line, opts, lc); - } else if (tokens.size() == 3) { - const char* opts[] = { "LIMITS", nullptr }; - arrayGenerator(text, line, opts, lc); - } - } -} - -std::vector throttleHintGenerator(std::vector const& tokens, bool inArgument) { - if (tokens.size() == 1) { - return { "", "[ARGS]" }; - } else if (tokencmp(tokens[1], "on")) { - std::vector opts = { "tag", "", "[RATE]", "[DURATION]", "[default|immediate|batch]" }; - if (tokens.size() == 2) { - return opts; - } else if (((tokens.size() == 3 && inArgument) || tokencmp(tokens[2], "tag")) && tokens.size() < 7) { - return std::vector(opts.begin() + tokens.size() - 2, opts.end()); - } - } else if (tokencmp(tokens[1], "off")) { - if (tokencmp(tokens[tokens.size() - 1], "tag")) { - return { "" }; - } else { - bool hasType = false; - bool hasTag = false; - bool hasPriority = false; - for (int i = 2; i < tokens.size(); ++i) { - if (tokencmp(tokens[i], "all") || tokencmp(tokens[i], "auto") || tokencmp(tokens[i], "manual")) { - hasType = true; - } else if (tokencmp(tokens[i], "default") || tokencmp(tokens[i], "immediate") || - tokencmp(tokens[i], "batch")) { - hasPriority = true; - } else if (tokencmp(tokens[i], "tag")) { - hasTag = true; - ++i; - } else { - return {}; - } - } - - std::vector options; - if (!hasType) { - options.push_back("[all|auto|manual]"); - } - if (!hasTag) { - options.push_back("[tag ]"); - } - if (!hasPriority) { - options.push_back("[default|immediate|batch]"); - } - - return options; - } - } else if ((tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) && tokens.size() == 2) { - return { "auto" }; - } else if (tokens.size() >= 2 && tokencmp(tokens[1], "list")) { - if (tokens.size() == 2) { - return { "[throttled|recommended|all]", "[LIMITS]" }; - } else if (tokens.size() == 3 && (tokencmp(tokens[2], "throttled") || tokencmp(tokens[2], "recommended") || - tokencmp(tokens[2], "all"))) { - return { "[LIMITS]" }; - } - } else if (tokens.size() == 2 && inArgument) { - return { "[ARGS]" }; - } - - return std::vector(); -} - -CommandFactory throttleFactory( - "throttle", - CommandHelp("throttle [ARGS]", - "view and control throttled tags", - "Use `on' and `off' to manually throttle or unthrottle tags. Use `enable auto' or `disable auto' " - "to enable or disable automatic tag throttling. Use `list' to print the list of throttled tags.\n"), - &throttleGenerator, - &throttleHintGenerator); -} // namespace fdb_cli diff --git a/fdbcli/ThrottleCommand.cpp b/fdbcli/ThrottleCommand.cpp new file mode 100644 index 00000000000..82531090a4b --- /dev/null +++ b/fdbcli/ThrottleCommand.cpp @@ -0,0 +1,412 @@ +/* + * ThrottleCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/IClientApi.h" +#include "fdbclient/TagThrottle.h" +#include "fdbclient/Knobs.h" +#include "fdbclient/SystemData.h" +#include "fdbclient/CommitTransaction.h" + +#include "flow/Arena.h" +#include "flow/ThreadHelper.actor.h" +#include "flow/genericactors.actor.h" + +namespace fdb_cli { + +static constexpr int defaultThrottleListLimit = 100; + +Future throttleCommandActor(Reference db, std::vector tokens) { + + if (tokens.size() == 1) { + printUsage(tokens[0]); + co_return false; + } else if (tokencmp(tokens[1], "list")) { + if (tokens.size() > 4) { + fmt::print("Usage: throttle list [throttled|recommended|all] [LIMIT]\n\n"); + fmt::print("Lists tags that are currently throttled.\n"); + fmt::print("The default LIMIT is {} tags.\n", defaultThrottleListLimit); + co_return false; + } + + bool reportThrottled = true; + bool reportRecommended = false; + if (tokens.size() >= 3) { + if (tokencmp(tokens[2], "recommended")) { + reportThrottled = false; + reportRecommended = true; + } else if (tokencmp(tokens[2], "all")) { + reportThrottled = true; + reportRecommended = true; + } else if (!tokencmp(tokens[2], "throttled")) { + printf("ERROR: failed to parse `%s'.\n", printable(tokens[2]).c_str()); + co_return false; + } + } + + int throttleListLimit = defaultThrottleListLimit; + if (tokens.size() >= 4) { + char* end; + throttleListLimit = std::strtol((const char*)tokens[3].begin(), &end, 10); + if ((tokens.size() > 4 && !std::isspace(*end)) || (tokens.size() == 4 && *end != '\0')) { + fprintf(stderr, "ERROR: failed to parse limit `%s'.\n", printable(tokens[3]).c_str()); + co_return false; + } + } + + std::vector tags; + if (reportThrottled && reportRecommended) { + tags = co_await ThrottleApi::getThrottledTags(db, throttleListLimit, ContainsRecommended::True); + } else if (reportThrottled) { + tags = co_await ThrottleApi::getThrottledTags(db, throttleListLimit); + } else if (reportRecommended) { + tags = co_await ThrottleApi::getRecommendedTags(db, throttleListLimit); + } + + bool anyLogged = false; + for (auto itr = tags.begin(); itr != tags.end(); ++itr) { + if (itr->expirationTime > now()) { + if (!anyLogged) { + printf("Throttled tags:\n\n"); + printf(" Rate (txn/s) | Expiration (s) | Priority | Type | Reason |Tag\n"); + printf(" --------------+----------------+-----------+--------+------------+------\n"); + + anyLogged = true; + } + + std::string reasonStr = "unset"; + if (itr->reason == TagThrottledReason::MANUAL) { + reasonStr = "manual"; + } else if (itr->reason == TagThrottledReason::BUSY_WRITE) { + reasonStr = "busy write"; + } else if (itr->reason == TagThrottledReason::BUSY_READ) { + reasonStr = "busy read"; + } + + printf(" %12d | %13ds | %9s | %6s | %10s |%s\n", + (int)(itr->tpsRate), + std::min((int)(itr->expirationTime - now()), (int)(itr->initialDuration)), + transactionPriorityToString(itr->priority, false), + itr->throttleType == TagThrottleType::AUTO ? "auto" : "manual", + reasonStr.c_str(), + itr->tag.toString().c_str()); + } + } + + if (tags.size() == throttleListLimit) { + printf("\nThe tag limit `%d' was reached. Use the [LIMIT] argument to view additional tags.\n", + throttleListLimit); + printf("Usage: throttle list [LIMIT]\n"); + } + if (!anyLogged) { + printf("There are no %s tags\n", reportThrottled ? "throttled" : "recommended"); + } + } else if (tokencmp(tokens[1], "on")) { + if (tokens.size() < 4 || !tokencmp(tokens[2], "tag") || tokens.size() > 7) { + printf("Usage: throttle on tag [RATE] [DURATION] [PRIORITY]\n"); + printf("\n"); + printf("Enables throttling for transactions with the specified tag.\n"); + printf("An optional transactions per second rate can be specified (default 0).\n"); + printf("An optional duration can be specified, which must include a time suffix (s, m, h, " + "d) (default 1h).\n"); + printf("An optional priority can be specified. Choices are `default', `immediate', and " + "`batch' (default `default').\n"); + co_return false; + } + + double tpsRate = 0.0; + uint64_t duration = 3600; + TransactionPriority priority = TransactionPriority::DEFAULT; + + if (tokens.size() >= 5) { + char* end; + tpsRate = std::strtod((const char*)tokens[4].begin(), &end); + if ((tokens.size() > 5 && !std::isspace(*end)) || (tokens.size() == 5 && *end != '\0')) { + fprintf(stderr, "ERROR: failed to parse rate `%s'.\n", printable(tokens[4]).c_str()); + co_return false; + } + if (tpsRate < 0) { + fprintf(stderr, "ERROR: rate cannot be negative `%f'\n", tpsRate); + co_return false; + } + } + if (tokens.size() == 6) { + Optional parsedDuration = parseDuration(tokens[5].toString()); + if (!parsedDuration.present()) { + fprintf(stderr, "ERROR: failed to parse duration `%s'.\n", printable(tokens[5]).c_str()); + co_return false; + } + duration = parsedDuration.get(); + + if (duration == 0) { + fprintf(stderr, "ERROR: throttle duration cannot be 0\n"); + co_return false; + } + } + if (tokens.size() == 7) { + if (tokens[6] == "default"_sr) { + priority = TransactionPriority::DEFAULT; + } else if (tokens[6] == "immediate"_sr) { + priority = TransactionPriority::IMMEDIATE; + } else if (tokens[6] == "batch"_sr) { + priority = TransactionPriority::BATCH; + } else { + fprintf(stderr, + "ERROR: unrecognized priority `%s'. Must be one of `default',\n `immediate', " + "or `batch'.\n", + tokens[6].toString().c_str()); + co_return false; + } + } + + TagSet tagSet; + tagSet.addTag(tokens[3]); + + co_await ThrottleApi::throttleTags(db, tagSet, tpsRate, duration, TagThrottleType::MANUAL, priority); + printf("Tag `%s' has been throttled\n", tokens[3].toString().c_str()); + } else if (tokencmp(tokens[1], "off")) { + int nextIndex = 2; + TagSet tagSet; + bool throttleTypeSpecified = false; + bool is_error = false; + Optional throttleType = TagThrottleType::MANUAL; + Optional priority; + + if (tokens.size() == 2) { + is_error = true; + } + + while (nextIndex < tokens.size() && !is_error) { + if (tokencmp(tokens[nextIndex], "all")) { + if (throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = Optional(); + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "auto")) { + if (throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = TagThrottleType::AUTO; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "manual")) { + if (throttleTypeSpecified) { + is_error = true; + continue; + } + throttleTypeSpecified = true; + throttleType = TagThrottleType::MANUAL; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "default")) { + if (priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::DEFAULT; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "immediate")) { + if (priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::IMMEDIATE; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "batch")) { + if (priority.present()) { + is_error = true; + continue; + } + priority = TransactionPriority::BATCH; + ++nextIndex; + } else if (tokencmp(tokens[nextIndex], "tag")) { + if (tagSet.size() > 0 || nextIndex == tokens.size() - 1) { + is_error = true; + continue; + } + tagSet.addTag(tokens[nextIndex + 1]); + nextIndex += 2; + } else { + is_error = true; + } + } + + if (!is_error) { + const char* throttleTypeString = + !throttleType.present() ? "" : (throttleType.get() == TagThrottleType::AUTO ? "auto-" : "manually "); + std::string priorityString = + priority.present() ? format(" at %s priority", transactionPriorityToString(priority.get(), false)) : ""; + + if (tagSet.size() > 0) { + bool success = co_await ThrottleApi::unthrottleTags(db, tagSet, throttleType, priority); + if (success) { + fmt::print("Unthrottled {0}{1}\n", tagSet.toString(), priorityString); + } else { + fmt::print("{0} was not {1}throttled{2}\n", + tagSet.toString(Capitalize::True), + throttleTypeString, + priorityString); + } + } else { + bool unthrottled = co_await ThrottleApi::unthrottleAll(db, throttleType, priority); + if (unthrottled) { + printf("Unthrottled all %sthrottled tags%s\n", throttleTypeString, priorityString.c_str()); + } else { + printf("There were no tags being %sthrottled%s\n", throttleTypeString, priorityString.c_str()); + } + } + } else { + printf("Usage: throttle off [all|auto|manual] [tag ] [PRIORITY]\n"); + printf("\n"); + printf("Disables throttling for throttles matching the specified filters. At least one " + "filter must be used.\n\n"); + printf("An optional qualifier `all', `auto', or `manual' can be used to specify the type " + "of throttle\n"); + printf("affected. `all' targets all throttles, `auto' targets those created by the " + "cluster, and\n"); + printf("`manual' targets those created manually (default `manual').\n\n"); + printf("The `tag' filter can be use to turn off only a specific tag.\n\n"); + printf("The priority filter can be used to turn off only throttles at specific priorities. " + "Choices are\n"); + printf("`default', `immediate', or `batch'. By default, all priorities are targeted.\n"); + } + } else if (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) { + if (tokens.size() != 3 || !tokencmp(tokens[2], "auto")) { + printf("Usage: throttle auto\n"); + printf("\n"); + printf("Enables or disable automatic tag throttling.\n"); + co_return false; + } + bool autoTagThrottlingEnabled = tokencmp(tokens[1], "enable"); + co_await ThrottleApi::enableAuto(db, autoTagThrottlingEnabled); + printf("Automatic tag throttling has been %s\n", autoTagThrottlingEnabled ? "enabled" : "disabled"); + } else { + printUsage(tokens[0]); + co_return false; + } + + co_return true; +} + +void throttleGenerator(const char* text, + const char* line, + std::vector& lc, + std::vector const& tokens) { + if (tokens.size() == 1) { + const char* opts[] = { "on tag", "off", "enable auto", "disable auto", "list", nullptr }; + arrayGenerator(text, line, opts, lc); + } else if (tokens.size() >= 2 && tokencmp(tokens[1], "on")) { + if (tokens.size() == 2) { + const char* opts[] = { "tag", nullptr }; + arrayGenerator(text, line, opts, lc); + } else if (tokens.size() == 6) { + const char* opts[] = { "default", "immediate", "batch", nullptr }; + arrayGenerator(text, line, opts, lc); + } + } else if (tokens.size() >= 2 && tokencmp(tokens[1], "off") && !tokencmp(tokens[tokens.size() - 1], "tag")) { + const char* opts[] = { "all", "auto", "manual", "tag", "default", "immediate", "batch", nullptr }; + arrayGenerator(text, line, opts, lc); + } else if (tokens.size() == 2 && (tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable"))) { + const char* opts[] = { "auto", nullptr }; + arrayGenerator(text, line, opts, lc); + } else if (tokens.size() >= 2 && tokencmp(tokens[1], "list")) { + if (tokens.size() == 2) { + const char* opts[] = { "throttled", "recommended", "all", nullptr }; + arrayGenerator(text, line, opts, lc); + } else if (tokens.size() == 3) { + const char* opts[] = { "LIMITS", nullptr }; + arrayGenerator(text, line, opts, lc); + } + } +} + +std::vector throttleHintGenerator(std::vector const& tokens, bool inArgument) { + if (tokens.size() == 1) { + return { "", "[ARGS]" }; + } else if (tokencmp(tokens[1], "on")) { + std::vector opts = { "tag", "", "[RATE]", "[DURATION]", "[default|immediate|batch]" }; + if (tokens.size() == 2) { + return opts; + } else if (((tokens.size() == 3 && inArgument) || tokencmp(tokens[2], "tag")) && tokens.size() < 7) { + return std::vector(opts.begin() + tokens.size() - 2, opts.end()); + } + } else if (tokencmp(tokens[1], "off")) { + if (tokencmp(tokens[tokens.size() - 1], "tag")) { + return { "" }; + } else { + bool hasType = false; + bool hasTag = false; + bool hasPriority = false; + for (int i = 2; i < tokens.size(); ++i) { + if (tokencmp(tokens[i], "all") || tokencmp(tokens[i], "auto") || tokencmp(tokens[i], "manual")) { + hasType = true; + } else if (tokencmp(tokens[i], "default") || tokencmp(tokens[i], "immediate") || + tokencmp(tokens[i], "batch")) { + hasPriority = true; + } else if (tokencmp(tokens[i], "tag")) { + hasTag = true; + ++i; + } else { + return {}; + } + } + + std::vector options; + if (!hasType) { + options.push_back("[all|auto|manual]"); + } + if (!hasTag) { + options.push_back("[tag ]"); + } + if (!hasPriority) { + options.push_back("[default|immediate|batch]"); + } + + return options; + } + } else if ((tokencmp(tokens[1], "enable") || tokencmp(tokens[1], "disable")) && tokens.size() == 2) { + return { "auto" }; + } else if (tokens.size() >= 2 && tokencmp(tokens[1], "list")) { + if (tokens.size() == 2) { + return { "[throttled|recommended|all]", "[LIMITS]" }; + } else if (tokens.size() == 3 && (tokencmp(tokens[2], "throttled") || tokencmp(tokens[2], "recommended") || + tokencmp(tokens[2], "all"))) { + return { "[LIMITS]" }; + } + } else if (tokens.size() == 2 && inArgument) { + return { "[ARGS]" }; + } + + return std::vector(); +} + +CommandFactory throttleFactory( + "throttle", + CommandHelp("throttle [ARGS]", + "view and control throttled tags", + "Use `on' and `off' to manually throttle or unthrottle tags. Use `enable auto' or `disable auto' " + "to enable or disable automatic tag throttling. Use `list' to print the list of throttled tags.\n"), + &throttleGenerator, + &throttleHintGenerator); +} // namespace fdb_cli diff --git a/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp b/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp deleted file mode 100644 index 25785fcdedb..00000000000 --- a/fdbcli/TriggerDDTeamInfoLogCommand.actor.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * TriggerDDTeamInfoLogCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/SystemData.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -ACTOR Future triggerddteaminfologCommandActor(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - std::string v = deterministicRandom()->randomUniqueID().toString(); - tr->set(triggerDDTeamInfoPrintKey, v); - wait(safeThreadFutureToFuture(tr->commit())); - printf("Triggered team info logging in data distribution.\n"); - return true; - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -CommandFactory triggerddteaminfologFactory( - "triggerddteaminfolog", - CommandHelp("triggerddteaminfolog", - "trigger the data distributor teams logging", - "Trigger the data distributor to log detailed information about its teams.")); - -} // namespace fdb_cli diff --git a/fdbcli/TriggerDDTeamInfoLogCommand.cpp b/fdbcli/TriggerDDTeamInfoLogCommand.cpp new file mode 100644 index 00000000000..00bfa7231a3 --- /dev/null +++ b/fdbcli/TriggerDDTeamInfoLogCommand.cpp @@ -0,0 +1,57 @@ +/* + * TriggerDDTeamInfoLogCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/SystemData.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" +namespace fdb_cli { + +Future triggerddteaminfologCommandActor(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + std::string v = deterministicRandom()->randomUniqueID().toString(); + tr->set(triggerDDTeamInfoPrintKey, v); + co_await safeThreadFutureToFuture(tr->commit()); + printf("Triggered team info logging in data distribution.\n"); + co_return true; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +CommandFactory triggerddteaminfologFactory( + "triggerddteaminfolog", + CommandHelp("triggerddteaminfolog", + "trigger the data distributor teams logging", + "Trigger the data distributor to log detailed information about its teams.")); + +} // namespace fdb_cli diff --git a/fdbcli/TssqCommand.actor.cpp b/fdbcli/TssqCommand.actor.cpp deleted file mode 100644 index a81df3df9e7..00000000000 --- a/fdbcli/TssqCommand.actor.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/* - * TssqCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/KeyBackedTypes.actor.h" -#include "fdbclient/SystemData.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace { - -ACTOR Future tssQuarantineList(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - // Hold the reference to the standalone's memory - state ThreadFuture resultFuture = tr->getRange(tssQuarantineKeys, CLIENT_KNOBS->TOO_MANY); - RangeResult result = wait(safeThreadFutureToFuture(resultFuture)); - // shouldn't have many quarantined TSSes - ASSERT(!result.more); - printf("Found %d quarantined TSS processes%s\n", result.size(), result.size() == 0 ? "." : ":"); - for (auto& it : result) { - printf(" %s\n", decodeTssQuarantineKey(it.key).toString().c_str()); - } - return Void(); - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future tssQuarantine(Reference db, bool enable, UID tssId) { - state Reference tr = db->createTransaction(); - state KeyBackedMap tssMapDB = KeyBackedMap(tssMappingKeys.begin); - - loop { - try { - tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - - // Do some validation first to make sure the command is valid - // hold the returned standalone object's memory - state ThreadFuture> serverListValueF = tr->get(serverListKeyFor(tssId)); - Optional serverListValue = wait(safeThreadFutureToFuture(serverListValueF)); - if (!serverListValue.present()) { - printf("No TSS %s found in cluster!\n", tssId.toString().c_str()); - return false; - } - state StorageServerInterface ssi = decodeServerListValue(serverListValue.get()); - if (!ssi.isTss()) { - printf("Cannot quarantine Non-TSS storage ID %s!\n", tssId.toString().c_str()); - return false; - } - - // hold the returned standalone object's memory - state ThreadFuture> currentQuarantineValueF = tr->get(tssQuarantineKeyFor(tssId)); - Optional currentQuarantineValue = wait(safeThreadFutureToFuture(currentQuarantineValueF)); - if (enable && currentQuarantineValue.present()) { - printf("TSS %s already in quarantine, doing nothing.\n", tssId.toString().c_str()); - return false; - } else if (!enable && !currentQuarantineValue.present()) { - printf("TSS %s is not in quarantine, cannot remove from quarantine!.\n", tssId.toString().c_str()); - return false; - } - - if (enable) { - tr->set(tssQuarantineKeyFor(tssId), ""_sr); - // remove server from TSS mapping when quarantine is enabled - tssMapDB.erase(tr, ssi.tssPairID.get()); - } else { - tr->clear(tssQuarantineKeyFor(tssId)); - } - - wait(safeThreadFutureToFuture(tr->commit())); - break; - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } - printf("Successfully %s TSS %s\n", enable ? "quarantined" : "removed", tssId.toString().c_str()); - return true; -} - -} // namespace - -namespace fdb_cli { - -ACTOR Future tssqCommandActor(Reference db, std::vector tokens) { - if (tokens.size() == 2) { - if (tokens[1] != "list"_sr) { - printUsage(tokens[0]); - return false; - } else { - wait(tssQuarantineList(db)); - } - } else if (tokens.size() == 3) { - if ((tokens[1] != "start"_sr && tokens[1] != "stop"_sr) || (tokens[2].size() != 32) || - !std::all_of(tokens[2].begin(), tokens[2].end(), &isxdigit)) { - printUsage(tokens[0]); - return false; - } else { - bool enable = tokens[1] == "start"_sr; - UID tssId = UID::fromString(tokens[2].toString()); - bool success = wait(tssQuarantine(db, enable, tssId)); - return success; - } - } else { - printUsage(tokens[0]); - return false; - } - return true; -} - -CommandFactory tssqFactory( - "tssq", - CommandHelp("tssq start|stop ", - "start/stop tss quarantine", - "Toggles Quarantine mode for a Testing Storage Server. Quarantine will happen automatically if the " - "TSS is detected to have incorrect data, but can also be initiated manually. You can also remove a " - "TSS from quarantine once your investigation is finished, which will destroy the TSS process.")); - -} // namespace fdb_cli diff --git a/fdbcli/TssqCommand.cpp b/fdbcli/TssqCommand.cpp new file mode 100644 index 00000000000..96453001503 --- /dev/null +++ b/fdbcli/TssqCommand.cpp @@ -0,0 +1,150 @@ +/* + * TssqCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/FDBOptions.g.h" +#include "fdbclient/IClientApi.h" +#include "fdbclient/KeyBackedTypes.h" +#include "fdbclient/SystemData.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace { + +Future tssQuarantineList(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + // Hold the reference to the standalone's memory + ThreadFuture resultFuture = tr->getRange(tssQuarantineKeys, CLIENT_KNOBS->TOO_MANY); + RangeResult result = co_await safeThreadFutureToFuture(resultFuture); + // shouldn't have many quarantined TSSes + ASSERT(!result.more); + printf("Found %d quarantined TSS processes%s\n", result.size(), result.empty() ? "." : ":"); + for (auto& it : result) { + printf(" %s\n", decodeTssQuarantineKey(it.key).toString().c_str()); + } + co_return; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future tssQuarantine(Reference db, bool enable, UID tssId) { + Reference tr = db->createTransaction(); + KeyBackedMap tssMapDB = KeyBackedMap(tssMappingKeys.begin); + + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + + // Do some validation first to make sure the command is valid + // hold the returned standalone object's memory + ThreadFuture> serverListValueF = tr->get(serverListKeyFor(tssId)); + Optional serverListValue = co_await safeThreadFutureToFuture(serverListValueF); + if (!serverListValue.present()) { + printf("No TSS %s found in cluster!\n", tssId.toString().c_str()); + co_return false; + } + StorageServerInterface ssi = decodeServerListValue(serverListValue.get()); + if (!ssi.isTss()) { + printf("Cannot quarantine Non-TSS storage ID %s!\n", tssId.toString().c_str()); + co_return false; + } + + // hold the returned standalone object's memory + ThreadFuture> currentQuarantineValueF = tr->get(tssQuarantineKeyFor(tssId)); + Optional currentQuarantineValue = co_await safeThreadFutureToFuture(currentQuarantineValueF); + if (enable && currentQuarantineValue.present()) { + printf("TSS %s already in quarantine, doing nothing.\n", tssId.toString().c_str()); + co_return false; + } else if (!enable && !currentQuarantineValue.present()) { + printf("TSS %s is not in quarantine, cannot remove from quarantine!.\n", tssId.toString().c_str()); + co_return false; + } + + if (enable) { + tr->set(tssQuarantineKeyFor(tssId), ""_sr); + // remove server from TSS mapping when quarantine is enabled + tssMapDB.erase(tr, ssi.tssPairID.get()); + } else { + tr->clear(tssQuarantineKeyFor(tssId)); + } + + co_await safeThreadFutureToFuture(tr->commit()); + break; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } + printf("Successfully %s TSS %s\n", enable ? "quarantined" : "removed", tssId.toString().c_str()); + co_return true; +} + +} // namespace + +namespace fdb_cli { + +Future tssqCommandActor(Reference db, std::vector tokens) { + if (tokens.size() == 2) { + if (tokens[1] != "list"_sr) { + printUsage(tokens[0]); + co_return false; + } else { + co_await tssQuarantineList(db); + } + } else if (tokens.size() == 3) { + if ((tokens[1] != "start"_sr && tokens[1] != "stop"_sr) || (tokens[2].size() != 32) || + !std::all_of(tokens[2].begin(), tokens[2].end(), &isxdigit)) { + printUsage(tokens[0]); + co_return false; + } else { + bool enable = tokens[1] == "start"_sr; + UID tssId = UID::fromString(tokens[2].toString()); + bool success = co_await tssQuarantine(db, enable, tssId); + co_return success; + } + } else { + printUsage(tokens[0]); + co_return false; + } + co_return true; +} + +CommandFactory tssqFactory( + "tssq", + CommandHelp("tssq start|stop ", + "start/stop tss quarantine", + "Toggles Quarantine mode for a Testing Storage Server. Quarantine will happen automatically if the " + "TSS is detected to have incorrect data, but can also be initiated manually. You can also remove a " + "TSS from quarantine once your investigation is finished, which will destroy the TSS process.")); + +} // namespace fdb_cli diff --git a/fdbcli/Util.actor.cpp b/fdbcli/Util.actor.cpp deleted file mode 100644 index de5556ab8d3..00000000000 --- a/fdbcli/Util.actor.cpp +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Util.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "fdbcli/fdbcli.actor.h" -#include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/Schemas.h" -#include "fdbclient/Status.h" - -#include "flow/Arena.h" - -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -bool tokencmp(StringRef token, const char* command) { - if (token.size() != strlen(command)) - return false; - - return !memcmp(token.begin(), command, token.size()); -} - -void printUsage(StringRef command) { - const auto& helpMap = CommandFactory::commands(); - auto i = helpMap.find(command.toString()); - if (i != helpMap.end()) - printf("Usage: %s\n", i->second.usage.c_str()); - else - fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str()); -} - -void printLongDesc(StringRef command) { - const auto& helpMap = CommandFactory::commands(); - auto i = helpMap.find(command.toString()); - if (i != helpMap.end()) - printf("%s\n", i->second.long_desc.c_str()); - else - fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str()); -} - -ACTOR Future getSpecialKeysFailureErrorMessage(Reference tr) { - // hold the returned standalone object's memory - state ThreadFuture> errorMsgF = tr->get(fdb_cli::errorMsgSpecialKey); - Optional errorMsg = wait(safeThreadFutureToFuture(errorMsgF)); - // Error message should be present - ASSERT(errorMsg.present()); - // Read the json string - auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); - // verify schema - auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); - std::string errorStr; - ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); - // return the error message - return valueObj["message"].get_str(); -} - -void addInterfacesFromKVs(RangeResult& kvs, - std::map>* address_interface) { - for (const auto& kv : kvs) { - ClientWorkerInterface workerInterf; - try { - // the interface is back-ward compatible, thus if parsing failed, it needs to upgrade cli version - workerInterf = BinaryReader::fromStringRef(kv.value, IncludeVersion()); - } catch (Error& e) { - fprintf(stderr, "Error: %s; CLI version is too old, please update to use a newer version\n", e.what()); - return; - } - ClientLeaderRegInterface leaderInterf(workerInterf.address()); - StringRef ip_port = (kv.key.endsWith(":tls"_sr) ? kv.key.removeSuffix(":tls"_sr) : kv.key) - .removePrefix("\xff\xff/worker_interfaces/"_sr); - (*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf); - - if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) { - Key full_ip_port2 = - StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString()); - StringRef ip_port2 = - full_ip_port2.endsWith(":tls"_sr) ? full_ip_port2.removeSuffix(":tls"_sr) : full_ip_port2; - (*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf); - } - } -} - -ACTOR Future getWorkerInterfaces(Reference tr, - std::map>* address_interface, - bool verify) { - if (verify) { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - tr->set(workerInterfacesVerifyOptionSpecialKey, ValueRef()); - } - // Hold the reference to the standalone's memory - state ThreadFuture kvsFuture = tr->getRange( - KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY); - state RangeResult kvs = wait(safeThreadFutureToFuture(kvsFuture)); - ASSERT(!kvs.more); - if (verify) { - // remove the option if set - tr->clear(workerInterfacesVerifyOptionSpecialKey); - } - addInterfacesFromKVs(kvs, address_interface); - return Void(); -} - -ACTOR Future getWorkers(Reference db, std::vector* workers) { - state Reference tr = db->createTransaction(); - loop { - try { - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state ThreadFuture processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY); - state ThreadFuture processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY); - - wait(success(safeThreadFutureToFuture(processClasses)) && success(safeThreadFutureToFuture(processData))); - ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY); - ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY); - - state std::map>, ProcessClass> id_class; - state int i; - for (i = 0; i < processClasses.get().size(); i++) { - try { - id_class[decodeProcessClassKey(processClasses.get()[i].key)] = - decodeProcessClassValue(processClasses.get()[i].value); - } catch (Error& e) { - fprintf(stderr, "Error: %s; Client version is too old, please use a newer version\n", e.what()); - return false; - } - } - - for (i = 0; i < processData.get().size(); i++) { - ProcessData data = decodeWorkerListValue(processData.get()[i].value); - ProcessClass processClass = id_class[data.locality.processId()]; - - if (processClass.classSource() == ProcessClass::DBSource || - data.processClass.classType() == ProcessClass::UnsetClass) - data.processClass = processClass; - - if (data.processClass.classType() != ProcessClass::TesterClass) - workers->push_back(data); - } - - return true; - } catch (Error& e) { - TraceEvent(SevWarn, "GetWorkersError").error(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future getStorageServerInterfaces(Reference db, - std::map* interfaces) { - state Reference tr = db->createTransaction(); - loop { - interfaces->clear(); - try { - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); - tr->setOption(FDBTransactionOptions::LOCK_AWARE); - state ThreadFuture serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY); - wait(success(safeThreadFutureToFuture(serverListF))); - ASSERT(!serverListF.get().more); - ASSERT_LT(serverListF.get().size(), CLIENT_KNOBS->TOO_MANY); - RangeResult serverList = serverListF.get(); - // decode server interfaces - for (int i = 0; i < serverList.size(); i++) { - auto ssi = decodeServerListValue(serverList[i].value); - (*interfaces)[ssi.address().toString()] = ssi; - } - return Void(); - } catch (Error& e) { - TraceEvent(SevWarn, "GetStorageServerInterfacesError").error(e); - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -} // namespace fdb_cli diff --git a/fdbcli/Util.cpp b/fdbcli/Util.cpp new file mode 100644 index 00000000000..39da058925d --- /dev/null +++ b/fdbcli/Util.cpp @@ -0,0 +1,465 @@ +/* + * Util.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "fdbcli/fdbcli.h" +#include "fdbclient/ManagementAPI.h" +#include "fdbclient/Schemas.h" +#include "fdbclient/Status.h" +#include "fdbclient/BulkDumping.h" +#include "fdbclient/BulkLoading.h" +#include "flow/Arena.h" +#include "flow/ThreadHelper.actor.h" +#include + +namespace fdb_cli { + +bool tokencmp(StringRef token, const char* command) { + if (token.size() != strlen(command)) + return false; + + return !memcmp(token.begin(), command, token.size()); +} + +void printUsage(StringRef command) { + const auto& helpMap = CommandFactory::commands(); + auto i = helpMap.find(command.toString()); + if (i != helpMap.end()) + printf("Usage: %s\n", i->second.usage.c_str()); + else + fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str()); +} + +void printLongDesc(StringRef command) { + const auto& helpMap = CommandFactory::commands(); + auto i = helpMap.find(command.toString()); + if (i != helpMap.end()) + printf("%s\n", i->second.long_desc.c_str()); + else + fprintf(stderr, "ERROR: Unknown command `%s'\n", command.toString().c_str()); +} + +Future getSpecialKeysFailureErrorMessage(Reference tr) { + // hold the returned standalone object's memory + ThreadFuture> errorMsgF = tr->get(fdb_cli::errorMsgSpecialKey); + Optional errorMsg = co_await safeThreadFutureToFuture(errorMsgF); + // Error message should be present + ASSERT(errorMsg.present()); + // Read the json string + auto valueObj = readJSONStrictly(errorMsg.get().toString()).get_obj(); + // verify schema + auto schema = readJSONStrictly(JSONSchemas::managementApiErrorSchema.toString()).get_obj(); + std::string errorStr; + ASSERT(schemaMatch(schema, valueObj, errorStr, SevError, true)); + // return the error message + co_return valueObj["message"].get_str(); +} + +void addInterfacesFromKVs(RangeResult& kvs, + std::map>* address_interface) { + for (const auto& kv : kvs) { + ClientWorkerInterface workerInterf; + try { + // the interface is back-ward compatible, thus if parsing failed, it needs to upgrade cli version + workerInterf = BinaryReader::fromStringRef(kv.value, IncludeVersion()); + } catch (Error& e) { + fprintf(stderr, "Error: %s; CLI version is too old, please update to use a newer version\n", e.what()); + return; + } + ClientLeaderRegInterface leaderInterf(workerInterf.address()); + StringRef ip_port = (kv.key.endsWith(":tls"_sr) ? kv.key.removeSuffix(":tls"_sr) : kv.key) + .removePrefix("\xff\xff/worker_interfaces/"_sr); + (*address_interface)[ip_port] = std::make_pair(kv.value, leaderInterf); + + if (workerInterf.reboot.getEndpoint().addresses.secondaryAddress.present()) { + Key full_ip_port2 = + StringRef(workerInterf.reboot.getEndpoint().addresses.secondaryAddress.get().toString()); + StringRef ip_port2 = + full_ip_port2.endsWith(":tls"_sr) ? full_ip_port2.removeSuffix(":tls"_sr) : full_ip_port2; + (*address_interface)[ip_port2] = std::make_pair(kv.value, leaderInterf); + } + } +} + +Future getWorkerInterfaces(Reference tr, + std::map>* address_interface, + bool verify) { + if (verify) { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->set(workerInterfacesVerifyOptionSpecialKey, ValueRef()); + } + // Hold the reference to the standalone's memory + ThreadFuture kvsFuture = tr->getRange( + KeyRangeRef("\xff\xff/worker_interfaces/"_sr, "\xff\xff/worker_interfaces0"_sr), CLIENT_KNOBS->TOO_MANY); + RangeResult kvs = co_await safeThreadFutureToFuture(kvsFuture); + ASSERT(!kvs.more); + if (verify) { + // remove the option if set + tr->clear(workerInterfacesVerifyOptionSpecialKey); + } + addInterfacesFromKVs(kvs, address_interface); +} + +Future getWorkers(Reference db, std::vector* workers) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + ThreadFuture processClasses = tr->getRange(processClassKeys, CLIENT_KNOBS->TOO_MANY); + ThreadFuture processData = tr->getRange(workerListKeys, CLIENT_KNOBS->TOO_MANY); + + co_await (success(safeThreadFutureToFuture(processClasses)) && + success(safeThreadFutureToFuture(processData))); + ASSERT(!processClasses.get().more && processClasses.get().size() < CLIENT_KNOBS->TOO_MANY); + ASSERT(!processData.get().more && processData.get().size() < CLIENT_KNOBS->TOO_MANY); + + std::map>, ProcessClass> id_class; + int i{ 0 }; + for (i = 0; i < processClasses.get().size(); i++) { + try { + id_class[decodeProcessClassKey(processClasses.get()[i].key)] = + decodeProcessClassValue(processClasses.get()[i].value); + } catch (Error& e) { + fprintf(stderr, "Error: %s; Client version is too old, please use a newer version\n", e.what()); + co_return false; + } + } + + for (i = 0; i < processData.get().size(); i++) { + ProcessData data = decodeWorkerListValue(processData.get()[i].value); + ProcessClass processClass = id_class[data.locality.processId()]; + + if (processClass.classSource() == ProcessClass::DBSource || + data.processClass.classType() == ProcessClass::UnsetClass) + data.processClass = processClass; + + if (data.processClass.classType() != ProcessClass::TesterClass) + workers->push_back(data); + } + + co_return true; + } catch (Error& e) { + err = e; + } + TraceEvent(SevWarn, "GetWorkersError").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future getStorageServerInterfaces(Reference db, + std::map* interfaces) { + Reference tr = db->createTransaction(); + while (true) { + interfaces->clear(); + Error err; + try { + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + tr->setOption(FDBTransactionOptions::LOCK_AWARE); + ThreadFuture serverListF = tr->getRange(serverListKeys, CLIENT_KNOBS->TOO_MANY); + co_await safeThreadFutureToFuture(serverListF); + ASSERT(!serverListF.get().more); + ASSERT_LT(serverListF.get().size(), CLIENT_KNOBS->TOO_MANY); + RangeResult serverList = serverListF.get(); + // decode server interfaces + for (int i = 0; i < serverList.size(); i++) { + auto ssi = decodeServerListValue(serverList[i].value); + (*interfaces)[ssi.address().toString()] = ssi; + } + co_return; + } catch (Error& e) { + err = e; + } + TraceEvent(SevWarn, "GetStorageServerInterfacesError").error(err); + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +// Shared UID validation for bulk operations (BulkDump/BulkLoad) +UID validateBulkJobId(StringRef token, const char* usage) { + UID jobId; + try { + jobId = UID::fromStringThrowsOnFailure(token.toString()); + } catch (Error&) { + fmt::println("ERROR: Invalid job id '{}' (expected 32 hex characters)", token.toString()); + fmt::println("{}", usage); + throw operation_failed(); + } + + if (!jobId.isValid()) { + fmt::println("ERROR: Invalid job id {}", token.toString()); + fmt::println("{}", usage); + throw operation_failed(); + } + + return jobId; +} + +Future getBulkOwnerSuffix(Database cx, UID jobId, bool isDumpJob) { + if (isDumpJob) { + Optional ownerInfo = co_await getBulkDumpOwner(cx, jobId); + if (ownerInfo.present()) { + co_return fmt::format(" (owned by {} '{}')", ownerInfo.get().ownerType, ownerInfo.get().ownerName); + } + } else { + Optional ownerInfo = co_await getBulkLoadOwner(cx, jobId); + if (ownerInfo.present()) { + co_return fmt::format(" (owned by {} '{}')", ownerInfo.get().ownerType, ownerInfo.get().ownerName); + } + } + co_return std::string(""); +} + +// Format common progress metrics (throughput, ETA, elapsed time) +void printProgressMetrics(double avgBytesPerSecond, Optional etaSeconds, double elapsedSeconds) { + // Throughput + if (avgBytesPerSecond > 0) { + fmt::println(" Throughput - {:.1f} MB/s", avgBytesPerSecond / 1048576.0); + } + + // ETA + if (etaSeconds.present()) { + fmt::println(" Estimated time remaining - {}", formatDurationHumanReadable((int)etaSeconds.get())); + } + + // Elapsed time + if (elapsedSeconds > 0) { + fmt::println(" Elapsed time - {}", formatDurationHumanReadable((int)elapsedSeconds)); + } +} + +// Print detailed task breakdown for bulk operations +void printTaskBreakdown(int submittedTasks, int triggeredTasks, int runningTasks, int completeTasks, int errorTasks) { + int totalTasks = submittedTasks + triggeredTasks + runningTasks + completeTasks + errorTasks; + if (totalTasks > 0) { + fmt::println(" Task Status Breakdown:"); + if (submittedTasks > 0) + fmt::println(" Submitted: {}", submittedTasks); + if (triggeredTasks > 0) + fmt::println(" Triggered: {}", triggeredTasks); + if (runningTasks > 0) + fmt::println(" Running: {}", runningTasks); + if (completeTasks > 0) + fmt::println(" Complete: {}", completeTasks); + if (errorTasks > 0) + fmt::println(" Error: {}", errorTasks); + } +} + +// Advanced performance monitoring for bulk operations +BulkHealthMetrics BulkHealthMetrics::analyze(double throughputMBps, + double efficiencyPercent, + int stalledTasks, + int errorTasks, + double elapsedMinutes) { + BulkHealthMetrics metrics; + + // Calculate health score (weighted average) + double throughputScore = std::min(100.0, throughputMBps * 2); // 50MB/s = 100 points + double efficiencyScore = efficiencyPercent; + double reliabilityScore = std::max(0.0, 100.0 - (stalledTasks * 10) - (errorTasks * 20)); + + metrics.healthScore = (throughputScore * 0.4) + (efficiencyScore * 0.4) + (reliabilityScore * 0.2); + + // Determine status + if (metrics.healthScore >= 90) { + metrics.healthStatus = "Excellent"; + } else if (metrics.healthScore >= 75) { + metrics.healthStatus = "Good"; + } else if (metrics.healthScore >= 50) { + metrics.healthStatus = "Fair"; + } else { + metrics.healthStatus = "Poor"; + } + + // Generate recommendations + if (throughputMBps < 5) { + metrics.recommendations.push_back("Consider increasing cluster resources for better throughput"); + } + if (errorTasks > 0) { + metrics.recommendations.push_back("Investigate error tasks for potential issues"); + } + if (stalledTasks > 0) { + metrics.recommendations.push_back("Monitor stalled tasks - may indicate network or storage issues"); + } + if (elapsedMinutes > 60 && efficiencyPercent < 50) { + metrics.recommendations.push_back("Long-running job with low efficiency - consider optimization"); + } + + return metrics; +} + +void printBulkHealthAnalysis(const BulkHealthMetrics& health) { + fmt::println(""); + fmt::println("Health Analysis:"); + fmt::println(" Overall Health: {:.1f}/100 ({})", health.healthScore, health.healthStatus); + + if (!health.recommendations.empty()) { + fmt::println(" Recommendations"); + for (const auto& rec : health.recommendations) { + fmt::println(" - {}", rec); + } + } +} + +BulkErrorAnalysis BulkErrorAnalysis::analyze(int errorTasks, + int stalledTasks, + const std::vector& recentErrors) { + BulkErrorAnalysis analysis; + analysis.totalErrors = errorTasks; + + // Categorize errors + for (const auto& error : recentErrors) { + if (error.find("timeout") != std::string::npos) { + analysis.errorCategories["Timeout Issues"]++; + } else if (error.find("network") != std::string::npos || error.find("connection") != std::string::npos) { + analysis.errorCategories["Network Issues"]++; + } else if (error.find("permission") != std::string::npos || error.find("access") != std::string::npos) { + analysis.errorCategories["Access Issues"]++; + } else if (error.find("disk") != std::string::npos || error.find("storage") != std::string::npos) { + analysis.errorCategories["Storage Issues"]++; + } else { + analysis.errorCategories["Other Issues"]++; + } + } + + return analysis; +} + +void printErrorDiagnostics(const BulkErrorAnalysis& analysis) { + if (analysis.totalErrors > 0) { + fmt::println(""); + fmt::println("Error Diagnostics:"); + + if (!analysis.errorCategories.empty()) { + fmt::println(" Error Categories:"); + for (const auto& [category, count] : analysis.errorCategories) { + fmt::println(" {} - {} occurrences", category, count); + } + } + } +} + +BulkOptimizationRecommendations BulkOptimizationRecommendations::generate(double throughputMBps, + double efficiency, + int stalledTasks, + int errorTasks, + double elapsedMinutes, + int totalTasks) { + BulkOptimizationRecommendations recs; + + // Performance recommendations + if (throughputMBps < 10 && totalTasks > 100) { + recs.performanceRecommendations.push_back( + "Low throughput detected. Consider increasing storage server resources or parallelism"); + } + if (elapsedMinutes > 30 && efficiency < 80) { + recs.performanceRecommendations.push_back( + "Long-running job with low efficiency. Consider optimizing shard sizes or task distribution"); + } + + return recs; +} + +void printOptimizationRecommendations(const BulkOptimizationRecommendations& recs) { + if (!recs.performanceRecommendations.empty()) { + fmt::println(""); + fmt::println("Optimization Recommendations:"); + fmt::println(" Performance:"); + for (const auto& rec : recs.performanceRecommendations) { + fmt::println(" - {}", rec); + } + } +} + +// Format bytes with progress percentage for bulk operations +std::string formatBytesProgress(int64_t completedBytes, Optional totalBytes) { + std::string result = format("%.2f MB", completedBytes / 1048576.0); + if (totalBytes.present()) { + result += fmt::format(" / {:.2f} MB ({:.1f}%)", + totalBytes.get() / 1048576.0, + totalBytes.get() > 0 ? 100.0 * completedBytes / totalBytes.get() : 0.0); + } + return result; +} + +// Print progress summary for bulk operations +void printProgressSummary(const char* operationName, + int completeTasks, + int totalTasks, + int64_t completedBytes, + int64_t totalBytes) { + fmt::println(""); + fmt::println("{} Progress Summary:", operationName); + fmt::println(" Tasks: {} / {} ({:.1f}%)", + completeTasks, + totalTasks, + totalTasks > 0 ? 100.0 * completeTasks / totalTasks : 0.0); + fmt::println(" Data: {}", formatBytesProgress(completedBytes, totalBytes)); +} + +void printStalledTasks(const std::vector& stalledTasks) { + if (!stalledTasks.empty()) { + fmt::println(""); + fmt::println("WARNING: {} stalled tasks (no progress > 60s):", stalledTasks.size()); + for (const auto& stalled : stalledTasks) { + fmt::println(" Task {}: {}, stalled {:.0f}s, {} restarts", + stalled.taskId.shortString(), + stalled.range.toString(), + stalled.stalledSeconds, + stalled.restartCount); + if (!stalled.lastError.empty()) { + fmt::println(" Last error: {}", stalled.lastError); + } + } + } +} + +void printBulkAnalysis(double avgBytesPerSecond, + double elapsedSeconds, + int completeTasks, + int totalTasks, + int errorTasks, + const std::vector& stalledTasks) { + double efficiency = totalTasks > 0 ? 100.0 * completeTasks / totalTasks : 0.0; + + auto healthMetrics = BulkHealthMetrics::analyze( + avgBytesPerSecond / 1048576.0, efficiency, stalledTasks.size(), errorTasks, elapsedSeconds / 60.0); + printBulkHealthAnalysis(healthMetrics); + + printStalledTasks(stalledTasks); + + std::vector recentErrors; + auto errorAnalysis = BulkErrorAnalysis::analyze(errorTasks, stalledTasks.size(), recentErrors); + printErrorDiagnostics(errorAnalysis); + + auto recommendations = BulkOptimizationRecommendations::generate( + avgBytesPerSecond / 1048576.0, efficiency, stalledTasks.size(), errorTasks, elapsedSeconds / 60.0, totalTasks); + printOptimizationRecommendations(recommendations); + + if (errorTasks > 0) { + fmt::println(""); + fmt::println("WARNING: {} tasks in error state", errorTasks); + } +} + +} // namespace fdb_cli diff --git a/fdbcli/VersionEpochCommand.actor.cpp b/fdbcli/VersionEpochCommand.actor.cpp deleted file mode 100644 index 41e4e268572..00000000000 --- a/fdbcli/VersionEpochCommand.actor.cpp +++ /dev/null @@ -1,174 +0,0 @@ -/* - * VersionEpochCommand.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/lexical_cast.hpp" - -#include "fdbcli/fdbcli.actor.h" - -#include "fdbclient/IClientApi.h" -#include "fdbclient/ManagementAPI.actor.h" - -#include "flow/Arena.h" -#include "flow/FastRef.h" -#include "flow/ThreadHelper.actor.h" -#include "flow/actorcompiler.h" // This must be the last #include. - -namespace fdb_cli { - -const KeyRef versionEpochSpecialKey = "\xff\xff/management/version_epoch"_sr; - -struct VersionInfo { - int64_t version; - int64_t expectedVersion; -}; - -ACTOR static Future> getVersionInfo(Reference db) { - state Reference tr = db->createTransaction(); - loop { - try { - tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); - state Version rv = wait(safeThreadFutureToFuture(tr->getReadVersion())); - state ThreadFuture> versionEpochValFuture = tr->get(versionEpochKey); - Optional versionEpochVal = wait(safeThreadFutureToFuture(versionEpochValFuture)); - if (!versionEpochVal.present()) { - return Optional(); - } - int64_t versionEpoch = BinaryReader::fromStringRef(versionEpochVal.get(), Unversioned()); - int64_t expected = g_network->timer() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND - versionEpoch; - return VersionInfo{ rv, expected }; - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR static Future> getVersionEpoch(Reference tr) { - loop { - try { - state ThreadFuture> versionEpochValFuture = tr->get(versionEpochSpecialKey); - Optional versionEpochVal = wait(safeThreadFutureToFuture(versionEpochValFuture)); - return versionEpochVal.present() ? boost::lexical_cast(versionEpochVal.get().toString()) - : Optional(); - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } -} - -ACTOR Future versionEpochCommandActor(Reference db, Database cx, std::vector tokens) { - if (tokens.size() <= 3) { - state Reference tr = db->createTransaction(); - if (tokens.size() == 1) { - Optional versionInfo = wait(getVersionInfo(db)); - if (versionInfo.present()) { - int64_t diff = versionInfo.get().expectedVersion - versionInfo.get().version; - printf("Version: %" PRId64 "\n", versionInfo.get().version); - printf("Expected: %" PRId64 "\n", versionInfo.get().expectedVersion); - printf("Difference: %" PRId64 " (%.2fs)\n", diff, 1.0 * diff / CLIENT_KNOBS->VERSIONS_PER_SECOND); - } else { - printf("Version epoch is unset\n"); - } - return true; - } else if (tokens.size() == 2 && tokencmp(tokens[1], "get")) { - Optional versionEpoch = wait(getVersionEpoch(db->createTransaction())); - if (versionEpoch.present()) { - printf("Current version epoch is %" PRId64 "\n", versionEpoch.get()); - } else { - printf("Version epoch is unset\n"); - } - return true; - } else if (tokens.size() == 2 && tokencmp(tokens[1], "disable")) { - // Clearing the version epoch means versions will no longer attempt - // to advance at the same rate as the clock. The current version - // will remain unchanged. - loop { - try { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - Optional versionEpoch = wait(getVersionEpoch(db->createTransaction())); - if (!versionEpoch.present()) { - return true; - } else { - tr->clear(versionEpochSpecialKey); - wait(safeThreadFutureToFuture(tr->commit())); - } - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } - } else if ((tokens.size() == 2 && tokencmp(tokens[1], "enable")) || - (tokens.size() == 3 && tokencmp(tokens[1], "set"))) { - state int64_t v; - if (tokens.size() == 3) { - int n = 0; - if (sscanf(tokens[2].toString().c_str(), "%" SCNd64 "%n", &v, &n) != 1 || n != tokens[2].size()) { - printUsage(tokens[0]); - return false; - } - } else { - v = 0; // default version epoch - } - - loop { - try { - tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); - Optional versionEpoch = wait(getVersionEpoch(tr)); - if (!versionEpoch.present() || (versionEpoch.get() != v && tokens.size() == 3)) { - tr->set(versionEpochSpecialKey, BinaryWriter::toValue(v, Unversioned())); - wait(safeThreadFutureToFuture(tr->commit())); - } else { - printf("Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target " - "version\n"); - return true; - } - } catch (Error& e) { - wait(safeThreadFutureToFuture(tr->onError(e))); - } - } - } else if (tokens.size() == 2 && tokencmp(tokens[1], "commit")) { - Optional versionInfo = wait(getVersionInfo(db)); - if (versionInfo.present()) { - wait(advanceVersion(cx, versionInfo.get().expectedVersion)); - } else { - printf("Must set the version epoch before committing it (see `versionepoch enable`)\n"); - } - return true; - } - } - - printUsage(tokens[0]); - return false; -} - -CommandFactory versionEpochFactory( - "versionepoch", - CommandHelp("versionepoch [ [EPOCH]]", - "Read or write the version epoch", - "If no arguments are specified, reports the offset between the expected version " - "and the actual version. Otherwise, enables, disables, or commits the version epoch. " - "Setting the version epoch can be irreversible since it can cause a large version jump. " - "Thus, the version epoch must first by enabled with the enable or set command. This " - "causes a recovery. Once the version epoch has been set, versions may be given out at " - "a faster or slower rate to attempt to match the actual version to the expected version, " - "based on the version epoch. After setting the version, run the commit command to perform " - "a one time jump to the expected version. This is useful when there is a very large gap " - "between the current version and the expected version. Note that once a version jump has " - "occurred, it cannot be undone. Run this command without any arguments to see the current " - "and expected version.")); -} // namespace fdb_cli diff --git a/fdbcli/VersionEpochCommand.cpp b/fdbcli/VersionEpochCommand.cpp new file mode 100644 index 00000000000..c10eefcdef5 --- /dev/null +++ b/fdbcli/VersionEpochCommand.cpp @@ -0,0 +1,181 @@ +/* + * VersionEpochCommand.cpp + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "boost/lexical_cast.hpp" + +#include "fdbcli/fdbcli.h" + +#include "fdbclient/IClientApi.h" +#include "fdbclient/ManagementAPI.h" + +#include "flow/Arena.h" +#include "flow/FastRef.h" +#include "flow/ThreadHelper.actor.h" + +namespace fdb_cli { + +const KeyRef versionEpochSpecialKey = "\xff\xff/management/version_epoch"_sr; + +struct VersionInfo { + int64_t version; + int64_t expectedVersion; +}; + +static Future> getVersionInfo(Reference db) { + Reference tr = db->createTransaction(); + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::READ_SYSTEM_KEYS); + Version rv = co_await safeThreadFutureToFuture(tr->getReadVersion()); + ThreadFuture> versionEpochValFuture = tr->get(versionEpochKey); + Optional versionEpochVal = co_await safeThreadFutureToFuture(versionEpochValFuture); + if (!versionEpochVal.present()) { + co_return Optional(); + } + int64_t versionEpoch = BinaryReader::fromStringRef(versionEpochVal.get(), Unversioned()); + int64_t expected = g_network->timer() * CLIENT_KNOBS->CORE_VERSIONSPERSECOND - versionEpoch; + co_return VersionInfo{ rv, expected }; + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +static Future> getVersionEpoch(Reference tr) { + while (true) { + Error err; + try { + ThreadFuture> versionEpochValFuture = tr->get(versionEpochSpecialKey); + Optional versionEpochVal = co_await safeThreadFutureToFuture(versionEpochValFuture); + co_return versionEpochVal.present() ? boost::lexical_cast(versionEpochVal.get().toString()) + : Optional(); + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } +} + +Future versionEpochCommandActor(Reference db, Database cx, std::vector tokens) { + if (tokens.size() <= 3) { + Reference tr = db->createTransaction(); + if (tokens.size() == 1) { + Optional versionInfo = co_await getVersionInfo(db); + if (versionInfo.present()) { + int64_t diff = versionInfo.get().expectedVersion - versionInfo.get().version; + printf("Version: %" PRId64 "\n", versionInfo.get().version); + printf("Expected: %" PRId64 "\n", versionInfo.get().expectedVersion); + printf("Difference: %" PRId64 " (%.2fs)\n", diff, 1.0 * diff / CLIENT_KNOBS->VERSIONS_PER_SECOND); + } else { + printf("Version epoch is unset\n"); + } + co_return true; + } else if (tokens.size() == 2 && tokencmp(tokens[1], "get")) { + Optional versionEpoch = co_await getVersionEpoch(db->createTransaction()); + if (versionEpoch.present()) { + printf("Current version epoch is %" PRId64 "\n", versionEpoch.get()); + } else { + printf("Version epoch is unset\n"); + } + co_return true; + } else if (tokens.size() == 2 && tokencmp(tokens[1], "disable")) { + // Clearing the version epoch means versions will no longer attempt + // to advance at the same rate as the clock. The current version + // will remain unchanged. + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Optional versionEpoch = co_await getVersionEpoch(db->createTransaction()); + if (!versionEpoch.present()) { + co_return true; + } else { + tr->clear(versionEpochSpecialKey); + co_await safeThreadFutureToFuture(tr->commit()); + } + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } + } else if ((tokens.size() == 2 && tokencmp(tokens[1], "enable")) || + (tokens.size() == 3 && tokencmp(tokens[1], "set"))) { + int64_t v; + if (tokens.size() == 3) { + int n = 0; + if (sscanf(tokens[2].toString().c_str(), "%" SCNd64 "%n", &v, &n) != 1 || n != tokens[2].size()) { + printUsage(tokens[0]); + co_return false; + } + } else { + v = 0; // default version epoch + } + + while (true) { + Error err; + try { + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + Optional versionEpoch = co_await getVersionEpoch(tr); + if (!versionEpoch.present() || (versionEpoch.get() != v && tokens.size() == 3)) { + tr->set(versionEpochSpecialKey, BinaryWriter::toValue(v, Unversioned())); + co_await safeThreadFutureToFuture(tr->commit()); + } else { + printf("Version epoch enabled. Run `versionepoch commit` to irreversibly jump to the target " + "version\n"); + co_return true; + } + } catch (Error& e) { + err = e; + } + co_await safeThreadFutureToFuture(tr->onError(err)); + } + } else if (tokens.size() == 2 && tokencmp(tokens[1], "commit")) { + Optional versionInfo = co_await getVersionInfo(db); + if (versionInfo.present()) { + co_await advanceVersion(cx, versionInfo.get().expectedVersion); + } else { + printf("Must set the version epoch before committing it (see `versionepoch enable`)\n"); + } + co_return true; + } + } + + printUsage(tokens[0]); + co_return false; +} + +CommandFactory versionEpochFactory( + "versionepoch", + CommandHelp("versionepoch [ [EPOCH]]", + "Read or write the version epoch", + "If no arguments are specified, reports the offset between the expected version " + "and the actual version. Otherwise, enables, disables, or commits the version epoch. " + "Setting the version epoch can be irreversible since it can cause a large version jump. " + "Thus, the version epoch must first by enabled with the enable or set command. This " + "causes a recovery. Once the version epoch has been set, versions may be given out at " + "a faster or slower rate to attempt to match the actual version to the expected version, " + "based on the version epoch. After setting the version, run the commit command to perform " + "a one time jump to the expected version. This is useful when there is a very large gap " + "between the current version and the expected version. Note that once a version jump has " + "occurred, it cannot be undone. Run this command without any arguments to see the current " + "and expected version.")); +} // namespace fdb_cli diff --git a/fdbcli/fdbcli.actor.cpp b/fdbcli/fdbcli.actor.cpp deleted file mode 100644 index f6184f13ca1..00000000000 --- a/fdbcli/fdbcli.actor.cpp +++ /dev/null @@ -1,2226 +0,0 @@ -/* - * fdbcli.actor.cpp - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2013-2026 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "boost/lexical_cast.hpp" -#include "fmt/format.h" -#include "fdbclient/ClusterConnectionFile.h" -#include "fdbclient/NativeAPI.actor.h" -#include "fdbclient/FDBTypes.h" -#include "fdbclient/IClientApi.h" -#include "fdbclient/MultiVersionTransaction.h" -#include "fdbclient/Status.h" -#include "fdbclient/KeyBackedTypes.actor.h" -#include "fdbclient/StatusClient.h" -#include "fdbclient/StorageServerInterface.h" -#include "fdbclient/DatabaseContext.h" -#include "fdbclient/GlobalConfig.actor.h" -#include "fdbclient/IKnobCollection.h" -#include "fdbclient/NativeAPI.actor.h" -#include "fdbclient/ClusterInterface.h" -#include "fdbclient/ManagementAPI.actor.h" -#include "fdbclient/Schemas.h" -#include "fdbclient/CoordinationInterface.h" -#include "fdbclient/FDBOptions.g.h" -#include "fdbclient/SystemData.h" -#include "fdbclient/TagThrottle.actor.h" -#include "fdbclient/Tuple.h" - -#include "fdbclient/ThreadSafeTransaction.h" -#include "flow/flow.h" -#include "flow/ApiVersion.h" -#include "flow/ArgParseUtil.h" -#include "flow/DeterministicRandom.h" -#include "flow/FastRef.h" -#include "flow/Platform.h" -#include "flow/SystemMonitor.h" -#include "flow/CodeProbe.h" - -#include "flow/TLSConfig.actor.h" -#include "flow/ThreadHelper.actor.h" -#include "SimpleOpt/SimpleOpt.h" - -#include "fdbcli/FlowLineNoise.h" -#include "fdbcli/fdbcli.actor.h" - -#include -#include -#include -#include - -#ifdef __unixish__ -#include -#include "linenoise/linenoise.h" -#endif - -#include "fdbclient/versions.h" -#include "fdbclient/BuildFlags.h" - -#include "flow/actorcompiler.h" // This must be the last #include. - -/* - * While we could just use the MultiVersionApi instance directly, this #define allows us to swap in any other IClientApi - * instance (e.g. from ThreadSafeApi) - */ -#define API ((IClientApi*)MultiVersionApi::api) - -extern const char* getSourceVersion(); - -std::vector validOptions; - -enum { - OPT_CONNFILE, - OPT_DATABASE, - OPT_HELP, - OPT_TRACE, - OPT_TRACE_DIR, - OPT_LOGGROUP, - OPT_TIMEOUT, - OPT_EXEC, - OPT_NO_STATUS, - OPT_NO_HINTS, - OPT_NO_HISTORY, - OPT_STATUS_FROM_JSON, - OPT_VERSION, - OPT_BUILD_FLAGS, - OPT_TRACE_FORMAT, - OPT_KNOB, - OPT_DEBUG_TLS, - OPT_API_VERSION, - OPT_MEMORY, - OPT_USE_FUTURE_PROTOCOL_VERSION, - OPT_ENCRYPT -}; - -CSimpleOpt::SOption g_rgOptions[] = { { OPT_CONNFILE, "-C", SO_REQ_SEP }, - { OPT_CONNFILE, "--cluster-file", SO_REQ_SEP }, - { OPT_DATABASE, "-d", SO_REQ_SEP }, - { OPT_TRACE, "--log", SO_NONE }, - { OPT_TRACE_DIR, "--log-dir", SO_REQ_SEP }, - { OPT_LOGGROUP, "--log-group", SO_REQ_SEP }, - { OPT_TIMEOUT, "--timeout", SO_REQ_SEP }, - { OPT_EXEC, "--exec", SO_REQ_SEP }, - { OPT_NO_STATUS, "--no-status", SO_NONE }, - { OPT_NO_HINTS, "--no-hints", SO_NONE }, - { OPT_NO_HISTORY, "--no-history", SO_NONE }, - { OPT_HELP, "-?", SO_NONE }, - { OPT_HELP, "-h", SO_NONE }, - { OPT_HELP, "--help", SO_NONE }, - { OPT_STATUS_FROM_JSON, "--status-from-json", SO_REQ_SEP }, - { OPT_VERSION, "--version", SO_NONE }, - { OPT_VERSION, "-v", SO_NONE }, - { OPT_BUILD_FLAGS, "--build-flags", SO_NONE }, - { OPT_TRACE_FORMAT, "--trace-format", SO_REQ_SEP }, - { OPT_KNOB, "--knob-", SO_REQ_SEP }, - { OPT_DEBUG_TLS, "--debug-tls", SO_NONE }, - { OPT_API_VERSION, "--api-version", SO_REQ_SEP }, - { OPT_MEMORY, "--memory", SO_REQ_SEP }, - { OPT_USE_FUTURE_PROTOCOL_VERSION, "--use-future-protocol-version", SO_NONE }, - { OPT_ENCRYPT, "--encrypt", SO_REQ_SEP }, - TLS_OPTION_FLAGS, - SO_END_OF_OPTIONS }; - -void printAtCol(const char* text, int col, FILE* stream = stdout) { - const char* iter = text; - const char* start = text; - const char* space = nullptr; - - do { - iter++; - if (*iter == '\n' || *iter == ' ' || *iter == '\0') - space = iter; - if (*iter == '\n' || *iter == '\0' || (iter - start == col)) { - if (!space) - space = iter; - fprintf(stream, "%.*s\n", (int)(space - start), start); - start = space; - if (*start == ' ' || *start == '\n') - start++; - space = nullptr; - } - } while (*iter); -} - -class FdbOptions { -public: - // Prints an error and throws invalid_option or invalid_option_value if the option could not be set - void setOption(Reference tr, - StringRef optionStr, - bool enabled, - Optional arg, - bool intrans) { - auto transactionItr = transactionOptions.legalOptions.find(optionStr.toString()); - if (transactionItr != transactionOptions.legalOptions.end()) - setTransactionOption(tr, transactionItr->second, enabled, arg, intrans); - else { - fprintf(stderr, - "ERROR: invalid option '%s'. Try `help options' for a list of available options.\n", - optionStr.toString().c_str()); - throw invalid_option(); - } - } - - // Applies all enabled transaction options to the given transaction - void apply(Reference tr) { - for (const auto& [name, value] : transactionOptions.options) { - tr->setOption(name, value.castTo()); - } - } - - // Returns true if any options have been set - bool hasAnyOptionsEnabled() const { return !transactionOptions.options.empty(); } - - // Prints a list of enabled options, along with their parameters (if any) - void print() const { - bool found = false; - found = found || transactionOptions.print(); - - if (!found) - printf("There are no options enabled\n"); - } - - // Returns a vector of the names of all documented options - std::vector getValidOptions() const { return transactionOptions.getValidOptions(); } - - // Prints the help string obtained by invoking `help options' - void printHelpString() const { transactionOptions.printHelpString(); } - -private: - // Sets a transaction option. If intrans == true, then this option is also applied to the passed in transaction. - void setTransactionOption(Reference tr, - FDBTransactionOptions::Option option, - bool enabled, - Optional arg, - bool intrans) { - // If the parameter type is an int, we will extract into this variable and reference its memory with a StringRef - int64_t parsedInt = 0; - - if (enabled) { - auto optionInfo = FDBTransactionOptions::optionInfo.getMustExist(option); - if (arg.present() != optionInfo.hasParameter) { - fprintf(stderr, "ERROR: option %s a parameter\n", arg.present() ? "did not expect" : "expected"); - throw invalid_option_value(); - } - if (arg.present() && optionInfo.paramType == FDBOptionInfo::ParamType::Int) { - try { - size_t nextIdx; - std::string value = arg.get().toString(); - parsedInt = std::stoll(value, &nextIdx); - if (nextIdx != value.length()) { - fprintf( - stderr, "ERROR: could not parse value `%s' as an integer\n", arg.get().toString().c_str()); - throw invalid_option_value(); - } - arg = StringRef(reinterpret_cast(&parsedInt), 8); - } catch (std::exception e) { - fprintf(stderr, "ERROR: could not parse value `%s' as an integer\n", arg.get().toString().c_str()); - throw invalid_option_value(); - } - } - } - - if (intrans) { - tr->setOption(option, arg); - } - - transactionOptions.setOption(option, enabled, arg.castTo()); - } - - // A group of enabled options (of type T::Option) as well as a legal options map from string to T::Option - template - struct OptionGroup { - std::map>> options; - std::map legalOptions; - - OptionGroup() {} - OptionGroup(OptionGroup& base) - : options(base.options.begin(), base.options.end()), legalOptions(base.legalOptions) {} - - // Enable or disable an option. Returns true if option value changed - bool setOption(typename T::Option option, bool enabled, Optional arg) { - auto optionItr = options.find(option); - if (enabled && (optionItr == options.end() || - Optional>(optionItr->second).castTo() != arg)) { - options[option] = arg.castTo>(); - return true; - } else if (!enabled && optionItr != options.end()) { - options.erase(optionItr); - return true; - } - - return false; - } - - // Prints a list of all enabled options in this group - bool print() const { - bool found = false; - - for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { - auto optionItr = options.find(itr->second); - if (optionItr != options.end()) { - if (optionItr->second.present()) - printf("%s: `%s'\n", itr->first.c_str(), formatStringRef(optionItr->second.get()).c_str()); - else - printf("%s\n", itr->first.c_str()); - - found = true; - } - } - - return found; - } - - // Returns true if the specified option is documented - bool isDocumented(typename T::Option option) const { - FDBOptionInfo info = T::optionInfo.getMustExist(option); - - std::string deprecatedStr = "Deprecated"; - return !info.comment.empty() && info.comment.substr(0, deprecatedStr.size()) != deprecatedStr; - } - - // Returns a vector of the names of all documented options - std::vector getValidOptions() const { - std::vector ret; - - for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) - if (isDocumented(itr->second)) - ret.push_back(itr->first); - - return ret; - } - - // Prints a help string for each option in this group. Any options with no comment - // are excluded from this help string. Lines are wrapped to 80 characters. - void printHelpString() const { - for (auto itr = legalOptions.begin(); itr != legalOptions.end(); ++itr) { - if (isDocumented(itr->second)) { - FDBOptionInfo info = T::optionInfo.getMustExist(itr->second); - std::string helpStr = info.name + " - " + info.comment; - if (info.hasParameter) - helpStr += " " + info.parameterComment; - helpStr += "\n"; - - printAtCol(helpStr.c_str(), 80); - } - } - } - }; - - OptionGroup transactionOptions; - -public: - FdbOptions() { - for (auto itr = FDBTransactionOptions::optionInfo.begin(); itr != FDBTransactionOptions::optionInfo.end(); - ++itr) - transactionOptions.legalOptions[itr->second.name] = itr->first; - } - - FdbOptions(FdbOptions& base) : transactionOptions(base.transactionOptions) {} -}; - -static std::string formatStringRef(StringRef item, bool fullEscaping = false) { - std::string ret; - - for (int i = 0; i < item.size(); i++) { - if (fullEscaping && item[i] == '\\') - ret += "\\\\"; - else if (fullEscaping && item[i] == '"') - ret += "\\\""; - else if (fullEscaping && item[i] == ' ') - ret += format("\\x%02x", item[i]); - else if (item[i] >= 32 && item[i] < 127) - ret += item[i]; - else - ret += format("\\x%02x", item[i]); - } - - return ret; -} - -static std::vector> parseLine(std::string& line, bool& err, bool& partial) { - err = false; - partial = false; - - bool quoted = false; - std::vector buf; - std::vector> ret; - - size_t i = line.find_first_not_of(' '); - size_t offset = i; - - bool forcetoken = false; - - while (i <= line.length()) { - switch (line[i]) { - case ';': - if (!quoted) { - if (i > offset || (forcetoken && i == offset)) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - ret.push_back(std::move(buf)); - offset = i = line.find_first_not_of(' ', i + 1); - forcetoken = false; - } else - i++; - break; - case '"': - quoted = !quoted; - line.erase(i, 1); - forcetoken = true; - break; - case ' ': - case '\n': - case '\t': - case '\r': - if (!quoted) { - if (i > offset || (forcetoken && i == offset)) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - offset = i = line.find_first_not_of(" \n\t\r", i); - forcetoken = false; - } else - i++; - break; - case '\\': - if (i + 2 > line.length()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - switch (line[i + 1]) { - char ent, save; - case '"': - case '\\': - case ' ': - case ';': - line.erase(i, 1); - break; - // Handle \uNNNN utf-8 characters from JSON strings but only as a single byte - // Return an error for a sequence out of range for a single byte - case 'u': { - if (i + 6 > line.length()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - char* pEnd; - save = line[i + 6]; - line[i + 6] = 0; - unsigned long val = strtoul(line.data() + i + 2, &pEnd, 16); - ent = char(val); - if (*pEnd || val > std::numeric_limits::max()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - line[i + 6] = save; - line.replace(i, 6, 1, ent); - break; - } - case 'x': - if (i + 4 > line.length()) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - char* pEnd; - save = line[i + 4]; - line[i + 4] = 0; - ent = char(strtoul(line.data() + i + 2, &pEnd, 16)); - if (*pEnd) { - err = true; - ret.push_back(std::move(buf)); - return ret; - } - line[i + 4] = save; - line.replace(i, 4, 1, ent); - break; - default: - err = true; - ret.push_back(std::move(buf)); - return ret; - } - default: - i++; - } - } - - i -= 1; - if (i > offset || (forcetoken && i == offset)) - buf.push_back(StringRef((uint8_t*)(line.data() + offset), i - offset)); - - ret.push_back(std::move(buf)); - - if (quoted) - partial = true; - - return ret; -} - -static void printProgramUsage(const char* name) { - printf("FoundationDB CLI " FDB_VT_PACKAGE_NAME " (v" FDB_VT_VERSION ")\n" - "usage: %s [OPTIONS]\n" - "\n", - name); - printf(" -C CONNFILE The path of a file containing the connection string for the\n" - " FoundationDB cluster. The default is first the value of the\n" - " FDB_CLUSTER_FILE environment variable, then `./fdb.cluster',\n" - " then `%s'.\n", - platform::getDefaultClusterFilePath().c_str()); - printf(" --log Enables trace file logging for the CLI session.\n" - " --log-dir PATH Specifies the output directory for trace files. If\n" - " unspecified, defaults to the current directory. Has\n" - " no effect unless --log is specified.\n" - " --log-group LOG_GROUP\n" - " Sets the LogGroup field with the specified value for all\n" - " events in the trace output (defaults to `default').\n" - " --trace-format FORMAT\n" - " Select the format of the log files. xml (the default) and json\n" - " are supported. Has no effect unless --log is specified.\n" - " --exec CMDS Immediately executes the semicolon separated CLI commands\n" - " and then exits.\n" - " --no-history Disables loading and saving the command history file.\n" - " --no-status Disables the initial status check done when starting\n" - " the CLI.\n" - " --api-version APIVERSION\n" - " Specifies the version of the API for the CLI to use.\n" TLS_HELP - " --knob-KNOBNAME KNOBVALUE\n" - " Changes a knob option. KNOBNAME should be lowercase.\n" - " --debug-tls Prints the TLS configuration and certificate chain, then exits.\n" - " Useful in reporting and diagnosing TLS issues.\n" - " --build-flags Print build information and exit.\n" - " --memory Resident memory limit of the CLI (defaults to 8GiB).\n" - " --use-future-protocol-version\n" - " Use the simulated future protocol version to connect to the cluster.\n" - " This option can be used testing purposes only!\n" - " --encrypt PASSWORD\n" - " Encrypts the specified password and prints the encrypted password\n" - " with the `encrypted:' prefix. The encrypted password can be used\n" - " with --tls-password option. This option causes fdbcli to encrypt\n" - " the password and exit.\n" - " -v, --version Print FoundationDB CLI version information and exit.\n" - " -h, --help Display this help and exit.\n"); -} - -#define ESCAPINGK "\n\nFor information on escaping keys, type `help escaping'." -#define ESCAPINGKV "\n\nFor information on escaping keys and values, type `help escaping'." - -using namespace fdb_cli; -std::map& helpMap = CommandFactory::commands(); -std::set& hiddenCommands = CommandFactory::hiddenCommands(); - -void initHelp() { - helpMap["begin"] = - CommandHelp("begin", - "begin a new transaction", - "By default, the fdbcli operates in autocommit mode. All operations are performed in their own " - "transaction, and are automatically committed for you. By explicitly beginning a transaction, " - "successive operations are all performed as part of a single transaction.\n\nTo commit the " - "transaction, use the commit command. To discard the transaction, use the reset command."); - helpMap["commit"] = CommandHelp("commit", - "commit the current transaction", - "Any sets or clears executed after the start of the current transaction will be " - "committed to the database. On success, the committed version number is displayed. " - "If commit fails, the error is displayed and the transaction must be retried."); - helpMap["clear"] = CommandHelp( - "clear ", - "clear a key from the database", - "Clear succeeds even if the specified key is not present, but may fail because of conflicts." ESCAPINGK); - helpMap["clearrange"] = CommandHelp( - "clearrange ", - "clear a range of keys from the database", - "All keys between BEGINKEY (inclusive) and ENDKEY (exclusive) are cleared from the database. This command will " - "succeed even if the specified range is empty, but may fail because of conflicts." ESCAPINGK); - helpMap["exit"] = CommandHelp("exit", "exit the CLI", ""); - helpMap["quit"] = CommandHelp(); - helpMap["waitconnected"] = CommandHelp(); - helpMap["waitopen"] = CommandHelp(); - helpMap["sleep"] = CommandHelp("sleep ", "sleep for a period of time", ""); - helpMap["get"] = - CommandHelp("get ", - "fetch the value for a given key", - "Displays the value of KEY in the database, or `not found' if KEY is not present." ESCAPINGK); - helpMap["getrange"] = - CommandHelp("getrange [ENDKEY] [LIMIT]", - "fetch key/value pairs in a range of keys", - "Displays up to LIMIT keys and values for keys between BEGINKEY (inclusive) and ENDKEY " - "(exclusive). If ENDKEY is omitted, then the range will include all keys starting with BEGINKEY. " - "LIMIT defaults to 25 if omitted." ESCAPINGK); - helpMap["getrangekeys"] = CommandHelp( - "getrangekeys [ENDKEY] [LIMIT]", - "fetch keys in a range of keys", - "Displays up to LIMIT keys for keys between BEGINKEY (inclusive) and ENDKEY (exclusive). If ENDKEY is omitted, " - "then the range will include all keys starting with BEGINKEY. LIMIT defaults to 25 if omitted." ESCAPINGK); - helpMap["getversion"] = - CommandHelp("getversion", - "Fetch the current read version", - "Displays the current read version of the database or currently running transaction."); - - helpMap["reset"] = - CommandHelp("reset", - "reset the current transaction", - "Any sets or clears executed after the start of the active transaction will be discarded."); - helpMap["rollback"] = CommandHelp("rollback", - "rolls back the current transaction", - "The active transaction will be discarded, including any sets or clears executed " - "since the transaction was started."); - helpMap["set"] = CommandHelp("set ", - "set a value for a given key", - "If KEY is not already present in the database, it will be created." ESCAPINGKV); - - helpMap["option"] = CommandHelp( - "option