Ow fork 7.1 update to apple-7.1.65

CMakeLists.txt

@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.63
+  VERSION 7.1.65
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

cmake/CompileBoost.cmake

@@ -49,7 +49,7 @@ function(compile_boost)
   include(ExternalProject)
   set(BOOST_INSTALL_DIR "${CMAKE_BINARY_DIR}/boost_install")
   ExternalProject_add("${COMPILE_BOOST_TARGET}Project"
-    URL "https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.bz2"
+    URL "https://archives.boost.io/release/1.78.0/source/boost_1_78_0.tar.bz2"
     URL_HASH SHA256=8681f175d4bdb26c52222665793eef08490d7758529330f98d3b29dd0735bccc
     CONFIGURE_COMMAND ${BOOTSTRAP_COMMAND} ${BOOTSTRAP_ARGS} --with-libraries=${BOOTSTRAP_LIBRARIES} --with-toolset=${BOOST_TOOLSET}
     BUILD_COMMAND ${B2_COMMAND} toolset=${BOOST_TOOLSET} link=static ${COMPILE_BOOST_BUILD_ARGS} --prefix=${BOOST_INSTALL_DIR} ${USER_CONFIG_FLAG} install

documentation/sphinx/extensions/rubydomain.py

@@ -711,4 +711,4 @@ def get_objects(self):
 
 
 def setup(app):
-    app.add_domain(RubyDomain)
+    app.add_domain(RubyDomain)

fdbbackup/backup.actor.cpp

@@ -4103,10 +4103,12 @@ int main(int argc, char* argv[]) {
 				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
 				// enough for all cases to finish and 5 retries should also be good enough for
 				// most cases.
+				int64_t timeout = 60000;
 				db->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT,
-				              Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				              Optional<StringRef>(StringRef((const uint8_t*)&timeout, sizeof(timeout))));
+				int64_t retryLimit = 5;
 				db->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT,
-				              Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
+				              Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit))));
 			}
 
 			return result.present();
@@ -4127,10 +4129,12 @@ int main(int argc, char* argv[]) {
 				// where the fdbbackup command hangs infinitely. 60 seconds should be more than
 				// enough for all cases to finish and 5 retries should also be good enough for
 				// most cases.
+				int64_t timeout = 60000;
 				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_TIMEOUT,
-				                    Optional<StringRef>(StringRef((const uint8_t*)60000, 8)));
+				                    Optional<StringRef>(StringRef((const uint8_t*)&timeout, sizeof(timeout))));
+				int64_t retryLimit = 5;
 				sourceDb->setOption(FDBDatabaseOptions::TRANSACTION_RETRY_LIMIT,
-				                    Optional<StringRef>(StringRef((const uint8_t*)5, 8)));
+				                    Optional<StringRef>(StringRef((const uint8_t*)&retryLimit, sizeof(retryLimit))));
 			}
 
 			return result.present();

fdbcli/ExcludeCommand.actor.cpp

@@ -42,6 +42,7 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 	state Reference<ITransaction> tr = db->createTransaction();
 	loop {
 		tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES);
+		tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE);
 		try {
 			if (force && servers.size())
 				tr->set(failed ? fdb_cli::failedForceOptionSpecialKey : fdb_cli::excludedForceOptionSpecialKey,
@@ -78,6 +79,7 @@ ACTOR Future<bool> excludeServersAndLocalities(Reference<IDatabase> db,
 				            : "Type `exclude FORCE failed <ADDRESS...>' to exclude without performing safety checks.");
 				return false;
 			}
+			TraceEvent(SevWarn, "ExcludeServersAndLocalitiesError").error(err);
 			wait(safeThreadFutureToFuture(tr->onError(err)));
 		}
 	}
@@ -107,6 +109,7 @@ ACTOR Future<std::vector<std::string>> getExcludedServers(Reference<IDatabase> d
 			}
 			return exclusions;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetExcludedServersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -137,6 +140,7 @@ ACTOR Future<std::vector<std::string>> getExcludedLocalities(Reference<IDatabase
 			}
 			return excludedLocalities;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetExcludedLocalitiesError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -186,6 +190,7 @@ ACTOR Future<std::set<NetworkAddress>> checkForExcludingServers(Reference<IDatab
 
 			wait(delayJittered(1.0)); // SOMEDAY: watches!
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "CheckForExcludingServersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -206,6 +211,7 @@ ACTOR Future<Void> checkForCoordinators(Reference<IDatabase> db, std::set<Addres
 			coordinatorList = NetworkAddress::parseList(coordinators.get().toString());
 			break;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "CheckForCoordinatorsError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}

fdbcli/Util.actor.cpp

@@ -155,6 +155,7 @@ ACTOR Future<bool> getWorkers(Reference<IDatabase> db, std::vector<ProcessData>*
 
 			return true;
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetWorkersError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}
@@ -181,6 +182,7 @@ ACTOR Future<Void> getStorageServerInterfaces(Reference<IDatabase> db,
 			}
 			return Void();
 		} catch (Error& e) {
+			TraceEvent(SevWarn, "GetStorageServerInterfacesError").error(e);
 			wait(safeThreadFutureToFuture(tr->onError(e)));
 		}
 	}

fdbclient/NativeAPI.actor.cpp

@@ -2043,6 +2043,7 @@ void DatabaseContext::setOption(FDBDatabaseOptions::Option option, Optional<Stri
 	if (defaultFor >= 0) {
 		ASSERT(FDBTransactionOptions::optionInfo.find((FDBTransactionOptions::Option)defaultFor) !=
 		       FDBTransactionOptions::optionInfo.end());
+		TraceEvent(SevDebug, "DatabaseContextSetPersistentOption").detail("Option", option).detail("Value", value);
 		transactionDefaults.addOption((FDBTransactionOptions::Option)defaultFor, value.castTo<Standalone<StringRef>>());
 	} else {
 		switch (option) {

fdbclient/ReadYourWrites.actor.cpp

@@ -2417,6 +2417,7 @@ void ReadYourWritesTransaction::setOption(FDBTransactionOptions::Option option,
 }
 
 void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option option, Optional<StringRef> value) {
+	TraceEvent(SevVerbose, "TransactionSetOption").detail("Option", option).detail("Value", value);
 	switch (option) {
 	case FDBTransactionOptions::READ_YOUR_WRITES_DISABLE:
 		validateOptionValueNotPresent(value);
@@ -2454,11 +2455,13 @@ void ReadYourWritesTransaction::setOptionImpl(FDBTransactionOptions::Option opti
 
 	case FDBTransactionOptions::TIMEOUT:
 		options.timeoutInSeconds = extractIntOption(value, 0, std::numeric_limits<int>::max()) / 1000.0;
+		TraceEvent(SevDebug, "TransactionTimeout").detail("TimeoutInSeconds", options.timeoutInSeconds);
 		resetTimeout();
 		break;
 
 	case FDBTransactionOptions::RETRY_LIMIT:
 		options.maxRetries = (int)extractIntOption(value, -1, std::numeric_limits<int>::max());
+		TraceEvent(SevDebug, "TransactionRetryLimit").detail("MaxRetries", options.maxRetries);
 		break;
 
 	case FDBTransactionOptions::DEBUG_RETRY_LOGGING:

fdbserver/RocksDBCheckpointUtils.actor.h

@@ -243,4 +243,4 @@ ACTOR Future<Void> deleteRocksCFCheckpoint(CheckpointMetaData checkpoint);
 ICheckpointReader* newRocksDBCheckpointReader(const CheckpointMetaData& checkpoint, UID logID);
 
 RocksDBColumnFamilyCheckpoint getRocksCF(const CheckpointMetaData& checkpoint);
-#endif
+#endif

fdbserver/Status.actor.cpp

@@ -875,9 +875,8 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 	}
 
 	state std::vector<OldTLogConf>::const_iterator oldTLogIter;
-	for (oldTLogIter = db->get().logSystemConfig.oldTLogs.begin();
-	     oldTLogIter != db->get().logSystemConfig.oldTLogs.end();
-	     ++oldTLogIter) {
+	state std::vector<OldTLogConf> oldTLogs = db->get().logSystemConfig.oldTLogs;
+	for (oldTLogIter = oldTLogs.begin(); oldTLogIter != oldTLogs.end(); ++oldTLogIter) {
 		for (auto& tLogSet : oldTLogIter->tLogs) {
 			for (auto& it : tLogSet.tLogs) {
 				if (it.present()) {

flow/WriteOnlySet.actor.cpp

@@ -23,7 +23,6 @@
 #include "flow/flow.h"
 #include "flow/UnitTest.h"
 
-#include <thread>
 #include <chrono>
 #include <random>
 #include <thread>

packaging/docker/Dockerfile

@@ -16,44 +16,33 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-FROM centos:7.9.2009 as base
-
-RUN sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo
-
-RUN yum install -y \
-    epel-release-7-11 \
-    centos-release-scl-2-3.el7.centos && \
-    sed -i s/mirror.centos.org/vault.centos.org/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^#.*baseurl=http/baseurl=http/g /etc/yum.repos.d/*.repo && \
-    sed -i s/^mirrorlist=http/#mirrorlist=http/g /etc/yum.repos.d/*.repo && \
-    yum install -y \
-    bind-utils-9.11.4-26.P2.el7_9.7 \
-    binutils-2.27-44.base.el7_9.1 \
-    curl-7.29.0-59.el7_9.1 \
-    gdb-7.6.1-120.el7 \
-    hostname-3.13-3.el7_7.1 \
-    jq-1.6-2.el7  \
-    less-458-9.el7 \
-    libubsan-7.3.1-5.16.el7 \
-    lsof-4.87-6.el7 \
-    net-tools-2.0-0.25.20131004git.el7 \
-    nmap-ncat-6.40-19.el7 \
-    perf-3.10.0-1160.45.1.el7 \
-    perl-5.16.3-299.el7_9 \
-    procps-ng-3.3.10-28.el7 \
-    strace-4.24-6.el7 \
-    sysstat-10.1.5-19.el7 \
-    tar-1.26-35.el7 \
-    tcpdump-4.9.2-4.el7_7.1 \
-    telnet-0.17-66.el7 \
-    traceroute-2.0.22-2.el7 \
-    unzip-6.0-22.el7_9 \
-    openssl-1.0.2k-24.el7_9.x86_64 \
-    vim-enhanced-7.4.629-8.el7_9 && \
-    yum clean all && \
-    rm -rf /var/cache/yum
+FROM docker.io/rockylinux/rockylinux:9.4-minimal AS base
+
+RUN microdnf install -y \
+    bind-utils \
+    binutils \
+    curl \
+    gdb \
+    hostname \
+    jq \
+    less \
+    libubsan \
+    lsof \
+    net-tools \
+    nmap-ncat \
+    perf \
+    perl \
+    procps-ng \
+    strace \
+    sysstat \
+    tar \
+    tcpdump \
+    telnet \
+    traceroute \
+    unzip \
+    openssl \
+    vim-enhanced && \
+    microdnf clean all
 
 WORKDIR /tmp
 
@@ -66,13 +55,13 @@ RUN curl -Ls https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd6
 
 WORKDIR /
 
-FROM golang:1.22.2-bullseye AS go-build
+FROM golang:1.22.5-bullseye AS go-build
 
 COPY fdbkubernetesmonitor/ /fdbkubernetesmonitor
 WORKDIR /fdbkubernetesmonitor
 RUN go build -o /fdb-kubernetes-monitor *.go
 
-FROM base as foundationdb-base
+FROM base AS foundationdb-base
 
 WORKDIR /tmp
 ARG FDB_VERSION=6.3.22
@@ -123,7 +112,7 @@ RUN curl -Ls $FDB_WEBSITE/$FDB_VERSION/libfdb_c.x86_64.so -o /usr/lib/libfdb_c.s
 RUN rm -rf /tmp/*
 WORKDIR /
 
-FROM foundationdb-base as fdb-kubernetes-monitor
+FROM foundationdb-base AS fdb-kubernetes-monitor
 
 # Install the kubernetes monitor binary
 COPY --from=go-build /fdb-kubernetes-monitor /usr/bin/
@@ -134,14 +123,20 @@ WORKDIR /var/fdb
 VOLUME /var/fdb/data
 ENTRYPOINT ["/usr/bin/fdb-kubernetes-monitor"]
 
-FROM foundationdb-base as foundationdb-kubernetes-sidecar
-
-RUN yum -y install \
-    rh-python38-2.0-4.el7 \
-    yum clean all && \
-    rm -rf /var/cache/yum && \
-    source /opt/rh/rh-python38/enable && \
-    pip3 install watchdog==0.9.0
+FROM foundationdb-base AS foundationdb-kubernetes-sidecar
+
+# We use python3.9 here as the tests with 3.12 resulted in the following error:
+# error:0A000126:SSL routines::unexpected eof while reading. This could be an issue
+# with the installed openssl version. Since the sidecar support will be deprecated in the
+# future it's fine to use python 3.9 here.
+# EOL for python 3.9 is 2025-10: https://devguide.python.org/versions/
+RUN microdnf -y install \
+    python3.9 \
+    python3.9-pip && \
+    microdnf clean all && \
+    update-alternatives --install /usr/bin/python python /usr/bin/python3.9 20 && \
+    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 20 && \
+    python3 -m pip install watchdog==4.0.1
 
 WORKDIR /
 ADD entrypoint.bash sidecar.py /
@@ -152,7 +147,7 @@ VOLUME /var/output-files
 ENV LISTEN_PORT 8080
 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/entrypoint.bash"]
 
-FROM foundationdb-base as foundationdb
+FROM foundationdb-base AS foundationdb
 
 WORKDIR /tmp
 RUN curl -LsO https://raw.githubusercontent.com/brendangregg/FlameGraph/90533539b75400297092f973163b8a7b067c66d3/stackcollapse-perf.pl && \
@@ -177,11 +172,11 @@ ENV FDB_CLUSTER_FILE_CONTENTS ""
 ENV FDB_PROCESS_CLASS unset
 ENTRYPOINT ["/usr/bin/tini", "-g", "--", "/var/fdb/scripts/fdb.bash"]
 
-FROM foundationdb-base as ycsb
+FROM foundationdb-base AS ycsb
 
-RUN yum -y install \
-    java-11-openjdk-11.0.13.0.8-1.el7_9 && \
-    yum clean all && \
+RUN microdnf -y install \
+    java-11-openjdk java-11-openjdk-devel && \
+    microdnf clean all && \
     rm -rf /var/cache/yum
 
 WORKDIR /tmp

packaging/docker/entrypoint.bash

@@ -24,8 +24,4 @@ if [[ -n "$ADDITIONAL_ENV_FILE" ]]; then
   source $ADDITIONAL_ENV_FILE
 fi
 
-if [[ -f "/opt/rh/rh-python38/enable" ]]; then
-    source /opt/rh/rh-python38/enable
-fi
-
 exec /sidecar.py $*

packaging/docker/kubernetes/test_config.yaml

@@ -18,7 +18,7 @@
 #
 
 # This file provides an example of using a statefulset to launch FDB processes
-# using the foundationdb-kubernetes image.
+# using the fdb-kubernetes-monitor image.
 #
 # This is not a recommended way to run production clusters, but it can be useful
 # to test the image in development.
@@ -43,7 +43,7 @@ spec:
     spec:
       containers:
         - name: foundationdb
-          image: foundationdb/foundationdb-kubernetes:6.3.13-local
+          image: foundationdb/fdb-kubernetes-monitor:7.1.5-local
           imagePullPolicy: IfNotPresent
           args:
             - --input-dir
@@ -91,13 +91,13 @@ spec:
             - name: logs
               mountPath: /var/fdb/logs
         - name: foundationdb-sidecar
-          image: foundationdb/foundationdb-kubernetes:6.3.15-local
+          image: foundationdb/fdb-kubernetes-monitor:7.1.6-local
           imagePullPolicy: IfNotPresent
           args:
             - --mode
             - sidecar
             - --main-container-version
-            - 6.3.13
+            - 7.1.5
             - --output-dir
             - /var/fdb/shared-binaries
             - --copy-binary
@@ -139,7 +139,7 @@ data:
   config.json: |
     {
       "runProcesses": false,
-      "version": "6.3.13",
+      "version": "7.1.5",
       "arguments": [
         {"value": "--cluster-file"},
         {"value": "/var/fdb/data/fdb.cluster"},
@@ -238,8 +238,8 @@ spec:
         - name: dynamic-conf
           emptyDir: {}
       initContainers:
-        - name: foundationdb-kubernetes-init
-          image: foundationdb/foundationdb-kubernetes:6.3.13-local
+        - name: fdb-kubernetes-monitor-init
+          image: foundationdb/fdb-kubernetes-monitor:7.1.5-local
           imagePullPolicy: IfNotPresent
           args:
             - "--mode"