Merged upstream 7.1.29

.github/workflows/build.yml

@@ -92,72 +92,26 @@ jobs:
   test_deploy:
     needs: [calc_ver, build]
     runs-on: ubuntu-latest
-    env:
-      RHEL_IMAGE_NAME: "registry.access.redhat.com/ubi8/ubi:latest"
-      UBUNTU_IMAGE_NAME: "ubuntu:18.04"
 
     steps:
-      - name: Download RPMs and DEBs packages
-        uses: actions/download-artifact@v3
-        id: download_packages
-
-      - name: Echo download path
-        run: echo ${{steps.download_packages.outputs.download-path}}
-
-      - name: Display structure of downloaded files
-        run: ls -R
-        working-directory: ${{github.workspace}}
+      - name: Checkout
+        uses: actions/checkout@v3
 
-      - name: Download RHEL and Ubuntu images
-        run: |
-          podman pull ${{ env.RHEL_IMAGE_NAME }}
-          podman pull ${{ env.UBUNTU_IMAGE_NAME }}
+      - name: Download the foundationdb distro
+        uses: actions/download-artifact@v3
+        with:
+          path: ${{github.workspace}}/bld/download
 
-      - name: Test RPMs deploy
-        shell: bash
-        run: |
-          MY_ARCH=`uname -m`
-          curr_path="${{ github.workspace }}"
-          server="foundationdb-server-${{ needs.calc_ver.outputs.full_ver }}.${MY_ARCH}.rpm"
-          clients="foundationdb-clients-${{ needs.calc_ver.outputs.full_ver }}.${MY_ARCH}.rpm"
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} ${{ env.RHEL_IMAGE_NAME }} /bin/bash -c "dnf install -y /home/${server}" && { echo "Installation $server have to fail"; exit 1; }
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.RHEL_IMAGE_NAME }} /bin/bash -c "dnf install -y /home/${server} /home/${clients}" || { echo "Installation $server and $clients have to success"; exit 2; }
-          podman run --rm -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.RHEL_IMAGE_NAME }} /bin/bash -c "dnf install -y /home/${clients} && grep -q foundationdb /etc/passwd" || { echo "User foundationdb is not created after $clients installation"; exit 3; }
-
-      - name: Test versioned RPMs deploy
-        shell: bash
-        run: |
-          MY_ARCH=`uname -m`
-          curr_path="${{ github.workspace }}"
-          server="foundationdb-${{ needs.calc_ver.outputs.full_ver }}-server-versioned-${{ needs.calc_ver.outputs.full_ver }}.${MY_ARCH}.rpm"
-          clients="foundationdb-${{ needs.calc_ver.outputs.full_ver }}-clients-versioned-${{ needs.calc_ver.outputs.full_ver }}.${MY_ARCH}.rpm"
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} ${{ env.RHEL_IMAGE_NAME }} /bin/bash -c "dnf install -y /home/${server}" && { echo "Installation $server have to fail"; exit 1; }
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.RHEL_IMAGE_NAME }} /bin/bash -c "dnf install -y /home/${server} /home/${clients}" || { echo "Installation $server and $clients have to success"; exit 2; }
-          podman run --rm -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.RHEL_IMAGE_NAME }} /bin/bash -c "dnf install -y /home/${clients} && grep -q foundationdb /etc/passwd" && { echo "User foundationdb was created after $clients installation"; exit 3; }
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.RHEL_IMAGE_NAME }} /bin/bash -c "dnf install -y /home/${server} /home/${clients} && grep -q foundationdb /etc/passwd" || { echo "User foundationdb is not created after $server installation"; exit 4; }
-
-      - name: Test DEBs deploy
+      - name: Make a normal distro directory from the downloaded artifacts
         shell: bash
         run: |
-          MY_ARCH=`dpkg-architecture -q DEB_BUILD_ARCH`
-          curr_path="${{ github.workspace }}"
-          server="foundationdb-server_${{ needs.calc_ver.outputs.full_ver }}_${MY_ARCH}.deb"
-          clients="foundationdb-clients_${{ needs.calc_ver.outputs.full_ver }}_${MY_ARCH}.deb"
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${server}" && { echo "Installation $server have to fail"; exit 1; }
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${server} /home/${clients}" || { echo "Installation $server and $clients have to success"; exit 2; }
-          podman run --rm -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${clients} && grep -q foundationdb /etc/passwd" || { echo "User foundationdb is not created after $clients installation"; exit 3; }
-
-      - name: Test versioned DEBs deploy
+          ls -Rl ${{github.workspace}}/bld/download
+          mkdir -p ${{github.workspace}}/bld/linux/packages
+          mv -v ${{github.workspace}}/bld/download/*/* ${{github.workspace}}/bld/linux/packages/
+
+      - name: Run tests
         shell: bash
-        run: |
-          MY_ARCH=`dpkg-architecture -q DEB_BUILD_ARCH`
-          curr_path="${{ github.workspace }}"
-          server="foundationdb-${{ needs.calc_ver.outputs.full_ver }}-server-versioned_${{ needs.calc_ver.outputs.full_ver }}_${MY_ARCH}.deb"
-          clients="foundationdb-${{ needs.calc_ver.outputs.full_ver }}-clients-versioned_${{ needs.calc_ver.outputs.full_ver }}_${MY_ARCH}.deb"
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${server}" && { echo "Installation $server have to fail"; exit 1; }
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${server} /home/${clients}" || { echo "Installation $server and $clients have to success"; exit 2; }
-          podman run --rm -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${clients} && grep -q foundationdb /etc/passwd" && { echo "User foundationdb was created after $clients installation"; exit 3; }
-          podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${server} /home/${clients} && grep -q foundationdb /etc/passwd" || { echo "User foundationdb is not created after $server installation"; exit 4; }
+        run: ${{github.workspace}}/build-scripts/for-linux/test-deploy.bash ${{ needs.calc_ver.outputs.full_ver }} ${{github.workspace}}/bld/linux/packages
 
   release:
     needs: [calc_ver, build]

CMakeLists.txt

@@ -23,7 +23,7 @@ else()
 endif()
 
 project(foundationdb
-  VERSION 7.1.25
+  VERSION 7.1.29
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM)

bindings/c/CMakeLists.txt

@@ -184,7 +184,11 @@ endif()
     # Make sure that fdb_c.h is compatible with c90
     add_executable(fdb_c90_test test/fdb_c90_test.c)
     set_property(TARGET fdb_c90_test PROPERTY C_STANDARD 90)
-    target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
+	if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+	  target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Wno-gnu-line-marker -Werror)
+	else ()
+	  target_compile_options(fdb_c90_test PRIVATE -Wall -Wextra -Wpedantic -Werror)
+	endif ()
     target_link_libraries(fdb_c90_test PRIVATE fdb_c)
   endif()
 

bindings/c/test/apitester/TesterOptions.h

@@ -42,6 +42,7 @@ class TesterOptions {
 	int numClients;
 	std::vector<std::pair<std::string, std::string>> knobs;
 	TestSpec testSpec;
+	bool retainClientLibCopies = false;
 };
 
 } // namespace FdbApiTester

bindings/c/test/apitester/fdb_c_api_tester.cpp

@@ -45,7 +45,8 @@ enum TesterOptionId {
 	OPT_TRACE_FORMAT,
 	OPT_KNOB,
 	OPT_EXTERNAL_CLIENT_LIBRARY,
-	OPT_TEST_FILE
+	OPT_TEST_FILE,
+	OPT_RETAIN_CLIENT_LIB_COPIES
 };
 
 CSimpleOpt::SOption TesterOptionDefs[] = //
@@ -61,6 +62,7 @@ CSimpleOpt::SOption TesterOptionDefs[] = //
 	  { OPT_EXTERNAL_CLIENT_LIBRARY, "--external-client-library", SO_REQ_SEP },
 	  { OPT_TEST_FILE, "-f", SO_REQ_SEP },
 	  { OPT_TEST_FILE, "--test-file", SO_REQ_SEP },
+	  { OPT_RETAIN_CLIENT_LIB_COPIES, "--retain-client-lib-copies", SO_NONE },
 	  SO_END_OF_OPTIONS };
 
 void printProgramUsage(const char* execName) {
@@ -144,6 +146,10 @@ bool processArg(TesterOptions& options, const CSimpleOpt& args) {
 		options.testFile = args.OptionArg();
 		options.testSpec = readTomlTestSpec(options.testFile);
 		break;
+
+	case OPT_RETAIN_CLIENT_LIB_COPIES:
+		options.retainClientLibCopies = true;
+		break;
 	}
 	return true;
 }
@@ -205,6 +211,10 @@ void applyNetworkOptions(TesterOptions& options) {
 		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_LOG_GROUP, options.logGroup));
 	}
 
+	if (options.retainClientLibCopies) {
+		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_RETAIN_CLIENT_LIBRARY_COPIES));
+	}
+
 	for (auto knob : options.knobs) {
 		fdb_check(FdbApi::setOption(FDBNetworkOption::FDB_NET_OPTION_KNOB,
 		                            fmt::format("{}={}", knob.first.c_str(), knob.second.c_str())));

bindings/c/test/apitester/run_c_api_tests.py

@@ -54,6 +54,9 @@ def run_tester(args, test_file):
     if args.external_client_library is not None:
         cmd += ["--external-client-library", args.external_client_library]
 
+    if args.retain_client_lib_copies:
+        cmd += ["--retain-client-lib-copies"]
+
     get_logger().info('\nRunning tester \'%s\'...' % ' '.join(cmd))
     proc = Popen(cmd, stdout=sys.stdout, stderr=sys.stderr)
     timed_out = False
@@ -111,7 +114,12 @@ def parse_args(argv):
                         help='The timeout in seconds for running each individual test. (default 300)')
     parser.add_argument('--logging-level', type=str, default='INFO',
                         choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], help='Specifies the level of detail in the tester output (default=\'INFO\').')
-
+    parser.add_argument(
+                        "--retain-client-lib-copies",
+                        action="store_true",
+                        default=False,
+                        help="Retain temporary external client library copies.",
+    )
     return parser.parse_args(argv)
 
 

bindings/go/src/fdb/generated.go

@@ -239,6 +239,11 @@ func (o NetworkOptions) SetClientThreadsPerVersion(param int64) error {
 	return o.setOpt(65, int64ToBytes(param))
 }
 
+// Retain temporary external client library copies that are created for enabling multi-threading.
+func (o NetworkOptions) SetRetainClientLibraryCopies() error {
+	return o.setOpt(67, nil)
+}
+
 // Disables logging of client statistics, such as sampled transaction activity.
 func (o NetworkOptions) SetDisableClientStatisticsLogging() error {
 	return o.setOpt(70, nil)
@@ -617,12 +622,12 @@ const (
 	StreamingModeWantAll StreamingMode = -1
 
 	// The default. The client doesn't know how much of the range it is likely
-	// to used and wants different performance concerns to be balanced. Only a
-	// small portion of data is transferred to the client initially (in order to
-	// minimize costs if the client doesn't read the entire range), and as the
-	// caller iterates over more items in the range larger batches will be
-	// transferred in order to minimize latency. After enough iterations, the
-	// iterator mode will eventually reach the same byte limit as “WANT_ALL“
+	// to used and wants different performance concerns to be balanced.
+	// Only a small portion of data is transferred to the client initially (in
+	// order to minimize costs if the client doesn't read the entire range), and
+	// as the caller iterates over more items in the range larger batches will
+	// be transferred in order to minimize latency. After enough iterations,
+	// the iterator mode will eventually reach the same byte limit as “WANT_ALL“
 	StreamingModeIterator StreamingMode = 0
 
 	// Infrequently used. The client has passed a specific row limit and wants
@@ -632,25 +637,25 @@ const (
 	// mode is used.
 	StreamingModeExact StreamingMode = 1
 
-	// Infrequently used. Transfer data in batches small enough to not be much
-	// more expensive than reading individual rows, to minimize cost if
+	// Infrequently used. Transfer data in batches small enough to not be
+	// much more expensive than reading individual rows, to minimize cost if
 	// iteration stops early.
 	StreamingModeSmall StreamingMode = 2
 
 	// Infrequently used. Transfer data in batches sized in between small and
 	// large.
 	StreamingModeMedium StreamingMode = 3
 
-	// Infrequently used. Transfer data in batches large enough to be, in a
-	// high-concurrency environment, nearly as efficient as possible. If the
-	// client stops iteration early, some disk and network bandwidth may be
-	// wasted. The batch size may still be too small to allow a single client to
-	// get high throughput from the database, so if that is what you need
+	// Infrequently used. Transfer data in batches large enough to be,
+	// in a high-concurrency environment, nearly as efficient as possible.
+	// If the client stops iteration early, some disk and network bandwidth may
+	// be wasted. The batch size may still be too small to allow a single client
+	// to get high throughput from the database, so if that is what you need
 	// consider the SERIAL StreamingMode.
 	StreamingModeLarge StreamingMode = 4
 
-	// Transfer data in batches large enough that an individual client can get
-	// reasonable read bandwidth from the database. If the client stops
+	// Transfer data in batches large enough that an individual client can
+	// get reasonable read bandwidth from the database. If the client stops
 	// iteration early, considerable disk and network bandwidth may be wasted.
 	StreamingModeSerial StreamingMode = 5
 )

build-scripts/for-linux/build-on-linux.bash

@@ -34,9 +34,6 @@ APP_PRMS="\
 [ ! -e /usr/lib64/libcrypto.a -a -e /opt/openssl/lib/libcrypto.a ] && \
   APP_PRMS="$APP_PRMS -DOPENSSL_ROOT_DIR=/opt/openssl"
 
-# find libatomic
-source $BASE_DIR/build-find-libatomic.bash
-
 echo "env CC=clang CXX=clang++ cmake -G Ninja $APP_PRMS . $SRC_DIR"
 env CC=clang CXX=clang++ cmake -G Ninja $APP_PRMS . $SRC_DIR