XDB-102 Moved the build logic from the github workflow to bash scripts

.github/workflows/build.yml

@@ -60,83 +60,47 @@ jobs:
           draft: true
           prerelease: false
 
-  build_linux:
-    runs-on: ubuntu-18.04
+  build:
     needs: [calc_ver]
 
-    steps:
-      - name: Install Dependencies
-        shell: bash
-        run: |
-          sudo apt update
-          sudo apt install -y sudo git cmake libboost-dev g++ build-essential libssl-dev default-jdk python3 python3-pip liblz4-dev ninja-build rpm
-          sudo pip3 install virtualenv
-          sudo apt install -y dirmngr gnupg apt-transport-https ca-certificates
-          sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-          sudo sh -c 'echo "deb https://download.mono-project.com/repo/ubuntu stable-bionic main" > /etc/apt/sources.list.d/mono-official-stable.list'
-          sudo apt update
-          sudo apt install -y mono-complete
-          sudo patch /usr/local/share/cmake-3.21/Modules/Internal/CPack/CPackRPM.cmake <${{github.workspace}}/build/patches/cpack.patch || true
+    strategy:
+      matrix:
+        include:
+          - run_on: ubuntu-18.04
+            for: linux
+            prepare: "debian-based"
+            build_on: linux
 
+    runs-on: ${{ matrix.run_on }}
+    steps:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Create Build Environment
-        # Some projects don't allow in-source building, so create a separate build directory
-        # We'll use this as our working directory for all subsequent commands
-        run: cmake -E make_directory ${{github.workspace}}/bld
-
-      - name: Configure CMake
-        # Use a bash shell so we can use the same syntax for environment variable
-        # access regardless of the host operating system
-        shell: bash
-        working-directory: ${{github.workspace}}/bld
-        # Note the current convention is to use the -S and -B options here to specify source
-        # and build directories, but this is only available with CMake 3.13 and higher.
-        # The CMake binaries on the Github Actions machines are (as of this writing) 3.12
-        run: cmake -D FDB_RELEASE=${{needs.calc_ver.outputs.release_flag}} -DBUILD_VERSION=${{needs.calc_ver.outputs.build_ver}} -DGENERATE_DEBUG_PACKAGES=OFF -DENABLE_SIMULATION_TESTS=ON -G Ninja ${{github.workspace}}
+      - name: Install Dependencies
+        run: ${{github.workspace}}/build-scripts/for-${{ matrix.for }}/prepare-${{ matrix.prepare }}.bash
 
       - name: Build
-        working-directory: ${{github.workspace}}/bld
-        shell: bash
-        run: ninja -j 2 all documentation/package_html
-
-      - name: Tests
-        working-directory: ${{github.workspace}}/bld
-        shell: bash
-        run: ctest --output-on-failure -V
-
-      - name: Build RPMs
-        working-directory: ${{github.workspace}}/bld
-        shell: bash
-        run: |
-          cpack -G RPM
-          for fn in packages/*.rpm; do echo "$fn:"; rpm -qpRv $fn; echo; done
-
-      - name: Build DEBs
-        working-directory: ${{github.workspace}}/bld
-        shell: bash
-        run: |
-          cpack -G DEB
-          for fn in packages/*.deb; do echo "$fn:"; dpkg -I $fn; done
+        working-directory: ${{github.workspace}}
+        run: ${{github.workspace}}/build-scripts/for-${{ matrix.for }}/build-on-${{ matrix.build_on }}.bash ${{needs.calc_ver.outputs.project_ver}} ${{needs.calc_ver.outputs.build_ver}} ${{needs.calc_ver.outputs.release_flag}} 3
 
-      # - name: Remove versioned
+      # - name: Minimal tests
+      #  working-directory: ${{github.workspace}}/bld
       #  shell: bash
-      #  run: rm -rf ${{github.workspace}}/bld/packages/*${{needs.calc_ver.outputs.full_ver}}.versioned*
+      #  run: ctest --output-on-failure -V
 
       - name: Upload result
         uses: nanoufo/action-upload-artifacts-and-release-assets@v1.5
         with:
           path: |
-            ${{github.workspace}}/bld/packages/*${{needs.calc_ver.outputs.full_ver}}*
+            ${{github.workspace}}/bld/linux/packages/*${{needs.calc_ver.outputs.full_ver}}*
           upload-release-files: ${{ needs.calc_ver.outputs.release_flag }}
           release-upload-url: ${{ needs.calc_ver.outputs.release_upload_url }}
           if-no-files-found: error
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   test_deploy:
-    needs: [calc_ver, build_linux]
+    needs: [calc_ver, build]
     runs-on: ubuntu-latest
     env:
       RHEL_IMAGE_NAME: "registry.access.redhat.com/ubi8/ubi:latest"
@@ -206,7 +170,7 @@ jobs:
           podman run --rm -v ${curr_path}/${server}/${server}:/home/${server} -v ${curr_path}/${clients}/${clients}:/home/${clients} ${{ env.UBUNTU_IMAGE_NAME }} /bin/bash -c "apt-get update && apt-get install -y /home/${server} /home/${clients} && grep -q foundationdb /etc/passwd" || { echo "User foundationdb is not created after $server installation"; exit 4; }
 
   release:
-    needs: [calc_ver, build_linux]
+    needs: [calc_ver, build]
     if: ${{ needs.calc_ver.outputs.release_flag == 'ON' }}
     runs-on: ubuntu-latest
     steps:
@@ -218,7 +182,7 @@ jobs:
           release_id: ${{ needs.calc_ver.outputs.release_id }}
 
   tests:
-    needs: [calc_ver, build_linux]
+    needs: [calc_ver, build]
     runs-on: ubuntu-latest
     env:
       JOSHUA_DB_VER: "6.3.15"

.gitignore

@@ -97,7 +97,7 @@ flow/coveragetool/obj
 
 # Openway fork build
 /bld/
-/nbproject/private/
+/ide-projects/*/nbproject/private/
 /Testing/Temporary/
 
 # Temporary and user configuration files

build-scripts/for-linux/build-find-libatomic.bash

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# should be called with "source" command
+# try to find libatomic library and set -DATOMIC_LIBRARY_FILE to the APP_PRMS
+
+set -e
+
+# try to find the static library
+LIBRARY_CANDIDATE=$(ls -t1 /usr/lib/gcc/x86_64*/*/libatomic.a | head -n 1) || true
+
+if [[ -n "$LIBRARY_CANDIDATE" ]]; then
+  # test if the library is linked relocatable
+  PROBE_O=glfree.o
+  TMP_DIR=`mktemp -d`
+  pushd $TMP_DIR
+  ar x $LIBRARY_CANDIDATE $PROBE_O
+  REL_TYPE=`objdump -r glfree.o | awk '/.rodata/{ print $2;exit;}'`
+  popd
+  rm -rf $TMP_DIR
+  # R_X86_64_32S is prohibited for linking shared libraries
+  [[ "$REL_TYPE" == "R_X86_64_32S" ]] && LIBRARY_CANDIDATE=""
+fi
+
+# try to find a dynamic library
+[[ -z "$LIBRARY_CANDIDATE" ]] \
+  && LIBRARY_CANDIDATE=$(ls -1 /usr/lib64/libatomic.so* /usr/lib/gcc/x86_64-*/*/libatomic.so* | sort | head -n 1) \
+  || true
+
+# if found then put in to APP_PRMS
+[[ -n "$LIBRARY_CANDIDATE" ]] && APP_PRMS="$APP_PRMS -DATOMIC_LIBRARY_FILE=$LIBRARY_CANDIDATE"

build-scripts/for-linux/build-on-linux.bash

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# $1 - Version
+# $2 - Build version
+# $3 - Release flag
+# $4 - Paralllel threads
+# $5 - Source Dir. If not set then relative to the script dir
+
+set -e
+
+BASE_DIR="$(readlink -f $(dirname $0))"
+source $BASE_DIR/../set-ver-prms.sh "$1" "$2"
+RELEASE_FLAG=${3:-OFF}
+PARALLEL_PRMS="-j ${4:-$(nproc)}"
+SRC_DIR=${5:-$(readlink -f $BASE_DIR/../..)}
+
+START_DIR=`pwd`
+BUILD_DIR=$START_DIR/bld/linux
+
+mkdir -p $BUILD_DIR
+pushd $BUILD_DIR
+
+rm -rf *
+export LANG=C
+
+APP_PRMS="\
+  $CMAKE_VERSION_PRMS \
+  -DFDB_RELEASE=$RELEASE_FLAG \
+  -DGENERATE_DEBUG_PACKAGES=OFF \
+  -DUSE_LIBCXX=OFF \
+  -DSTATIC_LINK_LIBCXX=ON \
+  -DENABLE_SIMULATION_TESTS=ON"
+
+[ ! -e /usr/lib64/libcrypto.a -a -e /opt/openssl/lib/libcrypto.a ] && \
+  APP_PRMS="$APP_PRMS -DOPENSSL_ROOT_DIR=/opt/openssl"
+
+# find libatomic
+source $BASE_DIR/build-find-libatomic.bash
+
+echo "env CC=clang CXX=clang++ cmake -G Ninja $APP_PRMS . $SRC_DIR"
+env CC=clang CXX=clang++ cmake -G Ninja $APP_PRMS . $SRC_DIR
+
+ninja $PARALLEL_PRMS -k 0
+
+cpack -G RPM
+for fn in packages/*.rpm; do echo "$fn:"; rpm -qpRv $fn; echo; done
+
+cpack -G DEB
+for fn in packages/*.deb; do echo "$fn:"; dpkg -I $fn; done
+
+ninja $PARALLEL_PRMS -k 0 documentation/package_html
+
+# make all-binaries archive
+# calculate filename
+BINS_FILENAME=`ls -1 packages/foundationdb-docs-* | sed s/-docs/-bins/`
+tar -I pigz -cvf $BINS_FILENAME bin
+
+popd
+

build-scripts/for-linux/prepare-debian-based-clang.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# install clang of the most recent version
+
+set -e
+
+sudo apt update
+RECENT_PKG=`apt search -o APT::Cache::Search::Version=1 '^clang-[0-9][0-9]$' | awk '{print $1;}' | sort | tail -n 1`
+
+sudo DEBIAN_FRONTEND=noninteractive apt install -y $RECENT_PKG
+
+[ -e /usr/local/bin/clang ] || sudo ln -s `ls -1 /usr/bin/clang-[0-9]* | tail -n 1` /usr/local/bin/clang
+[ -e /usr/local/bin/clang++ ] || sudo ln -s `ls -1 /usr/bin/clang++-[0-9]* | tail -n 1` /usr/local/bin/clang++

build-scripts/for-linux/prepare-debian-based-cmake.bash

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+# install cmake of a version not less than the minimal required
+# $1 - minimal required version
+
+set -e
+
+REQUIRED_VERSION=${1:-3.13}
+
+CMAKE_URL=https://github.com/Kitware/CMake/releases/download/v3.26.0/cmake-3.26.0-linux-x86_64.tar.gz
+
+EXISTING_VERSION=`cmake --version | awk '/version/ { print $3; }'` || true
+
+if dpkg --compare-versions "$EXISTING_VERSION" lt "$REQUIRED_VERSION"; then
+  echo "Existing cmake $EXISTING_VERSION is absent or too old."
+
+  sudo apt update
+  AVAILABLE_APT_VERSION=`apt-cache policy cmake | awk '/Candidate:/{print $2;}'`
+
+  if dpkg --compare-versions "$AVAILABLE_APT_VERSION" lt "$REQUIRED_VERSION"; then
+    echo "Available cmake $AVAILABLE_APT_VERSION is too old. Installing one from $CMAKE_URL."
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y wget
+    TMP_DIR=`mktemp -d`
+    wget -O $TMP_DIR/cmake.tar.gz $CMAKE_URL
+    sudo mkdir -p /opt
+    sudo tar -xvf $TMP_DIR/cmake.tar.gz -C /opt
+    rm -rf $TMP_DIR
+    CMAKE_DIR=`ls -d1 /opt/cmake-* | tail -n 1`
+    for F in $CMAKE_DIR/bin/*; do
+      sudo rm -rf /usr/local/bin/`basename $F`
+      sudo ln -s $F /usr/local/bin/
+    done
+    NEW_VERSION=`cmake --version | awk '/version/ { print $3; }'`
+  else
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cmake
+  fi
+  NEW_VERSION=`cmake --version | awk '/version/ { print $3; }'`
+  echo "Now cmake $NEW_VERSION has been installed."
+else
+  echo "Existing cmake $EXISTING_VERSION satisfies the minimum version $REQUIRED_VERSION."
+fi

build-scripts/for-linux/prepare-debian-based-jdk.bash

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# install openjdk of the most early longterm version
+
+set -e
+
+sudo apt update
+
+find_pkg() {
+  apt search -o APT::Cache::Search::Version=1 "^$1\$" | awk '{print $1;}'
+}
+
+PKG_AV=""
+for P in openjdk-{8,11,17}-jdk; do
+  PKG_AV=`find_pkg $P`
+  [ -n "$PKG_AV" ] && break;
+done
+
+if [ -n "$PKG_AV" ]; then
+  echo "Installing $PKG_AV"
+  sudo DEBIAN_FRONTEND=noninteractive apt install -y $PKG_AV
+else
+  echo "No openjdk-*-jdk package available." >1
+  return 1 2>&1 >/dev/null
+  exit 1
+fi

build-scripts/for-linux/prepare-debian-based.bash

@@ -0,0 +1,32 @@
+#!/bin/sh
+
+set -e
+
+BASE_DIR=`dirname $0`
+
+sudo apt update
+
+$BASE_DIR/prepare-debian-based-cmake.bash
+$BASE_DIR/prepare-debian-based-clang.sh
+$BASE_DIR/prepare-debian-based-jdk.bash
+
+# make is necessary for building Jemalloc on Astra Linux
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+  git \
+  mono-mcs \
+  make \
+  ninja-build \
+  pigz \
+  python3 \
+  python3-venv \
+  rpm
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+  libssl-dev \
+  liblz4-dev
+
+sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \
+  libmono-system-data-datasetextensions4.0-cil \
+  libmono-system-runtime-serialization4.0-cil \
+  libmono-system-xml-linq4.0-cil

build-scripts/for-linux/prepare-fedora-openssl.sh

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+# Fedora does not more provide the openssl static libraries so we need to build
+# them from sources
+
+set -e
+
+if [ ! -f /usr/lib64/libcrypto.a -a ! -f /opt/openssl/lib/libcrypto.a ]; then
+  # build cmake config for lz4
+  sudo dnf install -y \
+    wget \
+    make \
+    gcc \
+    coreutils \
+    perl-interpreter \
+    sed \
+    zlib-devel \
+    diffutils \
+    lksctp-tools-devel \
+    util-linux \
+    perl-podlators \
+    procps-ng \
+    'perl(Test::Harness)' \
+    'perl(Test::More)' \
+    'perl(Math::BigInt)' \
+    'perl(Module::Load::Conditional)' \
+    'perl(File::Temp)' \
+    'perl(Time::HiRes)' \
+    'perl(FindBin)' \
+    'perl(lib)' \
+    'perl(File::Compare)' \
+    'perl(File::Copy)' \
+    perl-Pod-Html
+
+    OPENSSL_FN=openssl-1.1.1t
+    OPENSSL_URL=https://github.com/openssl/openssl/releases/download/OpenSSL_1_1_1t/openssl-1.1.1t.tar.gz
+
+    OLD_DIR=`pwd`
+    BUILD_DIR=/tmp/openssl
+    mkdir -p $BUILD_DIR
+    rm -rf $BUILD_DIR/*
+
+    cd $BUILD_DIR
+
+    wget $OPENSSL_URL
+    tar -xvf $OPENSSL_FN.tar.gz
+    cd $OPENSSL_FN
+    ./config --prefix=/opt/$OPENSSL_FN --openssldir=/etc/pki/tls
+    make -j `nproc` -k
+    make -j `nproc` install DESTDIR=inst
+
+    sudo mkdir -p /opt
+    sudo cp -rv inst/opt/$OPENSSL_FN /opt/
+    [ -e /opt/openssl ] || sudo ln -s /opt/$OPENSSL_FN /opt/openssl
+    cd $OLD_DIR
+    rm -rf $BUILD_DIR
+fi