From f0350b14c9ab2d85b596979cdd1a447ef6292b77 Mon Sep 17 00:00:00 2001 From: Ray-Eldath Date: Fri, 3 Nov 2023 17:40:11 +0800 Subject: [PATCH] Provide gpdemo deployment script Customers have complained that the deployment procedures of CBDB is tedious. In this commit, gpdemo provides a default configuration to enable a one-click deployment experience. The default parameters are the same as "make create-demo-cluster". --- gpAux/gpdemo/Makefile | 36 +++--------------- gpAux/gpdemo/README | 32 +++++++++------- gpAux/gpdemo/demo_cluster.sh | 26 +++++-------- gpAux/gpdemo/gpdemo-defaults.sh | 38 +++++++++++++++++++ gpAux/gpdemo/probe_config.sh | 8 ++-- gpMgmt/bin/Makefile | 13 ++++++- gpMgmt/bin/gpdemo | 67 +++++++++++++++++++++++++++++++++ 7 files changed, 154 insertions(+), 66 deletions(-) create mode 100755 gpAux/gpdemo/gpdemo-defaults.sh create mode 100755 gpMgmt/bin/gpdemo diff --git a/gpAux/gpdemo/Makefile b/gpAux/gpdemo/Makefile index fb15c2cc11c..fb3be19f7bc 100644 --- a/gpAux/gpdemo/Makefile +++ b/gpAux/gpdemo/Makefile @@ -9,31 +9,7 @@ top_builddir = ../.. -include $(top_builddir)/src/Makefile.global -# -# VARIABLES -# - -export enable_gpfdist -export with_openssl - -PORT_BASE ?= 7000 -NUM_PRIMARY_MIRROR_PAIRS ?= 3 -BLDWRAP_POSTGRES_CONF_ADDONS := fsync=off $(BLDWRAP_POSTGRES_CONF_ADDONS) - -ifeq ($(WITH_MIRRORS), ) -WITH_MIRRORS = true -endif - -ifeq ($(WITH_MIRRORS), true) -WITH_STANDBY = true -endif - -export DEMO_PORT_BASE=$(PORT_BASE) -export NUM_PRIMARY_MIRROR_PAIRS -export WITH_MIRRORS -export WITH_STANDBY -export BLDWRAP_POSTGRES_CONF_ADDONS -export DEFAULT_QD_MAX_CONNECT=150 +SHELL := /usr/bin/bash all: $(MAKE) clean @@ -42,22 +18,22 @@ all: $(MAKE) probe cluster create-demo-cluster: - @./demo_cluster.sh + @source gpdemo-defaults.sh && ./demo_cluster.sh @echo "" probe: - @./probe_config.sh + @source gpdemo-defaults.sh && ./probe_config.sh @echo "" -check: - @./demo_cluster.sh -c +check: + @source gpdemo-defaults.sh && ./demo_cluster.sh -c @echo "" clean destroy-demo-cluster: @echo "======================================================================" @echo "Deleting cluster.... " @echo "======================================================================" - @./demo_cluster.sh -d + @source gpdemo-defaults.sh && ./demo_cluster.sh -d @echo "" .PHONY: all cluster create-demo-cluster create-segwalrep-cluster probe clean check clean destroy-demo-cluster diff --git a/gpAux/gpdemo/README b/gpAux/gpdemo/README index 41a7374364d..01ac455df00 100644 --- a/gpAux/gpdemo/README +++ b/gpAux/gpdemo/README @@ -31,11 +31,7 @@ RUNNING GP DEMO su - gpadmin -2. Go into the gpdemo directory: - - cd gpAux/gpdemo - -3. Source greenplum_path.sh +2. Source greenplum_path.sh . /usr/local/greenplum-db/greenplum_path.sh @@ -44,23 +40,28 @@ RUNNING GP DEMO The "." will source the greenplum_path.sh into the current shell, instead of starting a subprocess. -4. Create the cluster: +3. Create the cluster at the current directory: + + gpdemo - make cluster + By default, all data of the demo cluster will be stored in + "./datadirs" of the current directory. You may change this by + cd to another directory or using "DATADIRS" environment variable + described in the next section. -5. Source gpAux/gpdemo/gpdemo-env.sh +4. Source ./gpdemo-env.sh -6. If setup completes correctly, you will have a virtual Cloudberry +5. If setup completes correctly, you will have a virtual Cloudberry Database system running on the local host. You can connect to it using the psql client program as follows: psql postgres -7. To exit the psql client program: +6. To exit the psql client program: \q -8. See the Cloudberry Database User Guide for more information on +7. See the Cloudberry Database User Guide for more information on using your Cloudberry Database system. @@ -68,9 +69,9 @@ RUNNING GP DEMO REMOVING GP DEMO ****************************************************************** -1. From the gpdemo directory run make clean: +1. In the directory where you run "gpdemo" to create cluster: - make clean + gpdemo -d 2. This will stop all Cloudberry Database demo server processes and remove all files and directories created by the GP demo @@ -96,7 +97,10 @@ environment variables before initializing the cluster: export NUM_PRIMARY_MIRROR_PAIRS= It is recommended to specify an odd number of segments, in order - to catch problems with data distribution. + to catch problems with data distribution. When set to 0, a singlenode + deployment will be created. Under this mode, there are no segments, + only one coordinator (QD) and one coordinator standby. All data sit + on the two nodes and nothing gets distributed. * Data directory: diff --git a/gpAux/gpdemo/demo_cluster.sh b/gpAux/gpdemo/demo_cluster.sh index d04521480bb..866fbcf0641 100755 --- a/gpAux/gpdemo/demo_cluster.sh +++ b/gpAux/gpdemo/demo_cluster.sh @@ -102,16 +102,6 @@ checkDemoConfig(){ return 0 } -USAGE(){ - echo "" - echo " `basename $0` {-c | -d | -u} <-K>" - echo " -c : Check if demo is possible." - echo " -d : Delete the demo." - echo " -K : Create cluster without data checksums." - echo " -u : Usage, prints this message." - echo "" -} - # # Clean up the demo # @@ -140,6 +130,10 @@ cleanDemo(){ echo "Deleting clusterConfigFile" rm -f clusterConfigFile fi + if [ -f clusterConfigPostgresAddonsFile ]; then + echo "Deleting clusterConfigPostgresAddonsFile" + rm -f clusterConfigPostgresAddonsFile + fi if [ -d ${DATADIRS} ]; then echo "Deleting ${DATADIRS}" rm -rf ${DATADIRS} @@ -155,10 +149,9 @@ cleanDemo(){ # Main Section #***************************************************************************** -while getopts ":cdK'?'" opt +while getopts ":cdK" opt do - case $opt in - '?' ) USAGE ;; + case $opt in c) checkDemoConfig RETVAL=$? if [ $RETVAL -ne 0 ]; then @@ -173,9 +166,7 @@ do K) DATACHECKSUMS=0 shift ;; - *) USAGE - exit 0 - ;; + *) USAGE; exit 1;; esac done @@ -302,7 +293,7 @@ cat >> $CLUSTER_CONFIG <<-EOF COORDINATOR_PORT=${COORDINATOR_DEMO_PORT} # Shell to use to execute commands on all hosts - TRUSTED_SHELL="`pwd`/lalshell" + TRUSTED_SHELL="$(dirname "$0")/lalshell" ENCODING=UNICODE EOF @@ -487,6 +478,7 @@ cat > gpdemo-env.sh <<-EOF export PGPORT=${COORDINATOR_DEMO_PORT} export COORDINATOR_DATA_DIRECTORY=$QDDIR/${SEG_PREFIX}-1 export MASTER_DATA_DIRECTORY=$QDDIR/${SEG_PREFIX}-1 + export NUM_PRIMARY_MIRROR_PAIRS=${NUM_PRIMARY_MIRROR_PAIRS} EOF if [ "${RETURN}" -gt 1 ]; diff --git a/gpAux/gpdemo/gpdemo-defaults.sh b/gpAux/gpdemo/gpdemo-defaults.sh new file mode 100755 index 00000000000..95527c0a763 --- /dev/null +++ b/gpAux/gpdemo/gpdemo-defaults.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +USAGE() { + echo "" + echo " `basename $0` {-c | -d | -h} <-K>" + echo " -c : Check if demo is possible." + echo " -d : Delete the demo." + echo " -K : Create cluster without data checksums." + echo " -h : Usage, prints this message." + echo "" +} + +PORT_BASE=${PORT_BASE:=7000} +NUM_PRIMARY_MIRROR_PAIRS=${NUM_PRIMARY_MIRROR_PAIRS:=3} +if [ "${NUM_PRIMARY_MIRROR_PAIRS}" -eq "0" ]; then + BLDWRAP_POSTGRES_CONF_ADDONS="fsync=on ${BLDWRAP_POSTGRES_CONF_ADDONS}" # always enable fsync if singlenode +fi +if [[ ! ${BLDWRAP_POSTGRES_CONF_ADDONS} == *"fsync=on"* ]]; then + BLDWRAP_POSTGRES_CONF_ADDONS="fsync=off ${BLDWRAP_POSTGRES_CONF_ADDONS}" # only append fsync=off if it doesn't contains fsync=on +else + BLDWRAP_POSTGRES_CONF_ADDONS="${BLDWRAP_POSTGRES_CONF_ADDONS}" +fi +WITH_MIRRORS=${WITH_MIRRORS:="true"} +if [[ "${WITH_MIRRORS}" == "true" ]]; then + WITH_STANDBY="true" +fi + +export -f USAGE + +export enable_gpfdist +export with_openssl + +export DEMO_PORT_BASE="$PORT_BASE" +export NUM_PRIMARY_MIRROR_PAIRS +export WITH_MIRRORS +export WITH_STANDBY +export BLDWRAP_POSTGRES_CONF_ADDONS +export DEFAULT_QD_MAX_CONNECT=150 \ No newline at end of file diff --git a/gpAux/gpdemo/probe_config.sh b/gpAux/gpdemo/probe_config.sh index 6b7564417e2..760d31a5ae8 100755 --- a/gpAux/gpdemo/probe_config.sh +++ b/gpAux/gpdemo/probe_config.sh @@ -44,13 +44,13 @@ declare -a PORTS=(5432 10001 10002 10003) ((PORT_MIN=0)) ((PORT_MAX=$NUM_PRIMARY_MIRROR_PAIRS)) -if [ -z $DEMO_PORT_BASE ] ; then - echo "set DEMO_PORT_BASE" +if [ -z $PORT_BASE ] ; then + echo "set PORT_BASE" exit 1 fi -declare -a PORTS=(`expr $DEMO_PORT_BASE` \ - `expr $DEMO_PORT_BASE + 2` `expr $DEMO_PORT_BASE + 3 ` `expr $DEMO_PORT_BASE + 4`) +declare -a PORTS=(`expr $PORT_BASE` \ + `expr $PORT_BASE + 2` `expr $PORT_BASE + 3 ` `expr $PORT_BASE + 4`) # # Check tables on Coordinator diff --git a/gpMgmt/bin/Makefile b/gpMgmt/bin/Makefile index 54fe79ef39d..ea978e3f281 100644 --- a/gpMgmt/bin/Makefile +++ b/gpMgmt/bin/Makefile @@ -16,16 +16,24 @@ PROGRAMS= analyzedb gpactivatestandby gpaddmirrors gpcheckcat gpcheckperf \ gpcheckresgroupimpl gpconfig gpdeletesystem gpexpand gpinitstandby \ gpinitsystem gpload gpload.py gplogfilter gpmovemirrors \ gppkg gprecoverseg gpreload gpscp gpsd gpssh gpssh-exkeys gpstart \ - gpstate gpstop minirepro gpmemwatcher gpmemreport + gpstate gpstop minirepro gpmemwatcher gpmemreport gpdemo + +GPDEMO_LIBS = gpdemo-defaults.sh lalshell generate_certs.sh demo_cluster.sh \ + probe_config.sh README installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)/lib' + $(MKDIR_P) '$(DESTDIR)$(bindir)/lib/gpdemo' installprograms: installdirs for file in $(PROGRAMS); do \ $(INSTALL_SCRIPT) $$file '$(DESTDIR)$(bindir)/'$$file ; \ $(PERL) $(top_builddir)/putversion '$(DESTDIR)$(bindir)/'$$file ; \ done + # install dependencies of gpdemo + for file in $(GPDEMO_LIBS); do \ + $(INSTALL_SCRIPT) $(top_builddir)/gpAux/gpdemo/$$file '$(DESTDIR)$(bindir)/lib/gpdemo/'$$file ; \ + done # Symlink gpcheckcat from bin to bin/lib to maintain backward compatibility if [ ! -L $(DESTDIR)$(bindir)/lib/gpcheckcat ]; then \ cd $(DESTDIR)$(bindir)/lib/ && $(LN_S) ../gpcheckcat gpcheckcat; \ @@ -36,6 +44,9 @@ uninstall: for file in $(PROGRAMS); do \ rm -f '$(DESTDIR)$(bindir)/'$$file ; \ done + for file in $(GPDEMO_LIBS); do \ + rm -f '$(DESTDIR)$(bindir)/lib/gpdemo/'$$file ; \ + done rm -f '$(DESTDIR)$(bindir)/gpload.bat' # diff --git a/gpMgmt/bin/gpdemo b/gpMgmt/bin/gpdemo new file mode 100755 index 00000000000..502fa3004aa --- /dev/null +++ b/gpMgmt/bin/gpdemo @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +WORKDIR=$(dirname "$0") +GPDEMO_LIB=$WORKDIR/lib/gpdemo + +README() { + cat "$GPDEMO_LIB"/README +} + +PROBE_CONFIG() { + source_gpdemo_defaults + "$GPDEMO_LIB"/probe_config.sh +} + +DEMO_CLUSTER() { + source_gpdemo_defaults + "$GPDEMO_LIB"/demo_cluster.sh "$@" +} + +source_gpdemo_defaults() { + . "$GPDEMO_LIB"/gpdemo-defaults.sh + + # overwrite the shorter usage defined in gpdemo-defaults.sh + USAGE() { + echo "" + echo " gpdemo {-c | -d | -p | -h | -H | -v} <-K>" + echo " : Default behaviour (no argument) is to create demo cluster." + echo " -c : Check if demo is possible." + echo " -d : Delete the demo." + echo " -K : Create cluster without data checksums." + echo " -p : Probe configuration of an alive cluster." + echo " -h : Usage, prints this message." + echo " -H : Detailed usage." + echo " -v : Show version." + echo "" + echo " There are few environment variables (e.g., PORT_BASE, DATADIRS) to" + echo " further config the demo cluster. See \"gpdemo -H\" for more info." + echo "" + } +} + +CHECK_FILENAME=("demo_cluster.sh" "probe_config.sh" "gpdemo-defaults.sh" "lalshell" "README" "../gp_bash_version.sh") +for filename in "${CHECK_FILENAME[@]}"; do + checking="$GPDEMO_LIB"/"$filename" + if [[ ! -f "$checking" ]]; then + echo "Missing $checking" + echo "" + echo "This may be caused by incomplete installation or packaging error." + echo "Try to re-install CloudBerryDB. If it doesn't help, contact support." + exit 1 + fi +done + +. "$WORKDIR"/lib/gp_bash_version.sh + +while getopts ":pHv" opt +do + case $opt in + p) PROBE_CONFIG; exit 0;; + H) README; exit 0;; + v) print_version; exit 0;; + *) DEMO_CLUSTER "$@"; exit $?;; # fallback to demo_cluster.sh + esac +done + +# default behaviour is fallback to "demo_cluster.sh" (no argument) +DEMO_CLUSTER \ No newline at end of file