diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f171b1836a..abefa0be10a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ if(POLICY CMP0135) endif() project(foundationdb - VERSION 7.3.49 + VERSION 7.3.63 DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions." HOMEPAGE_URL "http://www.foundationdb.org/" LANGUAGES C CXX ASM) diff --git a/bindings/bindingtester/tests/test_util.py b/bindings/bindingtester/tests/test_util.py index 13d33c33700..a37e7e4fc04 100644 --- a/bindings/bindingtester/tests/test_util.py +++ b/bindings/bindingtester/tests/test_util.py @@ -150,11 +150,11 @@ def random_tuple_list(self, max_size, max_list_size): def random_range_params(self): if random.random() < 0.75: - limit = random.randint(1, 1e3) + limit = random.randint(1, int(1e3)) elif random.random() < 0.75: limit = 0 else: - limit = random.randint(1e8, (1 << 31) - 1) + limit = random.randint(int(1e8), (1 << 31) - 1) return (limit, random.randint(0, 1), random.randint(-2, 4)) diff --git a/cmake/FDBComponents.cmake b/cmake/FDBComponents.cmake index e1014de5ff0..cd27f952e67 100644 --- a/cmake/FDBComponents.cmake +++ b/cmake/FDBComponents.cmake @@ -21,7 +21,7 @@ if(USE_VALGRIND) endif() ################################################################################ -# SSL +# SSL & ZLIB ################################################################################ set(CMAKE_REQUIRED_INCLUDES ${OPENSSL_INCLUDE_DIR}) @@ -32,6 +32,8 @@ set(OPENSSL_USE_STATIC_LIBS TRUE) if (WIN32) set(OPENSSL_MSVC_STATIC_RT ON) endif() +# SSL requires ZLIB +find_package(ZLIB REQUIRED) find_package(OpenSSL REQUIRED) add_compile_options(-DHAVE_OPENSSL) diff --git a/cmake/InstallLayout.cmake b/cmake/InstallLayout.cmake index 5a9b24a28cf..2f3048ef2fb 100644 --- a/cmake/InstallLayout.cmake +++ b/cmake/InstallLayout.cmake @@ -315,6 +315,7 @@ set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION "/usr/lib/cmake" "/usr/lib/foundationdb-${FDB_VERSION}/etc/foundationdb" ) +set(CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX "/usr/src") set(CPACK_RPM_DEBUGINFO_PACKAGE ${GENERATE_DEBUG_PACKAGES}) #set(CPACK_RPM_BUILD_SOURCE_FDB_INSTALL_DIRS_PREFIX /usr/src) set(CPACK_RPM_COMPONENT_INSTALL ON) diff --git a/documentation/sphinx/extensions/rubydomain.py b/documentation/sphinx/extensions/rubydomain.py index 56670d280ac..e6c849cc413 100755 --- a/documentation/sphinx/extensions/rubydomain.py +++ b/documentation/sphinx/extensions/rubydomain.py @@ -711,4 +711,4 @@ def get_objects(self): def setup(app): - app.add_domain(RubyDomain) \ No newline at end of file + app.add_domain(RubyDomain) diff --git a/documentation/sphinx/source/release-notes/release-notes-730.rst b/documentation/sphinx/source/release-notes/release-notes-730.rst index 9517d71902c..f511806d4ac 100644 --- a/documentation/sphinx/source/release-notes/release-notes-730.rst +++ b/documentation/sphinx/source/release-notes/release-notes-730.rst @@ -4,6 +4,115 @@ Release Notes ############# +7.3.61 +====== +* Same as 7.3.60 release with AVX enabled. + +7.3.60 +====== +* Fixed a potential corruption of the destination storage server when shard_encode_location_metadata is enabled. `(PR #11934) `_ +* Paused storage wiggle when all storage servers do not have enough available space. `(PR #11911) `_ +* Updated Sharded RocksDB knobs. `(PR #11936) `_ +* Fixed storage migration to consider the perpetualStorageEngine setting. `(PR #11940) `_ +* Improved the coverage of location metadata auditing. `(PR #11897) `_ + +7.3.59 +====== +* Same as 7.3.58 release with AVX enabled. + +7.3.58 +====== +* Updated fdbmonitor to only delay the restart of fdbserver if the process exited with an exit code other than 0. `(PR #11812) `_ +* Added knob to pause perpetual storage wiggle when TSS count target is met. `(PR #11824) `_ +* Updated exclusion code to reduce the requests that are made by locality-based exclusions. `(PR #11848) `_ +* Added multiple improvements to reduce the false positive rate of gray failure triggered recoveries. `(PR #11850) `_, `(PR #11852) `_, `(PR #11885) `_ +* Added a knob to use direct IO for ShardedRocks storage engine. `(PR #11891) `_ +* Added a knob to compact based on number of range deletions in file for ShardedRocks storage engine. `(PR #11890) `_ +* Updated RocksDB force flush interval based on the last flush time. `(PR #11845) `_ + +7.3.57 +====== +* Same as 7.3.56 release with AVX enabled. + +7.3.56 +====== +* Added an option to force flush if RocksCb flush does not happen within a time interval. `(PR #11792) `_ +* Changed RocksDB histogram sample rate to 1. `(PR #11794) `_ +* Updated TLS input handling to allow multiple Criteria per NID. `(PR #11763) `_ +* Added Sharded RocksDB knobs and metrics for bloom filters. `(PR #11785) `_ +* Fixed a backup agent crash bug when receiving an invalid token error from S3. `(PR #11774) `_ +* Updated ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS Knob to prevent OOMs of RocksDB storage servers `(PR #11739) `_ + +7.3.55 +====== +* Same as 7.3.54 release with AVX enabled. + +7.3.54 +====== +* Addressed urgent consistency checker related issues. `(PR #11736) `_ +* Addressed a downgrade related incompatibility issue between 7.3 patch releases. `(PR #11732) `_ +* Added knob LOG_CONNECTION_ATTEMPTS_ENABLED to log all incoming connections. `(PR #11713) `_ + +7.3.53 +====== +* Same as 7.3.52 release with AVX enabled. + +7.3.52 +====== +* Improved Sharded Rocksdb to use a single iterator pool for all physical shards. `(PR #11694) `_ +* Changed the default values of various Sharded Rocksdb related knobs. `(PR #11706) `_ +* Removed CC_PAUSE_HEALTH_MONITOR knob. `(PR #11701) `_ +* Addressed protocol incompatibility issues between 7.3 patch releases. `(PR #11697) `_, `(PR #11705) `_ + +7.3.51 +====== +* Same as 7.3.50 release with AVX enabled. + +7.3.50 +====== +* Converted an assertion for Sharded Rocksdb in storage server to error logs. `(PR #11622) `_ +* Fixed backup agents from retrying uploading in the presence of S3 token errors. `(PR #11602) `_ +* Fixed an inconsistent location metadata issue and invalid shard IDs generated by seed shard servers when SHARD_ENCODE_LOCATION_METADATA is enabled. `(PR #11640) `_ and `(PR #11647) `_ +* Enabled direct_io for RocksDB storage engine and wiggle knobs. `(PR #11636) `_ +* Added knobs for caching index blocks for Sharded RocksDB. `(PR #11649) `_ and `(PR #11680) `_ +* Added various data distributor knobs for performance tuning. `(PR #11668) `_ and `(PR #11665) `_ +* Increased the minimum age to wiggle to avoid re-wiggling migrated storage servers. `(PR #11684) `_ +* Added various version vector improvements. `(PR #11590) `_, `(PR #11599) `_, `(PR #11605) `_, `(PR #11608) `_, `(PR #11620) `_, `(PR #11626) `_ and `(PR #11631) `_ +* Removed upgrade tests before FDB 6.3. `(PR #11662) `_ + +7.3.49 +====== +* Same as 7.3.48 release with AVX enabled. + +7.3.48 +====== +* Added various fdb-kubernetes-monitor improvements. `(PR #11456) `_, `(PR #11462) `_, and `(PR #11488) `_ +* Added more debug information for the fdbcli's checkall command. `(PR #11477) `_ +* Improved distributed consistency checker and updated the documentation. `(PR #11496) `_ +* Fixed a known committed version computation when version vector unicast is enabled. `(PR #11520) `_ +* Added PeerAddress to all PeerAddr/Peer TraceEvents. `(PR #11521) `_ +* Added a dynamic knob CC_PAUSE_HEALTH_MONITOR to disable gray failure recoveries. `(PR #11526) `_ +* Changed the default base Docker image to RockyLinux9. `(PR #11549) `_ +* Fixed a storage server crash when there are conflicting physical shard moves. `(PR #11485) `_ +* Fixed a rare cluster controller crash when serving status requests. `(PR #11580) `_ + +7.3.47 +====== +* Same as 7.3.46 release with AVX enabled. + +7.3.46 +====== +* Added exponential backoff when restarting fdbserver processes in fdb-kubernetes-monitor. `(PR #11453) `_ + +7.3.45 +====== +* Same as 7.3.44 release with AVX enabled. + +7.3.44 +====== +* Fixed negative free disk space in RkUpdate metrics. `(PR #11413) `_ +* Corrected the path where the fdb-kubernetes-monitor copies the binary into when running in sidecar mode. `(PR #11439) `_ + 7.3.43 ====== * Same as 7.3.42 release with AVX enabled. diff --git a/documentation/sphinx/source/special-keys.rst b/documentation/sphinx/source/special-keys.rst index 33987c1cc46..75877d922b5 100644 --- a/documentation/sphinx/source/special-keys.rst +++ b/documentation/sphinx/source/special-keys.rst @@ -275,7 +275,7 @@ Deprecated Keys Listed below are the special keys that have been deprecated. Special key(s) will no longer be accessible when the client specifies an API version equal to or larger than the version where they were deprecated. Clients specifying older API versions will be able to continue using the deprecated key(s). -#. ``\xff\xff/management/profiling/`` Deprecated as of API version 730. The corresponding functionalities are now covered by the global configuration module. For details, see :doc:`global-configuration`. Read/write. Changing these two keys will change the corresponding system keys ``\xff\x02/fdbClientInfo/``, respectively. The value of ``\xff\xff/management/client_txn_sample_rate`` is a literal text of ``double``, and the value of ``\xff\xff/management/client_txn_size_limit`` is a literal text of ``int64_t``. A special value ``default`` can be set to or read from these two keys, representing the client profiling is disabled. In addition, ``clear`` in this range is not allowed. For more details, see help text of ``fdbcli`` command ``profile client``. +#. ``\xff\xff/management/profiling/`` Deprecated as of API version 720. The corresponding functionalities are now covered by the global configuration module. For details, see :doc:`global-configuration`. Read/write. Changing these two keys will change the corresponding system keys ``\xff\x02/fdbClientInfo/``, respectively. The value of ``\xff\xff/management/client_txn_sample_rate`` is a literal text of ``double``, and the value of ``\xff\xff/management/client_txn_size_limit`` is a literal text of ``int64_t``. A special value ``default`` can be set to or read from these two keys, representing the client profiling is disabled. In addition, ``clear`` in this range is not allowed. For more details, see help text of ``fdbcli`` command ``profile client``. Versioning ========== diff --git a/fdbcli/ExcludeCommand.actor.cpp b/fdbcli/ExcludeCommand.actor.cpp index c8362e57885..48b8a61d1bd 100644 --- a/fdbcli/ExcludeCommand.actor.cpp +++ b/fdbcli/ExcludeCommand.actor.cpp @@ -178,6 +178,8 @@ ACTOR Future> checkForExcludingServers(Reference inProgressExclusion; state Reference tr = db->createTransaction(); + tr->setOption(FDBTransactionOptions::SPECIAL_KEY_SPACE_ENABLE_WRITES); + tr->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); loop { inProgressExclusion.clear(); try { diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index f48ad205a00..5d838ca640f 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -172,6 +172,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( PRIORITY_SPLIT_SHARD, 950 ); if( randomize && BUGGIFY ) PRIORITY_SPLIT_SHARD = 350; init( PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD, 960 ); if( randomize && BUGGIFY ) PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD = 360; // Set as the lowest priority + init( PROBABILITY_TEAM_REDUNDANT_DATAMOVE_CHOOSE_TRUE_BEST_DEST, 0.0 ); if (isSimulated) PROBABILITY_TEAM_REDUNDANT_DATAMOVE_CHOOSE_TRUE_BEST_DEST = deterministicRandom()->random01(); + init( PROBABILITY_TEAM_UNHEALTHY_DATAMOVE_CHOOSE_TRUE_BEST_DEST, 0.0 ); if (isSimulated) PROBABILITY_TEAM_UNHEALTHY_DATAMOVE_CHOOSE_TRUE_BEST_DEST = deterministicRandom()->random01(); + // Data distribution init( AVAILABLE_SPACE_PIVOT_RATIO, 0.5 ); init( CPU_PIVOT_RATIO, 0.9 ); @@ -184,10 +187,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ALLOW_LARGE_SHARD, false ); if( randomize && BUGGIFY ) ALLOW_LARGE_SHARD = true; init( MAX_LARGE_SHARD_BYTES, 1000000000 ); // 1G - init( SHARD_ENCODE_LOCATION_METADATA, false ); if( randomize && BUGGIFY ) SHARD_ENCODE_LOCATION_METADATA = true; + init( SHARD_ENCODE_LOCATION_METADATA, false ); if( isSimulated ) SHARD_ENCODE_LOCATION_METADATA = deterministicRandom()->coinflip(); init( ENABLE_DD_PHYSICAL_SHARD, false ); // EXPERIMENTAL; If true, SHARD_ENCODE_LOCATION_METADATA must be true; When true, optimization of data move between DCs is disabled - init( DD_PHYSICAL_SHARD_MOVE_PROBABILITY, 0.0 ); if( isSimulated ) DD_PHYSICAL_SHARD_MOVE_PROBABILITY = 0.5; - init( ENABLE_PHYSICAL_SHARD_MOVE_EXPERIMENT, false ); if( isSimulated ) ENABLE_PHYSICAL_SHARD_MOVE_EXPERIMENT = deterministicRandom()->coinflip(); + init( DD_PHYSICAL_SHARD_MOVE_PROBABILITY, 0.0 ); // EXPERIMENTAL + init( ENABLE_PHYSICAL_SHARD_MOVE_EXPERIMENT, false ); // EXPERIMENTAL init( MAX_PHYSICAL_SHARD_BYTES, 10000000 ); // 10 MB; for ENABLE_DD_PHYSICAL_SHARD; smaller leads to larger number of physicalShard per storage server init( PHYSICAL_SHARD_METRICS_DELAY, 300.0 ); // 300 seconds; for ENABLE_DD_PHYSICAL_SHARD init( ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME, 600.0 ); if( randomize && BUGGIFY ) ANONYMOUS_PHYSICAL_SHARD_TRANSITION_TIME = 0.0; // 600 seconds; for ENABLE_DD_PHYSICAL_SHARD @@ -269,6 +272,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi If this value is too small relative to SHARD_MIN_BYTES_PER_KSEC immediate merging work will be generated. */ + init( DD_SHARD_USABLE_REGION_CHECK_RATE, 0 ); if (isSimulated) DD_SHARD_USABLE_REGION_CHECK_RATE = deterministicRandom()->randomInt(1, 3); init( ENABLE_WRITE_BASED_SHARD_SPLIT, false ); if( randomize && BUGGIFY ) ENABLE_WRITE_BASED_SHARD_SPLIT = true; init( STORAGE_METRIC_TIMEOUT, isSimulated ? 60.0 : 600.0 ); if( randomize && BUGGIFY ) STORAGE_METRIC_TIMEOUT = deterministicRandom()->coinflip() ? 10.0 : 30.0; init( METRIC_DELAY, 0.1 ); if( randomize && BUGGIFY ) METRIC_DELAY = 1.0; @@ -280,8 +284,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( PERPETUAL_WIGGLE_DELAY, 60 ); init( PERPETUAL_WIGGLE_SMALL_LOAD_RATIO, 10 ); init( PERPETUAL_WIGGLE_MIN_BYTES_BALANCE_RATIO, 0.85 ); - init( PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO, 0 ); + init( PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO, 8 ); init( PERPETUAL_WIGGLE_DISABLE_REMOVER, true ); + init( PERPETUAL_WIGGLE_PAUSE_AFTER_TSS_TARGET_MET, true ); if (isSimulated) PERPETUAL_WIGGLE_PAUSE_AFTER_TSS_TARGET_MET = deterministicRandom()->coinflip(); + init( PERPETUAL_WIGGLE_MIN_AVAILABLE_SPACE_RATIO, 0.30 ); if (isSimulated) PERPETUAL_WIGGLE_MIN_AVAILABLE_SPACE_RATIO = 0.05; init( LOG_ON_COMPLETION_DELAY, DD_QUEUE_LOGGING_INTERVAL ); init( BEST_TEAM_MAX_TEAM_TRIES, 10 ); init( BEST_TEAM_OPTION_COUNT, 4 ); @@ -332,7 +338,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY, 120 ); if( randomize && BUGGIFY ) DD_TEAM_ZERO_SERVER_LEFT_LOG_DELAY = 5; init( DD_STORAGE_WIGGLE_PAUSE_THRESHOLD, 10 ); if( randomize && BUGGIFY ) DD_STORAGE_WIGGLE_PAUSE_THRESHOLD = 1000; init( DD_STORAGE_WIGGLE_STUCK_THRESHOLD, 20 ); - init( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC, isSimulated ? 2 : 21 * 60 * 60 * 24 ); if(randomize && BUGGIFY) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0: 120; + init( DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC, isSimulated ? 2 : 35 * 60 * 60 * 24 ); if(randomize && BUGGIFY) DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC = isSimulated ? 0: 120; init( DD_TENANT_AWARENESS_ENABLED, false ); init( STORAGE_QUOTA_ENABLED, true ); if(isSimulated) STORAGE_QUOTA_ENABLED = deterministicRandom()->coinflip(); init( TENANT_CACHE_LIST_REFRESH_INTERVAL, 5 ); if( randomize && BUGGIFY ) TENANT_CACHE_LIST_REFRESH_INTERVAL = deterministicRandom()->randomInt(1, 10); @@ -360,8 +366,10 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( DD_MAXIMUM_LARGE_TEAM_CLEANUP, 10000 ); if( randomize && BUGGIFY ) DD_MAXIMUM_LARGE_TEAM_CLEANUP = 10; init( DD_LARGE_TEAM_DELAY, 60.0 ); init( DD_FIX_WRONG_REPLICAS_DELAY, 60.0 ); + init (DD_VALIDATE_SERVER_TEAM_COUNT_AFTER_BUILD_TEAM, false ); if (isSimulated) DD_VALIDATE_SERVER_TEAM_COUNT_AFTER_BUILD_TEAM = true; // TeamRemover + init( TR_LOW_SPACE_PIVOT_DELAY_SEC, 0 ); if (isSimulated) TR_LOW_SPACE_PIVOT_DELAY_SEC = deterministicRandom()->randomInt(0, 3); init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER, false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true init( TR_REMOVE_MACHINE_TEAM_DELAY, 60.0 ); if( randomize && BUGGIFY ) TR_REMOVE_MACHINE_TEAM_DELAY = deterministicRandom()->random01() * 60.0; init( TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS, true ); if( randomize && BUGGIFY ) TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS = deterministicRandom()->random01() < 0.1 ? true : false; @@ -462,7 +470,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ROCKSDB_BLOOM_WHOLE_KEY_FILTERING, false ); init( ROCKSDB_MAX_AUTO_READAHEAD_SIZE, 65536 ); // If rocksdb block cache size is 0, the default 8MB is used. - int64_t blockCacheSize = isSimulated ? 16 * 1024 : 4LL * 1024 * 1024 * 1024 /* 4GB */; + int64_t blockCacheSize = isSimulated ? 1LL * 1024 * 1024 * 1024 : 4LL * 1024 * 1024 * 1024 /* 4GB */; init( ROCKSDB_BLOCK_CACHE_SIZE, blockCacheSize ); /* Datablocks cache + Index&filter blocks cache */ init( ROCKSDB_CACHE_HIGH_PRI_POOL_RATIO, 0.5 ); /* Share of high priority Index&filter blocks in cache */ init( ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS, true ); @@ -480,10 +488,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ROCKSDB_READ_QUEUE_SOFT_MAX, 500 ); init( ROCKSDB_FETCH_QUEUE_HARD_MAX, 100 ); init( ROCKSDB_FETCH_QUEUE_SOFT_MAX, 50 ); - init( ROCKSDB_HISTOGRAMS_SAMPLE_RATE, 0.001 ); if( randomize && BUGGIFY ) ROCKSDB_HISTOGRAMS_SAMPLE_RATE = 0; + init( ROCKSDB_HISTOGRAMS_SAMPLE_RATE, 1 ); if( isSimulated ) ROCKSDB_HISTOGRAMS_SAMPLE_RATE = deterministicRandom()->random01(); init( ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME, 30.0 ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME = 0.1; + init( ROCKSDB_PROBABILITY_REUSE_ITERATOR_SIM, 0.01 ); init( ROCKSDB_READ_RANGE_REUSE_ITERATORS, true ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_ITERATORS = deterministicRandom()->coinflip(); - init( SHARDED_ROCKSDB_REUSE_ITERATORS, false ); + init( SHARDED_ROCKSDB_REUSE_ITERATORS, false ); if (isSimulated) SHARDED_ROCKSDB_REUSE_ITERATORS = deterministicRandom()->coinflip(); init( ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS, false ); if( randomize && BUGGIFY ) ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS = deterministicRandom()->coinflip(); init( ROCKSDB_READ_RANGE_BOUNDED_ITERATORS_MAX_LIMIT, 200 ); // Set to 0 to disable rocksdb write rate limiting. Rate limiter unit: bytes per second. @@ -538,7 +547,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER, 4 ); // RocksDB default. init( ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER, 20 ); // RocksDB default. init( ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER, 36 ); // RocksDB default. - init( ROCKSDB_MAX_TOTAL_WAL_SIZE, isSimulated? 256 <<20 : 1 << 30 ); // 1GB. + init( ROCKSDB_MAX_TOTAL_WAL_SIZE, isSimulated? 256 <<20 : 512 << 20 ); // 512MB. init( ROCKSDB_MAX_BACKGROUND_JOBS, 2 ); // RocksDB default. init( ROCKSDB_DELETE_OBSOLETE_FILE_PERIOD, 21600 ); // 6h, RocksDB default. init( ROCKSDB_PHYSICAL_SHARD_CLEAN_UP_DELAY, isSimulated ? 300.0 : 15.0 ); // Delays shard clean up, must be larger than ROCKSDB_READ_VALUE_TIMEOUT to prevent reading deleted shard. @@ -554,17 +563,19 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ROCKSDB_WAL_RECOVERY_MODE, 2 ); // kPointInTimeRecovery, RocksDB default. init( ROCKSDB_TARGET_FILE_SIZE_BASE, 0 ); // If 0, pick RocksDB default. init( ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER, 1 ); // RocksDB default. - init( ROCKSDB_USE_DIRECT_READS, false ); - init( ROCKSDB_USE_DIRECT_IO_FLUSH_COMPACTION, false ); + init( ROCKSDB_USE_DIRECT_READS, true ); + init( ROCKSDB_USE_DIRECT_IO_FLUSH_COMPACTION, true ); init( ROCKSDB_MAX_OPEN_FILES, -1 ); // RocksDB default. init( ROCKSDB_USE_POINT_DELETE_FOR_SYSTEM_KEYS, false ); init( ROCKSDB_CF_RANGE_DELETION_LIMIT, 0 ); - init( ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS, 0 ); + init( ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS, 10000 ); init( ROCKSDB_WAIT_ON_CF_FLUSH, false ); - init( ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH, false ); + init( ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH, true ); init( ROCKSDB_CF_METRICS_DELAY, 900.0 ); init( ROCKSDB_MAX_LOG_FILE_SIZE, 10485760 ); // 10MB. init( ROCKSDB_KEEP_LOG_FILE_NUM, 100 ); // Keeps 1GB log per storage server. + // Does manual flushes at regular intervals(seconds), incase rocksdb did not flush. Feature disabled if the value is 0. + init( ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL, 600 ); if( isSimulated ) ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL = deterministicRandom()->randomInt(4, 1200); init( ROCKSDB_SKIP_STATS_UPDATE_ON_OPEN, true ); init( ROCKSDB_SKIP_FILE_SIZE_CHECK_ON_OPEN, true ); init( ROCKSDB_FULLFILE_CHECKSUM, false ); if ( randomize && BUGGIFY ) ROCKSDB_FULLFILE_CHECKSUM = true; @@ -575,23 +586,28 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ROCKSDB_MEMTABLE_PROTECTION_BYTES_PER_KEY, 0 ); if ( randomize && BUGGIFY ) ROCKSDB_MEMTABLE_PROTECTION_BYTES_PER_KEY = 8; // Default: 0 (disabled). Supported values: 0, 1, 2, 4, 8. // Block cache key-value checksum. Checksum is validated during read, so has non-trivial impact on read performance. init( ROCKSDB_BLOCK_PROTECTION_BYTES_PER_KEY, 0 ); if ( randomize && BUGGIFY ) ROCKSDB_BLOCK_PROTECTION_BYTES_PER_KEY = 8; // Default: 0 (disabled). Supported values: 0, 1, 2, 4, 8. + init( SHARDED_ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH, false ); init( SHARDED_ROCKSDB_VALIDATE_MAPPING_RATIO, 0.01 ); if (isSimulated) SHARDED_ROCKSDB_VALIDATE_MAPPING_RATIO = deterministicRandom()->random01(); init( SHARD_METADATA_SCAN_BYTES_LIMIT, 10485760 ); // 10MB init( ROCKSDB_MAX_MANIFEST_FILE_SIZE, 100 << 20 ); if (isSimulated) ROCKSDB_MAX_MANIFEST_FILE_SIZE = 500 << 20; // 500MB in simulation - init( SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS, 100 ); + init( SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS, 500 ); if (isSimulated) SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS = 50; init( SHARDED_ROCKSDB_AVERAGE_FILE_SIZE, 8 << 20 ); // 8MB init( SHARDED_ROCKSDB_COMPACTION_PERIOD, isSimulated? 3600 : 2592000 ); // 30d init( SHARDED_ROCKSDB_COMPACTION_ACTOR_DELAY, 3600 ); // 1h init( SHARDED_ROCKSDB_COMPACTION_SHARD_LIMIT, -1 ); init( SHARDED_ROCKSDB_WRITE_BUFFER_SIZE, 16 << 20 ); // 16MB - init( SHARDED_ROCKSDB_TOTAL_WRITE_BUFFER_SIZE, 1 << 30 ); // 1GB + init( SHARDED_ROCKSDB_TOTAL_WRITE_BUFFER_SIZE, 2LL << 30 ); // 2GB init( SHARDED_ROCKSDB_MEMTABLE_BUDGET, 64 << 20 ); // 64MB init( SHARDED_ROCKSDB_MAX_WRITE_BUFFER_NUMBER, 6 ); // RocksDB default. init( SHARDED_ROCKSDB_TARGET_FILE_SIZE_BASE, 16 << 20 ); // 16MB init( SHARDED_ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER, 1 ); // RocksDB default. - init( SHARDED_ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE, true ); + bool suggestCompactRange = deterministicRandom()->coinflip(); + init( SHARDED_ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE, true ); if (isSimulated) SHARDED_ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE = suggestCompactRange; + init( SHARDED_ROCKSDB_COMPACT_ON_RANGE_DELETION_THRESHOLD, 0 ); if (isSimulated) SHARDED_ROCKSDB_COMPACT_ON_RANGE_DELETION_THRESHOLD = suggestCompactRange? 0:50; init( SHARDED_ROCKSDB_MAX_BACKGROUND_JOBS, 4 ); - init( SHARDED_ROCKSDB_BLOCK_CACHE_SIZE, isSimulated? 16 * 1024 : 134217728 /* 128MB */); + init( SHARDED_ROCKSDB_BLOCK_CACHE_SIZE, isSimulated? 128 << 20 : 3LL << 30); // 3GB + init( SHARDED_ROCKSDB_CACHE_HIGH_PRI_POOL_RATIO, 0.5 ); /* Share of high priority Index&filter blocks in cache */ + init( SHARDED_ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS, true ); // Set to 0 to disable rocksdb write rate limiting. Rate limiter unit: bytes per second. init( SHARDED_ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC, 300 << 20 ); init( SHARDED_ROCKSDB_RATE_LIMITER_MODE, 2 ); @@ -603,7 +619,11 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( SHARDED_ROCKSDB_MAX_OPEN_FILES, 50000 ); // Should be smaller than OS's fd limit. init( SHARDED_ROCKSDB_DELAY_COMPACTION_FOR_DATA_MOVE, false); if (isSimulated) SHARDED_ROCKSDB_DELAY_COMPACTION_FOR_DATA_MOVE = deterministicRandom()->coinflip(); init (SHARDED_ROCKSDB_READ_ASYNC_IO, false ); if (isSimulated) SHARDED_ROCKSDB_READ_ASYNC_IO = deterministicRandom()->coinflip(); - init( SHARDED_ROCKSDB_PREFIX_LEN, 0 ); if( randomize && BUGGIFY ) SHARDED_ROCKSDB_PREFIX_LEN = deterministicRandom()->randomInt(1, 20); + init( SHARDED_ROCKSDB_PREFIX_LEN, 11 ); if( randomize && BUGGIFY ) SHARDED_ROCKSDB_PREFIX_LEN = deterministicRandom()->randomInt(1, 20); + init( SHARDED_ROCKSDB_BLOOM_FILTER_BITS, 3 ); if( randomize && BUGGIFY ) SHARDED_ROCKSDB_BLOOM_FILTER_BITS = deterministicRandom()->randomInt(3, 10); + init (SHARDED_ROCKSDB_MEMTABLE_BLOOM_FILTER_RATIO, 0.1 ); + init( SHARDED_ROCKSDB_HISTOGRAMS_SAMPLE_RATE, 0.001 ); if( randomize && BUGGIFY ) SHARDED_ROCKSDB_HISTOGRAMS_SAMPLE_RATE = 0; + init( SHARDED_ROCKSDB_USE_DIRECT_IO, false ); if (isSimulated) SHARDED_ROCKSDB_USE_DIRECT_IO = deterministicRandom()->coinflip(); // Leader election @@ -636,6 +656,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi // FIXME Remove this when client latency issue is resolved init( WRITE_CLIENT_LATENCY_TRACEEVENT, false ); + init( COMMIT_BATCH_RANDOMIZE_INTERVAL, false ); + init( COMMIT_BATCH_MAX_IN_PROGRESS, 0 ); init( COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE, 0.0005 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_FROM_IDLE = 0.005; init( COMMIT_TRANSACTION_BATCH_INTERVAL_MIN, 0.001 ); if( randomize && BUGGIFY ) COMMIT_TRANSACTION_BATCH_INTERVAL_MIN = 0.1; init( COMMIT_TRANSACTION_BATCH_INTERVAL_MAX, 0.200 ); @@ -756,23 +778,24 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( COORDINATOR_REGISTER_INTERVAL, 5.0 ); init( CLIENT_REGISTER_INTERVAL, 600.0 ); init( CC_ENABLE_WORKER_HEALTH_MONITOR, false ); - init( CC_PAUSE_HEALTH_MONITOR, false, Atomic::NO ); init( CC_WORKER_HEALTH_CHECKING_INTERVAL, 60.0 ); init( CC_DEGRADED_LINK_EXPIRATION_INTERVAL, 300.0 ); init( CC_MIN_DEGRADATION_INTERVAL, 120.0 ); init( ENCRYPT_KEY_PROXY_FAILURE_TIME, 0.1 ); if ( isSimulated ) ENCRYPT_KEY_PROXY_FAILURE_TIME = 1.0 + deterministicRandom()->random01(); init( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, 3 ); + init( CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE_MIN, 1 ); init( CC_MAX_EXCLUSION_DUE_TO_HEALTH, 2 ); - init( CC_HEALTH_TRIGGER_RECOVERY, false ); + init( CC_HEALTH_TRIGGER_RECOVERY, false, Atomic::NO ); init( CC_TRACKING_HEALTH_RECOVERY_INTERVAL, 3600.0 ); init( CC_MAX_HEALTH_RECOVERY_COUNT, 5 ); - init( CC_HEALTH_TRIGGER_FAILOVER, false ); + init( CC_HEALTH_TRIGGER_FAILOVER, false, Atomic::NO ); init( CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION, 5 ); init( CC_FAILOVER_DUE_TO_HEALTH_MAX_DEGRADATION, 10 ); init( CC_ENABLE_ENTIRE_SATELLITE_MONITORING, false ); init( CC_SATELLITE_DEGRADATION_MIN_COMPLAINER, 3 ); init( CC_SATELLITE_DEGRADATION_MIN_BAD_SERVER, 3 ); init( CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING, true ); + init( CC_INVALIDATE_EXCLUDED_PROCESSES, false); if (isSimulated && deterministicRandom()->coinflip()) CC_INVALIDATE_EXCLUDED_PROCESSES = true; init( CC_THROTTLE_SINGLETON_RERECRUIT_INTERVAL, 0.5 ); init( INCOMPATIBLE_PEERS_LOGGING_INTERVAL, 600 ); if( randomize && BUGGIFY ) INCOMPATIBLE_PEERS_LOGGING_INTERVAL = 60.0; @@ -843,6 +866,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( STORAGE_DURABILITY_LAG_SOFT_MAX, 250e6 ); if( smallStorageTarget ) STORAGE_DURABILITY_LAG_SOFT_MAX = 10e6; init( STORAGE_INCLUDE_FEED_STORAGE_QUEUE, true ); if ( randomize && BUGGIFY ) STORAGE_INCLUDE_FEED_STORAGE_QUEUE = false; init( STORAGE_SHARD_CONSISTENCY_CHECK_INTERVAL, 0.0); if ( isSimulated ) STORAGE_SHARD_CONSISTENCY_CHECK_INTERVAL = 5.0; + init( CONSISTENCY_CHECK_BACKWARD_READ, false ); if (isSimulated) CONSISTENCY_CHECK_BACKWARD_READ = deterministicRandom()->coinflip(); init (STORAGE_FETCH_KEYS_DELAY, 0.0 ); if ( randomize && BUGGIFY ) { STORAGE_FETCH_KEYS_DELAY = deterministicRandom()->random01() * 5.0; } init (STORAGE_FETCH_KEYS_USE_COMMIT_BUDGET, false ); if (isSimulated) STORAGE_FETCH_KEYS_USE_COMMIT_BUDGET = deterministicRandom()->coinflip(); init (STORAGE_ROCKSDB_LOG_CLEAN_UP_DELAY, 3600 * 2 ); if (isSimulated) STORAGE_ROCKSDB_LOG_CLEAN_UP_DELAY = 20.0; @@ -1078,6 +1102,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi // Test harness init( WORKER_POLL_DELAY, 1.0 ); + init( PROBABILITY_FACTOR_MEMORY_ENGINE_SELECTED_SIM, 2 ); // default is 1 + init( PROBABILITY_FACTOR_ROCKSDB_ENGINE_SELECTED_SIM, 2 ); // default is 1 + init( PROBABILITY_FACTOR_SQLITE_ENGINE_SELECTED_SIM, 2 ); // default is 1 // Coordination init( COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL, 1.0 ); if( randomize && BUGGIFY ) COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL = 10.0; @@ -1190,6 +1217,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ENABLE_BLOB_GRANULE_COMPRESSION, false ); if ( randomize && BUGGIFY ) { ENABLE_BLOB_GRANULE_COMPRESSION = deterministicRandom()->coinflip(); } init( BLOB_GRANULE_COMPRESSION_FILTER, "NONE" ); if ( randomize && BUGGIFY ) { BLOB_GRANULE_COMPRESSION_FILTER = CompressionUtils::toString(CompressionUtils::getRandomFilter()); } init( ENCRYPTION_LOGGING_INTERVAL, 5.0 ); + init( DISABLED_ENCRYPTION_PROBABILITY_SIM, 0.90 ); // KMS connector type init( KMS_CONNECTOR_TYPE, "RESTKmsConnector" ); diff --git a/fdbclient/SpecialKeySpace.actor.cpp b/fdbclient/SpecialKeySpace.actor.cpp index 739060287ba..703503f3247 100644 --- a/fdbclient/SpecialKeySpace.actor.cpp +++ b/fdbclient/SpecialKeySpace.actor.cpp @@ -945,33 +945,11 @@ bool parseNetWorkAddrFromKeys(ReadYourWritesTransaction* ryw, } ACTOR Future checkExclusion(Database db, - std::vector* addresses, - std::set* exclusions, + std::vector addresses, + std::set exclusions, + std::unordered_set localities, bool markFailed, Optional* msg) { - - if (markFailed) { - state bool safe; - try { - bool _safe = wait(checkSafeExclusions(db, *addresses)); - safe = _safe; - } catch (Error& e) { - if (e.code() == error_code_actor_cancelled) - throw; - TraceEvent("CheckSafeExclusionsError").error(e); - safe = false; - } - if (!safe) { - std::string temp = "ERROR: It is unsafe to exclude the specified servers at this time.\n" - "Please check that this exclusion does not bring down an entire storage team.\n" - "Please also ensure that the exclusion will keep a majority of coordinators alive.\n" - "You may add more storage processes or coordinators to make the operation safe.\n" - "Call set(\"0xff0xff/management/failed/\", ...) to exclude without " - "performing safety checks.\n"; - *msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp); - return false; - } - } StatusObject status = wait(StatusClient::statusFetcher(db)); state std::string errorString = "ERROR: Could not calculate the impact of this exclude on the total available space in the cluster.\n" @@ -1002,24 +980,84 @@ ACTOR Future checkExclusion(Database db, state int64_t totalKvStoreUsedBytes = 0; state int64_t totalKvStoreUsedBytesNotExcluded = 0; state int64_t totalKvStoreAvailableBytes = 0; - // Keep track if we exclude any storage process with the provided adddresses - state bool excludedAddressesContainsStorageRole = false; + state std::map> parsedLocalities; + // Convert the passed localities into a map of vectors to make it easier to check if a process + // is excluded by locality. + for (const auto& locality : localities) { + std::pair locality_key_value = decodeLocality(locality); + if (locality_key_value.first == "") { + continue; + } + + parsedLocalities[locality_key_value.first].push_back(locality_key_value.second); + } + + // If the exclusion marks the excluded processes as failed and the provided addresses are not empty, + // we will copy the input addresses into a new vector to not modify the input vector when additional + // addresses are added. We do this to get the excluded addresses when locality based exclusions are used + // with the failed flag, in this case the checkSafeExclusions will be called with the excluded addresses + // to verify if the exclusion can be done. In case of locality based exclusions the addresses should + // be empty and if the addresses are not empty, we are not using the locality based exclusion. So this + // step is a safeguard for future changes where the provided addresses are not empty and locality based + // exclusions are used. + state std::vector excludedAddresses; + if (markFailed && !addresses.empty()) { + excludedAddresses = addresses; + } try { for (auto proc : processesMap.obj()) { StatusObjectReader process(proc.second); - std::string addrStr; - if (!process.get("address", addrStr)) { - *msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString); - return false; + StatusObjectReader localityObj; + bool hasLocalities = process.get("locality", localityObj); + + bool excluded = process.has("excluded") && process.last().get_bool(); + // If the process is already excluded based on the process status, we don't have to check if the process is + // also excluded by address or locality. + if (!excluded) { + // First check if the process is excluded based on the exclusions. + std::string addrStr; + if (!process.get("address", addrStr)) { + *msg = + ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", errorString); + return false; + } + NetworkAddress addr = NetworkAddress::parse(addrStr); + excluded = addressExcluded(exclusions, addr); + + // If the process is not already excluded and the parsed localites have at least one entry, + // check if the process is excluded by localities. + if (!excluded && hasLocalities && !parsedLocalities.empty()) { + // Iterate over all excluded localites and check if the process is excluded based on the locality. + for (const auto& [localityKey, localityVec] : parsedLocalities) { + std::string localityValue; + if (!localityObj.get(localityKey, localityValue)) { + // If the locality doesn't exist in the locality object skip over it. + continue; + } + + // When the process has a matching locality field, add it to the exclusion list. + if (std::find(localityVec.begin(), localityVec.end(), localityValue) != localityVec.end()) { + excluded = true; + // If the exclusion will exclude the processes as failed, we have to store the addresses of + // the excluded processes in this list, as the checkSafeExclusions check will work on this + // information. + if (markFailed) { + auto addrExclusion = AddressExclusion(addr.ip, addr.port); + if (std::find(excludedAddresses.begin(), excludedAddresses.end(), addrExclusion) != + excludedAddresses.end()) { + excludedAddresses.push_back(addrExclusion); + } + } + + break; + } + } + } } - NetworkAddress addr = NetworkAddress::parse(addrStr); - bool includedInExclusion = addressExcluded(*exclusions, addr); - bool excluded = (process.has("excluded") && process.last().get_bool()) || includedInExclusion; - StatusObjectReader localityObj; std::string disk_id; - if (process.get("locality", localityObj)) { + if (hasLocalities) { process.get("disk_id", disk_id); // its ok if we don't have this field } @@ -1031,12 +1069,6 @@ ACTOR Future checkExclusion(Database db, ssExcludedCount++; } - // Check if we are excluding a process that serves the storage role. We only have to check the free - // capacity if we are excluding at least one process that serves the storage role. - if (!excludedAddressesContainsStorageRole && includedInExclusion) { - excludedAddressesContainsStorageRole = true; - } - int64_t used_bytes; if (!role.get("kvstore_used_bytes", used_bytes)) { *msg = ManagementAPIError::toJsonString( @@ -1083,33 +1115,56 @@ ACTOR Future checkExclusion(Database db, // If the exclusion command only contains processes that serve a non storage role we can skip the free capacity // check in order to not block those exclusions. - if (!excludedAddressesContainsStorageRole) { - return true; - } - - // The numerator is the total space in use by FDB that is not immediately reusable. - // This is calculated as: used + free - available = used + free - (free - reusable) = used - reusable. - // The denominator is the total capacity usable by FDB (either used or unused currently) on non-excluded servers. - double finalUnavailableRatio = - (double)(totalKvStoreUsedBytes + totalKvStoreFreeBytes - totalKvStoreAvailableBytes) / - std::max((double)(totalKvStoreUsedBytesNotExcluded + totalKvStoreFreeBytesNotExcluded), (double)1); - - TraceEvent(SevInfo, "CheckExclusionDetails") - .detail("SsTotalCount", ssTotalCount) - .detail("SsExcludedCount", ssExcludedCount) - .detail("FinalUnavailableRatio", finalUnavailableRatio) - .detail("TotalKvStoreUsedBytes", totalKvStoreUsedBytes) - .detail("TotalKvStoreFreeBytes", totalKvStoreFreeBytes) - .detail("TotalKvStoreAvailableBytes", totalKvStoreAvailableBytes) - .detail("TotalKvStoreUsedBytesNotExcluded", totalKvStoreUsedBytesNotExcluded) - .detail("TotalKvStoreFreeBytesNotExcluded", totalKvStoreFreeBytesNotExcluded); - - if (ssExcludedCount == ssTotalCount || finalUnavailableRatio > 0.9) { - std::string temp = "ERROR: This exclude may cause the total available space in the cluster to drop below 10%.\n" - "Call set(\"0xff0xff/management/options/exclude/force\", ...) first to exclude without " - "checking available space.\n"; - *msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp); - return false; + if (ssExcludedCount > 0) { + // The numerator is the total space in use by FDB that is not immediately reusable. + // This is calculated as: used + free - available = used + free - (free - reusable) = used - reusable. + // The denominator is the total capacity usable by FDB (either used or unused currently) on non-excluded + // servers. + double finalUnavailableRatio = + (double)(totalKvStoreUsedBytes + totalKvStoreFreeBytes - totalKvStoreAvailableBytes) / + std::max((double)(totalKvStoreUsedBytesNotExcluded + totalKvStoreFreeBytesNotExcluded), (double)1); + + TraceEvent(SevInfo, "CheckExclusionDetails") + .detail("SsTotalCount", ssTotalCount) + .detail("SsExcludedCount", ssExcludedCount) + .detail("FinalUnavailableRatio", finalUnavailableRatio) + .detail("TotalKvStoreUsedBytes", totalKvStoreUsedBytes) + .detail("TotalKvStoreFreeBytes", totalKvStoreFreeBytes) + .detail("TotalKvStoreAvailableBytes", totalKvStoreAvailableBytes) + .detail("TotalKvStoreUsedBytesNotExcluded", totalKvStoreUsedBytesNotExcluded) + .detail("TotalKvStoreFreeBytesNotExcluded", totalKvStoreFreeBytesNotExcluded); + + if (ssExcludedCount == ssTotalCount || finalUnavailableRatio > 0.9) { + std::string temp = + "ERROR: This exclude may cause the total available space in the cluster to drop below 10%.\n" + "Call set(\"0xff0xff/management/options/exclude/force\", ...) first to exclude without " + "checking available space.\n"; + *msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp); + return false; + } + } + + if (markFailed) { + state bool safe; + try { + bool _safe = wait(checkSafeExclusions(db, addresses)); + safe = _safe; + } catch (Error& e) { + if (e.code() == error_code_actor_cancelled) + throw; + TraceEvent("CheckSafeExclusionsError").error(e); + safe = false; + } + if (!safe) { + std::string temp = "ERROR: It is unsafe to exclude the specified servers at this time.\n" + "Please check that this exclusion does not bring down an entire storage team.\n" + "Please also ensure that the exclusion will keep a majority of coordinators alive.\n" + "You may add more storage processes or coordinators to make the operation safe.\n" + "Call set(\"0xff0xff/management/failed/\", ...) to exclude without " + "performing safety checks.\n"; + *msg = ManagementAPIError::toJsonString(false, markFailed ? "exclude failed" : "exclude", temp); + return false; + } } return true; @@ -1157,14 +1212,16 @@ ACTOR Future> excludeCommitActor(ReadYourWritesTransaction state Optional result; state std::vector addresses; state std::set exclusions; + state std::unordered_set localities; + if (!parseNetWorkAddrFromKeys(ryw, failed, addresses, exclusions, result)) return result; // If force option is not set, we need to do safety check auto force = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey( failed ? "failed" : "excluded", "force")]; - // only do safety check when we have servers to be excluded and the force option key is not set + // Only do safety check when we have servers to be excluded and the force option key is not set if (addresses.size() && !(force.first && force.second.present())) { - bool safe = wait(checkExclusion(ryw->getDatabase(), &addresses, &exclusions, failed, &result)); + bool safe = wait(checkExclusion(ryw->getDatabase(), addresses, exclusions, localities, failed, &result)); if (!safe) return result; } @@ -2751,9 +2808,6 @@ void includeLocalities(ReadYourWritesTransaction* ryw) { bool parseLocalitiesFromKeys(ReadYourWritesTransaction* ryw, bool failed, std::unordered_set& localities, - std::vector& addresses, - std::set& exclusions, - std::vector& workers, Optional& msg) { KeyRangeRef range = failed ? SpecialKeySpace::getManagementApiCommandRange("failedlocality") : SpecialKeySpace::getManagementApiCommandRange("excludedlocality"); @@ -2770,12 +2824,6 @@ bool parseLocalitiesFromKeys(ReadYourWritesTransaction* ryw, Key locality = iter->begin().removePrefix(range.begin); if (locality.startsWith(LocalityData::ExcludeLocalityPrefix) && locality.toString().find(':') != std::string::npos) { - std::set localityAddresses = getAddressesByLocality(workers, locality.toString()); - if (!localityAddresses.empty()) { - std::copy(localityAddresses.begin(), localityAddresses.end(), back_inserter(addresses)); - exclusions.insert(localityAddresses.begin(), localityAddresses.end()); - } - localities.insert(locality.toString()); } else { std::string error = "ERROR: \'" + locality.toString() + "\' is not a valid locality\n"; @@ -2798,16 +2846,21 @@ ACTOR Future> excludeLocalityCommitActor(ReadYourWritesTra state std::set exclusions; ryw->setOption(FDBTransactionOptions::RAW_ACCESS); + ryw->setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); + ryw->setOption(FDBTransactionOptions::LOCK_AWARE); - state std::vector workers = wait(getWorkers(&ryw->getTransaction())); - if (!parseLocalitiesFromKeys(ryw, failed, localities, addresses, exclusions, workers, result)) + // We are not parsing here over the worker list as the checkExclusion step will iterate over the status + // and the status will contain the same information for workers (actually a bit more information). + // If we fetch the workers here and in the status, we are bascially fetching the same information twice + // which is fine under normal circumstances, but there is no need to do that. + if (!parseLocalitiesFromKeys(ryw, failed, localities, result)) return result; // If force option is not set, we need to do safety check auto force = ryw->getSpecialKeySpaceWriteMap()[SpecialKeySpace::getManagementApiCommandOptionSpecialKey( failed ? "failed_locality" : "excluded_locality", "force")]; // only do safety check when we have localities to be excluded and the force option key is not set if (localities.size() && !(force.first && force.second.present())) { - bool safe = wait(checkExclusion(ryw->getDatabase(), &addresses, &exclusions, failed, &result)); + bool safe = wait(checkExclusion(ryw->getDatabase(), addresses, exclusions, localities, failed, &result)); if (!safe) return result; } diff --git a/fdbclient/SystemData.cpp b/fdbclient/SystemData.cpp index b344925bc84..f4f9df9d4cf 100644 --- a/fdbclient/SystemData.cpp +++ b/fdbclient/SystemData.cpp @@ -594,6 +594,13 @@ void decodeDataMoveId(const UID& id, bool& assigned, bool& emptyRange, DataMoveT emptyRange = id.second() == emptyShardId; if (assigned && !emptyRange && id != anonymousShardId) { dataMoveType = static_cast(0xFF & id.second()); + if (dataMoveType < DataMoveType::LOGICAL || dataMoveType >= DataMoveType::NUMBER_OF_TYPES) { + TraceEvent(SevError, "DecodeDataMoveIdInvalid") + .detail("DataMoveID", id.toString()) + .detail("DataMoveType", dataMoveType) + .detail("Assigned", assigned) + .detail("EmptyRange", emptyRange); + } } } @@ -627,10 +634,6 @@ void decodeServerKeysValue(const ValueRef& value, } } -bool physicalShardMoveEnabled(const UID& dataMoveId) { - return (dataMoveId.second() & 1U); -} - const KeyRef cacheKeysPrefix = "\xff\x02/cacheKeys/"_sr; const Key cacheKeysKey(uint16_t idx, const KeyRef& key) { diff --git a/fdbclient/include/fdbclient/FDBTypes.h b/fdbclient/include/fdbclient/FDBTypes.h index 6179da19379..6e83cfc089f 100644 --- a/fdbclient/include/fdbclient/FDBTypes.h +++ b/fdbclient/include/fdbclient/FDBTypes.h @@ -370,6 +370,8 @@ struct KeyRangeRef { return KeyRangeRef(begin.removePrefix(prefix), end.removePrefix(prefix)); } + int prefixLength() const { return commonPrefixLength(begin, end); } + const KeyRangeRef& operator=(const KeyRangeRef& rhs) { const_cast(begin) = rhs.begin; const_cast(end) = rhs.end; @@ -1668,24 +1670,30 @@ struct StorageMetadataType { KeyValueStoreType storeType; // no need to serialize part (should be assigned after initialization) - bool wrongConfigured = false; + // Used only during wiggling to find out if the SS has incorrect storageType + // compared to perpetualStorageWiggleType. If perpetualStorageWiggleType is not + // set, configuredStorageType is compared to SS storageType. + bool wrongConfiguredForWiggle = false; StorageMetadataType() : createdTime(0) {} - StorageMetadataType(double t, KeyValueStoreType storeType = KeyValueStoreType::END, bool wrongConfigured = false) - : createdTime(t), storeType(storeType), wrongConfigured(wrongConfigured) {} + StorageMetadataType(double t, + KeyValueStoreType storeType = KeyValueStoreType::END, + bool wrongConfiguredForWiggle = false) + : createdTime(t), storeType(storeType), wrongConfiguredForWiggle(wrongConfiguredForWiggle) {} static double currentTime() { return g_network->timer(); } bool operator==(const StorageMetadataType& b) const { - return createdTime == b.createdTime && storeType == b.storeType && wrongConfigured == b.wrongConfigured; + return createdTime == b.createdTime && storeType == b.storeType && + wrongConfiguredForWiggle == b.wrongConfiguredForWiggle; } bool operator<(const StorageMetadataType& b) const { - if (wrongConfigured == b.wrongConfigured) { + if (wrongConfiguredForWiggle == b.wrongConfiguredForWiggle) { // the older SS has smaller createdTime return createdTime < b.createdTime; } - return wrongConfigured > b.wrongConfigured; + return wrongConfiguredForWiggle > b.wrongConfiguredForWiggle; } bool operator>(const StorageMetadataType& b) const { return b < *this; } diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index 0879ee102ae..25f9b7d8ad9 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -193,6 +193,11 @@ class ServerKnobs : public KnobsImpl { // it are default to be 'anonymous' for compatibility. int PRIORITY_ENFORCE_MOVE_OUT_OF_PHYSICAL_SHARD; + // Probability that a team redundant data move set TrueBest when get destination team + double PROBABILITY_TEAM_REDUNDANT_DATAMOVE_CHOOSE_TRUE_BEST_DEST; + // Probability that a team unhealthy data move set TrueBest when get destination team + double PROBABILITY_TEAM_UNHEALTHY_DATAMOVE_CHOOSE_TRUE_BEST_DEST; + // Data distribution // DD use AVAILABLE_SPACE_PIVOT_RATIO to calculate pivotAvailableSpaceRatio. Given a array that's descend // sorted by available space ratio, the pivot position is AVAILABLE_SPACE_PIVOT_RATIO * team count. @@ -241,6 +246,8 @@ class ServerKnobs : public KnobsImpl { // shard metrics will update immediately int64_t SHARD_READ_OPS_CHANGE_THRESHOLD; bool ENABLE_WRITE_BASED_SHARD_SPLIT; // Experimental. Enable to enforce shard split when write traffic is high + int DD_SHARD_USABLE_REGION_CHECK_RATE; // Assuming all shards need to repair, the (rough) number of shards moving + // for usable region per second. Set 0 to disable shard usable region check double SHARD_MAX_READ_DENSITY_RATIO; int64_t SHARD_READ_HOT_BANDWIDTH_MIN_PER_KSECONDS; @@ -262,6 +269,10 @@ class ServerKnobs : public KnobsImpl { // balanced/filledup before starting the next wiggle. double PERPETUAL_WIGGLE_DELAY; // The max interval between the last wiggle finish and the next wiggle start bool PERPETUAL_WIGGLE_DISABLE_REMOVER; // Whether the start of perpetual wiggle replace team remover + bool PERPETUAL_WIGGLE_PAUSE_AFTER_TSS_TARGET_MET; + double PERPETUAL_WIGGLE_MIN_AVAILABLE_SPACE_RATIO; // Pause wiggle until all Storage servers have minimumn + // of PERPETUAL_WIGGLE_MIN_AVAILABLE_SPACE_RATIO disk + // space available. double LOG_ON_COMPLETION_DELAY; int BEST_TEAM_MAX_TEAM_TRIES; int BEST_TEAM_OPTION_COUNT; @@ -337,8 +348,15 @@ class ServerKnobs : public KnobsImpl { bool DD_ENABLE_REBALANCE_STORAGE_QUEUE_WITH_LIGHT_WRITE_SHARD; // Enable to allow storage queue rebalancer to move // light-traffic shards out of the overloading server double DD_WAIT_TSS_DATA_MOVE_DELAY; + bool DD_VALIDATE_SERVER_TEAM_COUNT_AFTER_BUILD_TEAM; // Enable to validate server team count per server after build + // team // TeamRemover to remove redundant teams + double TR_LOW_SPACE_PIVOT_DELAY_SEC; // teamRedundant data moves can make the min SS available % smaller in + // particular when the majority of SSes have low available %. So, when the + // pivot is below the target, teamRemover wait for the specified time to check + // the pivot again. teamRemover triggers teamRedundant data moves only when the + // pivot is above the target. bool TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER; // disable the machineTeamRemover actor double TR_REMOVE_MACHINE_TEAM_DELAY; // wait for the specified time before try to remove next machine team bool TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS; // guard to select which machineTeamRemover logic to use @@ -456,6 +474,7 @@ class ServerKnobs : public KnobsImpl { // Set to 0 to disable histograms. double ROCKSDB_HISTOGRAMS_SAMPLE_RATE; double ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME; + double ROCKSDB_PROBABILITY_REUSE_ITERATOR_SIM; // Probability that RocksDB reuses iterator in simulation bool ROCKSDB_READ_RANGE_REUSE_ITERATORS; bool SHARDED_ROCKSDB_REUSE_ITERATORS; bool ROCKSDB_READ_RANGE_REUSE_BOUNDED_ITERATORS; @@ -526,6 +545,7 @@ class ServerKnobs : public KnobsImpl { double ROCKSDB_CF_METRICS_DELAY; int ROCKSDB_MAX_LOG_FILE_SIZE; int ROCKSDB_KEEP_LOG_FILE_NUM; + int ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL; bool ROCKSDB_SKIP_STATS_UPDATE_ON_OPEN; bool ROCKSDB_SKIP_FILE_SIZE_CHECK_ON_OPEN; bool ROCKSDB_FULLFILE_CHECKSUM; // For validate sst files when compaction and producing backup files. TODO: set @@ -536,6 +556,7 @@ class ServerKnobs : public KnobsImpl { int ROCKSDB_WRITEBATCH_PROTECTION_BYTES_PER_KEY; int ROCKSDB_MEMTABLE_PROTECTION_BYTES_PER_KEY; int ROCKSDB_BLOCK_PROTECTION_BYTES_PER_KEY; + bool SHARDED_ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH; int SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS; double SHARDED_ROCKSDB_VALIDATE_MAPPING_RATIO; int SHARD_METADATA_SCAN_BYTES_LIMIT; @@ -551,8 +572,11 @@ class ServerKnobs : public KnobsImpl { int SHARDED_ROCKSDB_TARGET_FILE_SIZE_BASE; int SHARDED_ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER; bool SHARDED_ROCKSDB_SUGGEST_COMPACT_CLEAR_RANGE; + int SHARDED_ROCKSDB_COMPACT_ON_RANGE_DELETION_THRESHOLD; int SHARDED_ROCKSDB_MAX_BACKGROUND_JOBS; int64_t SHARDED_ROCKSDB_BLOCK_CACHE_SIZE; + double SHARDED_ROCKSDB_CACHE_HIGH_PRI_POOL_RATIO; + bool SHARDED_ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS; int64_t SHARDED_ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC; int64_t SHARDED_ROCKSDB_RATE_LIMITER_MODE; int SHARDED_ROCKSDB_BACKGROUND_PARALLELISM; @@ -564,6 +588,10 @@ class ServerKnobs : public KnobsImpl { int SHARDED_ROCKSDB_MAX_OPEN_FILES; bool SHARDED_ROCKSDB_READ_ASYNC_IO; int SHARDED_ROCKSDB_PREFIX_LEN; + int SHARDED_ROCKSDB_BLOOM_FILTER_BITS; + double SHARDED_ROCKSDB_MEMTABLE_BLOOM_FILTER_RATIO; + double SHARDED_ROCKSDB_HISTOGRAMS_SAMPLE_RATE; + bool SHARDED_ROCKSDB_USE_DIRECT_IO; // Leader election int MAX_NOTIFICATIONS; @@ -709,7 +737,6 @@ class ServerKnobs : public KnobsImpl { double REPLACE_INTERFACE_CHECK_DELAY; double COORDINATOR_REGISTER_INTERVAL; double CLIENT_REGISTER_INTERVAL; - bool CC_PAUSE_HEALTH_MONITOR; bool CC_ENABLE_WORKER_HEALTH_MONITOR; double CC_WORKER_HEALTH_CHECKING_INTERVAL; // The interval of refreshing the degraded server list. double CC_DEGRADED_LINK_EXPIRATION_INTERVAL; // The time period from the last degradation report after which a @@ -720,6 +747,8 @@ class ServerKnobs : public KnobsImpl { int CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE; // The maximum number of degraded peers when excluding a server. When the // number of degraded peers is more than this value, we will not exclude // this server since it may because of server overload. + int CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE_MIN; // Similar to CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE which is an upper + // bound, this is a lower bound. int CC_MAX_EXCLUSION_DUE_TO_HEALTH; // The max number of degraded servers to exclude by Cluster Controller due to // degraded health. bool CC_HEALTH_TRIGGER_RECOVERY; // If true, cluster controller will kill the master to trigger recovery when @@ -727,8 +756,9 @@ class ServerKnobs : public KnobsImpl { double CC_TRACKING_HEALTH_RECOVERY_INTERVAL; // The number of recovery count should not exceed // CC_MAX_HEALTH_RECOVERY_COUNT within // CC_TRACKING_HEALTH_RECOVERY_INTERVAL. - int CC_MAX_HEALTH_RECOVERY_COUNT; // The max number of recoveries can be triggered due to worker health within - // CC_TRACKING_HEALTH_RECOVERY_INTERVAL + int CC_MAX_HEALTH_RECOVERY_COUNT; // The max number recoveries that can be triggered due to worker + // health within CC_TRACKING_HEALTH_RECOVERY_INTERVAL. This count accounts for any + // recovery trigger including non-gray failure ones. bool CC_HEALTH_TRIGGER_FAILOVER; // Whether to enable health triggered failover in CC. int CC_FAILOVER_DUE_TO_HEALTH_MIN_DEGRADATION; // The minimum number of degraded servers that can trigger a // failover. @@ -744,6 +774,8 @@ class ServerKnobs : public KnobsImpl { // determined as degraded satellite. bool CC_ENABLE_REMOTE_LOG_ROUTER_MONITORING; // When enabled, gray failure tries to detect whether the remote log // router is degraded and may use trigger recovery to recover from it. + bool CC_INVALIDATE_EXCLUDED_PROCESSES; // When enabled, invalidate the complaints by processes that were excluded + // in gray failure triggered recoveries. double CC_THROTTLE_SINGLETON_RERECRUIT_INTERVAL; // The interval to prevent re-recruiting the same singleton if a // recruiting fight between two cluster controllers occurs. @@ -1030,6 +1062,7 @@ class ServerKnobs : public KnobsImpl { std::string STORAGESERVER_READTYPE_PRIORITY_MAP; int SPLIT_METRICS_MAX_ROWS; double STORAGE_SHARD_CONSISTENCY_CHECK_INTERVAL; + bool CONSISTENCY_CHECK_BACKWARD_READ; int PHYSICAL_SHARD_MOVE_LOG_SEVERITY; int FETCH_SHARD_BUFFER_BYTE_LIMIT; int FETCH_SHARD_UPDATES_BYTE_LIMIT; @@ -1081,6 +1114,9 @@ class ServerKnobs : public KnobsImpl { // Test harness double WORKER_POLL_DELAY; + int PROBABILITY_FACTOR_MEMORY_ENGINE_SELECTED_SIM; + int PROBABILITY_FACTOR_ROCKSDB_ENGINE_SELECTED_SIM; + int PROBABILITY_FACTOR_SQLITE_ENGINE_SELECTED_SIM; // Coordination double COORDINATED_STATE_ONCONFLICT_POLL_INTERVAL; @@ -1208,6 +1244,7 @@ class ServerKnobs : public KnobsImpl { int SIM_KMS_MAX_KEYS; int ENCRYPT_PROXY_MAX_DBG_TRACE_LENGTH; double ENCRYPTION_LOGGING_INTERVAL; + double DISABLED_ENCRYPTION_PROBABILITY_SIM; // Probability that encryption is forced to be disabled in simulation // Compression bool ENABLE_BLOB_GRANULE_COMPRESSION; diff --git a/fdbmonitor/fdbmonitor.cpp b/fdbmonitor/fdbmonitor.cpp index eeae06a1acd..e011e7e31c1 100644 --- a/fdbmonitor/fdbmonitor.cpp +++ b/fdbmonitor/fdbmonitor.cpp @@ -1920,7 +1920,16 @@ int main(int argc, char** argv) { int delay = cmd->get_and_update_current_restart_delay(); if (!cmd->quiet) { if (WIFEXITED(child_status)) { - Severity priority = (WEXITSTATUS(child_status) == 0) ? SevWarn : SevError; + Severity priority; + // If the process exited successfully, e.g. because someone restarted the process + // with fdbcli kill we don't want to delay the restart. We only want to delay + // the restart if the process was exited unsuccessfully (exit code different from 0). + if (WEXITSTATUS(child_status) == 0) { + priority = SevWarn; + delay = 0; + } else { + priority = SevError; + } log_process_msg(priority, cmd->ssection.c_str(), "Process %d exited %d, restarting in %d seconds\n", diff --git a/fdbrpc/FlowTransport.actor.cpp b/fdbrpc/FlowTransport.actor.cpp index e4a3ae1ca10..1da9c942e50 100644 --- a/fdbrpc/FlowTransport.actor.cpp +++ b/fdbrpc/FlowTransport.actor.cpp @@ -20,9 +20,14 @@ #include "fdbrpc/FlowTransport.h" #include "flow/Arena.h" +#include "flow/IThreadPool.h" +#include "flow/Knobs.h" +#include "flow/NetworkAddress.h" #include "flow/network.h" #include +#include +#include #include #if VALGRIND #include @@ -352,8 +357,112 @@ class TransportData { Future publicKeyFileWatch; std::unordered_map, PublicKey> publicKeys; + + struct ConnectionHistoryEntry { + int64_t time; + NetworkAddress addr; + bool failed; + }; + std::deque connectionHistory; + Future connectionHistoryLoggerF; + Reference connectionLogWriterThread; +}; + +struct ConnectionLogWriter : IThreadPoolReceiver { + const std::string baseDir; + std::string fileName; + std::fstream file; + + ConnectionLogWriter(const std::string baseDir) : baseDir(baseDir) {} + + virtual ~ConnectionLogWriter() { + if (file.is_open()) + file.close(); + } + + struct AppendAction : TypedAction { + std::string localAddr; + std::deque entries; + AppendAction(std::string localAddr, std::deque&& entries) + : localAddr(localAddr), entries(std::move(entries)) {} + + double getTimeEstimate() const { return 2; } + }; + + std::string newFileName() const { return baseDir + "fdb-connection-log-" + time_str() + ".csv"; } + + void init() { fileName = newFileName(); } + + std::string time_str() const { return std::to_string(now()); } + + void openOrRoll() { + if (fileName.empty()) { + fileName = newFileName(); + } + + if (!file.is_open()) { + TraceEvent("OpenConnectionLog").detail("FileName", fileName); + file = std::fstream(fileName, std::ios::in | std::ios::out | std::ios::app); + } + + if (!file.is_open()) { + TraceEvent(SevError, "ErrorOpenConnectionLog").detail("FileName", fileName); + throw io_error(); + } + + if (file.tellg() > 100 * 1024 * 1024 /* 100 MB */) { + file.close(); + fileName = newFileName(); + TraceEvent("RollConnectionLog").detail("FileName", fileName); + openOrRoll(); + } + } + + void action(AppendAction& a) { + openOrRoll(); + + std::string output; + for (const auto& entry : a.entries) { + output += std::to_string(entry.time) + ","; + output += a.localAddr + ","; + output += entry.failed ? "failed," : "success,"; + output += entry.addr.toString() + "\n"; + } + file << output; + file.flush(); + } }; +ACTOR Future connectionHistoryLogger(TransportData* self) { + if (!FLOW_KNOBS->LOG_CONNECTION_ATTEMPTS_ENABLED) { + return Void(); + } + + state Future next = Void(); + + // One thread ensures async serialized execution on the log file. + if (g_network->isSimulated()) { + self->connectionLogWriterThread = Reference(new DummyThreadPool()); + } else { + self->connectionLogWriterThread = createGenericThreadPool(); + } + + self->connectionLogWriterThread->addThread(new ConnectionLogWriter(FLOW_KNOBS->CONNECTION_LOG_DIRECTORY)); + loop { + wait(next); + next = delay(FLOW_KNOBS->LOG_CONNECTION_INTERVAL_SECS); + if (self->connectionHistory.size() == 0) { + continue; + } + std::string localAddr = FlowTransport::getGlobalLocalAddress().toString(); + auto action = new ConnectionLogWriter::AppendAction(localAddr, std::move(self->connectionHistory)); + ASSERT(action != nullptr); + self->connectionLogWriterThread->post(action); + wait(delay(1)); + ASSERT(self->connectionHistory.size() == 0); + } +} + ACTOR Future pingLatencyLogger(TransportData* self) { state NetworkAddress lastAddress = NetworkAddress(); loop { @@ -422,6 +531,8 @@ TransportData::TransportData(uint64_t transportId, int maxWellKnownEndpoints, IP allowList(allowList == nullptr ? IPAllowList() : *allowList) { degraded = makeReference>(false); pingLogger = pingLatencyLogger(this); + + connectionHistoryLoggerF = connectionHistoryLogger(this); } #define CONNECT_PACKET_V0 0x0FDB00A444020001LL @@ -1492,10 +1603,17 @@ ACTOR static Future connectionReader(TransportData* transport, } ACTOR static Future connectionIncoming(TransportData* self, Reference conn) { + state TransportData::ConnectionHistoryEntry entry; + entry.time = now(); + entry.addr = conn->getPeerAddress(); try { wait(conn->acceptHandshake()); state Promise> onConnected; state Future reader = connectionReader(self, conn, Reference(), onConnected); + if (FLOW_KNOBS->LOG_CONNECTION_ATTEMPTS_ENABLED) { + entry.failed = false; + self->connectionHistory.push_back(entry); + } choose { when(wait(reader)) { ASSERT(false); @@ -1509,17 +1627,21 @@ ACTOR static Future connectionIncoming(TransportData* self, ReferencegetDebugID()) .errorUnsuppressed(e) .suppressFor(1.0) .detail("FromAddress", conn->getPeerAddress()); + if (FLOW_KNOBS->LOG_CONNECTION_ATTEMPTS_ENABLED) { + entry.failed = true; + self->connectionHistory.push_back(entry); + } } conn->close(); - return Void(); } + + return Void(); } ACTOR static Future listen(TransportData* self, NetworkAddress listenAddr) { diff --git a/fdbserver/ClusterController.actor.cpp b/fdbserver/ClusterController.actor.cpp index 255a071b912..adcd2c90f7a 100644 --- a/fdbserver/ClusterController.actor.cpp +++ b/fdbserver/ClusterController.actor.cpp @@ -273,6 +273,7 @@ ACTOR Future clusterWatchDatabase(ClusterControllerData* cluster, collection = actorCollection(db->recoveryData->addActor.getFuture()); recoveryCore = clusterRecoveryCore(db->recoveryData); + cluster->recentHealthTriggeredRecoveryTime.push(now()); // Master failure detection is pretty sensitive, but if we are in the middle of a very long recovery we // really don't want to have to start over @@ -2917,6 +2918,18 @@ ACTOR Future dbInfoUpdater(ClusterControllerData* self) { } } +// If we are excluding processes and triggering recovery because of gray failure, also +// invalidate the past complaints from such processes because that signal is no longer +// reliable. +static void invalidateExcludedProcessComplaints(ClusterControllerData* self) { + if (!SERVER_KNOBS->CC_INVALIDATE_EXCLUDED_PROCESSES) { + return; + } + for (const auto& addr : self->excludedDegradedServers) { + self->workerHealth.erase(addr); + } +} + // The actor that periodically monitors the health of tracked workers. ACTOR Future workerHealthMonitor(ClusterControllerData* self) { loop { @@ -2957,18 +2970,20 @@ ACTOR Future workerHealthMonitor(ClusterControllerData* self) { // Check if the cluster controller should trigger a recovery to exclude any degraded servers from // the transaction system. - if (SERVER_KNOBS->CC_PAUSE_HEALTH_MONITOR) { - TraceEvent(SevWarnAlways, "HealthMonitorPaused"); - } else if (self->shouldTriggerRecoveryDueToDegradedServers()) { + if (self->shouldTriggerRecoveryDueToDegradedServers()) { if (SERVER_KNOBS->CC_HEALTH_TRIGGER_RECOVERY) { if (self->recentRecoveryCountDueToHealth() < SERVER_KNOBS->CC_MAX_HEALTH_RECOVERY_COUNT) { - self->recentHealthTriggeredRecoveryTime.push(now()); self->excludedDegradedServers = self->degradationInfo.degradedServers; self->excludedDegradedServers.insert(self->degradationInfo.disconnectedServers.begin(), self->degradationInfo.disconnectedServers.end()); + invalidateExcludedProcessComplaints(self); TraceEvent(SevWarnAlways, "DegradedServerDetectedAndTriggerRecovery") .detail("RecentRecoveryCountDueToHealth", self->recentRecoveryCountDueToHealth()); self->db.forceMasterFailure.trigger(); + } else { + TraceEvent(SevWarnAlways, "RecentRecoveryCountHigh") + .suppressFor(1.0) + .detail("RecentRecoveryCountDueToHealth", self->recentRecoveryCountDueToHealth()); } } else { self->excludedDegradedServers.clear(); @@ -3940,4 +3955,79 @@ TEST_CASE("/fdbserver/clustercontroller/shouldTriggerFailoverDueToDegradedServer return Void(); } +TEST_CASE("/fdbserver/clustercontroller/invalidateExcludedProcessComplaints") { + ClusterControllerData data(ClusterControllerFullInterface(), + LocalityData(), + ServerCoordinators(Reference( + new ClusterConnectionMemoryRecord(ClusterConnectionString()))), + makeReference>>()); + NetworkAddress worker1(IPAddress::parse("1.1.1.0").get(), 1); + NetworkAddress worker2(IPAddress::parse("1.1.1.1").get(), 1); + NetworkAddress worker3(IPAddress::parse("1.1.1.2").get(), 1); + NetworkAddress badPeer(IPAddress::parse("1.1.1.3").get(), 1); + + ASSERT(data.workerHealth.empty()); + + // {worker1, worker2, worker3} complain about badPeer + data.workerHealth[worker1].degradedPeers[badPeer] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[worker2].degradedPeers[badPeer] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[worker3].degradedPeers[badPeer] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + + // badPeer complains about {worker1, worker2, worker3} + data.workerHealth[badPeer].degradedPeers[worker1] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer].degradedPeers[worker2] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + data.workerHealth[badPeer].degradedPeers[worker3] = { now() - SERVER_KNOBS->CC_MIN_DEGRADATION_INTERVAL - 1, + now() }; + + // At this point, we should have 4 complaints total + ASSERT(data.workerHealth.contains(worker1)); + ASSERT(data.workerHealth.contains(worker2)); + ASSERT(data.workerHealth.contains(worker3)); + ASSERT(data.workerHealth.contains(badPeer)); + ASSERT(data.workerHealth.size() == 4); + + // Compute degraded processes + data.degradationInfo = data.getDegradationInfo(); + data.excludedDegradedServers = data.degradationInfo.degradedServers; + + // Ensure badPeer is successfully added to excluded list + // At this point, recovery would also be triggered in a production setting + // We would also invalidate complaints by the excluded process at this point + ASSERT(data.degradationInfo.degradedServers.size() == 1); + ASSERT(data.degradationInfo.degradedServers.contains(badPeer)); + invalidateExcludedProcessComplaints(&data); + + // Now it's possible because of various factors (e.g. timing, expiration) that + // the complaints against badPeer disappear, but the initial complaints that badPeer + // made against others are still there. + data.workerHealth[worker1].degradedPeers.erase(badPeer); + data.workerHealth[worker2].degradedPeers.erase(badPeer); + data.workerHealth[worker3].degradedPeers.erase(badPeer); + + // Compute degraded processes again + data.degradationInfo = data.getDegradationInfo(); + data.excludedDegradedServers = data.degradationInfo.degradedServers; + + if (SERVER_KNOBS->CC_INVALIDATE_EXCLUDED_PROCESSES) { + // With CC_INVALIDATE_EXCLUDE_PROCESSES, we should got 0 degraded processes + // because the original complaints by the now excluded badPeer were invalidated + ASSERT(data.degradationInfo.degradedServers.empty()); + } else { + // However, without CC_INVALIDATE_EXCLUDE_PROCESSES, we would get 3 degraded processes + // which are: worker1, worker2, worker3. This is because we did not invalidate workerHealth + // to remove badPeer complaints when badPeer was excluded and recovery was triggered. + ASSERT(data.degradationInfo.degradedServers.size() == 3); + ASSERT(data.degradationInfo.degradedServers.contains(worker1)); + ASSERT(data.degradationInfo.degradedServers.contains(worker2)); + ASSERT(data.degradationInfo.degradedServers.contains(worker3)); + } + + return Void(); +} + } // namespace diff --git a/fdbserver/DDRelocationQueue.actor.cpp b/fdbserver/DDRelocationQueue.actor.cpp index 31204346cd1..240e5ad6b95 100644 --- a/fdbserver/DDRelocationQueue.actor.cpp +++ b/fdbserver/DDRelocationQueue.actor.cpp @@ -1282,6 +1282,21 @@ static std::string destServersString(std::vectorPRIORITY_TEAM_UNHEALTHY) { + return deterministicRandom()->random01() < + SERVER_KNOBS->PROBABILITY_TEAM_UNHEALTHY_DATAMOVE_CHOOSE_TRUE_BEST_DEST; + } else if (priority == SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT) { + return deterministicRandom()->random01() < + SERVER_KNOBS->PROBABILITY_TEAM_REDUNDANT_DATAMOVE_CHOOSE_TRUE_BEST_DEST; + } else { + return false; + } +} + // This actor relocates the specified keys to a good place. // The inFlightActor key range map stores the actor for each RelocateData ACTOR Future dataDistributionRelocator(DDQueue* self, @@ -1306,6 +1321,7 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, state double startTime = now(); state std::vector destIds; state WantTrueBest wantTrueBest(isValleyFillerPriority(rd.priority)); + state WantTrueBestIfMoveout wantTrueBestIfMoveout(getWantTrueBestIfMoveout(rd.priority)); state uint64_t debugID = deterministicRandom()->randomUInt64(); state bool enableShardMove = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD; @@ -1319,6 +1335,7 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, .detail("KeyBegin", rd.keys.begin) .detail("KeyEnd", rd.keys.end) .detail("Priority", rd.priority) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) .detail("SuppressedEventCount", self->suppressIntervals); if (relocateShardInterval.severity != SevDebug) { @@ -1443,6 +1460,7 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, req.src = rd.src; req.completeSources = rd.completeSources; req.storageQueueAware = SERVER_KNOBS->ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION; + req.wantTrueBestIfMoveout = wantTrueBestIfMoveout; if (enableShardMove && tciIndex == 1) { ASSERT(physicalShardIDCandidate != UID().first() && @@ -1451,8 +1469,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, self->physicalShardCollection->tryGetAvailableRemoteTeamWith( physicalShardIDCandidate, metrics, debugID); if (!remoteTeamWithPhysicalShard.second) { - // Physical shard with `physicalShardIDCandidate` is not available. Retry selecting new - // dst physical shard. + // Physical shard with `physicalShardIDCandidate` is not available. Retry selecting + // new dst physical shard. self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++; foundTeams = false; break; @@ -1465,8 +1483,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, } } - // bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have any - // server that hosts the relocateData. This is possible, for example, in a fearless + // bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have + // any server that hosts the relocateData. This is possible, for example, in a fearless // configuration when the remote DC is just brought up. Future>, bool>> fbestTeam = brokenPromiseToNever(self->teamCollections[tciIndex].getTeam.getReply(req)); @@ -1503,11 +1521,11 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, if (enableShardMove) { if (tciIndex == 1 && !forceToUseNewPhysicalShard) { // critical to the correctness of team selection by PhysicalShardCollection - // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a primary - // team Thus, tryGetAvailableRemoteTeamWith() may select an almost full remote team In - // this case, we must re-select a remote team We set foundTeams = false to avoid - // finishing team selection Then, forceToUseNewPhysicalShard is set, which enforce to - // use getTeam to select a remote team + // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a + // primary team Thus, tryGetAvailableRemoteTeamWith() may select an almost full + // remote team In this case, we must re-select a remote team We set foundTeams = + // false to avoid finishing team selection Then, forceToUseNewPhysicalShard is set, + // which enforce to use getTeam to select a remote team bool minAvailableSpaceRatio = bestTeam.first.get()->getMinAvailableSpaceRatio(true); if (minAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) { self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteTeamIsFull]++; @@ -1516,11 +1534,11 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, } // critical to the correctness of team selection by PhysicalShardCollection - // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a primary - // team Thus, tryGetAvailableRemoteTeamWith() may select an unhealthy remote team In - // this case, we must re-select a remote team We set foundTeams = false to avoid - // finishing team selection Then, forceToUseNewPhysicalShard is set, which enforce to - // use getTeam to select a remote team + // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a + // primary team Thus, tryGetAvailableRemoteTeamWith() may select an unhealthy remote + // team In this case, we must re-select a remote team We set foundTeams = false to + // avoid finishing team selection Then, forceToUseNewPhysicalShard is set, which + // enforce to use getTeam to select a remote team if (!bestTeam.first.get()->isHealthy()) { self->retryFindDstReasonCount [DDQueue::RetryFindDstReason::RemoteTeamIsNotHealthy]++; @@ -1530,11 +1548,11 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, } bestTeams.emplace_back(bestTeam.first.get(), true); - // Always set bestTeams[i].second = true to disable optimization in data move between DCs - // for the correctness of PhysicalShardCollection - // Currently, enabling the optimization will break the invariant of PhysicalShardCollection - // Invariant: once a physical shard is created with a specific set of SSes, this SS set will - // never get changed. + // Always set bestTeams[i].second = true to disable optimization in data move between + // DCs for the correctness of PhysicalShardCollection Currently, enabling the + // optimization will break the invariant of PhysicalShardCollection Invariant: once a + // physical shard is created with a specific set of SSes, this SS set will never get + // changed. if (tciIndex == 0) { ASSERT(foundTeams); @@ -1554,9 +1572,9 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, self->retryFindDstReasonCount [DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++; if (wantTrueBest) { - // Next retry will likely get the same team, and we know that we can't reuse - // any existing physical shard in this team. So force to create new physical - // shard. + // Next retry will likely get the same team, and we know that we can't + // reuse any existing physical shard in this team. So force to create + // new physical shard. forceToUseNewPhysicalShard = true; } foundTeams = false; @@ -1595,6 +1613,9 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, destOverloadedCount++; TraceEvent(destOverloadedCount > 50 ? SevInfo : SevDebug, "DestSSBusy", distributorId) .suppressFor(1.0) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) + .detail("IsRestore", rd.isRestore()) + .detail("Priority", rd.priority) .detail("StuckCount", stuckCount) .detail("DestOverloadedCount", destOverloadedCount) .detail("TeamCollectionId", tciIndex) @@ -1612,6 +1633,9 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, stuckCount++; TraceEvent(stuckCount > 50 ? SevWarnAlways : SevWarn, "BestTeamStuck", distributorId) .suppressFor(1.0) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) + .detail("IsRestore", rd.isRestore()) + .detail("Priority", rd.priority) .detail("StuckCount", stuckCount) .detail("DestOverloadedCount", destOverloadedCount) .detail("TeamCollectionId", tciIndex) @@ -1690,10 +1714,11 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, if (allHealthy && anyWithSource && !bestTeams[i].second) { // When all servers in bestTeams[i] do not hold the shard (!bestTeams[i].second), it indicates // the bestTeams[i] is in a new DC where data has not been replicated to. - // To move data (specified in RelocateShard) to bestTeams[i] in the new DC AND reduce data movement - // across DC, we randomly choose a server in bestTeams[i] as the shard's destination, and - // move the shard to the randomly chosen server (in the remote DC), which will later - // propogate its data to the servers in the same team. This saves data movement bandwidth across DC + // To move data (specified in RelocateShard) to bestTeams[i] in the new DC AND reduce data + // movement across DC, we randomly choose a server in bestTeams[i] as the shard's destination, + // and move the shard to the randomly chosen server (in the remote DC), which will later + // propogate its data to the servers in the same team. This saves data movement bandwidth across + // DC int idx = deterministicRandom()->randomInt(0, serverIds.size()); destIds.push_back(serverIds[idx]); healthyIds.push_back(serverIds[idx]); @@ -1741,6 +1766,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, .detail("Priority", rd.priority) .detail("KeyBegin", rd.keys.begin) .detail("KeyEnd", rd.keys.end) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) + .detail("IsRestore", rd.isRestore()) .detail("StorageMetrics", metrics.toString()) .detail("SourceServers", describe(rd.src)) .detail("DestinationTeam", describe(destIds)) @@ -1749,6 +1776,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, TraceEvent(relocateShardInterval.severity, "RelocateShardHasDestination", distributorId) .detail("PairId", relocateShardInterval.pairID) .detail("Priority", rd.priority) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) + .detail("IsRestore", rd.isRestore()) .detail("KeyBegin", rd.keys.begin) .detail("KeyEnd", rd.keys.end) .detail("SourceServers", describe(rd.src)) @@ -1938,8 +1967,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, auto serverIds = bestTeams[i].first->getServerIDs(); selectedTeams.push_back(ShardsAffectedByTeamFailure::Team(serverIds, i == 0)); } - // The update of PhysicalShardToTeams, PhysicalShardInstances, keyRangePhysicalShardIDMap should - // be atomic + // The update of PhysicalShardToTeams, PhysicalShardInstances, keyRangePhysicalShardIDMap + // should be atomic self->physicalShardCollection->updatePhysicalShardCollection( rd.keys, rd.isRestore(), selectedTeams, rd.dataMoveId.first(), metrics, debugID); } @@ -2012,8 +2041,8 @@ inline double getWorstCpu(const HealthMetrics& metrics, const std::vector& return cpu; } -// Move the shard with the top K highest read density of sourceTeam's to destTeam if sourceTeam has much more read load -// than destTeam +// Move the shard with the top K highest read density of sourceTeam's to destTeam if sourceTeam has much more read +// load than destTeam ACTOR Future rebalanceReadLoad(DDQueue* self, DataMovementReason moveReason, Reference sourceTeam, @@ -2556,8 +2585,8 @@ struct DDQueueImpl { } } } catch (Error& e) { - if (e.code() != error_code_broken_promise && // FIXME: Get rid of these broken_promise errors every time we - // are killed by the master dying + if (e.code() != error_code_broken_promise && // FIXME: Get rid of these broken_promise errors every time + // we are killed by the master dying e.code() != error_code_movekeys_conflict && e.code() != error_code_data_move_cancelled && e.code() != error_code_data_move_dest_team_not_found) TraceEvent(SevError, "DataDistributionQueueError", self->distributorId).error(e); diff --git a/fdbserver/DDShardTracker.actor.cpp b/fdbserver/DDShardTracker.actor.cpp index b282db01925..d18f1438ee2 100644 --- a/fdbserver/DDShardTracker.actor.cpp +++ b/fdbserver/DDShardTracker.actor.cpp @@ -210,6 +210,29 @@ std::pair calculateShardSizeBounds( return { bounds, readHotShard }; } +ACTOR Future shardUsableRegions(DataDistributionTracker::SafeAccessor self, KeyRange keys) { + ASSERT(SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA); + ASSERT(SERVER_KNOBS->DD_SHARD_USABLE_REGION_CHECK_RATE > 0); + wait(yieldedFuture(self()->readyToStart.getFuture())); + double expectedCompletionSeconds = self()->shards->size() * 1.0 / SERVER_KNOBS->DD_SHARD_USABLE_REGION_CHECK_RATE; + double delayTime = deterministicRandom()->random01() * expectedCompletionSeconds; + wait(delayJittered(delayTime)); + auto [newTeam, previousTeam] = self()->shardsAffectedByTeamFailure->getTeamsForFirstShard(keys); + if (newTeam.size() < self()->usableRegions) { + TraceEvent(SevWarn, "ShardUsableRegionMismatch", self()->distributorId) + .suppressFor(5.0) + .detail("NewTeamSize", newTeam.size()) + .detail("PreviousTeamSize", previousTeam.size()) + .detail("NewServers", describe(newTeam)) + .detail("PreviousServers", describe(previousTeam)) + .detail("UsableRegion", self()->usableRegions) + .detail("Shard", keys); + RelocateShard rs(keys, DataMovementReason::POPULATE_REGION, RelocateReason::OTHER); + self()->output.send(rs); + } + return Void(); +} + ACTOR Future trackShardMetrics(DataDistributionTracker::SafeAccessor self, KeyRange keys, Reference>> shardMetrics, @@ -936,9 +959,6 @@ Future shardMerger(DataDistributionTracker* self, const UID actionId = deterministicRandom()->randomUniqueID(); const Severity stSev = static_cast(SERVER_KNOBS->DD_SHARD_TRACKING_LOG_SEVERITY); int64_t maxShardSize = self->maxShardSize->get().get(); - if (SERVER_KNOBS->ALLOW_LARGE_SHARD) { - maxShardSize = SERVER_KNOBS->MAX_LARGE_SHARD_BYTES; - } auto prevIter = self->shards->rangeContaining(keys.begin); auto nextIter = self->shards->rangeContaining(keys.begin); @@ -1128,10 +1148,6 @@ ACTOR Future shardEvaluator(DataDistributionTracker* self, StorageMetrics const& stats = shardSize->get().get().metrics; auto bandwidthStatus = getBandwidthStatus(stats); - if (SERVER_KNOBS->ALLOW_LARGE_SHARD) { - shardBounds.max.bytes = SERVER_KNOBS->MAX_LARGE_SHARD_BYTES; - } - bool sizeSplit = stats.bytes > shardBounds.max.bytes, writeSplit = bandwidthStatus == BandwidthStatusHigh && keys.begin < keyServersKeys.begin; bool shouldSplit = sizeSplit || writeSplit; @@ -1259,6 +1275,10 @@ void restartShardTrackers(DataDistributionTracker* self, data.trackShard = shardTracker(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics); data.trackBytes = trackShardMetrics(DataDistributionTracker::SafeAccessor(self), ranges[i], shardMetrics, whenDDInit); + if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->DD_SHARD_USABLE_REGION_CHECK_RATE > 0 && + self->usableRegions != -1) { + data.trackUsableRegion = shardUsableRegions(DataDistributionTracker::SafeAccessor(self), ranges[i]); + } self->shards->insert(ranges[i], data); } } @@ -1517,7 +1537,7 @@ DataDistributionTracker::DataDistributionTracker(DataDistributionTrackerInitPara output(params.output), shardsAffectedByTeamFailure(params.shardsAffectedByTeamFailure), physicalShardCollection(params.physicalShardCollection), readyToStart(params.readyToStart), anyZeroHealthyTeams(params.anyZeroHealthyTeams), trackerCancelled(params.trackerCancelled), - ddTenantCache(params.ddTenantCache) {} + ddTenantCache(params.ddTenantCache), usableRegions(params.usableRegions) {} DataDistributionTracker::~DataDistributionTracker() { if (trackerCancelled) { diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp index 55e1ee19837..b4af2ee5d0a 100644 --- a/fdbserver/DDTeamCollection.actor.cpp +++ b/fdbserver/DDTeamCollection.actor.cpp @@ -25,6 +25,7 @@ #include "fdbserver/DDTeamCollection.h" #include "fdbserver/ExclusionTracker.actor.h" #include "fdbserver/DataDistributionTeam.h" +#include "fdbserver/Knobs.h" #include "flow/IRandom.h" #include "flow/Trace.h" #include "flow/network.h" @@ -432,7 +433,7 @@ class DDTeamCollectionImpl { if (req.storageQueueAware) { storageQueueThreshold = calculateTeamStorageQueueThreshold(self->teams); } - if (req.teamSelect == TeamSelect::WANT_TRUE_BEST) { + if (req.teamSelect == TeamSelect::WANT_TRUE_BEST || req.wantTrueBestIfMoveout) { ASSERT(!bestOption.present()); if (SERVER_KNOBS->ENFORCE_SHARD_COUNT_PER_TEAM && req.preferWithinShardLimit) { bestOption = getBestTeam(self, @@ -554,11 +555,14 @@ class DDTeamCollectionImpl { // self->traceAllInfo(true); } - if (req.storageQueueAware && !bestOption.present()) { + if (!bestOption.present() && (req.storageQueueAware || req.wantTrueBestIfMoveout)) { + // re-run getTeam without storageQueueAware and wantTrueBestIfMoveout req.storageQueueAware = false; - TraceEvent(SevWarn, "StorageQueueAwareGetTeamFailed", self->distributorId) - .detail("Reason", "bestOption not present"); - wait(getTeam(self, req)); // re-run getTeam without storageQueueAware + req.wantTrueBestIfMoveout = false; + TraceEvent(SevWarn, "GetTeamRetry", self->distributorId) + .detail("OldStorageQueueAware", req.storageQueueAware) + .detail("OldWantTrueBestIfMoveout", req.wantTrueBestIfMoveout); + wait(getTeam(self, req)); } else { req.reply.send(std::make_pair(bestOption, foundSrc)); } @@ -814,6 +818,8 @@ class DDTeamCollectionImpl { .detail("AddedTeams", 0) .detail("TeamsToBuild", teamsToBuild) .detail("CurrentServerTeams", self->teams.size()) + .detail("Servers", self->server_info.size()) + .detail("HealthyServers", serverCount) .detail("DesiredTeams", desiredTeams) .detail("MaxTeams", maxTeams) .detail("StorageTeamSize", self->configuration.storageTeamSize) @@ -833,7 +839,8 @@ class DDTeamCollectionImpl { // If there are too few machines to even build teams or there are too few represented datacenters, can't // build any team. self->lastBuildTeamsFailed = true; - TraceEvent(SevWarnAlways, "BuildTeamsNotEnoughUniqueMachines", self->distributorId) + TraceEvent(SevWarnAlways, "BuildTeamsLastBuildTeamsFailed", self->distributorId) + .detail("Reason", "Do not have enough unique machines") .detail("Primary", self->primary) .detail("UniqueMachines", uniqueMachines) .detail("Replication", self->configuration.storageTeamSize); @@ -1896,6 +1903,16 @@ class DDTeamCollectionImpl { // tracker) and remove bad team (cancel the team tracker). wait(self->badTeamRemover); + if (SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC > 0 && + self->teamPivots.pivotAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) { + TraceEvent(SevWarn, "MachineTeamRemoverDelayedForLowSpacePivot", self->distributorId) + .detail("IsPrimary", self->primary) + .detail("CurrentSpacePivot", self->teamPivots.pivotAvailableSpaceRatio) + .detail("TargetSpacePivot", SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO); + wait(delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC)); + continue; + } + state int healthyMachineCount = self->calculateHealthyMachineCount(); // Check if all machines are healthy, if not, we wait for 1 second and loop back. // Eventually, all machines will become healthy. @@ -2024,6 +2041,16 @@ class DDTeamCollectionImpl { // adding the bad team (add the team tracker) and remove bad team (cancel the team tracker). wait(self->badTeamRemover); + if (SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC > 0 && + self->teamPivots.pivotAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) { + TraceEvent(SevWarn, "ServerTeamRemoverDelayedForLowSpacePivot", self->distributorId) + .detail("IsPrimary", self->primary) + .detail("CurrentSpacePivot", self->teamPivots.pivotAvailableSpaceRatio) + .detail("TargetSpacePivot", SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO); + wait(delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC)); + continue; + } + // From this point, all server teams should be healthy, because we wait above // until processingUnhealthy is done, and all machines are healthy int desiredServerTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * self->server_info.size(); @@ -2183,7 +2210,7 @@ class DDTeamCollectionImpl { self->getAverageShardBytes.send(avgShardBytes); int64_t avgBytes = wait(avgShardBytes.getFuture()); double ratio; - bool imbalance; + bool imbalance, noMinAvailSpace; int numSSToBeLoadBytesBalanced; if (SERVER_KNOBS->PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO) { @@ -2203,23 +2230,36 @@ class DDTeamCollectionImpl { } CODE_PROBE(imbalance, "Perpetual Wiggle pause because cluster is imbalance."); + noMinAvailSpace = + !self->allServersHaveMinAvailableSpace(SERVER_KNOBS->PERPETUAL_WIGGLE_MIN_AVAILABLE_SPACE_RATIO); + // there must not have other teams to place wiggled data takeRest = self->server_info.size() <= self->configuration.storageTeamSize || - self->machine_info.size() < self->configuration.storageTeamSize || imbalance; + self->machine_info.size() < self->configuration.storageTeamSize || imbalance || noMinAvailSpace; + + if (SERVER_KNOBS->PERPETUAL_WIGGLE_PAUSE_AFTER_TSS_TARGET_MET && + self->configuration.storageMigrationType == StorageMigrationType::DEFAULT) { + takeRest = takeRest || (self->getTargetTSSInDC() > 0 && self->reachTSSPairTarget()); + } // log the extra delay and change the wiggler state if (takeRest) { self->storageWiggler->setWiggleState(StorageWiggler::PAUSE); - if (self->configuration.storageMigrationType == StorageMigrationType::GRADUAL) { - TraceEvent(SevWarn, "PerpetualStorageWiggleSleep", self->distributorId) - .suppressFor(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY * 4) - .detail("ImbalanceFactor", - SERVER_KNOBS->PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO ? numSSToBeLoadBytesBalanced - : ratio) - .detail("ServerSize", self->server_info.size()) - .detail("MachineSize", self->machine_info.size()) - .detail("StorageTeamSize", self->configuration.storageTeamSize); - } + Severity sev = + self->configuration.storageMigrationType == StorageMigrationType::GRADUAL ? SevWarn : SevInfo; + TraceEvent(sev, "PerpetualStorageWiggleSleep", self->distributorId) + .suppressFor(SERVER_KNOBS->PERPETUAL_WIGGLE_DELAY * 4) + .detail("Primary", self->primary) + .detail("ImbalanceFactor", + SERVER_KNOBS->PW_MAX_SS_LESSTHAN_MIN_BYTES_BALANCE_RATIO ? numSSToBeLoadBytesBalanced + : ratio) + .detail("ServerSize", self->server_info.size()) + .detail("MachineSize", self->machine_info.size()) + .detail("StorageTeamSize", self->configuration.storageTeamSize) + .detail("TargetTSSInDC", self->getTargetTSSInDC()) + .detail("ReachTSSPairTarget", self->reachTSSPairTarget()) + .detail("NoMinAvailableSpace", noMinAvailSpace) + .detail("MigrationType", self->configuration.storageMigrationType.toString()); } } return Void(); @@ -3211,9 +3251,9 @@ class DDTeamCollectionImpl { StorageMetadataType::currentTime(), server->getStoreType(), !(server->isCorrectStoreType(isTss ? self->configuration.testingStorageServerStoreType - : self->configuration.storageServerStoreType) || - server->isCorrectStoreType(isTss ? self->configuration.testingStorageServerStoreType - : self->configuration.perpetualStoreType))); + : (self->configuration.perpetualStoreType.isValid() + ? self->configuration.perpetualStoreType + : self->configuration.storageServerStoreType)))); // read storage metadata loop { @@ -3956,6 +3996,23 @@ Future DDTeamCollection::removeBadTeams() { return DDTeamCollectionImpl::removeBadTeams(this); } +bool DDTeamCollection::allServersHaveMinAvailableSpace(double minAvailableSpaceRatio) const { + for (auto& [id, s] : server_info) { + // If a healthy SS don't have storage metrics, skip this round + if (server_status.get(s->getId()).isUnhealthy() || !s->metricsPresent()) { + TraceEvent(SevDebug, "AllServersHaveMinAvailableSpaceNoMetrics").detail("Server", id); + return false; + } + + if (!s->hasHealthyAvailableSpace(minAvailableSpaceRatio)) { + TraceEvent(SevDebug, "AllServersHaveMinAvailableSpaceNotTrue").detail("Server", id); + return false; + } + } + + return true; +} + double DDTeamCollection::loadBytesBalanceRatio(int64_t smallLoadThreshold) const { double minLoadBytes = std::numeric_limits::max(); double totalLoadBytes = 0; @@ -4398,7 +4455,8 @@ bool DDTeamCollection::isValidLocality(Reference storagePoli void DDTeamCollection::evaluateTeamQuality() const { int teamCount = teams.size(), serverCount = allServers.size(); double teamsPerServer = (double)teamCount * configuration.storageTeamSize / serverCount; - + const int targetTeamNumPerServer = + (SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (configuration.storageTeamSize + 1)) / 2; ASSERT_EQ(serverCount, server_info.size()); int minTeams = std::numeric_limits::max(); @@ -4414,6 +4472,16 @@ void DDTeamCollection::evaluateTeamQuality() const { varTeams += (stc - teamsPerServer) * (stc - teamsPerServer); // Use zoneId as server's machine id machineTeams[info->getLastKnownInterface().locality.zoneId()] += stc; + // Check invariant: if latest buildTeam succeeds, then each server must have at least + // targetTeamNumPerServer serverTeams + // lastBuildTeamsFailed is set only when (1) machine count is less than configured team size; + // (2) Not find any server team candidates when creating server team; (3) failed to add machine team + if (SERVER_KNOBS->DD_VALIDATE_SERVER_TEAM_COUNT_AFTER_BUILD_TEAM && !lastBuildTeamsFailed && + stc < targetTeamNumPerServer) { + TraceEvent(SevError, "NewAddServerNotMatchTargetSTCount", distributorId) + .detail("CurrentServerTeams", stc) + .detail("TargetServerTeams", targetTeamNumPerServer); + } } } varTeams /= teamsPerServer * teamsPerServer; @@ -5037,9 +5105,9 @@ int DDTeamCollection::addBestMachineTeams(int machineTeamsToBuild) { // When too many teams exist in simulation, traceAllInfo will buffer too many trace logs before // trace has a chance to flush its buffer, which causes assertion failure. traceAllInfo(!g_network->isSimulated()); - TraceEvent(SevWarn, "DataDistributionBuildTeams", distributorId) + TraceEvent(SevWarn, "BuildTeamsLastBuildTeamsFailed", distributorId) .detail("Primary", primary) - .detail("Reason", "Unable to make desired machine Teams") + .detail("Reason", "Unable to make desired machineTeams") .detail("Hint", "Check TraceAllInfo event"); lastBuildTeamsFailed = true; break; @@ -5449,6 +5517,11 @@ int DDTeamCollection::addTeamsBestOf(int teamsToBuild, int desiredTeams, int max if (bestServerTeam.size() != configuration.storageTeamSize) { // Not find any team and will unlikely find a team lastBuildTeamsFailed = true; + TraceEvent(SevWarn, "BuildTeamsLastBuildTeamsFailed", distributorId) + .detail("Reason", "Unable to find any valid serverTeam") + .detail("Primary", primary) + .detail("BestServerTeam", describe(bestServerTeam)) + .detail("ConfigStorageTeamSize", configuration.storageTeamSize); break; } diff --git a/fdbserver/DataDistribution.actor.cpp b/fdbserver/DataDistribution.actor.cpp index 6e66455bc3c..82ed65edbe0 100644 --- a/fdbserver/DataDistribution.actor.cpp +++ b/fdbserver/DataDistribution.actor.cpp @@ -199,7 +199,8 @@ void StorageWiggler::updateMetadata(const UID& serverId, const StorageMetadataTy } bool StorageWiggler::necessary(const UID& serverId, const StorageMetadataType& metadata) const { - return metadata.wrongConfigured || (now() - metadata.createdTime > SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC); + return metadata.wrongConfiguredForWiggle || + (now() - metadata.createdTime > SERVER_KNOBS->DD_STORAGE_WIGGLE_MIN_SS_AGE_SEC); } Optional StorageWiggler::getNextServerId(bool necessaryOnly) { @@ -983,7 +984,8 @@ ACTOR Future dataDistribution(Reference self, .anyZeroHealthyTeams = anyZeroHealthyTeams, .shards = &shards, .trackerCancelled = &trackerCancelled, - .ddTenantCache = self->ddTenantCache }); + .ddTenantCache = self->ddTenantCache, + .usableRegions = self->configuration.usableRegions }); actors.push_back(reportErrorsExcept(DataDistributionTracker::run(shardTracker, self->initData, getShardMetrics.getFuture(), diff --git a/fdbserver/KeyValueStoreRocksDB.actor.cpp b/fdbserver/KeyValueStoreRocksDB.actor.cpp index fc022151000..b052212b7df 100644 --- a/fdbserver/KeyValueStoreRocksDB.actor.cpp +++ b/fdbserver/KeyValueStoreRocksDB.actor.cpp @@ -105,22 +105,36 @@ class SharedRocksDBState { rocksdb::ColumnFamilyOptions getCfOptions() const { return this->cfOptions; } rocksdb::Options getOptions() const { return rocksdb::Options(this->dbOptions, this->cfOptions); } rocksdb::ReadOptions getReadOptions() { return this->readOptions; } + rocksdb::FlushOptions getFlushOptions() { return this->flushOptions; } + double getLastFlushTime() const { return this->lastFlushTime_; } + void setLastFlushTime(double lastFlushTime) { this->lastFlushTime_ = lastFlushTime; } private: const UID id; rocksdb::ColumnFamilyOptions initialCfOptions(); rocksdb::DBOptions initialDbOptions(); rocksdb::ReadOptions initialReadOptions(); + rocksdb::FlushOptions initialFlushOptions(); bool closing; rocksdb::DBOptions dbOptions; rocksdb::ColumnFamilyOptions cfOptions; rocksdb::ReadOptions readOptions; + rocksdb::FlushOptions flushOptions; + std::atomic lastFlushTime_; }; SharedRocksDBState::SharedRocksDBState(UID id) : id(id), closing(false), dbOptions(initialDbOptions()), cfOptions(initialCfOptions()), - readOptions(initialReadOptions()) {} + readOptions(initialReadOptions()), flushOptions(initialFlushOptions()) {} + +rocksdb::FlushOptions SharedRocksDBState::initialFlushOptions() { + rocksdb::FlushOptions fOptions; + fOptions.wait = SERVER_KNOBS->ROCKSDB_WAIT_ON_CF_FLUSH; + fOptions.allow_write_stall = SERVER_KNOBS->ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH; + + return fOptions; +} rocksdb::ColumnFamilyOptions SharedRocksDBState::initialCfOptions() { rocksdb::ColumnFamilyOptions options; @@ -260,9 +274,6 @@ rocksdb::DBOptions SharedRocksDBState::initialDbOptions() { // checks will be performed with ttl being first. options.WAL_ttl_seconds = SERVER_KNOBS->ROCKSDB_WAL_TTL_SECONDS; options.WAL_size_limit_MB = SERVER_KNOBS->ROCKSDB_WAL_SIZE_LIMIT_MB; - if (g_network->isSimulated()) { // Used to fix external timeout in simulation - options.max_manifest_file_size = SERVER_KNOBS->ROCKSDB_MAX_MANIFEST_FILE_SIZE; - } options.statistics = rocksdb::CreateDBStatistics(); options.statistics->set_stats_level(rocksdb::StatsLevel(SERVER_KNOBS->ROCKSDB_STATS_LEVEL)); @@ -362,6 +373,19 @@ class RocksDBErrorListener : public rocksdb::EventListener { std::mutex mutex; UID id; }; + +class RocksDBEventListener : public rocksdb::EventListener { +public: + RocksDBEventListener(std::shared_ptr sharedState) : sharedState(sharedState){}; + + void OnFlushCompleted(rocksdb::DB* db, const rocksdb::FlushJobInfo& info) override { + sharedState->setLastFlushTime(now()); + } + +private: + std::shared_ptr sharedState; +}; + using DB = rocksdb::DB*; using CF = rocksdb::ColumnFamilyHandle*; @@ -556,6 +580,15 @@ class ReadIteratorPool { // Called on every read operation. ReadIterator getIterator(KeyRange keyRange) { + // Reusing iterator in simulation can cause slow down + // We avoid to always reuse iterator in simulation to speed up the simulation + if (g_network->isSimulated() && + deterministicRandom()->random01() > SERVER_KNOBS->ROCKSDB_PROBABILITY_REUSE_ITERATOR_SIM) { + index++; + ReadIterator iter(cf, index, db, sharedState, keyRange); + return iter; + } + if (SERVER_KNOBS->ROCKSDB_READ_RANGE_REUSE_ITERATORS) { mutex.lock(); for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) { @@ -957,6 +990,29 @@ ACTOR Future flowLockLogger(UID id, const FlowLock* readLock, const FlowLo } } +ACTOR Future manualFlush(UID id, rocksdb::DB* db, std::shared_ptr sharedState, CF cf) { + if (SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL) { + state rocksdb::FlushOptions fOptions = sharedState->getFlushOptions(); + state double waitTime = SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL; + state double currTime = 0; + state int timeElapsedAfterLastFlush = 0; + loop { + wait(delay(waitTime)); + + currTime = now(); + timeElapsedAfterLastFlush = currTime - sharedState->getLastFlushTime(); + if (timeElapsedAfterLastFlush >= SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL) { + db->Flush(fOptions, cf); + waitTime = SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL; + TraceEvent("RocksDBManualFlush", id).detail("TimeElapsedAfterLastFlush", timeElapsedAfterLastFlush); + } else { + waitTime = SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL - timeElapsedAfterLastFlush; + } + } + } + return Void(); +} + ACTOR Future rocksDBMetricLogger(UID id, std::shared_ptr sharedState, std::shared_ptr statistics, @@ -1232,15 +1288,18 @@ struct RocksDBKeyValueStore : IKeyValueStore { const FlowLock* readLock; const FlowLock* fetchLock; std::shared_ptr errorListener; + std::shared_ptr eventListener; + std::shared_ptr lastFlushTime; Counters& counters; OpenAction(std::string path, Optional>& metrics, const FlowLock* readLock, const FlowLock* fetchLock, std::shared_ptr errorListener, + std::shared_ptr eventListener, Counters& counters) : path(std::move(path)), metrics(metrics), readLock(readLock), fetchLock(fetchLock), - errorListener(errorListener), counters(counters) {} + errorListener(errorListener), eventListener(eventListener), counters(counters) {} double getTimeEstimate() const override { return SERVER_KNOBS->COMMIT_TIME_ESTIMATE; } }; @@ -1261,6 +1320,9 @@ struct RocksDBKeyValueStore : IKeyValueStore { } options.listeners.push_back(a.errorListener); + if (SERVER_KNOBS->ROCKSDB_MANUAL_FLUSH_TIME_INTERVAL > 0) { + options.listeners.push_back(a.eventListener); + } if (SERVER_KNOBS->ROCKSDB_WRITE_RATE_LIMITER_BYTES_PER_SEC > 0) { options.rate_limiter = rateLimiter; } @@ -1305,7 +1367,8 @@ struct RocksDBKeyValueStore : IKeyValueStore { a.metrics = rocksDBMetricLogger( id, sharedState, options.statistics, perfContextMetrics, db, readIterPool, &a.counters, cf) && - flowLockLogger(id, a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool); + flowLockLogger(id, a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool) && + manualFlush(id, db, sharedState, cf); } else { onMainThread([&] { a.metrics = rocksDBMetricLogger(id, @@ -1316,7 +1379,8 @@ struct RocksDBKeyValueStore : IKeyValueStore { readIterPool, &a.counters, cf) && - flowLockLogger(id, a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool); + flowLockLogger(id, a.readLock, a.fetchLock) && refreshReadIteratorPool(readIterPool) && + manualFlush(id, db, sharedState, cf); return Future(true); }).blockUntilReady(); } @@ -1395,15 +1459,8 @@ struct RocksDBKeyValueStore : IKeyValueStore { rocksdb::WriteOptions options; options.sync = !SERVER_KNOBS->ROCKSDB_UNSAFE_AUTO_FSYNC; - - double writeBeginTime = timer_monotonic(); rocksdb::Status s = db->Write(options, a.batchToCommit.get()); readIterPool->update(); - double currTime = timer_monotonic(); - if (a.getHistograms) { - metricPromiseStream->send( - std::make_pair(ROCKSDB_WRITE_HISTOGRAM.toString(), currTime - writeBeginTime)); - } if (!s.ok()) { logRocksDBError(id, s, "Commit"); @@ -1425,7 +1482,7 @@ struct RocksDBKeyValueStore : IKeyValueStore { } } } - currTime = timer_monotonic(); + double currTime = timer_monotonic(); if (a.getHistograms) { metricPromiseStream->send( std::make_pair(ROCKSDB_COMMIT_ACTION_HISTOGRAM.toString(), currTime - commitBeginTime)); @@ -1585,7 +1642,6 @@ struct RocksDBKeyValueStore : IKeyValueStore { readOptions.deadline = std::chrono::duration_cast(deadlineSeconds); } - double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0; auto s = db->Get(readOptions, cf, toSlice(a.key), &value); if (!s.ok() && !s.IsNotFound()) { logRocksDBError(id, s, "ReadValue"); @@ -1593,11 +1649,6 @@ struct RocksDBKeyValueStore : IKeyValueStore { return; } - if (a.getHistograms) { - metricPromiseStream->send( - std::make_pair(ROCKSDB_READVALUE_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime)); - } - if (a.debugID.present()) { traceBatch.get().addEvent("GetValueDebug", a.debugID.get().first(), "Reader.After"); traceBatch.get().dump(); @@ -1674,12 +1725,7 @@ struct RocksDBKeyValueStore : IKeyValueStore { readOptions.deadline = std::chrono::duration_cast(deadlineSeconds); } - double dbGetBeginTime = a.getHistograms ? timer_monotonic() : 0; auto s = db->Get(readOptions, cf, toSlice(a.key), &value); - if (a.getHistograms) { - metricPromiseStream->send( - std::make_pair(ROCKSDB_READPREFIX_GET_HISTOGRAM.toString(), timer_monotonic() - dbGetBeginTime)); - } if (a.debugID.present()) { traceBatch.get().addEvent("GetValuePrefixDebug", @@ -1715,15 +1761,12 @@ struct RocksDBKeyValueStore : IKeyValueStore { double startTime; bool getHistograms; ThreadReturnPromise result; - Counters& counters; - ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, ReadType type, Counters& counters) + ReadRangeAction(KeyRange keys, int rowLimit, int byteLimit, ReadType type) : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), type(type), startTime(timer_monotonic()), - counters(counters), getHistograms(deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) {} double getTimeEstimate() const override { return SERVER_KNOBS->READ_RANGE_TIME_ESTIMATE; } }; void action(ReadRangeAction& a) { - ++a.counters.rocksdbReadRangeQueries; bool doPerfContextMetrics = SERVER_KNOBS->ROCKSDB_PERFCONTEXT_ENABLE && (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE); @@ -1824,14 +1867,13 @@ struct RocksDBKeyValueStore : IKeyValueStore { result.more = (result.size() == a.rowLimit) || (result.size() == -a.rowLimit) || (accumulatedBytes >= a.byteLimit); a.result.send(result); + + const double endTime = timer_monotonic(); if (a.getHistograms) { metricPromiseStream->send( std::make_pair(ROCKSDB_READ_RANGE_BYTES_RETURNED_HISTOGRAM.toString(), result.logicalSize())); metricPromiseStream->send( std::make_pair(ROCKSDB_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM.toString(), result.size())); - } - const double endTime = timer_monotonic(); - if (a.getHistograms) { metricPromiseStream->send( std::make_pair(ROCKSDB_READRANGE_ACTION_HISTOGRAM.toString(), endTime - readBeginTime)); metricPromiseStream->send( @@ -1852,6 +1894,7 @@ struct RocksDBKeyValueStore : IKeyValueStore { numReadWaiters(SERVER_KNOBS->ROCKSDB_READ_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_READ_QUEUE_SOFT_MAX), numFetchWaiters(SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_HARD_MAX - SERVER_KNOBS->ROCKSDB_FETCH_QUEUE_SOFT_MAX), errorListener(std::make_shared(id)), errorFuture(errorListener->getFuture()) { + eventListener = std::make_shared(sharedState); // In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage engine // is still multi-threaded as background compaction threads are still present. Reads/writes to disk will also // block the network thread in a way that would be unacceptable in production but is a necessary evil here. When @@ -2038,21 +2081,14 @@ struct RocksDBKeyValueStore : IKeyValueStore { void close() override { doClose(this, false); } - KeyValueStoreType getType() const override { - if (SERVER_KNOBS->ENABLE_SHARDED_ROCKSDB) - // KVSRocks pretends as KVSShardedRocksDB - // TODO: to remove when the ShardedRocksDB KVS implementation is added in the future - return KeyValueStoreType(KeyValueStoreType::SSD_SHARDED_ROCKSDB); - else - return KeyValueStoreType(KeyValueStoreType::SSD_ROCKSDB_V1); - } + KeyValueStoreType getType() const override { return KeyValueStoreType(KeyValueStoreType::SSD_ROCKSDB_V1); } Future init() override { if (openFuture.isValid()) { return openFuture; } auto a = std::make_unique( - path, metrics, &readSemaphore, &fetchSemaphore, errorListener, counters); + path, metrics, &readSemaphore, &fetchSemaphore, errorListener, eventListener, counters); openFuture = a->done.getFuture(); writeThread->post(a.release()); return openFuture; @@ -2289,7 +2325,8 @@ struct RocksDBKeyValueStore : IKeyValueStore { } if (!shouldThrottle(type, keys.begin)) { - auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit, type, counters); + ++counters.rocksdbReadRangeQueries; + auto a = new Reader::ReadRangeAction(keys, rowLimit, byteLimit, type); auto res = a->result.getFuture(); readThreads->post(a); return res; @@ -2299,7 +2336,8 @@ struct RocksDBKeyValueStore : IKeyValueStore { int maxWaiters = (type == ReadType::FETCH) ? numFetchWaiters : numReadWaiters; checkWaiters(semaphore, maxWaiters); - auto a = std::make_unique(keys, rowLimit, byteLimit, type, counters); + ++counters.rocksdbReadRangeQueries; + auto a = std::make_unique(keys, rowLimit, byteLimit, type); return read(a.release(), &semaphore, readThreads.getPtr(), &counters.failedToAcquire); } @@ -2367,6 +2405,7 @@ struct RocksDBKeyValueStore : IKeyValueStore { Reference writeThread; Reference readThreads; std::shared_ptr errorListener; + std::shared_ptr eventListener; Future errorFuture; Promise closePromise; Future openFuture; @@ -2389,7 +2428,6 @@ struct RocksDBKeyValueStore : IKeyValueStore { std::vector>>> metricPromiseStreams; // ThreadReturnPromiseStream pair.first stores the histogram name and // pair.second stores the corresponding measured latency (seconds) - Future actorErrorListener; Future collection; PromiseStream> addActor; Counters counters; diff --git a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp b/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp index a904b831e74..65f4ce10cbb 100644 --- a/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp +++ b/fdbserver/KeyValueStoreShardedRocksDB.actor.cpp @@ -67,7 +67,6 @@ const StringRef ROCKSDB_COMMIT_LATENCY_HISTOGRAM = "RocksDBCommitLatency"_sr; const StringRef ROCKSDB_COMMIT_ACTION_HISTOGRAM = "RocksDBCommitAction"_sr; const StringRef ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM = "RocksDBCommitQueueWait"_sr; const StringRef ROCKSDB_WRITE_HISTOGRAM = "RocksDBWrite"_sr; -const StringRef ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM = "RocksDBDeleteCompactRange"_sr; const StringRef ROCKSDB_READRANGE_LATENCY_HISTOGRAM = "RocksDBReadRangeLatency"_sr; const StringRef ROCKSDB_READVALUE_LATENCY_HISTOGRAM = "RocksDBReadValueLatency"_sr; const StringRef ROCKSDB_READPREFIX_LATENCY_HISTOGRAM = "RocksDBReadPrefixLatency"_sr; @@ -80,6 +79,10 @@ const StringRef ROCKSDB_READPREFIX_QUEUEWAIT_HISTOGRAM = "RocksDBReadPrefixQueue const StringRef ROCKSDB_READRANGE_NEWITERATOR_HISTOGRAM = "RocksDBReadRangeNewIterator"_sr; const StringRef ROCKSDB_READVALUE_GET_HISTOGRAM = "RocksDBReadValueGet"_sr; const StringRef ROCKSDB_READPREFIX_GET_HISTOGRAM = "RocksDBReadPrefixGet"_sr; + +// RocksDB Internal stats. +const KeyRef ROCKSDB_STATS_HISTOGRAM_GROUP = "RocksDBInternalStats"_sr; + // Flush reason code: // https://github.com/facebook/rocksdb/blob/63a5125a5220d953bf504daf33694f038403cc7c/include/rocksdb/listener.h#L164-L181 // This function needs to be updated when flush code changes. @@ -140,10 +143,57 @@ int getWriteStallState(const rocksdb::WriteStallCondition& condition) { return 3; } +// RocksDB's reason string contains spaces and will break trace events. +// Reference https://sourcegraph.com/github.com/facebook/rocksdb/-/blob/db/flush_job.cc?L52 +const char* getFlushReasonString(FlushReason flush_reason) { + switch (flush_reason) { + case FlushReason::kOthers: + return "ReasonOthers"; + case FlushReason::kGetLiveFiles: + return "ReasonGetLiveFiles"; + case FlushReason::kShutDown: + return "ReasonShutdown"; + case FlushReason::kExternalFileIngestion: + return "ReasonExternalFileIngestion"; + case FlushReason::kManualCompaction: + return "ReasonManualCompaction"; + case FlushReason::kWriteBufferManager: + return "ReasonWriteBufferManager"; + case FlushReason::kWriteBufferFull: + return "ReasonWriteBufferFull"; + case FlushReason::kTest: + return "ReasonTest"; + case FlushReason::kDeleteFiles: + return "ReasonDeleteFiles"; + case FlushReason::kAutoCompaction: + return "ReasonAutoCompaction"; + case FlushReason::kManualFlush: + return "ReasonManualFlush"; + case FlushReason::kErrorRecovery: + return "ReasonErrorRecovery"; + case FlushReason::kErrorRecoveryRetryFlush: + return "ReasonErrorRecoveryRetryFlush"; + case FlushReason::kWalFull: + return "ReasonWALFull"; + case FlushReason::kCatchUpAfterErrorRecovery: + return "ReasonCatchUpAfterErrorRecovery"; + default: + return "ReasonInvalid"; + } +} + class RocksDBEventListener : public rocksdb::EventListener { public: RocksDBEventListener(UID id) : logId(id), compactionReasons((int)CompactionReason::kNumOfReasons), flushReasons(ROCKSDB_NUM_FLUSH_REASONS), + numRangeDeletionsInTableFile(Histogram::getHistogram(ROCKSDB_STATS_HISTOGRAM_GROUP, + "NumRangeDeletionsInTableFile"_sr, + Histogram::Unit::bytes)), + numPointDeletionsInTableFile(Histogram::getHistogram(ROCKSDB_STATS_HISTOGRAM_GROUP, + "NumPointDeletionsInTableFile"_sr, + Histogram::Unit::bytes)), + numEntriesInTableFile( + Histogram::getHistogram(ROCKSDB_STATS_HISTOGRAM_GROUP, "NumEntriesInTableFile"_sr, Histogram::Unit::bytes)), lastResetTime(now()) {} void OnStallConditionsChanged(const rocksdb::WriteStallInfo& info) override { @@ -182,6 +232,12 @@ class RocksDBEventListener : public rocksdb::EventListener { compactionReasons[index]++; } + void OnTableFileCreated(const rocksdb::TableFileCreationInfo& info) override { + numRangeDeletionsInTableFile->sample(info.table_properties.num_range_deletions); + numPointDeletionsInTableFile->sample(info.table_properties.num_deletions); + numEntriesInTableFile->sample(info.table_properties.num_entries); + } + void logRecentRocksDBBackgroundWorkStats(UID ssId, std::string logReason = "PeriodicLog") { int flushCount = flushTotal.load(std::memory_order_relaxed); int compactionCount = compactionTotal.load(std::memory_order_relaxed); @@ -193,7 +249,7 @@ class RocksDBEventListener : public rocksdb::EventListener { e.detail("DurationSeconds", now() - lastResetTime); e.detail("FlushCountTotal", flushCount); for (int i = 0; i < ROCKSDB_NUM_FLUSH_REASONS; ++i) { - e.detail(rocksdb::GetFlushReasonString((rocksdb::FlushReason)i), flushReasons[i]); + e.detail(getFlushReasonString((rocksdb::FlushReason)i), flushReasons[i]); } } if (compactionCount > 0) { @@ -232,6 +288,11 @@ class RocksDBEventListener : public rocksdb::EventListener { std::atomic_int flushTotal; std::atomic_int compactionTotal; + // Histograms. + Reference numRangeDeletionsInTableFile; + Reference numPointDeletionsInTableFile; + Reference numEntriesInTableFile; + double lastResetTime; }; @@ -282,12 +343,283 @@ class RocksDBErrorListener : public rocksdb::EventListener { std::mutex mutex; }; +// Collects range deletion count during table file creation. Marks table file for compaction if range deletion count +// exceeds threshold. +class CompactOnRangeDeletionCollector : public rocksdb::TablePropertiesCollector { +public: + CompactOnRangeDeletionCollector(uint64_t numRangeDeletionsAllowed, std::atomic_uint* numFilesMarkedForCompaction) + : threshold(numRangeDeletionsAllowed), numFilesMarkedForCompaction(numFilesMarkedForCompaction) { + ASSERT(numFilesMarkedForCompaction); + } + + // AddUserKey() will be called when a new key/value pair is inserted into the + // table. + // @params key the user key that is inserted into the table. + // @params value the value that is inserted into the table. + // @params file_size file size up to now + // TODO: Consider collecting point deletion info. + rocksdb::Status AddUserKey(const rocksdb::Slice& key, + const rocksdb::Slice& value, + rocksdb::EntryType type, + rocksdb::SequenceNumber seq, + uint64_t file_size) override { + if (type == rocksdb::EntryType::kEntryRangeDeletion) { + ++numRangeDeletions; + } + return rocksdb::Status::OK(); + } + + // Finish() will be called when a table has already been built and is ready for writing the properties block. + // @params properties User will add their collected statistics to + // `properties`. + // Number of range deletion is included in rocksdb's table properties. No need to add a custom entry. + rocksdb::Status Finish(rocksdb::UserCollectedProperties* /*properties*/) override { + if (numRangeDeletions > threshold) { + ++(*numFilesMarkedForCompaction); + } + return rocksdb::Status::OK(); + }; + + // Return the human-readable properties, where the key is property name and + // the value is the human-readable form of value. No additional properties needed + // for this collector. + rocksdb::UserCollectedProperties GetReadableProperties() const override { + return rocksdb::UserCollectedProperties(); + } + + // The name of the properties collector can be used for debugging purpose. + const char* Name() const override { return "CompactOnRangeDeletionCollector"; } + + // EXPERIMENTAL Return whether the output file should be further compacted based on the threshold. + bool NeedCompact() const override { return threshold > 0 && numRangeDeletions > threshold; } + +private: + uint64_t threshold = 0; + uint64_t numRangeDeletions = 0; + std::atomic_uint* numFilesMarkedForCompaction; +}; + +class CompactOnRangeDeletionCollectorFactory : public rocksdb::TablePropertiesCollectorFactory { +public: + // A factory of a table property collector that marks a SST file as need-compaction when the number of range + // deletions exceeds the threshold. + // @param numRangeDeletionsAllowed, triggers compaction range deletion count exceeds numRangeDeletionsAllowed. + CompactOnRangeDeletionCollectorFactory(uint64_t numRangeDeletionsAllowed) + : threshold(numRangeDeletionsAllowed), numFilesMarkedForCompaction(0) {} + + ~CompactOnRangeDeletionCollectorFactory() {} + + rocksdb::TablePropertiesCollector* CreateTablePropertiesCollector( + rocksdb::TablePropertiesCollectorFactory::Context context) override { + return new CompactOnRangeDeletionCollector(threshold, &numFilesMarkedForCompaction); + } + + uint getNumFilesMarkedForCompaction() { return numFilesMarkedForCompaction; } + + static const char* kClassName() { return "CompactOnRangeDeletionCollectorFactory"; } + const char* Name() const override { return kClassName(); } + + std::string ToString() const override { + return "CompactOnRangeDeletionCollectorFactory: " + std::to_string(threshold); + } + + void logMetrics(bool reset) { + TraceEvent("CompactOnRangeDeletionMetrics") + .detail("NumFilesMarkedForCompaction", numFilesMarkedForCompaction) + .detail("Threshold", threshold); + if (reset) { + numFilesMarkedForCompaction = 0; + } + } + +private: + const uint64_t threshold; + std::atomic_uint numFilesMarkedForCompaction; +}; +struct Counters { + CounterCollection cc; + Counter immediateThrottle; + Counter failedToAcquire; + Counter convertedRangeDeletions; + + Counters() + : cc("RocksDBCounters"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("FailedToAcquire", cc), + convertedRangeDeletions("ConvertedRangeDeletions", cc) {} +}; + +rocksdb::CompactionPri getCompactionPriority() { + switch (SERVER_KNOBS->ROCKSDB_COMPACTION_PRI) { + case 0: + return rocksdb::CompactionPri::kByCompensatedSize; + case 1: + return rocksdb::CompactionPri::kOldestLargestSeqFirst; + case 2: + return rocksdb::CompactionPri::kOldestSmallestSeqFirst; + case 3: + return rocksdb::CompactionPri::kMinOverlappingRatio; + case 4: + return rocksdb::CompactionPri::kRoundRobin; + default: + TraceEvent(SevWarn, "InvalidCompactionPriority").detail("KnobValue", SERVER_KNOBS->ROCKSDB_COMPACTION_PRI); + return rocksdb::CompactionPri::kMinOverlappingRatio; + } +} + +rocksdb::WALRecoveryMode getWalRecoveryMode() { + switch (SERVER_KNOBS->ROCKSDB_WAL_RECOVERY_MODE) { + case 0: + return rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords; + case 1: + return rocksdb::WALRecoveryMode::kAbsoluteConsistency; + case 2: + return rocksdb::WALRecoveryMode::kPointInTimeRecovery; + case 3: + return rocksdb::WALRecoveryMode::kSkipAnyCorruptedRecords; + default: + TraceEvent(SevWarn, "InvalidWalRecoveryMode").detail("KnobValue", SERVER_KNOBS->ROCKSDB_WAL_RECOVERY_MODE); + return rocksdb::WALRecoveryMode::kPointInTimeRecovery; + } +} + // Encapsulation of shared states. struct ShardedRocksDBState { bool closing = false; -}; + Counters counters; + std::shared_ptr blockCache = nullptr; + std::shared_ptr compactOnRangeDeletionFactory = nullptr; + + ShardedRocksDBState() { + if (SERVER_KNOBS->SHARDED_ROCKSDB_BLOCK_CACHE_SIZE > 0) { + blockCache = + rocksdb::NewLRUCache(SERVER_KNOBS->SHARDED_ROCKSDB_BLOCK_CACHE_SIZE, + -1, /* num_shard_bits, default value:-1*/ + false, /* strict_capacity_limit, default value:false */ + SERVER_KNOBS->SHARDED_ROCKSDB_CACHE_HIGH_PRI_POOL_RATIO /* high_pri_pool_ratio */); + } + if (SERVER_KNOBS->SHARDED_ROCKSDB_COMPACT_ON_RANGE_DELETION_THRESHOLD > 0) { + compactOnRangeDeletionFactory = std::make_shared( + SERVER_KNOBS->SHARDED_ROCKSDB_COMPACT_ON_RANGE_DELETION_THRESHOLD); + } + } -std::shared_ptr rocksdb_block_cache = nullptr; + rocksdb::ColumnFamilyOptions getCFOptions() { + rocksdb::ColumnFamilyOptions options; + + if (SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES) { + options.level_compaction_dynamic_level_bytes = SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES; + options.OptimizeLevelStyleCompaction(SERVER_KNOBS->SHARDED_ROCKSDB_MEMTABLE_BUDGET); + } + options.write_buffer_size = SERVER_KNOBS->SHARDED_ROCKSDB_WRITE_BUFFER_SIZE; + options.max_write_buffer_number = SERVER_KNOBS->SHARDED_ROCKSDB_MAX_WRITE_BUFFER_NUMBER; + options.target_file_size_base = SERVER_KNOBS->SHARDED_ROCKSDB_TARGET_FILE_SIZE_BASE; + options.target_file_size_multiplier = SERVER_KNOBS->SHARDED_ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER; + + if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) { + options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS; + } + + options.disable_auto_compactions = SERVER_KNOBS->ROCKSDB_DISABLE_AUTO_COMPACTIONS; + options.memtable_protection_bytes_per_key = SERVER_KNOBS->ROCKSDB_MEMTABLE_PROTECTION_BYTES_PER_KEY; + options.block_protection_bytes_per_key = SERVER_KNOBS->ROCKSDB_BLOCK_PROTECTION_BYTES_PER_KEY; + options.paranoid_file_checks = SERVER_KNOBS->ROCKSDB_PARANOID_FILE_CHECKS; + options.memtable_max_range_deletions = SERVER_KNOBS->SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS; + if (SERVER_KNOBS->SHARD_SOFT_PENDING_COMPACT_BYTES_LIMIT > 0) { + options.soft_pending_compaction_bytes_limit = SERVER_KNOBS->SHARD_SOFT_PENDING_COMPACT_BYTES_LIMIT; + } + if (SERVER_KNOBS->SHARD_HARD_PENDING_COMPACT_BYTES_LIMIT > 0) { + options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->SHARD_HARD_PENDING_COMPACT_BYTES_LIMIT; + } + + if (compactOnRangeDeletionFactory) { + options.table_properties_collector_factories.emplace_back(compactOnRangeDeletionFactory); + } + + // Compact sstables when there's too much deleted stuff. + if (SERVER_KNOBS->ROCKSDB_ENABLE_COMPACT_ON_DELETION) { + // Creates a factory of a table property collector that marks a SST + // file as need-compaction when it observe at least "D" deletion + // entries in any "N" consecutive entries, or the ratio of tombstone + // entries >= deletion_ratio. + + // @param sliding_window_size "N". Note that this number will be + // round up to the smallest multiple of 128 that is no less + // than the specified size. + // @param deletion_trigger "D". Note that even when "N" is changed, + // the specified number for "D" will not be changed. + // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction + // based on deletion ratio. Disabled by default. + options.table_properties_collector_factories.emplace_back( + rocksdb::NewCompactOnDeletionCollectorFactory(SERVER_KNOBS->ROCKSDB_CDCF_SLIDING_WINDOW_SIZE, + SERVER_KNOBS->ROCKSDB_CDCF_DELETION_TRIGGER, + SERVER_KNOBS->ROCKSDB_CDCF_DELETION_RATIO)); + } + + rocksdb::BlockBasedTableOptions bbOpts; + if (SERVER_KNOBS->SHARDED_ROCKSDB_PREFIX_LEN > 0) { + // Prefix blooms are used during Seek. + options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(SERVER_KNOBS->SHARDED_ROCKSDB_PREFIX_LEN)); + + // Also turn on bloom filters in the memtable. + // TODO: Make a knob for this as well. + options.memtable_prefix_bloom_size_ratio = 0.1; + + // 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned + // filters use a generally faster and more accurate Bloom filter + // implementation, with a different schema. + // https://github.com/facebook/rocksdb/blob/b77569f18bfc77fb1d8a0b3218f6ecf571bc4988/include/rocksdb/table.h#L391 + bbOpts.format_version = 5; + + // Create and apply a bloom filter using the 10 bits + // which should yield a ~1% false positive rate: + // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#full-filters-new-format + bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(SERVER_KNOBS->SHARDED_ROCKSDB_BLOOM_FILTER_BITS)); + + // The whole key blooms are only used for point lookups. + // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#prefix-vs-whole-key + bbOpts.whole_key_filtering = false; + } + + options.level0_file_num_compaction_trigger = SERVER_KNOBS->SHARDED_ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER; + options.level0_slowdown_writes_trigger = SERVER_KNOBS->SHARDED_ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER; + options.level0_stop_writes_trigger = SERVER_KNOBS->SHARDED_ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER; + + bbOpts.cache_index_and_filter_blocks = SERVER_KNOBS->SHARDED_ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS; + bbOpts.pin_l0_filter_and_index_blocks_in_cache = SERVER_KNOBS->SHARDED_ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS; + bbOpts.cache_index_and_filter_blocks_with_high_priority = + SERVER_KNOBS->SHARDED_ROCKSDB_CACHE_INDEX_AND_FILTER_BLOCKS; + bbOpts.block_cache = blockCache; + + options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts)); + + options.compaction_pri = getCompactionPriority(); + + return options; + } + + rocksdb::ColumnFamilyOptions getCFOptionsForInactiveShard() { + auto options = getCFOptions(); + // never slowdown ingest. + options.level0_file_num_compaction_trigger = (1 << 30); + options.level0_slowdown_writes_trigger = (1 << 30); + options.level0_stop_writes_trigger = (1 << 30); + options.soft_pending_compaction_bytes_limit = 0; + options.hard_pending_compaction_bytes_limit = 0; + + // no auto compactions please. The application should issue a + // manual compaction after all data is loaded into L0. + options.disable_auto_compactions = true; + // A manual compaction run should pick all files in L0 in + // a single compaction run. + options.max_compaction_bytes = (static_cast(1) << 60); + + // It is better to have only 2 levels, otherwise a manual + // compaction would compact at every possible level, thereby + // increasing the total time needed for compactions. + options.num_levels = 2; + + return options; + } +}; rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpoint) { rocksdb::ExportImportFilesMetaData metaData; @@ -453,159 +785,12 @@ Error statusToError(const rocksdb::Status& s) { } } -rocksdb::CompactionPri getCompactionPriority() { - switch (SERVER_KNOBS->ROCKSDB_COMPACTION_PRI) { - case 0: - return rocksdb::CompactionPri::kByCompensatedSize; - case 1: - return rocksdb::CompactionPri::kOldestLargestSeqFirst; - case 2: - return rocksdb::CompactionPri::kOldestSmallestSeqFirst; - case 3: - return rocksdb::CompactionPri::kMinOverlappingRatio; - case 4: - return rocksdb::CompactionPri::kRoundRobin; - default: - TraceEvent(SevWarn, "InvalidCompactionPriority").detail("KnobValue", SERVER_KNOBS->ROCKSDB_COMPACTION_PRI); - return rocksdb::CompactionPri::kMinOverlappingRatio; - } -} - -rocksdb::WALRecoveryMode getWalRecoveryMode() { - switch (SERVER_KNOBS->ROCKSDB_WAL_RECOVERY_MODE) { - case 0: - return rocksdb::WALRecoveryMode::kTolerateCorruptedTailRecords; - case 1: - return rocksdb::WALRecoveryMode::kAbsoluteConsistency; - case 2: - return rocksdb::WALRecoveryMode::kPointInTimeRecovery; - case 3: - return rocksdb::WALRecoveryMode::kSkipAnyCorruptedRecords; - default: - TraceEvent(SevWarn, "InvalidWalRecoveryMode").detail("KnobValue", SERVER_KNOBS->ROCKSDB_WAL_RECOVERY_MODE); - return rocksdb::WALRecoveryMode::kPointInTimeRecovery; - } -} - -rocksdb::ColumnFamilyOptions getCFOptions() { - rocksdb::ColumnFamilyOptions options; - - if (SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES) { - options.level_compaction_dynamic_level_bytes = SERVER_KNOBS->ROCKSDB_LEVEL_COMPACTION_DYNAMIC_LEVEL_BYTES; - options.OptimizeLevelStyleCompaction(SERVER_KNOBS->SHARDED_ROCKSDB_MEMTABLE_BUDGET); - } - options.write_buffer_size = SERVER_KNOBS->SHARDED_ROCKSDB_WRITE_BUFFER_SIZE; - options.max_write_buffer_number = SERVER_KNOBS->SHARDED_ROCKSDB_MAX_WRITE_BUFFER_NUMBER; - options.target_file_size_base = SERVER_KNOBS->SHARDED_ROCKSDB_TARGET_FILE_SIZE_BASE; - options.target_file_size_multiplier = SERVER_KNOBS->SHARDED_ROCKSDB_TARGET_FILE_SIZE_MULTIPLIER; - - if (SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS > 0) { - options.periodic_compaction_seconds = SERVER_KNOBS->ROCKSDB_PERIODIC_COMPACTION_SECONDS; - } - - options.disable_auto_compactions = SERVER_KNOBS->ROCKSDB_DISABLE_AUTO_COMPACTIONS; - options.memtable_protection_bytes_per_key = SERVER_KNOBS->ROCKSDB_MEMTABLE_PROTECTION_BYTES_PER_KEY; - options.block_protection_bytes_per_key = SERVER_KNOBS->ROCKSDB_BLOCK_PROTECTION_BYTES_PER_KEY; - options.paranoid_file_checks = SERVER_KNOBS->ROCKSDB_PARANOID_FILE_CHECKS; - options.memtable_max_range_deletions = SERVER_KNOBS->SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS; - if (SERVER_KNOBS->SHARD_SOFT_PENDING_COMPACT_BYTES_LIMIT > 0) { - options.soft_pending_compaction_bytes_limit = SERVER_KNOBS->SHARD_SOFT_PENDING_COMPACT_BYTES_LIMIT; - } - if (SERVER_KNOBS->SHARD_HARD_PENDING_COMPACT_BYTES_LIMIT > 0) { - options.hard_pending_compaction_bytes_limit = SERVER_KNOBS->SHARD_HARD_PENDING_COMPACT_BYTES_LIMIT; - } - - // Compact sstables when there's too much deleted stuff. - if (SERVER_KNOBS->ROCKSDB_ENABLE_COMPACT_ON_DELETION) { - // Creates a factory of a table property collector that marks a SST - // file as need-compaction when it observe at least "D" deletion - // entries in any "N" consecutive entries, or the ratio of tombstone - // entries >= deletion_ratio. - - // @param sliding_window_size "N". Note that this number will be - // round up to the smallest multiple of 128 that is no less - // than the specified size. - // @param deletion_trigger "D". Note that even when "N" is changed, - // the specified number for "D" will not be changed. - // @param deletion_ratio, if <= 0 or > 1, disable triggering compaction - // based on deletion ratio. Disabled by default. - options.table_properties_collector_factories = { rocksdb::NewCompactOnDeletionCollectorFactory( - SERVER_KNOBS->ROCKSDB_CDCF_SLIDING_WINDOW_SIZE, - SERVER_KNOBS->ROCKSDB_CDCF_DELETION_TRIGGER, - SERVER_KNOBS->ROCKSDB_CDCF_DELETION_RATIO) }; - } - - rocksdb::BlockBasedTableOptions bbOpts; - // TODO: Add a knob for the block cache size. (Default is 8 MB) - if (SERVER_KNOBS->SHARDED_ROCKSDB_PREFIX_LEN > 0) { - // Prefix blooms are used during Seek. - options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(SERVER_KNOBS->SHARDED_ROCKSDB_PREFIX_LEN)); - - // Also turn on bloom filters in the memtable. - // TODO: Make a knob for this as well. - options.memtable_prefix_bloom_size_ratio = 0.1; - - // 5 -- Can be read by RocksDB's versions since 6.6.0. Full and partitioned - // filters use a generally faster and more accurate Bloom filter - // implementation, with a different schema. - // https://github.com/facebook/rocksdb/blob/b77569f18bfc77fb1d8a0b3218f6ecf571bc4988/include/rocksdb/table.h#L391 - bbOpts.format_version = 5; - - // Create and apply a bloom filter using the 10 bits - // which should yield a ~1% false positive rate: - // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#full-filters-new-format - bbOpts.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10)); - - // The whole key blooms are only used for point lookups. - // https://github.com/facebook/rocksdb/wiki/RocksDB-Bloom-Filter#prefix-vs-whole-key - bbOpts.whole_key_filtering = false; - } - - options.level0_file_num_compaction_trigger = SERVER_KNOBS->SHARDED_ROCKSDB_LEVEL0_FILENUM_COMPACTION_TRIGGER; - options.level0_slowdown_writes_trigger = SERVER_KNOBS->SHARDED_ROCKSDB_LEVEL0_SLOWDOWN_WRITES_TRIGGER; - options.level0_stop_writes_trigger = SERVER_KNOBS->SHARDED_ROCKSDB_LEVEL0_STOP_WRITES_TRIGGER; - - if (rocksdb_block_cache == nullptr && SERVER_KNOBS->SHARDED_ROCKSDB_BLOCK_CACHE_SIZE > 0) { - rocksdb_block_cache = rocksdb::NewLRUCache(SERVER_KNOBS->SHARDED_ROCKSDB_BLOCK_CACHE_SIZE); - } - bbOpts.block_cache = rocksdb_block_cache; - - options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(bbOpts)); - - options.compaction_pri = getCompactionPriority(); - - return options; -} - -rocksdb::ColumnFamilyOptions getCFOptionsForInactiveShard() { - auto options = getCFOptions(); - // never slowdown ingest. - options.level0_file_num_compaction_trigger = (1 << 30); - options.level0_slowdown_writes_trigger = (1 << 30); - options.level0_stop_writes_trigger = (1 << 30); - options.soft_pending_compaction_bytes_limit = 0; - options.hard_pending_compaction_bytes_limit = 0; - - // no auto compactions please. The application should issue a - // manual compaction after all data is loaded into L0. - options.disable_auto_compactions = true; - // A manual compaction run should pick all files in L0 in - // a single compaction run. - options.max_compaction_bytes = (static_cast(1) << 60); - - // It is better to have only 2 levels, otherwise a manual - // compaction would compact at every possible level, thereby - // increasing the total time needed for compactions. - options.num_levels = 2; - - return options; -} - rocksdb::DBOptions getOptions() { rocksdb::DBOptions options; options.avoid_unnecessary_blocking_io = true; options.create_if_missing = true; options.atomic_flush = SERVER_KNOBS->ROCKSDB_ATOMIC_FLUSH; + options.use_direct_io_for_flush_and_compaction = SERVER_KNOBS->SHARDED_ROCKSDB_USE_DIRECT_IO; if (SERVER_KNOBS->SHARDED_ROCKSDB_BACKGROUND_PARALLELISM > 0) { options.IncreaseParallelism(SERVER_KNOBS->SHARDED_ROCKSDB_BACKGROUND_PARALLELISM); } @@ -641,7 +826,6 @@ rocksdb::DBOptions getOptions() { if (SERVER_KNOBS->ROCKSDB_LOG_LEVEL_DEBUG) { options.info_log_level = rocksdb::InfoLogLevel::DEBUG_LEVEL; } - options.skip_stats_update_on_db_open = SERVER_KNOBS->ROCKSDB_SKIP_STATS_UPDATE_ON_OPEN; options.skip_checking_sst_file_sizes_on_db_open = SERVER_KNOBS->ROCKSDB_SKIP_FILE_SIZE_CHECK_ON_OPEN; options.max_manifest_file_size = SERVER_KNOBS->ROCKSDB_MAX_MANIFEST_FILE_SIZE; @@ -664,121 +848,103 @@ rocksdb::ReadOptions getReadOptions() { } struct ReadIterator { - uint64_t index; // incrementing counter to uniquely identify read iterator. - bool inUse; - std::shared_ptr iter; + std::unique_ptr iter; double creationTime; KeyRange keyRange; - std::shared_ptr beginSlice, endSlice; + std::unique_ptr beginSlice, endSlice; - ReadIterator(rocksdb::ColumnFamilyHandle* cf, uint64_t index, rocksdb::DB* db) - : index(index), inUse(true), creationTime(now()), iter(db->NewIterator(getReadOptions(), cf)) {} - ReadIterator(rocksdb::ColumnFamilyHandle* cf, uint64_t index, rocksdb::DB* db, const KeyRange& range) - : index(index), inUse(true), creationTime(now()), keyRange(range) { + ReadIterator(rocksdb::ColumnFamilyHandle* cf, rocksdb::DB* db) + : creationTime(now()), iter(db->NewIterator(getReadOptions(), cf)) {} + + ReadIterator(rocksdb::ColumnFamilyHandle* cf, rocksdb::DB* db, const KeyRange& range) + : creationTime(now()), keyRange(range) { auto options = getReadOptions(); - beginSlice = std::shared_ptr(new rocksdb::Slice(toSlice(keyRange.begin))); + beginSlice = std::unique_ptr(new rocksdb::Slice(toSlice(keyRange.begin))); options.iterate_lower_bound = beginSlice.get(); - endSlice = std::shared_ptr(new rocksdb::Slice(toSlice(keyRange.end))); + endSlice = std::unique_ptr(new rocksdb::Slice(toSlice(keyRange.end))); options.iterate_upper_bound = endSlice.get(); - iter = std::shared_ptr(db->NewIterator(options, cf)); + iter = std::unique_ptr(db->NewIterator(options, cf)); } }; -/* -ReadIteratorPool: Collection of iterators. Reuses iterators on non-concurrent multiple read operations, -instead of creating and deleting for every read. - -Read: IteratorPool provides an unused iterator if exists or creates and gives a new iterator. -Returns back the iterator after the read is done. - -Write: Iterators in the pool are deleted, forcing new iterator creation on next reads. The iterators -which are currently used by the reads can continue using the iterator as it is a shared_ptr. Once -the read is processed, shared_ptr goes out of scope and gets deleted. Eventually the iterator object -gets deleted as the ref count becomes 0. -*/ -class ReadIteratorPool { +// Stores iterators for all shards for future reuse. One iterator is stored per shard. +class IteratorPool { public: - ReadIteratorPool(rocksdb::DB* db, rocksdb::ColumnFamilyHandle* cf, const std::string& path) - : db(db), cf(cf), index(0), iteratorsReuseCount(0) { - ASSERT(db); - ASSERT(cf); - TraceEvent(SevVerbose, "ShardedRocksReadIteratorPool") - .detail("Path", path) - .detail("KnobRocksDBReadRangeReuseIterators", SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) - .detail("KnobRocksDBPrefixLen", SERVER_KNOBS->SHARDED_ROCKSDB_PREFIX_LEN); - } - - // Called on every db commit. - void update() { - if (SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) { - std::lock_guard lock(mutex); - iteratorsMap.clear(); - } - } - - // Called on every read operation. - ReadIterator getIterator(const KeyRange& range) { - // Shared iterators are not bounded. - if (SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) { - std::lock_guard lock(mutex); - for (it = iteratorsMap.begin(); it != iteratorsMap.end(); it++) { - if (!it->second.inUse) { - it->second.inUse = true; - iteratorsReuseCount++; - return it->second; - } - } - index++; - ReadIterator iter(cf, index, db); - iteratorsMap.insert({ index, iter }); - return iter; + IteratorPool() {} + + std::shared_ptr getIterator(const std::string& id) { + std::unique_lock lock(mu); + auto it = pool.find(id); + if (it == pool.end()) { + ++numNewIterators; + return nullptr; } else { - index++; - ReadIterator iter(cf, index, db, range); - return iter; - } - } - - // Called on every read operation, after the keys are collected. - void returnIterator(ReadIterator& iter) { - if (SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) { - std::lock_guard lock(mutex); - it = iteratorsMap.find(iter.index); - // iterator found: put the iterator back to the pool(inUse=false). - // iterator not found: update would have removed the iterator from pool, so nothing to do. - if (it != iteratorsMap.end()) { - ASSERT(it->second.inUse); - it->second.inUse = false; - } + auto ret = it->second; + pool.erase(it); + ++numReusedIters; + return ret; } } - // Called for every ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME seconds in a loop. - void refreshIterators() { - std::lock_guard lock(mutex); - it = iteratorsMap.begin(); - while (it != iteratorsMap.end()) { - if (now() - it->second.creationTime > SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME) { - it = iteratorsMap.erase(it); + void returnIterator(const std::string& id, std::shared_ptr iterator) { + std::unique_lock lock(mu); + auto it = pool.find(id); + if (it != pool.end()) { + // An iterator already exist in the pool, replace it any way. + ++numReplacedIters; + } + pool[id] = iterator; + } + + void refresh() { + std::unique_lock lock(mu); + auto poolSize = pool.size(); + auto it = pool.begin(); + auto currTime = now(); + int refreshedIterCount = 0; + while (it != pool.end()) { + if (currTime - it->second->creationTime > SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME) { + it = pool.erase(it); + ++refreshedIterCount; } else { - it++; + ++it; } } + TraceEvent("RefreshIterators") + .detail("NumReplacedIterators", numReplacedIters) + .detail("NumReusedIterators", numReusedIters) + .detail("NumNewIterators", numNewIterators) + .detail("PoolSize", poolSize) + .detail("RefreshedIterators", refreshedIterCount); + numReplacedIters = 0; + numReusedIters = 0; + numNewIterators = 0; + } + + void clear() { + std::unique_lock lock(mu); + pool.clear(); + } + + void update(const std::string& id) { + std::unique_lock lock(mu); + auto it = pool.find(id); + if (it != pool.end()) { + it->second->iter->Refresh(); + } } - uint64_t numReadIteratorsCreated() { return index; } - - uint64_t numTimesReadIteratorsReused() { return iteratorsReuseCount; } + void erase(const std::string& id) { + std::unique_lock lock(mu); + pool.erase(id); + } private: - std::unordered_map iteratorsMap; - std::unordered_map::iterator it; - rocksdb::DB* db; - rocksdb::ColumnFamilyHandle* cf; - std::mutex mutex; - // incrementing counter for every new iterator creation, to uniquely identify the iterator in returnIterator(). - uint64_t index; - uint64_t iteratorsReuseCount; + std::mutex mu; + std::unordered_map> pool; + uint64_t numReplacedIters = 0; + uint64_t numReusedIters = 0; + uint64_t numNewIterators = 0; }; ACTOR Future flowLockLogger(const FlowLock* readLock, const FlowLock* fetchLock) { @@ -812,7 +978,6 @@ struct PhysicalShard { PhysicalShard(rocksdb::DB* db, std::string id, rocksdb::ColumnFamilyHandle* handle) : db(db), id(id), cf(handle), isInitialized(true) { ASSERT(cf); - readIterPool = std::make_shared(db, cf, id); } rocksdb::Status init() { @@ -825,7 +990,6 @@ struct PhysicalShard { return status; } logShardEvent(id, ShardOp::OPEN); - readIterPool = std::make_shared(db, cf, id); this->isInitialized.store(true); return status; } @@ -840,12 +1004,12 @@ struct PhysicalShard { TraceEvent(SevInfo, "RocksDBRestoreEmptyShard") .detail("ShardId", id) .detail("CheckpointID", checkpoint.checkpointID); - status = db->CreateColumnFamily(getCFOptions(), id, &cf); + status = db->CreateColumnFamily(cfOptions, id, &cf); } else { TraceEvent(SevInfo, "RocksDBRestoreCF"); rocksdb::ImportColumnFamilyOptions importOptions; importOptions.move_files = SERVER_KNOBS->ROCKSDB_IMPORT_MOVE_FILES; - status = db->CreateColumnFamilyWithImport(getCFOptions(), id, importOptions, metaData, &cf); + status = db->CreateColumnFamilyWithImport(cfOptions, id, importOptions, metaData, &cf); TraceEvent(SevInfo, "RocksDBRestoreCFEnd").detail("Status", status.ToString()); } } else if (format == RocksDBKeyValues) { @@ -890,10 +1054,7 @@ struct PhysicalShard { .detail("Checkpoint", checkpoint.toString()); if (status.ok()) { if (!this->isInitialized) { - readIterPool = std::make_shared(db, cf, id); this->isInitialized.store(true); - } else if (SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) { - this->readIterPool->update(); } } @@ -902,11 +1063,6 @@ struct PhysicalShard { bool initialized() { return this->isInitialized.load(); } - void refreshReadIteratorPool() { - ASSERT(this->readIterPool != nullptr); - this->readIterPool->refreshIterators(); - } - std::vector getAllRanges() const { std::vector res; for (const auto& [key, shard] : dataShards) { @@ -935,7 +1091,6 @@ struct PhysicalShard { ~PhysicalShard() { logShardEvent(id, ShardOp::CLOSE); isInitialized.store(false); - readIterPool.reset(); // Deleting default column family is not allowed. if (deletePending && id != DEFAULT_CF_NAME) { @@ -960,7 +1115,6 @@ struct PhysicalShard { rocksdb::ColumnFamilyOptions cfOptions; rocksdb::ColumnFamilyHandle* cf = nullptr; std::unordered_map> dataShards; - std::shared_ptr readIterPool; bool deletePending = false; std::atomic isInitialized; uint64_t numRangeDeletions = 0; @@ -968,19 +1122,37 @@ struct PhysicalShard { double lastCompactionTime = 0.0; }; -int readRangeInDb(PhysicalShard* shard, const KeyRangeRef range, int rowLimit, int byteLimit, RangeResult* result) { +int readRangeInDb(PhysicalShard* shard, + const KeyRangeRef range, + int rowLimit, + int byteLimit, + RangeResult* result, + std::shared_ptr iteratorPool) { if (rowLimit == 0 || byteLimit == 0) { return 0; } int accumulatedBytes = 0; rocksdb::Status s; - + bool reuseIterator = SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS && iteratorPool != nullptr; + if (g_network->isSimulated() && + deterministicRandom()->random01() > SERVER_KNOBS->ROCKSDB_PROBABILITY_REUSE_ITERATOR_SIM) { + // Reduce probability of reusing iterators in simulation. + reuseIterator = false; + } + std::shared_ptr readIter = nullptr; + if (reuseIterator) { + readIter = iteratorPool->getIterator(shard->id); + if (readIter == nullptr) { + readIter = std::make_shared(shard->cf, shard->db); + } + } else { + readIter = std::make_shared(shard->cf, shard->db, range); + } // When using a prefix extractor, ensure that keys are returned in order even if they cross // a prefix boundary. if (rowLimit >= 0) { - ReadIterator readIter = shard->readIterPool->getIterator(range); - auto cursor = readIter.iter; + auto* cursor = readIter->iter.get(); cursor->Seek(toSlice(range.begin)); while (cursor->Valid() && toStringRef(cursor->key()) < range.end) { KeyValueRef kv(toStringRef(cursor->key()), toStringRef(cursor->value())); @@ -993,10 +1165,8 @@ int readRangeInDb(PhysicalShard* shard, const KeyRangeRef range, int rowLimit, i cursor->Next(); } s = cursor->status(); - shard->readIterPool->returnIterator(readIter); } else { - ReadIterator readIter = shard->readIterPool->getIterator(range); - auto cursor = readIter.iter; + auto* cursor = readIter->iter.get(); cursor->SeekForPrev(toSlice(range.end)); if (cursor->Valid() && toStringRef(cursor->key()) == range.end) { cursor->Prev(); @@ -1012,7 +1182,6 @@ int readRangeInDb(PhysicalShard* shard, const KeyRangeRef range, int rowLimit, i cursor->Prev(); } s = cursor->status(); - shard->readIterPool->returnIterator(readIter); } if (!s.ok()) { @@ -1021,31 +1190,25 @@ int readRangeInDb(PhysicalShard* shard, const KeyRangeRef range, int rowLimit, i // should never be returned to user. return -1; } + if (reuseIterator) { + iteratorPool->returnIterator(shard->id, readIter); + } return accumulatedBytes; } -struct Counters { - CounterCollection cc; - Counter immediateThrottle; - Counter failedToAcquire; - Counter convertedRangeDeletions; - - Counters() - : cc("RocksDBCounters"), immediateThrottle("ImmediateThrottle", cc), failedToAcquire("FailedToAcquire", cc), - convertedRangeDeletions("ConvertedRangeDeletions", cc) {} -}; - // Manages physical shards and maintains logical shard mapping. class ShardManager { public: ShardManager(std::string path, UID logId, + std::shared_ptr rState, const rocksdb::DBOptions& options, std::shared_ptr errorListener, std::shared_ptr eventListener, - Counters* cc) - : path(path), logId(logId), dbOptions(options), cfOptions(getCFOptions()), dataShardMap(nullptr, specialKeys.end), - counters(cc) { + Counters* cc, + std::shared_ptr iteratorPool) + : path(path), logId(logId), rState(rState), dbOptions(options), dataShardMap(nullptr, specialKeys.end), + counters(cc), iteratorPool(iteratorPool) { if (!g_network->isSimulated()) { // Generating trace events in non-FDB thread will cause errors. The event listener is tested with local FDB // cluster. @@ -1156,12 +1319,12 @@ class ShardManager { if (name == METADATA_SHARD_ID) { foundMetadata = true; } - descriptors.push_back(rocksdb::ColumnFamilyDescriptor(name, cfOptions)); + descriptors.push_back(rocksdb::ColumnFamilyDescriptor(name, rState->getCFOptions())); } // Add default column family if it's a newly opened database. if (descriptors.size() == 0) { - descriptors.push_back(rocksdb::ColumnFamilyDescriptor("default", cfOptions)); + descriptors.push_back(rocksdb::ColumnFamilyDescriptor("default", rState->getCFOptions())); } std::vector handles; @@ -1199,7 +1362,8 @@ class ShardManager { keyRange, std::max(2, SERVER_KNOBS->ROCKSDB_READ_RANGE_ROW_LIMIT), SERVER_KNOBS->SHARD_METADATA_SCAN_BYTES_LIMIT, - &metadata); + &metadata, + iteratorPool); if (bytes <= 0) { break; } @@ -1276,7 +1440,7 @@ class ShardManager { physicalShards[defaultShard->id] = defaultShard; // Create metadata shard. - auto metadataShard = std::make_shared(db, METADATA_SHARD_ID, cfOptions); + auto metadataShard = std::make_shared(db, METADATA_SHARD_ID, rState->getCFOptions()); metadataShard->init(); columnFamilyMap[metadataShard->cf->GetID()] = metadataShard->cf; physicalShards[METADATA_SHARD_ID] = metadataShard; @@ -1293,7 +1457,6 @@ class ShardManager { if (!status.ok()) { return status; } - metadataShard->readIterPool->update(); TraceEvent(SevInfo, "ShardedRocksInitializeMetaDataShard", this->logId) .detail("MetadataShardCF", metadataShard->cf->GetID()); } @@ -1304,6 +1467,7 @@ class ShardManager { SERVER_KNOBS->ROCKSDB_WRITEBATCH_PROTECTION_BYTES_PER_KEY, // protection_bytes_per_key 0 /* default_cf_ts_sz default:0 */); dirtyShards = std::make_unique>(); + iteratorPool->update(getMetaDataShard()->id); TraceEvent(SevInfo, "ShardedRocksDBInitEnd", this->logId) .detail("DataPath", path) @@ -1401,8 +1565,8 @@ class ShardManager { } } - auto cfOptions = active ? getCFOptions() : getCFOptionsForInactiveShard(); - auto [it, inserted] = physicalShards.emplace(id, std::make_shared(db, id, cfOptions)); + auto currentCfOptions = active ? rState->getCFOptions() : rState->getCFOptionsForInactiveShard(); + auto [it, inserted] = physicalShards.emplace(id, std::make_shared(db, id, currentCfOptions)); std::shared_ptr& shard = it->second; activePhysicalShardIds.emplace(id); @@ -1416,7 +1580,8 @@ class ShardManager { TraceEvent(SevInfo, "ShardedRocksDBRangeAdded", this->logId) .detail("Range", range) .detail("ShardId", id) - .detail("Active", active); + .detail("Active", active) + .detail("PrefixLength", range.prefixLength()); return shard.get(); } @@ -1442,7 +1607,7 @@ class ShardManager { if (SERVER_KNOBS->ROCKSDB_EMPTY_RANGE_CHECK && existingShard->initialized()) { // Enable consistency validation. RangeResult rangeResult; - auto bytesRead = readRangeInDb(existingShard, range, 1, UINT16_MAX, &rangeResult); + auto bytesRead = readRangeInDb(existingShard, range, 1, UINT16_MAX, &rangeResult, iteratorPool); if (bytesRead > 0) { TraceEvent(SevError, "ShardedRocksDBRangeNotEmpty") .detail("ShardId", existingShard->toString()) @@ -1731,7 +1896,7 @@ class ShardManager { } rocksdb::FlushOptions fOptions; fOptions.wait = SERVER_KNOBS->ROCKSDB_WAIT_ON_CF_FLUSH; - fOptions.allow_write_stall = SERVER_KNOBS->ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH; + fOptions.allow_write_stall = SERVER_KNOBS->SHARDED_ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH; db->Flush(fOptions, it->second->cf); } @@ -1763,7 +1928,7 @@ class ShardManager { logRocksDBError(s, "Close"); return; } - s = rocksdb::DestroyDB(path, rocksdb::Options(dbOptions, getCFOptions())); + s = rocksdb::DestroyDB(path, rocksdb::Options(dbOptions, rState->getCFOptions())); if (!s.ok()) { logRocksDBError(s, "DestroyDB"); } @@ -1850,8 +2015,8 @@ class ShardManager { private: const std::string path; const UID logId; + std::shared_ptr rState; rocksdb::DBOptions dbOptions; - rocksdb::ColumnFamilyOptions cfOptions; rocksdb::DB* db = nullptr; std::unordered_map> physicalShards; std::unordered_set activePhysicalShardIds; @@ -1862,11 +2027,14 @@ class ShardManager { KeyRangeMap dataShardMap; std::deque pendingDeletionShards; Counters* counters; + std::shared_ptr iteratorPool; }; class RocksDBMetrics { public: - RocksDBMetrics(UID debugID, std::shared_ptr stats); + RocksDBMetrics(UID debugID, + std::shared_ptr rState, + std::shared_ptr stats); void logStats(rocksdb::DB* db, std::string manifestDirectory); // PerfContext void resetPerfContext(); @@ -1890,15 +2058,13 @@ class RocksDBMetrics { Reference getCommitActionHistogram(); Reference getCommitQueueWaitHistogram(); Reference getWriteHistogram(); - Reference getDeleteCompactRangeHistogram(); - // Stat for Memory Usage - void logMemUsage(rocksdb::DB* db); std::vector> getManifestBytes(std::string manifestDirectory); private: const UID debugID; // Global Statistic Input to RocksDB DB instance std::shared_ptr stats; + std::shared_ptr rState; // Statistic Output from RocksDB std::vector> tickerStats; std::vector> intPropertyStats; @@ -1924,7 +2090,6 @@ class RocksDBMetrics { Reference commitActionHistogram; Reference commitQueueWaitHistogram; Reference writeHistogram; - Reference deleteCompactRangeHistogram; uint64_t getRocksdbPerfcontextMetric(int metric); }; @@ -1991,12 +2156,11 @@ Reference RocksDBMetrics::getCommitQueueWaitHistogram() { Reference RocksDBMetrics::getWriteHistogram() { return writeHistogram; } -Reference RocksDBMetrics::getDeleteCompactRangeHistogram() { - return deleteCompactRangeHistogram; -} -RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr stats) - : debugID(debugID), stats(stats) { +RocksDBMetrics::RocksDBMetrics(UID debugID, + std::shared_ptr rState, + std::shared_ptr stats) + : debugID(debugID), rState(rState), stats(stats) { tickerStats = { { "StallMicros", rocksdb::STALL_MICROS, 0 }, { "BytesRead", rocksdb::BYTES_READ, 0 }, @@ -2034,6 +2198,8 @@ RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr { "RowCacheHit", rocksdb::ROW_CACHE_HIT, 0 }, { "RowCacheMiss", rocksdb::ROW_CACHE_MISS, 0 }, { "CountIterSkippedKeys", rocksdb::NUMBER_ITER_SKIP, 0 }, + { "NoIteratorCreated", rocksdb::NO_ITERATOR_CREATED, 0 }, + { "NoIteratorDeleted", rocksdb::NO_ITERATOR_DELETED, 0 }, }; @@ -2175,8 +2341,6 @@ RocksDBMetrics::RocksDBMetrics(UID debugID, std::shared_ptr ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_COMMIT_QUEUEWAIT_HISTOGRAM, Histogram::Unit::milliseconds); writeHistogram = Histogram::getHistogram(ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_WRITE_HISTOGRAM, Histogram::Unit::milliseconds); - deleteCompactRangeHistogram = Histogram::getHistogram( - ROCKSDBSTORAGE_HISTOGRAM_GROUP, ROCKSDB_DELETE_COMPACTRANGE_HISTOGRAM, Histogram::Unit::milliseconds); } void RocksDBMetrics::logStats(rocksdb::DB* db, std::string manifestDirectory) { @@ -2207,25 +2371,11 @@ void RocksDBMetrics::logStats(rocksdb::DB* db, std::string manifestDirectory) { ASSERT(db->GetProperty(rocksdb::DB::Properties::kDBWriteStallStats, &propValue)); TraceEvent(SevInfo, "DBWriteStallStats", debugID).detail("Stats", propValue); - if (rocksdb_block_cache) { - e.detail("CacheUsage", rocksdb_block_cache->GetUsage()); + if (rState->blockCache) { + e.detail("CacheUsage", rState->blockCache->GetUsage()); } } -void RocksDBMetrics::logMemUsage(rocksdb::DB* db) { - TraceEvent e(SevInfo, "ShardedRocksDBMemMetrics", debugID); - uint64_t stat; - ASSERT(db != nullptr); - ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCacheUsage, &stat)); - e.detail("BlockCacheUsage", stat); - ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kEstimateTableReadersMem, &stat)); - e.detail("EstimateSstReaderBytes", stat); - ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kCurSizeAllMemTables, &stat)); - e.detail("AllMemtablesBytes", stat); - ASSERT(db->GetAggregatedIntProperty(rocksdb::DB::Properties::kBlockCachePinnedUsage, &stat)); - e.detail("BlockCachePinnedUsage", stat); -} - void RocksDBMetrics::resetPerfContext() { rocksdb::SetPerfLevel(rocksdb::PerfLevel::kEnableCount); rocksdb::get_perf_context()->Reset(); @@ -2409,7 +2559,9 @@ ACTOR Future rocksDBAggregatedMetricsLogger(std::shared_ptrlogStats(db, manifestDirectory); - rocksDBMetrics->logMemUsage(db); + if (rState->compactOnRangeDeletionFactory) { + rState->compactOnRangeDeletionFactory->logMetrics(/*reset=*/true); + } if (SERVER_KNOBS->ROCKSDB_PERFCONTEXT_SAMPLE_RATE != 0) { rocksDBMetrics->logPerfContext(true); } @@ -2425,33 +2577,30 @@ ACTOR Future rocksDBAggregatedMetricsLogger(std::shared_ptr refreshReadIteratorPools( - std::shared_ptr rState, - Future readyToStart, - std::unordered_map>* physicalShards) { + ACTOR static Future refreshIteratorPool(std::shared_ptr rState, + std::shared_ptr iteratorPool, + Future readyToStart) { + if (!SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) { + return Void(); + } state Reference histogram = Histogram::getHistogram( ROCKSDBSTORAGE_HISTOGRAM_GROUP, "TimeSpentRefreshIterators"_sr, Histogram::Unit::milliseconds); - if (SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) { - try { - wait(readyToStart); - loop { - wait(delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME)); - if (rState->closing) { - break; - } - double startTime = timer_monotonic(); - for (auto& [_, shard] : *physicalShards) { - if (shard->initialized()) { - shard->refreshReadIteratorPool(); - } - } - histogram->sample(timer_monotonic() - startTime); - } - } catch (Error& e) { - if (e.code() != error_code_actor_cancelled) { - TraceEvent(SevError, "RefreshReadIteratorPoolError").errorUnsuppressed(e); + try { + wait(readyToStart); + loop { + wait(delay(SERVER_KNOBS->ROCKSDB_READ_RANGE_ITERATOR_REFRESH_TIME)); + if (rState->closing) { + break; } + double startTime = timer_monotonic(); + + iteratorPool->refresh(); + histogram->sample(timer_monotonic() - startTime); + } + } catch (Error& e) { + if (e.code() != error_code_actor_cancelled) { + TraceEvent(SevError, "RefreshReadIteratorPoolError").errorUnsuppressed(e); } } @@ -2597,14 +2746,16 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { int threadIndex; std::unordered_map* columnFamilyMap; std::shared_ptr rocksDBMetrics; + std::shared_ptr iteratorPool; double sampleStartTime; explicit Writer(UID logId, int threadIndex, std::unordered_map* columnFamilyMap, - std::shared_ptr rocksDBMetrics) + std::shared_ptr rocksDBMetrics, + std::shared_ptr iteratorPool) : logId(logId), threadIndex(threadIndex), columnFamilyMap(columnFamilyMap), rocksDBMetrics(rocksDBMetrics), - sampleStartTime(now()) {} + iteratorPool(iteratorPool), sampleStartTime(now()) {} ~Writer() override {} @@ -2685,6 +2836,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { columnFamilyMap->erase(shard->cf->GetID()); a.metadataShard->db->Delete( rocksdb::WriteOptions(), a.metadataShard->cf, compactionTimestampPrefix.toString() + shard->id); + iteratorPool->erase(shard->id); } TraceEvent("RemoveShardTime").detail("Duration", now() - start).detail("Size", a.shards.size()); a.shards.clear(); @@ -2707,7 +2859,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { std::unordered_map* columnFamilyMap) : db(db), writeBatch(std::move(writeBatch)), dirtyShards(std::move(dirtyShards)), columnFamilyMap(columnFamilyMap) { - if (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) { + if (deterministicRandom()->random01() < SERVER_KNOBS->SHARDED_ROCKSDB_HISTOGRAMS_SAMPLE_RATE) { getHistograms = true; startTime = timer_monotonic(); } else { @@ -2798,7 +2950,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { if (SERVER_KNOBS->SHARDED_ROCKSDB_REUSE_ITERATORS) { for (auto shard : *(a.dirtyShards)) { - shard->readIterPool->update(); + iteratorPool->update(shard->id); } } @@ -2818,7 +2970,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { if (SERVER_KNOBS->ROCKSDB_CF_RANGE_DELETION_LIMIT > 0) { rocksdb::FlushOptions fOptions; fOptions.wait = SERVER_KNOBS->ROCKSDB_WAIT_ON_CF_FLUSH; - fOptions.allow_write_stall = SERVER_KNOBS->ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH; + fOptions.allow_write_stall = SERVER_KNOBS->SHARDED_ROCKSDB_ALLOW_WRITE_STALL_ON_FLUSH; for (auto shard : (*a.dirtyShards)) { if (shard->shouldFlush()) { @@ -2952,7 +3104,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { rocksdb::ColumnFamilyHandle* handle; const std::string cfName = deterministicRandom()->randomAlphaNumeric(8); s = a.shardManager->getDb()->CreateColumnFamilyWithImport( - getCFOptions(), cfName, importOptions, metadata, &handle); + rocksdb::ColumnFamilyOptions(), cfName, importOptions, metadata, &handle); if (!s.ok()) { TraceEvent(SevError, "ShardedRocksCheckpointValidateImportError", logId) .detail("Status", s.ToString()) @@ -3256,10 +3408,15 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { double readRangeTimeout; int threadIndex; std::shared_ptr rocksDBMetrics; + std::shared_ptr iteratorPool; double sampleStartTime; - explicit Reader(UID logId, int threadIndex, std::shared_ptr rocksDBMetrics) - : logId(logId), threadIndex(threadIndex), rocksDBMetrics(rocksDBMetrics), sampleStartTime(now()) { + explicit Reader(UID logId, + int threadIndex, + std::shared_ptr rocksDBMetrics, + std::shared_ptr iteratorPool) + : logId(logId), threadIndex(threadIndex), rocksDBMetrics(rocksDBMetrics), iteratorPool(iteratorPool), + sampleStartTime(now()) { readValueTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_TIMEOUT; readValuePrefixTimeout = SERVER_KNOBS->ROCKSDB_READ_VALUE_PREFIX_TIMEOUT; readRangeTimeout = SERVER_KNOBS->ROCKSDB_READ_RANGE_TIMEOUT; @@ -3287,9 +3444,9 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { ReadValueAction(KeyRef key, PhysicalShard* shard, ReadType type, Optional debugID) : key(key), shard(shard), type(type), debugID(debugID), startTime(timer_monotonic()), - getHistograms( - (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) { - } + getHistograms((deterministicRandom()->random01() < SERVER_KNOBS->SHARDED_ROCKSDB_HISTOGRAMS_SAMPLE_RATE) + ? true + : false) {} double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; } }; @@ -3372,7 +3529,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { ReadValuePrefixAction(Key key, int maxLength, PhysicalShard* shard, ReadType type, Optional debugID) : key(key), maxLength(maxLength), shard(shard), type(type), debugID(debugID), startTime(timer_monotonic()), - getHistograms((deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) + getHistograms((deterministicRandom()->random01() < SERVER_KNOBS->SHARDED_ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false){}; double getTimeEstimate() const override { return SERVER_KNOBS->READ_VALUE_TIME_ESTIMATE; } @@ -3459,8 +3616,9 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { ThreadReturnPromise result; ReadRangeAction(KeyRange keys, std::vector shards, int rowLimit, int byteLimit, ReadType type) : keys(keys), rowLimit(rowLimit), byteLimit(byteLimit), type(type), startTime(timer_monotonic()), - getHistograms( - (deterministicRandom()->random01() < SERVER_KNOBS->ROCKSDB_HISTOGRAMS_SAMPLE_RATE) ? true : false) { + getHistograms((deterministicRandom()->random01() < SERVER_KNOBS->SHARDED_ROCKSDB_HISTOGRAMS_SAMPLE_RATE) + ? true + : false) { std::set usedShards; for (const DataShard* shard : shards) { ASSERT(shard); @@ -3521,7 +3679,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { .detail("Reason", shard == nullptr ? "Not Exist" : "Not Initialized"); continue; } - auto bytesRead = readRangeInDb(shard, range, rowLimit, byteLimit, &result); + auto bytesRead = readRangeInDb(shard, range, rowLimit, byteLimit, &result, iteratorPool); if (bytesRead < 0) { // Error reading an instance. a.result.sendError(internal_error()); @@ -3576,8 +3734,9 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { errorListener(std::make_shared()), eventListener(std::make_shared(id)), errorFuture(forwardError(errorListener->getFuture())), dbOptions(getOptions()), - shardManager(path, id, dbOptions, errorListener, eventListener, &counters), - rocksDBMetrics(std::make_shared(id, dbOptions.statistics)) { + iteratorPool(std::make_shared()), + shardManager(path, id, rState, dbOptions, errorListener, eventListener, &counters, iteratorPool), + rocksDBMetrics(std::make_shared(id, rState, dbOptions.statistics)) { // In simluation, run the reader/writer threads as Coro threads (i.e. in the network thread. The storage // engine is still multi-threaded as background compaction threads are still present. Reads/writes to disk // will also block the network thread in a way that would be unacceptable in production but is a necessary @@ -3599,12 +3758,13 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { compactionThread = createGenericThreadPool(0, SERVER_KNOBS->ROCKSDB_COMPACTION_THREAD_PRIORITY); readThreads = createGenericThreadPool(/*stackSize=*/0, SERVER_KNOBS->ROCKSDB_READER_THREAD_PRIORITY); } - writeThread->addThread(new Writer(id, 0, shardManager.getColumnFamilyMap(), rocksDBMetrics), "fdb-rocksdb-wr"); + writeThread->addThread(new Writer(id, 0, shardManager.getColumnFamilyMap(), rocksDBMetrics, iteratorPool), + "fdb-rocksdb-wr"); compactionThread->addThread(new CompactionWorker(id), "fdb-rocksdb-cw"); TraceEvent("ShardedRocksDBReadThreads", id) .detail("KnobRocksDBReadParallelism", SERVER_KNOBS->ROCKSDB_READ_PARALLELISM); for (unsigned i = 0; i < SERVER_KNOBS->ROCKSDB_READ_PARALLELISM; ++i) { - readThreads->addThread(new Reader(id, i, rocksDBMetrics), "fdb-rocksdb-re"); + readThreads->addThread(new Reader(id, i, rocksDBMetrics, iteratorPool), "fdb-rocksdb-re"); } } @@ -3626,6 +3786,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { } TraceEvent("CloseKeyValueStore").detail("DeleteKVS", deleteOnClose); + self->iteratorPool->clear(); auto a = new Writer::CloseAction(&self->shardManager, deleteOnClose); auto f = a->done.getFuture(); self->writeThread->post(a); @@ -3676,7 +3837,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { ShardManager::shardMetricsLogger(this->rState, openFuture, &shardManager) && rocksDBAggregatedMetricsLogger(this->rState, openFuture, rocksDBMetrics, &shardManager, this->path); this->compactionJob = compactShards(this->rState, openFuture, &shardManager, compactionThread); - this->refreshHolder = refreshReadIteratorPools(this->rState, openFuture, shardManager.getAllShards()); + this->refreshHolder = refreshIteratorPool(this->rState, iteratorPool, openFuture); this->refreshRocksDBBackgroundWorkHolder = refreshRocksDBBackgroundEventCounter(this->id, this->eventListener); this->cleanUpJob = emptyShardCleaner(this->rState, openFuture, &shardManager, writeThread); @@ -4031,6 +4192,7 @@ struct ShardedRocksDBKeyValueStore : IKeyValueStore { rocksdb::DBOptions dbOptions; std::shared_ptr errorListener; std::shared_ptr eventListener; + std::shared_ptr iteratorPool; ShardManager shardManager; std::shared_ptr rocksDBMetrics; std::string path; diff --git a/fdbserver/MoveKeys.actor.cpp b/fdbserver/MoveKeys.actor.cpp index 2b0460919a8..47092ccfcd6 100644 --- a/fdbserver/MoveKeys.actor.cpp +++ b/fdbserver/MoveKeys.actor.cpp @@ -505,7 +505,11 @@ ACTOR Future auditLocationMetadataPreCheck(Database occ, ACTOR Future auditLocationMetadataPostCheck(Database occ, KeyRange range, std::string context, UID dataMoveId) { if (range.empty()) { - TraceEvent(SevWarn, "CheckLocationMetadataEmptyInputRange").detail("By", "PostCheck").detail("Range", range); + TraceEvent(g_network->isSimulated() ? SevError : SevWarnAlways, "CheckLocationMetadataEmptyInputRange") + .detail("By", "PostCheck") + .detail("Range", range) + .detail("Context", context) + .detail("DataMoveId", dataMoveId.toString()); return Void(); } state std::vector> actors; @@ -1604,8 +1608,8 @@ ACTOR static Future startMoveShards(Database occ, loop { state Key begin = keys.begin; state KeyRange currentKeys = keys; - state bool complete = false; + state std::vector> actors; // Note this clears ongoing actors from the previous iteration state Transaction tr(occ); try { @@ -1687,7 +1691,6 @@ ACTOR static Future startMoveShards(Database occ, } currentKeys = KeyRangeRef(begin, keys.end); - state std::vector> actors; if (!currentKeys.empty()) { const int rowLimit = SERVER_KNOBS->MOVE_SHARD_KRM_ROW_LIMIT; @@ -1837,7 +1840,6 @@ ACTOR static Future startMoveShards(Database occ, if (currentKeys.end == keys.end) { dataMove.setPhase(DataMoveMetaData::Running); - complete = true; TraceEvent(sevDm, "StartMoveShardsDataMoveComplete", dataMoveId) .detail("DataMoveID", dataMoveId) .detail("DataMove", dataMove.toString()); @@ -1864,15 +1866,12 @@ ACTOR static Future startMoveShards(Database occ, .detail("Range", describe(dataMove.ranges)) .detail("DataMove", dataMove.toString()); - // Post validate consistency of update of keyServers and serverKeys - if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) { - if (!currentKeys.empty()) { - wait(auditLocationMetadataPostCheck(occ, currentKeys, "startMoveShards_postcheck", dataMoveId)); - } - } - dataMove = DataMoveMetaData(); - if (complete) { + if (currentKeys.end == keys.end) { + // Post validate consistency of update of keyServers and serverKeys + if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) { + wait(auditLocationMetadataPostCheck(occ, keys, "startMoveShards_postcheck", dataMoveId)); + } break; } } catch (Error& e) { @@ -1990,7 +1989,6 @@ ACTOR static Future finishMoveShards(Database occ, state Future warningLogger = logWarningAfter("FinishMoveShardsTooLong", 600, destinationTeam); state int retries = 0; state DataMoveMetaData dataMove; - state bool complete = false; state bool cancelDataMove = false; state Severity sevDm = static_cast(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY); @@ -2240,7 +2238,6 @@ ACTOR static Future finishMoveShards(Database occ, if (range.end == dataMove.ranges.front().end) { wait(deleteCheckpoints(&tr, dataMove.checkpoints, dataMoveId)); tr.clear(dataMoveKeyFor(dataMoveId)); - complete = true; TraceEvent(sevDm, "FinishMoveShardsDeleteMetaData", dataMoveId) .detail("DataMove", dataMove.toString()); } else { @@ -2253,13 +2250,12 @@ ACTOR static Future finishMoveShards(Database occ, } wait(tr.commit()); - - // Post validate consistency of update of keyServers and serverKeys - if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) { - wait(auditLocationMetadataPostCheck(occ, range, "finishMoveShards_postcheck", dataMoveId)); - } - - if (complete) { + if (range.end == dataMove.ranges.front().end) { + // Post validate consistency of update of keyServers and serverKeys + if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) { + wait(auditLocationMetadataPostCheck( + occ, dataMove.ranges.front(), "finishMoveShards_postcheck", relocationIntervalId)); + } break; } } else { @@ -2889,7 +2885,6 @@ ACTOR Future cleanUpDataMoveCore(Database occ, state KeyRange range; state Severity sevDm = static_cast(SERVER_KNOBS->PHYSICAL_SHARD_MOVE_LOG_SEVERITY); TraceEvent(sevDm, "CleanUpDataMoveBegin", dataMoveId).detail("DataMoveID", dataMoveId).detail("Range", keys); - state bool complete = false; state Error lastError; state bool runPreCheck = true; @@ -2906,7 +2901,6 @@ ACTOR Future cleanUpDataMoveCore(Database occ, range = KeyRange(); try { - complete = false; tr.trState->taskID = TaskPriority::MoveKeys; tr.setOption(FDBTransactionOptions::PRIORITY_SYSTEM_IMMEDIATE); tr.setOption(FDBTransactionOptions::ACCESS_SYSTEM_KEYS); @@ -3015,7 +3009,6 @@ ACTOR Future cleanUpDataMoveCore(Database occ, if (range.end == dataMove.ranges.front().end) { wait(deleteCheckpoints(&tr, dataMove.checkpoints, dataMoveId)); tr.clear(dataMoveKeyFor(dataMoveId)); - complete = true; TraceEvent(sevDm, "CleanUpDataMoveDeleteMetaData", dataMoveId) .detail("DataMoveID", dataMove.toString()); @@ -3041,12 +3034,12 @@ ACTOR Future cleanUpDataMoveCore(Database occ, .detail("DataMoveID", dataMoveId) .detail("Range", range); - // Post validate consistency of update of keyServers and serverKeys - if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) { - wait(auditLocationMetadataPostCheck(occ, range, "cleanUpDataMoveCore_postcheck", dataMoveId)); - } - - if (complete) { + if (range.end == dataMove.ranges.front().end) { + // Post validate consistency of update of keyServers and serverKeys + if (SERVER_KNOBS->AUDIT_DATAMOVE_POST_CHECK) { + wait(auditLocationMetadataPostCheck( + occ, dataMove.ranges.front(), "cleanUpDataMoveCore_postcheck", dataMoveId)); + } break; } } catch (Error& e) { @@ -3258,7 +3251,8 @@ void seedShardServers(Arena& arena, CommitTransactionRef& tr, std::vectorSHARD_ENCODE_LOCATION_METADATA) { - const UID shardId = deterministicRandom()->randomUniqueID(); + const UID shardId = + newDataMoveId(deterministicRandom()->randomUInt64(), AssignEmptyRange::False, DataMoveType::LOGICAL); ksValue = keyServersValue(serverSrcUID, /*dest=*/std::vector(), shardId, UID()); krmSetPreviouslyEmptyRange(tr, arena, keyServersPrefix, KeyRangeRef(KeyRef(), allKeys.end), ksValue, Value()); diff --git a/fdbserver/RocksDBCheckpointUtils.actor.cpp b/fdbserver/RocksDBCheckpointUtils.actor.cpp index 2814ff8be28..91481d8ef58 100644 --- a/fdbserver/RocksDBCheckpointUtils.actor.cpp +++ b/fdbserver/RocksDBCheckpointUtils.actor.cpp @@ -93,9 +93,6 @@ rocksdb::ExportImportFilesMetaData getMetaData(const CheckpointMetaData& checkpo liveFileMetaData.oldest_blob_file_number = fileMetaData.oldest_blob_file_number; liveFileMetaData.oldest_ancester_time = fileMetaData.oldest_ancester_time; liveFileMetaData.file_creation_time = fileMetaData.file_creation_time; - liveFileMetaData.smallest = fileMetaData.smallest; - liveFileMetaData.largest = fileMetaData.largest; - liveFileMetaData.file_type = rocksdb::kTableFile; liveFileMetaData.epoch_number = fileMetaData.epoch_number; liveFileMetaData.name = fileMetaData.name; liveFileMetaData.db_path = fileMetaData.db_path; diff --git a/fdbserver/SimulatedCluster.actor.cpp b/fdbserver/SimulatedCluster.actor.cpp index c8a3601082c..c78a2cb90db 100644 --- a/fdbserver/SimulatedCluster.actor.cpp +++ b/fdbserver/SimulatedCluster.actor.cpp @@ -1645,6 +1645,18 @@ void SimulationConfig::setTenantMode(const TestConfig& testConfig) { } void SimulationConfig::setEncryptionAtRestMode(const TestConfig& testConfig) { + // Enable encryption siginificantly reduces chance that a storage engine other than redwood got selected + // We want storage servers are selected by simulation tests with roughly equal chance + // If encryptMode is not specified explicitly, with high probability, we disable encryption + if (testConfig.encryptModes.empty() && + deterministicRandom()->random01() < SERVER_KNOBS->DISABLED_ENCRYPTION_PROBABILITY_SIM) { + EncryptionAtRestMode encryptionMode = EncryptionAtRestMode::DISABLED; + TraceEvent("SimulatedClusterEncryptionMode").detail("Mode", encryptionMode.toString()); + CODE_PROBE(true, "Enforce to disable encryption in simulation"); + set_config("encryption_at_rest_mode=" + encryptionMode.toString()); + return; + } + std::vector available; std::vector probability; if (!testConfig.encryptModes.empty()) { @@ -1707,10 +1719,34 @@ void SimulationConfig::setStorageEngine(const TestConfig& testConfig) { } else if (testConfig.storageEngineType.present()) { storage_engine_type = testConfig.storageEngineType.get(); } else { - // Continuously re-pick the storage engine type if it's the one we want to exclude - while (testConfig.excludedStorageEngineType(storage_engine_type)) { - storage_engine_type = deterministicRandom()->randomInt(0, 6); + // Randomly choose one from available set of storage engines + std::unordered_set storageEngineAvailable = { 0, 1, 2, 3, 4, 5 }; + for (const auto& storageEngineExcluded : testConfig.storageEngineExcludeTypes) { + storageEngineAvailable.erase(storageEngineExcluded); + } + ASSERT(storageEngineAvailable.size() > 0); + std::vector storageEngineCandidates; + for (const auto& storageEngine : storageEngineAvailable) { + if (storageEngine == 1) { // Memory engine + for (int i = 0; i < SERVER_KNOBS->PROBABILITY_FACTOR_MEMORY_ENGINE_SELECTED_SIM; i++) { + storageEngineCandidates.push_back(storageEngine); + // Adjust the chance that Memory is selected + } + } else if (storageEngine == 0) { // SQLite engine + for (int i = 0; i < SERVER_KNOBS->PROBABILITY_FACTOR_SQLITE_ENGINE_SELECTED_SIM; i++) { + storageEngineCandidates.push_back(storageEngine); + // Adjust the chance that SQLite is selected + } + } else if (storageEngine == 4) { // RocksDB engine + for (int i = 0; i < SERVER_KNOBS->PROBABILITY_FACTOR_ROCKSDB_ENGINE_SELECTED_SIM; i++) { + storageEngineCandidates.push_back(storageEngine); + // Adjust the chance that RocksDB is selected + } + } else { + storageEngineCandidates.push_back(storageEngine); + } } + storage_engine_type = deterministicRandom()->randomChoice(storageEngineCandidates); } if (storage_engine_type == 5) { @@ -2721,10 +2757,6 @@ ACTOR void setupAndRun(std::string dataFolder, state bool allowDefaultTenant = testConfig.allowDefaultTenant; state bool allowCreatingTenants = testConfig.allowCreatingTenants; - if (!SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) { - testConfig.storageEngineExcludeTypes.push_back(5); - } - // The RocksDB engine is not always built with the rest of fdbserver. Don't try to use it if it is not included // in the build. if (!rocksDBEnabled) { @@ -2732,6 +2764,21 @@ ACTOR void setupAndRun(std::string dataFolder, testConfig.storageEngineExcludeTypes.push_back(5); } + if (!SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && + // NOTE: PhysicalShardMove and BulkLoading are required to have SHARDED_ROCKSDB storage engine working. + // Inside the TOML file, the SHARD_ENCODE_LOCATION_METADATA is overridden, however, the + // override will not take effect until the test starts. Here, we do an additional check + // for this special simulation test. + (std::string_view(testFile).find("PhysicalShardMove") == std::string_view::npos && + std::string_view(testFile).find("BulkLoading") == std::string_view::npos)) { + testConfig.storageEngineExcludeTypes.push_back(5); + } else { + // Only use sqlite, rocksdb, shardeded rocksdb when location metadata is enabled. + testConfig.storageEngineExcludeTypes.push_back(1); + testConfig.storageEngineExcludeTypes.push_back(2); + testConfig.storageEngineExcludeTypes.push_back(3); + } + if (std::string_view(testFile).find("Encrypt") != std::string_view::npos) { testConfig.storageEngineExcludeTypes.push_back(5); } diff --git a/fdbserver/StorageMetrics.actor.cpp b/fdbserver/StorageMetrics.actor.cpp index ace7fa40f8d..b73e5a4ab64 100644 --- a/fdbserver/StorageMetrics.actor.cpp +++ b/fdbserver/StorageMetrics.actor.cpp @@ -286,8 +286,7 @@ void StorageServerMetrics::splitMetrics(SplitMetricsRequest req) const { lastKey, key, hasUsed); - if (used.bytes < minSplitBytes && (!SERVER_KNOBS->ENABLE_WRITE_BASED_SHARD_SPLIT || - remaining.bytesWrittenPerKSecond < minSplitWriteTraffic)) + if (used.bytes < minSplitBytes) key = std::max( key, byteSample.splitEstimate(KeyRangeRef(lastKey, req.keys.end), minSplitBytes - used.bytes)); key = getSplitKey(remaining.iosPerKSecond, diff --git a/fdbserver/TLogServer.actor.cpp b/fdbserver/TLogServer.actor.cpp index 2bb124c4059..d2a15e2ecc5 100644 --- a/fdbserver/TLogServer.actor.cpp +++ b/fdbserver/TLogServer.actor.cpp @@ -317,7 +317,10 @@ struct TLogData : NonCopyable { // interface should work without directly accessing rawPersistentQueue TLogQueue* persistentQueue; // Logical queue the log operates on and persist its data. - std::deque> unknownCommittedVersions; + // For each version above knownCommittedVersion, track: + // + std::deque unknownCommittedVersions; int64_t diskQueueCommitBytes; AsyncVar @@ -336,7 +339,7 @@ struct TLogData : NonCopyable { int64_t overheadBytesInput; int64_t overheadBytesDurable; int activePeekStreams = 0; - + Optional clusterRecoveryVersion; WorkerCache tlogCache; FlowLock peekMemoryLimiter; @@ -849,8 +852,9 @@ ACTOR Future tLogLock(TLogData* self, ReplyPromise reply, TLogLockResult result; result.end = stopVersion; result.knownCommittedVersion = logData->knownCommittedVersion; - result.unknownCommittedVersions = self->unknownCommittedVersions; result.id = self->dbgid; + result.logId = logData->logId; + result.unknownCommittedVersions = self->unknownCommittedVersions; TraceEvent("TLogStop2", self->dbgid) .detail("LogId", logData->logId) @@ -1799,7 +1803,16 @@ Future tLogPeekMessages(PromiseType replyPromise, state double blockStart = now(); - if (reqReturnIfBlocked && logData->version.get() < reqBegin) { + // if tLog locked for recovery, return an empty message at the cluster recovery version + // if requested version is greater than any received. + state Optional clusterRecoveryVersion = Optional(); + ASSERT(!clusterRecoveryVersion.present() || reqBegin <= clusterRecoveryVersion.get()); + if (logData->stopped() && logData->version.get() < reqBegin && self->clusterRecoveryVersion.present()) { + clusterRecoveryVersion = self->clusterRecoveryVersion.get(); + TraceEvent("TLogPeekMessagesClusterRecoveryVersion").detail("Version", clusterRecoveryVersion.get()); + } + + if (!clusterRecoveryVersion.present() && reqReturnIfBlocked && logData->version.get() < reqBegin) { replyPromise.sendError(end_of_stream()); if (reqSequence.present()) { auto& trackerData = logData->peekTracker[peekId]; @@ -1819,7 +1832,7 @@ Future tLogPeekMessages(PromiseType replyPromise, .detail("Version", logData->version.get()) .detail("RecoveredAt", logData->recoveredAt); // Wait until we have something to return that the caller doesn't already have - if (logData->version.get() < reqBegin) { + if (!clusterRecoveryVersion.present() && logData->version.get() < reqBegin) { wait(logData->version.whenAtLeast(reqBegin)); wait(delay(SERVER_KNOBS->TLOG_PEEK_DELAY, g_network->getCurrentTask())); } @@ -2096,7 +2109,7 @@ Future tLogPeekMessages(PromiseType replyPromise, auto messagesValue = messages.toValue(); reply.arena.dependsOn(messagesValue.arena()); reply.messages = messagesValue; - reply.end = endVersion; + reply.end = clusterRecoveryVersion.present() ? clusterRecoveryVersion.get() : endVersion; reply.onlySpilled = onlySpilled; DebugLogTraceEvent("TLogPeekMessages4", self->dbgid) @@ -2344,7 +2357,6 @@ ACTOR Future tLogCommit(TLogData* self, logData->minKnownCommittedVersion = std::max(logData->minKnownCommittedVersion, req.minKnownCommittedVersion); wait(logData->version.whenAtLeast(req.prevVersion)); - // Time until now has been spent waiting in the queue to do actual work. state double queueWaitEndTime = g_network->timer(); self->queueWaitLatencyDist->sampleSeconds(queueWaitEndTime - req.requestTime()); @@ -2397,9 +2409,9 @@ ACTOR Future tLogCommit(TLogData* self, // Notifies the commitQueue actor to commit persistentQueue, and also unblocks tLogPeekMessages actors logData->version.set(req.version); if (SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) { - self->unknownCommittedVersions.push_front(std::make_tuple(req.version, req.tLogCount)); + self->unknownCommittedVersions.emplace_front(req.version, req.seqPrevVersion, req.tLogLocIds); while (!self->unknownCommittedVersions.empty() && - std::get<0>(self->unknownCommittedVersions.back()) <= req.knownCommittedVersion) { + self->unknownCommittedVersions.back().version <= req.knownCommittedVersion) { self->unknownCommittedVersions.pop_back(); } } @@ -2847,6 +2859,11 @@ ACTOR Future serveTLogInterface(TLogData* self, when(TLogEnablePopRequest enablePopReq = waitNext(tli.enablePopRequest.getFuture())) { logData->addActor.send(tLogEnablePopReq(enablePopReq, self, logData)); } + when(setClusterRecoveryVersionRequest req = waitNext(tli.setClusterRecoveryVersion.getFuture())) { + ASSERT(logData->stopped()); + self->clusterRecoveryVersion = req.recoveryVersion; + req.reply.send(Void()); + } } } diff --git a/fdbserver/TagPartitionedLogSystem.actor.cpp b/fdbserver/TagPartitionedLogSystem.actor.cpp index 05a2f47b87a..7fe55887b3c 100644 --- a/fdbserver/TagPartitionedLogSystem.actor.cpp +++ b/fdbserver/TagPartitionedLogSystem.actor.cpp @@ -19,6 +19,7 @@ */ #include "fdbserver/TagPartitionedLogSystem.actor.h" +#include #include "flow/actorcompiler.h" // This must be the last #include. @@ -557,9 +558,11 @@ Future TagPartitionedLogSystem::push(Version prevVersion, std::vector> quorumResults; std::vector> allReplies; int location = 0; + Version seqPrevVersion = prevVersion; // prevVersion provided by the sequencer Span span("TPLS:push"_loc, spanContext); - std::unordered_map tLogCount; + std::unordered_map tLogCount; + std::unordered_map> tLogLocIds; if (SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) { int location = 0; int logGroupLocal = 0; @@ -570,6 +573,7 @@ Future TagPartitionedLogSystem::push(Version prevVersion, for (int loc = 0; loc < it->logServers.size(); loc++) { if (tpcvMap.get().find(location) != tpcvMap.get().end()) { tLogCount[logGroupLocal]++; + tLogLocIds[logGroupLocal].push_back(location); } location++; } @@ -615,8 +619,10 @@ Future TagPartitionedLogSystem::push(Version prevVersion, version, knownCommittedVersion, minKnownCommittedVersion, + seqPrevVersion, msg, tLogCount[logGroupLocal], + tLogLocIds[logGroupLocal], debugID), TaskPriority::ProxyTLogCommitReply))); Future commitSuccess = success(allReplies.back()); @@ -2041,41 +2047,141 @@ ACTOR Future TagPartitionedLogSystem::getDurableVersionChanged(LogLockInfo return Void(); } -// If VERSION_VECTOR_UNICAST is enabled, one tLog's DV may advance beyond the min(DV) over all tLogs. +void getTLogLocIds(std::vector>& tLogs, + std::vector>>& logGroupResults, + std::vector>& tLogLocIds, + uint16_t& maxTLogLocId) { + // Initialization. + tLogLocIds.clear(); + tLogLocIds.resize(logGroupResults.size()); + maxTLogLocId = 0; + + // Map the interfaces of all (local) tLogs to their corresponding locations in LogSets. + std::map interfLocMap; + uint16_t location = 0; + for (auto& it : tLogs) { + if (!it->isLocal) { + continue; + } + for (uint16_t i = 0; i < it->logServers.size(); i++) { + if (it->logServers[i]->get().present()) { + interfLocMap[it->logServers[i]->get().interf().id()] = location; + } + location++; + } + } + + // Set maxTLogLocId. + maxTLogLocId = location; + + // Find the locations of tLogs in "logGroupResults". + uint8_t logGroupId = 0; + for (auto& logGroupResult : logGroupResults) { + for (auto& tLogResult : std::get<1>(logGroupResult)) { + ASSERT(interfLocMap.find(tLogResult.logId) != interfLocMap.end()); + tLogLocIds[logGroupId].push_back(interfLocMap[tLogResult.logId]); + } + logGroupId++; + } +} + +void populateBitset(boost::dynamic_bitset<>& bs, std::vector& ids) { + for (auto& id : ids) { + ASSERT(id < bs.size()); + bs.set(id); + } +} + +// If ENABLE_VERSION_VECTOR_TLOG_UNICAST is set, one tLog's DV may advance beyond the min(DV) over all tLogs. // This function finds the highest recoverable version for each tLog group over all log groups. // All prior versions to the chosen RV must also be recoverable. // TODO: unit tests to stress UNICAST -Version getRecoverVersionUnicast(std::vector>>& logGroupResults, - Version minEnd) { +Version getRecoverVersionUnicast(std::vector>& logServers, + std::vector>>& logGroupResults, + Version minDVEnd, + Version minKCVEnd) { + std::vector> tLogLocIds; + uint16_t maxTLogLocId; // maximum possible id, not maximum of id's of available log servers + getTLogLocIds(logServers, logGroupResults, tLogLocIds, maxTLogLocId); + uint16_t bsSize = maxTLogLocId + 1; // bitset size, used below + + // NOTE: We think the unicast recovery version is always greater than or equal to + // "min(DV)" (= "minDVEnd"). To be conservative we use "min(KCV)" (= "minKCVEnd") + // as the default (starting) recovery version and later verify that the computed + // recovery version is greater than or equal to "minDVEnd". + // @todo modify code to use "minDVEnd" as the default (starting) recovery version + Version minEnd = minKCVEnd; + std::vector RVs(maxTLogLocId + 1, minEnd); // recovery versions of various tLogs + + uint8_t tLogGroupIdx = 0; Version minLogGroup = std::numeric_limits::max(); for (auto& logGroupResult : logGroupResults) { - std::unordered_map versionRepCount; - std::map versionTLogCount; + uint16_t tLogIdx = 0; + boost::dynamic_bitset<> availableTLogs(bsSize); + // version -> tLogs (that are avaiable) that have received the version + std::unordered_map> versionAvailableTLogs; + // version -> all tLogs that the version was sent to (by the commit proxy) + std::map> versionAllTLogs; + // version -> prevVersion (that was given out by the sequencer) map + std::map prevVersionMap; int replicationFactor = std::get<0>(logGroupResult); for (auto& tLogResult : std::get<1>(logGroupResult)) { + uint16_t tLogLocId = tLogLocIds[tLogGroupIdx][tLogIdx]; + availableTLogs.set(tLogLocId); bool logGroupCandidate = false; for (auto& unknownCommittedVersion : tLogResult.unknownCommittedVersions) { - Version k = std::get<0>(unknownCommittedVersion); + Version k = unknownCommittedVersion.version; if (k > minEnd) { - versionRepCount[k]++; - versionTLogCount[k] = std::get<1>(unknownCommittedVersion); + if (versionAvailableTLogs[k].empty()) { + versionAvailableTLogs[k].resize(bsSize); + } + versionAvailableTLogs[k].set(tLogLocId); + prevVersionMap[k] = unknownCommittedVersion.prev; + if (versionAllTLogs[k].empty()) { + versionAllTLogs[k].resize(bsSize); + } + populateBitset(versionAllTLogs[k], unknownCommittedVersion.tLogLocIds); logGroupCandidate = true; } } if (!logGroupCandidate) { return minEnd; } + tLogIdx++; } Version minTLogs = minEnd; - for (auto const& [version, tLogCount] : versionTLogCount) { - if (versionRepCount[version] >= tLogCount - replicationFactor + 1) { - minTLogs = version; - } else { + Version prevVersion = minEnd; + for (auto const& [version, tLogs] : versionAllTLogs) { + if (!(prevVersion == minEnd || prevVersion == prevVersionMap[version])) { break; } + // This version is not recoverable if there is a log server (LS) such that: + // - the commit proxy sent this version to LS (i.e., LS is present in "versionAllTLogs[version]") + // - LS is available (i.e., LS is present in "availableTLogs") + // - LS didn't receive this version (i.e., LS is not present in "versionAvailableTLogs[version]") + if (((tLogs & availableTLogs) & ~versionAvailableTLogs[version]).any()) { + break; + } + // If the commit proxy sent this version to "N" log servers then at least + // (N - replicationFactor + 1) log servers must be available. + if (!(versionAvailableTLogs[version].size() >= tLogs.size() - replicationFactor + 1)) { + break; + } + // Update RV. + minTLogs = version; + // Update recovery version vector. + for (boost::dynamic_bitset<>::size_type id = 0; id < versionAvailableTLogs[version].size(); id++) { + if (versionAvailableTLogs[version][id]) { + RVs[id] = version; + } + } + // Update prevVersion. + prevVersion = version; } minLogGroup = std::min(minLogGroup, minTLogs); + tLogGroupIdx++; } + ASSERT_WE_THINK(minLogGroup >= minDVEnd && minLogGroup != std::numeric_limits::max()); return minLogGroup; } @@ -2214,6 +2320,7 @@ ACTOR Future TagPartitionedLogSystem::epochEnd(Reference lockResults; + state Reference lockResultsInterf = makeReference(); state std::vector>>, Reference>> allLogServers; state std::vector> logServers; @@ -2255,7 +2362,8 @@ ACTOR Future TagPartitionedLogSystem::epochEnd(ReferencelogServers.size(); t++) { - lockResults[i].replies.push_back(TagPartitionedLogSystem::lockTLog(dbgid, logServers[i]->logServers[t])); + lockResults[i].replies.push_back( + TagPartitionedLogSystem::lockTLog(dbgid, logServers[i]->logServers[t], lockResultsInterf)); } } @@ -2276,7 +2384,8 @@ ACTOR Future TagPartitionedLogSystem::epochEnd(ReferencelogServers.size(); t++) { - lockResult.replies.push_back(TagPartitionedLogSystem::lockTLog(dbgid, log->logServers[t])); + lockResult.replies.push_back( + TagPartitionedLogSystem::lockTLog(dbgid, log->logServers[t], lockResultsInterf)); } lockResults.push_back(lockResult); } @@ -2292,7 +2401,8 @@ ACTOR Future TagPartitionedLogSystem::epochEnd(ReferencelogServers.size(); t++) { - lockResult.replies.push_back(TagPartitionedLogSystem::lockTLog(dbgid, old.tLogs[0]->logServers[t])); + lockResult.replies.push_back( + TagPartitionedLogSystem::lockTLog(dbgid, old.tLogs[0]->logServers[t], lockResultsInterf)); } allLockResults.push_back(lockResult); } @@ -2336,9 +2446,10 @@ ACTOR Future TagPartitionedLogSystem::epochEnd(Reference::max(); + Version minKCVEnd = std::numeric_limits::max(); Version maxEnd = 0; - std::vector> changes; - std::vector>> logGroupResults; + state std::vector> changes; + state std::vector>> logGroupResults; for (int log = 0; log < logServers.size(); log++) { if (!logServers[log]->isLocal) { continue; @@ -2350,22 +2461,18 @@ ACTOR Future TagPartitionedLogSystem::epochEnd(ReferencetLogReplicationFactor, std::get<2>(versions.get())); maxEnd = std::max(maxEnd, std::get<1>(versions.get())); minEnd = std::min(minEnd, std::get<1>(versions.get())); + minKCVEnd = std::min(minKCVEnd, std::get<0>(versions.get())); } changes.push_back(TagPartitionedLogSystem::getDurableVersionChanged(lockResults[log], logFailed[log])); } if (maxEnd > 0 && (!lastEnd.present() || maxEnd < lastEnd.get())) { CODE_PROBE(lastEnd.present(), "Restarting recovery at an earlier point"); - auto logSystem = makeReference(dbgid, locality, prevState.recoveryCount); + state Reference logSystem = + makeReference(dbgid, locality, prevState.recoveryCount); logSystem->recoverAt = minEnd; - if (SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) { - logSystem->recoverAt = getRecoverVersionUnicast(logGroupResults, minEnd); - TraceEvent("RecoveryVersionInfo").detail("RecoverAt", logSystem->recoverAt); - } - lastEnd = minEnd; - logSystem->tLogs = logServers; logSystem->logRouterTags = prevState.logRouterTags; logSystem->txsTags = prevState.txsTags; @@ -2384,6 +2491,25 @@ ACTOR Future TagPartitionedLogSystem::epochEnd(Referencestopped = true; logSystem->pseudoLocalities = prevState.pseudoLocalities; + if (SERVER_KNOBS->ENABLE_VERSION_VECTOR_TLOG_UNICAST) { + logSystem->recoverAt = getRecoverVersionUnicast(logServers, logGroupResults, minEnd, minKCVEnd); + TraceEvent("RecoveryVersionInfo").detail("RecoverAt", logSystem->recoverAt); + // When a new log system is created, inform the surviving tLogs of the RV. + // SOMEDAY: Assert surviving tLogs use the RV from the latest log system. + for (auto logGroupResult : logGroupResults) { + state std::vector tLogResults = std::get<1>(logGroupResult); + for (auto& tLogResult : tLogResults) { + wait(transformErrors( + throwErrorOr(lockResultsInterf->lockInterf[tLogResult.id] + .setClusterRecoveryVersion.getReplyUnlessFailedFor( + setClusterRecoveryVersionRequest(logSystem->recoverAt.get()), + SERVER_KNOBS->TLOG_TIMEOUT, + SERVER_KNOBS->MASTER_FAILURE_SLOPE_DURING_RECOVERY)), + cluster_recovery_failed())); + } + } + } + outLogSystem->set(logSystem); } @@ -3114,7 +3240,7 @@ ACTOR Future> TagPartitionedLogSystem::newEpoch( } wait(waitForAll(satelliteInitializationReplies) || oldRouterRecruitment); - TraceEvent("PrimarySatelliteTLogInitializationComplete").log(); + TraceEvent("PrimarySatelliteTLogInitializationComplete", logSystem->getDebugID()).log(); for (int i = 0; i < satelliteInitializationReplies.size(); i++) { logSystem->tLogs[1]->logServers[i] = makeReference>>( @@ -3252,8 +3378,8 @@ ACTOR Future TagPartitionedLogSystem::trackRejoins( ACTOR Future TagPartitionedLogSystem::lockTLog( UID myID, - Reference>> tlog) { - + Reference>> tlog, + Optional> lockInterf) { TraceEvent("TLogLockStarted", myID).detail("TLog", tlog->get().id()).detail("InfPresent", tlog->get().present()); loop { choose { @@ -3261,6 +3387,9 @@ ACTOR Future TagPartitionedLogSystem::lockTLog( tlog->get().present() ? brokenPromiseToNever(tlog->get().interf().lock.getReply()) : Never())) { TraceEvent("TLogLocked", myID).detail("TLog", tlog->get().id()).detail("End", data.end); + if (lockInterf.present()) { + lockInterf.get()->lockInterf[data.id] = tlog->get().interf(); + } return data; } when(wait(tlog->onChange())) {} @@ -3301,4 +3430,4 @@ ACTOR Future TagPartitionedLogSystem::trackTLogRecoveryActor( TraceEvent("TLogRecoveredVersion").detail("RecoveredVersion", currentRecoveredVersion); recoveredVersion->set(currentRecoveredVersion); } -} \ No newline at end of file +} diff --git a/fdbserver/include/fdbserver/ClusterController.actor.h b/fdbserver/include/fdbserver/ClusterController.actor.h index 42ecb046ab4..60e05e251a3 100644 --- a/fdbserver/include/fdbserver/ClusterController.actor.h +++ b/fdbserver/include/fdbserver/ClusterController.actor.h @@ -3255,7 +3255,8 @@ class ClusterControllerData { // For degraded server that are complained by more than SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE, we // don't know if it is a hot server, or the network is bad. We remove from the returned degraded server list. for (const auto& badServer : currentDegradedServers) { - if (degradedLinkDst2Src[badServer].size() <= SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE) { + if (degradedLinkDst2Src[badServer].size() >= SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE_MIN && + degradedLinkDst2Src[badServer].size() <= SERVER_KNOBS->CC_DEGRADED_PEER_DEGREE_TO_EXCLUDE) { currentDegradationInfo.degradedServers.insert(badServer); } } diff --git a/fdbserver/include/fdbserver/DDShardTracker.h b/fdbserver/include/fdbserver/DDShardTracker.h index e88f4b30290..755662d239e 100644 --- a/fdbserver/include/fdbserver/DDShardTracker.h +++ b/fdbserver/include/fdbserver/DDShardTracker.h @@ -47,6 +47,7 @@ struct DataDistributionTrackerInitParams { KeyRangeMap* shards = nullptr; bool* trackerCancelled = nullptr; Optional> ddTenantCache; + int32_t usableRegions = -1; }; // track the status of shards @@ -65,6 +66,7 @@ class DataDistributionTracker : public IDDShardTracker, public ReferenceCounted< Reference> dbSizeEstimate; Reference>> maxShardSize; Future maxShardSizeUpdater; + int32_t usableRegions = -1; // CapacityTracker PromiseStream output; diff --git a/fdbserver/include/fdbserver/DDTeamCollection.h b/fdbserver/include/fdbserver/DDTeamCollection.h index a90aae48f1f..333194e6d82 100644 --- a/fdbserver/include/fdbserver/DDTeamCollection.h +++ b/fdbserver/include/fdbserver/DDTeamCollection.h @@ -484,6 +484,8 @@ class DDTeamCollection : public ReferenceCounted { Future waitForAllDataRemoved(UID serverID, Version addedVersion) const; + bool allServersHaveMinAvailableSpace(double minAvailableSpaceRatio) const; + // calculate minLoadBytes / avgLoadBytes among servers. An unhealthy server's load is considered as 0. If the // average load of each storage server is less than smallLoadThreshold, return 1 always. double loadBytesBalanceRatio(int64_t smallLoadThreshold) const; diff --git a/fdbserver/include/fdbserver/DataDistribution.actor.h b/fdbserver/include/fdbserver/DataDistribution.actor.h index bc151ec434d..3d3facab4ae 100644 --- a/fdbserver/include/fdbserver/DataDistribution.actor.h +++ b/fdbserver/include/fdbserver/DataDistribution.actor.h @@ -270,6 +270,7 @@ struct ShardMetrics { struct ShardTrackedData { Future trackShard; Future trackBytes; + Future trackUsableRegion; Reference>> stats; }; diff --git a/fdbserver/include/fdbserver/DataDistributionTeam.h b/fdbserver/include/fdbserver/DataDistributionTeam.h index 639e2ba3877..59c1b5d484f 100644 --- a/fdbserver/include/fdbserver/DataDistributionTeam.h +++ b/fdbserver/include/fdbserver/DataDistributionTeam.h @@ -94,6 +94,7 @@ struct IDataDistributionTeam { FDB_BOOLEAN_PARAM(WantNewServers); FDB_BOOLEAN_PARAM(WantTrueBest); +FDB_BOOLEAN_PARAM(WantTrueBestIfMoveout); FDB_BOOLEAN_PARAM(PreferLowerDiskUtil); FDB_BOOLEAN_PARAM(TeamMustHaveShards); FDB_BOOLEAN_PARAM(ForReadBalance); @@ -140,7 +141,8 @@ struct GetTeamRequest { double inflightPenalty; bool findTeamByServers; Optional keys; - bool storageQueueAware; + bool storageQueueAware = false; + bool wantTrueBestIfMoveout = false; // completeSources have all shards in the key range being considered for movement, src have at least 1 shard in the // key range for movement. From the point of set, completeSources is the Intersection set of several , @@ -166,13 +168,14 @@ struct GetTeamRequest { : teamSelect(teamSelectRequest), storageQueueAware(false), preferLowerDiskUtil(preferLowerDiskUtil), teamMustHaveShards(teamMustHaveShards), forReadBalance(forReadBalance), preferLowerReadUtil(preferLowerReadUtil), preferWithinShardLimit(preferWithinShardLimit), - inflightPenalty(inflightPenalty), findTeamByServers(FindTeamByServers::False), keys(keys) {} + inflightPenalty(inflightPenalty), findTeamByServers(FindTeamByServers::False), keys(keys), + wantTrueBestIfMoveout(false) {} GetTeamRequest(std::vector servers) : teamSelect(TeamSelect::WANT_COMPLETE_SRCS), storageQueueAware(false), preferLowerDiskUtil(PreferLowerDiskUtil::False), teamMustHaveShards(TeamMustHaveShards::False), forReadBalance(ForReadBalance::False), preferLowerReadUtil(PreferLowerReadUtil::False), preferWithinShardLimit(PreferWithinShardLimit::False), inflightPenalty(1.0), - findTeamByServers(FindTeamByServers::True), src(std::move(servers)) {} + findTeamByServers(FindTeamByServers::True), src(std::move(servers)), wantTrueBestIfMoveout(false) {} // return true if a.score < b.score [[nodiscard]] bool lessCompare(TeamRef a, TeamRef b, int64_t aLoadBytes, int64_t bLoadBytes) const { @@ -187,10 +190,10 @@ struct GetTeamRequest { std::stringstream ss; ss << "TeamSelect:" << teamSelect.toString() << " StorageQueueAware:" << storageQueueAware - << " PreferLowerDiskUtil:" << preferLowerDiskUtil << " PreferLowerReadUtil:" << preferLowerReadUtil - << " PreferWithinShardLimit:" << preferWithinShardLimit << " teamMustHaveShards:" << teamMustHaveShards - << " forReadBalance:" << forReadBalance << " inflightPenalty:" << inflightPenalty - << " findTeamByServers:" << findTeamByServers << ";"; + << " WantTrueBestIfMoveout:" << wantTrueBestIfMoveout << " PreferLowerDiskUtil:" << preferLowerDiskUtil + << " PreferLowerReadUtil:" << preferLowerReadUtil << " PreferWithinShardLimit:" << preferWithinShardLimit + << " teamMustHaveShards:" << teamMustHaveShards << " forReadBalance:" << forReadBalance + << " inflightPenalty:" << inflightPenalty << " findTeamByServers:" << findTeamByServers << ";"; ss << "CompleteSources:"; for (const auto& cs : completeSources) { ss << cs.toString() << ","; diff --git a/fdbserver/include/fdbserver/StorageMetrics.actor.h b/fdbserver/include/fdbserver/StorageMetrics.actor.h index b58e53c046f..a9b526dccb1 100644 --- a/fdbserver/include/fdbserver/StorageMetrics.actor.h +++ b/fdbserver/include/fdbserver/StorageMetrics.actor.h @@ -48,6 +48,7 @@ const StringRef SS_DURABLE_VERSION_UPDATE_LATENCY_HISTOGRAM = "SSDurableVersionU const StringRef SS_READ_RANGE_BYTES_RETURNED_HISTOGRAM = "SSReadRangeBytesReturned"_sr; const StringRef SS_READ_RANGE_BYTES_LIMIT_HISTOGRAM = "SSReadRangeBytesLimit"_sr; const StringRef SS_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM = "SSReadRangeKVPairsReturned"_sr; +const StringRef SS_READ_RANGE_PREFIX_LENGTH_HISTOGRAM = "SSReadRangePrefixLength"_sr; struct StorageMetricSample { IndexedSet sample; diff --git a/fdbserver/include/fdbserver/TLogInterface.h b/fdbserver/include/fdbserver/TLogInterface.h index 9bd26c84044..34833748615 100644 --- a/fdbserver/include/fdbserver/TLogInterface.h +++ b/fdbserver/include/fdbserver/TLogInterface.h @@ -53,6 +53,7 @@ struct TLogInterface { RequestStream enablePopRequest; RequestStream snapRequest; RequestStream trackRecovery; + RequestStream setClusterRecoveryVersion; TLogInterface() {} explicit TLogInterface(const LocalityData& locality) @@ -87,6 +88,7 @@ struct TLogInterface { streams.push_back(snapRequest.getReceiver()); streams.push_back(peekStreamMessages.getReceiver(TaskPriority::TLogPeek)); streams.push_back(trackRecovery.getReceiver()); + streams.push_back(setClusterRecoveryVersion.getReceiver()); FlowTransport::transport().addEndpoints(streams); } @@ -117,6 +119,8 @@ struct TLogInterface { RequestStream(peekMessages.getEndpoint().getAdjustedEndpoint(11)); trackRecovery = RequestStream(peekMessages.getEndpoint().getAdjustedEndpoint(12)); + setClusterRecoveryVersion = RequestStream( + peekMessages.getEndpoint().getAdjustedEndpoint(13)); } } }; @@ -133,16 +137,32 @@ struct TLogRecoveryFinishedRequest { } }; +struct UnknownCommittedVersions { + constexpr static FileIdentifier file_identifier = 11822137; + Version version; // version made durable on recovering tLog + Version prev; // previous to version; to ensure no gaps in chain + std::vector tLogLocIds; // locations version was sent to + UnknownCommittedVersions() {} + UnknownCommittedVersions(Version version, Version prev, std::vector tLogLocIds) + : version(version), prev(prev), tLogLocIds(tLogLocIds) {} + template + void serialize(Ar& ar) { + serializer(ar, version, prev, tLogLocIds); + } +}; + struct TLogLockResult { constexpr static FileIdentifier file_identifier = 11822027; Version end; Version knownCommittedVersion; - std::deque> unknownCommittedVersions; - UID id; + std::deque> unknownCommittedVersionTuples; // unused, kept for compatibility + std::deque unknownCommittedVersions; + UID id; // captures TLogData::dbgid + UID logId; // captures LogData::logId template void serialize(Ar& ar) { - serializer(ar, end, knownCommittedVersion, unknownCommittedVersions, id); + serializer(ar, end, knownCommittedVersion, unknownCommittedVersionTuples, id, unknownCommittedVersions, logId); } }; @@ -303,12 +323,13 @@ struct TLogCommitRequest : TimedRequest { constexpr static FileIdentifier file_identifier = 4022206; SpanContext spanContext; Arena arena; - Version prevVersion, version, knownCommittedVersion, minKnownCommittedVersion; + Version prevVersion, version, knownCommittedVersion, minKnownCommittedVersion, seqPrevVersion; StringRef messages; // Each message prefixed by a 4-byte length ReplyPromise reply; - int tLogCount; + uint16_t tLogCount; + std::vector tLogLocIds; Optional debugID; TLogCommitRequest() {} @@ -318,12 +339,15 @@ struct TLogCommitRequest : TimedRequest { Version version, Version knownCommittedVersion, Version minKnownCommittedVersion, + Version seqPrevVersion, StringRef messages, - int tLogCount, + uint16_t tLogCount, + std::vector& tLogLocIds, Optional debugID) : spanContext(context), arena(a), prevVersion(prevVersion), version(version), knownCommittedVersion(knownCommittedVersion), minKnownCommittedVersion(minKnownCommittedVersion), - messages(messages), tLogCount(tLogCount), debugID(debugID) {} + seqPrevVersion(seqPrevVersion), messages(messages), tLogCount(tLogCount), tLogLocIds(tLogLocIds), + debugID(debugID) {} template void serialize(Ar& ar) { serializer(ar, @@ -336,6 +360,8 @@ struct TLogCommitRequest : TimedRequest { debugID, tLogCount, spanContext, + seqPrevVersion, + tLogLocIds, arena); } }; @@ -445,4 +471,19 @@ struct TrackTLogRecoveryRequest { } }; +struct setClusterRecoveryVersionRequest { + constexpr static FileIdentifier file_identifier = 6876464; + + Version recoveryVersion; + ReplyPromise reply; + + setClusterRecoveryVersionRequest() = default; + setClusterRecoveryVersionRequest(Version recoveryVersion) : recoveryVersion(recoveryVersion) {} + + template + void serialize(Ar& ar) { + serializer(ar, recoveryVersion, reply); + } +}; + #endif diff --git a/fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h b/fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h index 04c1cce3527..4fb786e5a02 100644 --- a/fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h +++ b/fdbserver/include/fdbserver/TagPartitionedLogSystem.actor.h @@ -65,6 +65,10 @@ struct OldLogData { } }; +struct IdToInterf : ReferenceCounted { + std::map lockInterf; +}; + struct LogLockInfo { Version epochEnd; bool isCurrent; @@ -389,8 +393,10 @@ struct TagPartitionedLogSystem final : ILogSystem, ReferenceCounted>>> tlogs, Reference> recoveredVersion); - ACTOR static Future lockTLog(UID myID, Reference>> tlog); - + ACTOR static Future lockTLog( + UID myID, + Reference>> tlog, + Optional> lockInterf = Optional>()); template static std::vector getReadyNonError(std::vector> const& futures); }; diff --git a/fdbserver/storageserver.actor.cpp b/fdbserver/storageserver.actor.cpp index 0d0e7a1fb7b..6d0fdb03e85 100644 --- a/fdbserver/storageserver.actor.cpp +++ b/fdbserver/storageserver.actor.cpp @@ -1047,6 +1047,7 @@ struct StorageServer : public IStorageMetricsService { Reference readRangeBytesReturnedHistogram; Reference readRangeBytesLimitHistogram; Reference readRangeKVPairsReturnedHistogram; + Reference readRangePrefixLengthHistogram; // watch map operations Reference getWatchMetadata(KeyRef key, int64_t tenantId) const; @@ -1655,6 +1656,11 @@ struct StorageServer : public IStorageMetricsService { readRangeKVPairsReturnedHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP, SS_READ_RANGE_KV_PAIRS_RETURNED_HISTOGRAM, Histogram::Unit::bytes)), + readRangePrefixLengthHistogram(Histogram::getHistogram(STORAGESERVER_HISTOGRAM_GROUP, + SS_READ_RANGE_PREFIX_LENGTH_HISTOGRAM, + Histogram::Unit::countLinear, + /*lower=*/0, + /*upper*/ 50)), tag(invalidTag), poppedAllAfter(std::numeric_limits::max()), cpuUsage(0.0), diskUsage(0.0), storage(this, storage), shardChangeCounter(0), lastTLogVersion(0), lastVersionWithData(0), restoredVersion(0), prevVersion(0), rebootAfterDurableVersion(std::numeric_limits::max()), @@ -4373,6 +4379,7 @@ ACTOR Future readRange(StorageServer* data, } } } + data->readRangePrefixLengthHistogram->sampleRecordCounter(range.prefixLength()); data->readRangeBytesReturnedHistogram->sample(resultLogicalSize); data->readRangeKVPairsReturnedHistogram->sample(result.data.size()); @@ -10010,9 +10017,28 @@ void changeServerKeysWithPhysicalShards(StorageServer* data, .detail("PhysicalShard", currentShard->toStorageServerShard().toString()); } if (!currentShard.isValid()) { - ASSERT(currentRange == keys); // there shouldn't be any nulls except for the range being inserted + if (currentRange != keys) { + TraceEvent(SevError, "PhysicalShardStateError") + .detail("SubError", "RangeDifferent") + .detail("CurrentRange", currentRange) + .detail("ModifiedRange", keys) + .detail("Assigned", nowAssigned) + .detail("DataMoveId", dataMoveId) + .detail("Version", version); + throw internal_error(); + } } else if (currentShard->notAssigned()) { - ASSERT(nowAssigned); // Adding a new range to the server. + if (!nowAssigned) { + TraceEvent(SevError, "PhysicalShardStateError") + .detail("SubError", "UnassignEmptyRange") + .detail("Assigned", nowAssigned) + .detail("ModifiedRange", keys) + .detail("DataMoveId", dataMoveId) + .detail("Version", version) + .detail("ConflictingShard", currentShard->shardId) + .detail("DesiredShardId", currentShard->desiredShardId); + throw internal_error(); + } StorageServerShard newShard = currentShard->toStorageServerShard(); newShard.range = currentRange; data->addShard(ShardInfo::newShard(data, newShard)); @@ -10031,7 +10057,17 @@ void changeServerKeysWithPhysicalShards(StorageServer* data, .detail("Version", cVer) .detail("ResultingShard", newShard.toString()); } else if (currentShard->adding) { - ASSERT(!nowAssigned); + if (nowAssigned) { + TraceEvent(SevError, "PhysicalShardStateError") + .detail("SubError", "UpdateAddingShard") + .detail("Assigned", nowAssigned) + .detail("ModifiedRange", keys) + .detail("DataMoveId", dataMoveId) + .detail("Version", version) + .detail("ConflictingShard", currentShard->shardId) + .detail("DesiredShardId", currentShard->desiredShardId); + throw internal_error(); + } StorageServerShard newShard = currentShard->toStorageServerShard(); newShard.range = currentRange; data->addShard(ShardInfo::newShard(data, newShard)); @@ -10041,7 +10077,17 @@ void changeServerKeysWithPhysicalShards(StorageServer* data, .detail("Version", cVer) .detail("ResultingShard", newShard.toString()); } else if (currentShard->moveInShard) { - ASSERT(!nowAssigned); + if (nowAssigned) { + TraceEvent(SevError, "PhysicalShardStateError") + .detail("SubError", "UpdateMoveInShard") + .detail("Assigned", nowAssigned) + .detail("ModifiedRange", keys) + .detail("DataMoveId", dataMoveId) + .detail("Version", version) + .detail("ConflictingShard", currentShard->shardId) + .detail("DesiredShardId", currentShard->desiredShardId); + throw internal_error(); + } currentShard->moveInShard->cancel(); updatedMoveInShards.emplace(currentShard->moveInShard->id(), currentShard->moveInShard); StorageServerShard newShard = currentShard->toStorageServerShard(); @@ -10080,6 +10126,9 @@ void changeServerKeysWithPhysicalShards(StorageServer* data, .detail("Version", cVer); newEmptyRanges.push_back(range); updatedShards.emplace_back(range, cVer, desiredId, desiredId, StorageServerShard::ReadWrite); + if (data->physicalShards.find(desiredId) == data->physicalShards.end()) { + data->pendingAddRanges[cVer].emplace_back(desiredId, range); + } } else if (!nowAssigned) { if (dataAvailable) { ASSERT(data->newestAvailableVersion[range.begin] == @@ -10150,7 +10199,13 @@ void changeServerKeysWithPhysicalShards(StorageServer* data, .detailf("CurrentShard", "%016llx", shard->desiredShardId) .detail("IsTSS", data->isTss()) .detail("Version", cVer); - throw data_move_conflict(); + if (data->isTss() && g_network->isSimulated()) { + // Tss data move conflicts are expected in simulation, and can be safely ignored + // by restarting the server. + throw please_reboot(); + } else { + throw data_move_conflict(); + } } else { TraceEvent(SevInfo, "CSKMoveInToSameShard", data->thisServerID) .detail("DataMoveID", dataMoveId) diff --git a/fdbserver/tester.actor.cpp b/fdbserver/tester.actor.cpp index 05fa57c9226..73a0cf1abcf 100644 --- a/fdbserver/tester.actor.cpp +++ b/fdbserver/tester.actor.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include "flow/ActorCollection.h" @@ -969,6 +970,7 @@ ACTOR Future testerServerCore(TesterInterface interf, .detail("ConsistencyCheckerId", work.sharedRandomNumber) .detail("ClientId", work.clientId) .detail("ClientCount", work.clientCount); + work.reply.sendError(consistency_check_urgent_duplicate_request()); } else if (consistencyCheckerUrgentTester.second.isValid() && !consistencyCheckerUrgentTester.second.isReady()) { TraceEvent(SevWarnAlways, "ConsistencyCheckUrgent_TesterWorkloadConflict", interf.id()) @@ -976,13 +978,15 @@ ACTOR Future testerServerCore(TesterInterface interf, .detail("ArrivingConsistencyCheckerId", work.sharedRandomNumber) .detail("ClientId", work.clientId) .detail("ClientCount", work.clientCount); + work.reply.sendError(consistency_check_urgent_conflicting_request()); + } else { + consistencyCheckerUrgentTester = std::make_pair( + work.sharedRandomNumber, testerServerConsistencyCheckerUrgentWorkload(work, ccr, dbInfo)); + TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadInitialized", interf.id()) + .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first) + .detail("ClientId", work.clientId) + .detail("ClientCount", work.clientCount); } - consistencyCheckerUrgentTester = std::make_pair( - work.sharedRandomNumber, testerServerConsistencyCheckerUrgentWorkload(work, ccr, dbInfo)); - TraceEvent(SevInfo, "ConsistencyCheckUrgent_TesterWorkloadInitialized", interf.id()) - .detail("ConsistencyCheckerId", consistencyCheckerUrgentTester.first) - .detail("ClientId", work.clientId) - .detail("ClientCount", work.clientCount); } else { addWorkload.send(testerServerWorkload(work, ccr, dbInfo, locality)); } @@ -1737,7 +1741,13 @@ std::unordered_map> makeTaskAssignment(Database cx, std::vector shardsToCheck, int testersCount, int round) { + ASSERT(testersCount >= 1); std::unordered_map> assignment; + + std::vector shuffledIndices(testersCount); + std::iota(shuffledIndices.begin(), shuffledIndices.end(), 0); // creates [0, 1, ..., testersCount - 1] + deterministicRandom()->randomShuffle(shuffledIndices); + int batchSize = CLIENT_KNOBS->CONSISTENCY_CHECK_URGENT_BATCH_SHARD_COUNT; int startingPoint = 0; if (shardsToCheck.size() > batchSize * testersCount) { @@ -1752,7 +1762,17 @@ std::unordered_map> makeTaskAssignment(Database cx, if (testerIdx > testersCount - 1) { break; // Have filled up all testers } - assignment[testerIdx].push_back(shardsToCheck[i]); + // When assigning a shards/batch to a tester idx, there are certain edge cases which can result in urgent + // consistency checker being infinetely stuck in a loop. Examples: + // 1. if there is 1 remaining shard, and tester 0 consistently fails, we will still always pick tester 0 + // 2. if there are 10 remaining shards, and batch size is 10, and tester 0 consistently fails, we will + // still always pick tester 0 + // 3. if there are 20 remaining shards, and batch size is 10, and testers {0, 1} consistently fail, we will + // keep picking testers {0, 1} + // To avoid repeatedly picking the same testers even though they could be failing, shuffledIndices provides an + // indirection to a random tester idx. That way, each invocation of makeTaskAssignment won't + // result in the same task assignment for the class of edge cases mentioned above. + assignment[shuffledIndices[testerIdx]].push_back(shardsToCheck[i]); } std::unordered_map>::iterator assignIt; for (assignIt = assignment.begin(); assignIt != assignment.end(); assignIt++) { @@ -1840,7 +1860,7 @@ ACTOR Future runConsistencyCheckerUrgentCore(Reference writeKeysAndGetLatencyVersion(Database cx, ClientMetricWorkload* self, int numKeys, - int previousVS) { + uint64_t previousVS) { state int retry = 0; state int max_retry = 10; state int keysLimit = 1; diff --git a/fdbserver/workloads/ConsistencyCheckUrgent.actor.cpp b/fdbserver/workloads/ConsistencyCheckUrgent.actor.cpp index a1821224c92..7511e20f6c3 100644 --- a/fdbserver/workloads/ConsistencyCheckUrgent.actor.cpp +++ b/fdbserver/workloads/ConsistencyCheckUrgent.actor.cpp @@ -283,7 +283,8 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload { } wait(delay(5.0)); retryCount++; - if (retryCount > 50) { + if (retryCount > 10) { + // SS could be removed from the cluster throw timed_out(); } } @@ -304,6 +305,9 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload { req.begin = begin; req.end = firstGreaterOrEqual(range.end); req.limit = 1e4; + if (g_network->isSimulated() && SERVER_KNOBS->CONSISTENCY_CHECK_BACKWARD_READ) { + req.limit = -1e4; + } req.limitBytes = CLIENT_KNOBS->REPLY_BYTE_LIMIT; req.version = version; req.tags = TagSet(); @@ -324,6 +328,7 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload { valueAvailableToCheck = false; TraceEvent e(SevInfo, "ConsistencyCheckUrgent_TesterGetRangeError"); e.detail("ResultPresent", rangeResult.present()); + e.detail("StorageServer", storageServerInterfaces[j].uniqueID); if (rangeResult.present()) { e.detail("ErrorPresent", rangeResult.get().error.present()); if (rangeResult.get().error.present()) { @@ -331,6 +336,11 @@ struct ConsistencyCheckUrgentWorkload : TestWorkload { } } else { e.detail("ResultNotPresentWithError", rangeResult.getError().what()); + if (g_network->isSimulated() && + g_simulator->getProcessByAddress(storageServerInterfaces[j].address())->failed) { + e.detail("MachineFailed", "True"); + // skipCheck = true; + } } break; } diff --git a/fdbserver/workloads/Cycle.actor.cpp b/fdbserver/workloads/Cycle.actor.cpp index c522e8c86d7..dc05b23a04d 100644 --- a/fdbserver/workloads/Cycle.actor.cpp +++ b/fdbserver/workloads/Cycle.actor.cpp @@ -124,6 +124,11 @@ struct CycleWorkload : TestWorkload, CycleMembers { Key keyForIndex(int n) { return key(n); } Key key(int n) { return doubleToTestKey((double)n / nodeCount, keyPrefix); } Value value(int n) { return doubleToTestKey(n, keyPrefix); } + KeyRange keyRange(int n) { + Key beginKey = doubleToTestKey((double)n / nodeCount, keyPrefix); + Key endKey = beginKey.withSuffix(" end"_sr); + return KeyRangeRef(beginKey, endKey); + } int fromValue(const ValueRef& v) { return testKeyToDouble(v, keyPrefix); } Standalone operator()(int n) { return KeyValueRef(key(n), value((n + 1) % nodeCount)); } @@ -177,7 +182,11 @@ struct CycleWorkload : TestWorkload, CycleMembers { self->badRead("KeyR3", r3, tr); int r4 = self->fromValue(v3.get()); - tr.clear(self->key(r)); //< Shouldn't have an effect, but will break with wrong ordering + // Single key clear range op will be converted to point delete inside storage engine. Generating + // a larger range here to increase test coverage. + tr.clear( + self->keyRange(r), + AddConflictRange::True); //< Shouldn't have an effect, but will break with wrong ordering tr.set(self->key(r), self->value(r3)); tr.set(self->key(r2), self->value(r4)); tr.set(self->key(r3), self->value(r2)); diff --git a/fdbserver/workloads/DataLossRecovery.actor.cpp b/fdbserver/workloads/DataLossRecovery.actor.cpp index 62fc78fd2e2..b073df4c066 100644 --- a/fdbserver/workloads/DataLossRecovery.actor.cpp +++ b/fdbserver/workloads/DataLossRecovery.actor.cpp @@ -221,7 +221,9 @@ struct DataLossRecoveryWorkload : TestWorkload { TraceEvent("DataLossRecovery").detail("Phase", "StartMoveKeys"); std::unique_ptr params; if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) { - params = std::make_unique(deterministicRandom()->randomUniqueID(), + const UID dataMoveId = newDataMoveId( + deterministicRandom()->randomUInt64(), AssignEmptyRange::False, DataMoveType::LOGICAL); + params = std::make_unique(dataMoveId, std::vector{ keys }, dest, dest, diff --git a/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp b/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp index a7c29c0f04b..2d52518f0ab 100644 --- a/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp +++ b/fdbserver/workloads/IDDTxnProcessorApiCorrectness.actor.cpp @@ -345,7 +345,9 @@ struct IDDTxnProcessorApiWorkload : TestWorkload { std::vector destTeam = self->getRandomTeam(); std::sort(destTeam.begin(), destTeam.end()); if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) { - return MoveKeysParams(deterministicRandom()->randomUniqueID(), + const UID dataMoveId = + newDataMoveId(deterministicRandom()->randomUInt64(), AssignEmptyRange::False, DataMoveType::LOGICAL); + return MoveKeysParams(dataMoveId, std::vector{ keys }, destTeam, destTeam, diff --git a/fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp b/fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp index 81c3f302e76..7c364a1a627 100644 --- a/fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp +++ b/fdbserver/workloads/MockDDTrackerShardEvaluator.actor.cpp @@ -166,7 +166,8 @@ struct MockDDTrackerShardEvaluatorWorkload : public MockDDTestWorkload { .anyZeroHealthyTeams = zeroHealthyTeams, .shards = &shards, .trackerCancelled = &ddcx.trackerCancelled, - .ddTenantCache = {} }); + .ddTenantCache = {}, + .usableRegions = -1 }); actors.add(DataDistributionTracker::run(shardTracker, initData, getShardMetrics.getFuture(), diff --git a/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp b/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp index ae431e60152..06ade8c8efb 100644 --- a/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp +++ b/fdbserver/workloads/MutationLogReaderCorrectness.actor.cpp @@ -60,7 +60,7 @@ struct MutationLogReaderCorrectnessWorkload : TestWorkload { beginVersion = deterministicRandom()->randomInt64( 0, std::numeric_limits::max()); // intentionally not max of int64 - records = deterministicRandom()->randomInt(0, 1e6); + records = deterministicRandom()->randomInt(0, 1e5); versionRange = deterministicRandom()->randomInt64(records, std::numeric_limits::max()); versionIncrement = versionRange / (records + 1); diff --git a/fdbserver/workloads/PhysicalShardMove.actor.cpp b/fdbserver/workloads/PhysicalShardMove.actor.cpp index c901b36316a..24561c772a8 100644 --- a/fdbserver/workloads/PhysicalShardMove.actor.cpp +++ b/fdbserver/workloads/PhysicalShardMove.actor.cpp @@ -134,8 +134,14 @@ struct PhysicalShardMoveWorkLoad : TestWorkload { // Move range [TestKeyD, TestKeyF) to sh0; includes.insert(teamA.begin(), teamA.end()); currentRange = KeyRangeRef("TestKeyD"_sr, "TestKeyF"_sr); - state std::vector teamE = wait(self->moveShard( - self, cx, UID(sh0, deterministicRandom()->randomUInt64()), currentRange, teamSize, includes, excludes)); + state std::vector teamE = + wait(self->moveShard(self, + cx, + newDataMoveId(sh0, AssignEmptyRange::False, DataMoveType::PHYSICAL), + currentRange, + teamSize, + includes, + excludes)); TraceEvent(SevDebug, "TestMovedRange3").detail("Range", currentRange).detail("Team", describe(teamE)); ASSERT(std::equal(teamA.begin(), teamA.end(), teamE.begin())); @@ -160,8 +166,14 @@ struct PhysicalShardMoveWorkLoad : TestWorkload { // Move range [TestKeyB, TestKeyC) to sh1, on the same server. currentRange = KeyRangeRef("TestKeyB"_sr, "TestKeyC"_sr); includes.insert(teamA.begin(), teamA.end()); - state std::vector teamB = wait(self->moveShard( - self, cx, UID(sh1, deterministicRandom()->randomUInt64()), currentRange, teamSize, includes, excludes)); + state std::vector teamB = + wait(self->moveShard(self, + cx, + newDataMoveId(sh1, AssignEmptyRange::False, DataMoveType::PHYSICAL), + currentRange, + teamSize, + includes, + excludes)); TraceEvent(SevDebug, "TestMovedRange4").detail("Range", currentRange).detail("Team", describe(teamB)); ASSERT(std::equal(teamA.begin(), teamA.end(), teamB.begin())); diff --git a/fdbserver/workloads/RandomMoveKeys.actor.cpp b/fdbserver/workloads/RandomMoveKeys.actor.cpp index ac387f5ef27..7e31ac380aa 100644 --- a/fdbserver/workloads/RandomMoveKeys.actor.cpp +++ b/fdbserver/workloads/RandomMoveKeys.actor.cpp @@ -158,7 +158,9 @@ struct MoveKeysWorkload : FailureInjectionWorkload { state DDEnabledState ddEnabledState; std::unique_ptr params; if (SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA) { - params = std::make_unique(deterministicRandom()->randomUniqueID(), + const UID dataMoveId = newDataMoveId( + deterministicRandom()->randomUInt64(), AssignEmptyRange::False, DataMoveType::LOGICAL); + params = std::make_unique(dataMoveId, std::vector{ keys }, destinationTeamIDs, destinationTeamIDs, diff --git a/fdbserver/workloads/ValidateStorage.actor.cpp b/fdbserver/workloads/ValidateStorage.actor.cpp index 9cdc70a2c67..4ece80aee74 100644 --- a/fdbserver/workloads/ValidateStorage.actor.cpp +++ b/fdbserver/workloads/ValidateStorage.actor.cpp @@ -781,4 +781,4 @@ struct ValidateStorage : TestWorkload { void getMetrics(std::vector& m) override {} }; -WorkloadFactory ValidateStorageFactory; +WorkloadFactory ValidateStorageFactory; \ No newline at end of file diff --git a/flow/FastAlloc.cpp b/flow/FastAlloc.cpp index 61d4cbdfdc6..a105a55d246 100644 --- a/flow/FastAlloc.cpp +++ b/flow/FastAlloc.cpp @@ -664,6 +664,7 @@ template class FastAllocator<8192>; template class FastAllocator<16384>; #ifdef USE_JEMALLOC +#include TEST_CASE("/jemalloc/4k_aligned_usable_size") { void* ptr; try { diff --git a/flow/Knobs.cpp b/flow/Knobs.cpp index 4207127c75e..0f1b5618f1f 100644 --- a/flow/Knobs.cpp +++ b/flow/Knobs.cpp @@ -121,6 +121,9 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) { init( INCOMPATIBLE_PEER_DELAY_BEFORE_LOGGING, 5.0 ); init( PING_LOGGING_INTERVAL, 3.0 ); init( PING_SKETCH_ACCURACY, 0.1 ); + init( LOG_CONNECTION_ATTEMPTS_ENABLED, false ); + init( CONNECTION_LOG_DIRECTORY, "" ); + init( LOG_CONNECTION_INTERVAL_SECS, 3 ); init( TLS_CERT_REFRESH_DELAY_SECONDS, 12*60*60 ); init( TLS_SERVER_CONNECTION_THROTTLE_TIMEOUT, 9.0 ); @@ -202,7 +205,7 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) { //IAsyncFile init( INCREMENTAL_DELETE_TRUNCATE_AMOUNT, 5e8 ); //500MB init( INCREMENTAL_DELETE_INTERVAL, 1.0 ); //every 1 second - + //Net2 and FlowTransport init( MIN_COALESCE_DELAY, 10e-6 ); if( randomize && BUGGIFY ) MIN_COALESCE_DELAY = 0; init( MAX_COALESCE_DELAY, 20e-6 ); if( randomize && BUGGIFY ) MAX_COALESCE_DELAY = 0; diff --git a/flow/TLSConfig.actor.cpp b/flow/TLSConfig.actor.cpp index 591960b724a..c23685bdffa 100644 --- a/flow/TLSConfig.actor.cpp +++ b/flow/TLSConfig.actor.cpp @@ -567,7 +567,7 @@ TLSPolicy::Rule::Rule(std::string input) { s = eq + 3; } else { - std::map* criteria = &subject_criteria; + std::set>* criteria = &subject_criteria; if (term.find('.') != term.npos) { auto scoped = splitPair(term, '.'); @@ -592,7 +592,15 @@ TLSPolicy::Rule::Rule(std::string input) { NID termNID = abbrevToNID(term); const X509Location loc = locationForNID(termNID); - criteria->insert(std::make_pair(termNID, Criteria(unesc, mt, loc))); + auto criteriaToInsert = Criteria(unesc, mt, loc); + auto res = criteria->insert(std::make_pair(termNID, criteriaToInsert)); + if (!res.second) { + TraceEvent(SevWarn, "TLSKeyValueDuplicated") + .suppressFor(60.0) + .detail("TermNID", termNID) + .detail("NewCriteria", criteriaToInsert.criteria) + .detail("ExistingCriteria", res.first->second.criteria); + } if (remain != input.size() && input[remain] != ',') throw std::runtime_error("parse_verify"); diff --git a/flow/WriteOnlySet.actor.cpp b/flow/WriteOnlySet.actor.cpp index c4a0b785382..96d77922096 100644 --- a/flow/WriteOnlySet.actor.cpp +++ b/flow/WriteOnlySet.actor.cpp @@ -23,7 +23,6 @@ #include "flow/flow.h" #include "flow/UnitTest.h" -#include #include #include #include diff --git a/flow/include/flow/Knobs.h b/flow/include/flow/Knobs.h index c96ae005a6e..816135236c5 100644 --- a/flow/include/flow/Knobs.h +++ b/flow/include/flow/Knobs.h @@ -187,6 +187,9 @@ class FlowKnobs : public KnobsImpl { double INCOMPATIBLE_PEER_DELAY_BEFORE_LOGGING; double PING_LOGGING_INTERVAL; double PING_SKETCH_ACCURACY; + bool LOG_CONNECTION_ATTEMPTS_ENABLED; + int LOG_CONNECTION_INTERVAL_SECS; + std::string CONNECTION_LOG_DIRECTORY; int TLS_CERT_REFRESH_DELAY_SECONDS; double TLS_SERVER_CONNECTION_THROTTLE_TIMEOUT; diff --git a/flow/include/flow/TLSConfig.actor.h b/flow/include/flow/TLSConfig.actor.h index b80ac7a7ee7..fb171154a81 100644 --- a/flow/include/flow/TLSConfig.actor.h +++ b/flow/include/flow/TLSConfig.actor.h @@ -40,6 +40,9 @@ #include "flow/flow.h" #include + +#include "flow/actorcompiler.h" // This must be the last #include. + typedef int NID; enum class MatchType { @@ -68,9 +71,18 @@ struct Criteria { bool operator==(const Criteria& c) const { return criteria == c.criteria && match_type == c.match_type && location == c.location; } -}; -#include "flow/actorcompiler.h" // This must be the last #include. + bool operator<(const Criteria& c) const { + if (criteria != c.criteria) { + return criteria < c.criteria; + } else if (match_type != c.match_type) { + return match_type < c.match_type; + } else if (location != c.location) { + return location < c.location; + } + return false; + } +}; enum class TLSEndpointType { UNSET = 0, CLIENT, SERVER }; @@ -232,9 +244,9 @@ class TLSPolicy : ReferenceCounted { std::string toString() const; - std::map subject_criteria; - std::map issuer_criteria; - std::map root_criteria; + std::set> subject_criteria; // deduped and sorted + std::set> issuer_criteria; + std::set> root_criteria; bool verify_cert = true; bool verify_time = true; diff --git a/flow/include/flow/error_definitions.h b/flow/include/flow/error_definitions.h index e6b65458e5a..43c1edd925d 100755 --- a/flow/include/flow/error_definitions.h +++ b/flow/include/flow/error_definitions.h @@ -108,6 +108,8 @@ ERROR( duplicate_snapshot_request, 1083, "A duplicate snapshot request has been ERROR( dd_config_changed, 1084, "DataDistribution configuration changed." ) ERROR( consistency_check_urgent_task_failed, 1085, "Consistency check urgent task is failed") ERROR( data_move_conflict, 1086, "Data move conflict in SS") +ERROR( consistency_check_urgent_duplicate_request, 1087, "Consistency check urgent got a duplicate request") +ERROR( consistency_check_urgent_conflicting_request, 1088, "Consistency check urgent can process 1 workload at a time") ERROR( broken_promise, 1100, "Broken promise" ) ERROR( operation_cancelled, 1101, "Asynchronous operation cancelled" ) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 08490cf4e01..99ca09aab1e 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -288,26 +288,14 @@ if(WITH_PYTHON) add_fdb_test(TEST_FILES rare/StatusBuilderPerf.toml) add_fdb_test(TEST_FILES rare/TLogVersionMessagesOverheadFactor.toml) add_fdb_test( - TEST_FILES restarting/from_5.0.0_until_6.3.0/CycleTestRestart-1.txt - restarting/from_5.0.0_until_6.3.0/CycleTestRestart-2.txt) + TEST_FILES restarting/from_6.3.13_until_7.3.0/ClientTransactionProfilingCorrectness-1.txt + restarting/from_6.3.13_until_7.3.0/ClientTransactionProfilingCorrectness-2.txt) add_fdb_test( - TEST_FILES restarting/from_5.0.0_until_6.3.0/StorefrontTestRestart-1.txt - restarting/from_5.0.0_until_6.3.0/StorefrontTestRestart-2.txt) + TEST_FILES restarting/from_6.3.13_until_7.3.0/CycleTestRestart-1.txt + restarting/from_6.3.13_until_7.3.0/CycleTestRestart-2.txt) add_fdb_test( - TEST_FILES restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-1.txt - restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt IGNORE) - add_fdb_test( - TEST_FILES restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-1.txt - restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-2.txt) - add_fdb_test( - TEST_FILES restarting/from_6.3.13/ClientTransactionProfilingCorrectness-1.txt - restarting/from_6.3.13/ClientTransactionProfilingCorrectness-2.txt) - add_fdb_test( - TEST_FILES restarting/from_6.3.13/CycleTestRestart-1.txt - restarting/from_6.3.13/CycleTestRestart-2.txt) - add_fdb_test( - TEST_FILES restarting/from_6.3.13/StorefrontTestRestart-1.txt - restarting/from_6.3.13/StorefrontTestRestart-2.txt) + TEST_FILES restarting/from_6.3.13_until_7.3.0/StorefrontTestRestart-1.txt + restarting/from_6.3.13_until_7.3.0/StorefrontTestRestart-2.txt) add_fdb_test( TEST_FILES restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-1.txt restarting/from_6.3.13_until_7.2.0/DrUpgradeRestart-2.txt IGNORE) @@ -354,50 +342,107 @@ if(WITH_PYTHON) TEST_FILES restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-1.toml restarting/from_7.2.4_until_7.3.0/UpgradeAndBackupRestore-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/ConfigureTestRestart-1.toml - restarting/from_7.3.0/ConfigureTestRestart-2.toml) + TEST_FILES restarting/from_7.3.0_until_7.3.50/ConfigureTestRestart-1.toml + restarting/from_7.3.0_until_7.3.50/ConfigureTestRestart-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-1.toml - restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-2.toml) + TEST_FILES restarting/from_7.3.0_until_7.3.50/ConfigureStorageMigrationTestRestart-1.toml + restarting/from_7.3.0_until_7.3.50/ConfigureStorageMigrationTestRestart-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/DrUpgradeRestart-1.toml - restarting/from_7.3.0/DrUpgradeRestart-2.toml) + TEST_FILES restarting/from_7.3.0_until_7.3.50/DrUpgradeRestart-1.toml + restarting/from_7.3.0_until_7.3.50/DrUpgradeRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.0_until_7.3.50/UpgradeAndBackupRestore-1.toml + restarting/from_7.3.0_until_7.3.50/UpgradeAndBackupRestore-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.0_until_7.3.50/VersionVectorDisableRestart-1.toml + restarting/from_7.3.0_until_7.3.50/VersionVectorDisableRestart-2.toml IGNORE) + add_fdb_test( + TEST_FILES restarting/from_7.3.0_until_7.3.50/VersionVectorEnableRestart-1.toml + restarting/from_7.3.0_until_7.3.50/VersionVectorEnableRestart-2.toml IGNORE) add_fdb_test( - TEST_FILES restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml - restarting/from_7.3.0/UpgradeAndBackupRestore-2.toml) + TEST_FILES restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-1.toml + restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/VersionVectorDisableRestart-1.toml - restarting/from_7.3.0/VersionVectorDisableRestart-2.toml IGNORE) + TEST_FILES restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-1.toml + restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/VersionVectorEnableRestart-1.toml - restarting/from_7.3.0/VersionVectorEnableRestart-2.toml IGNORE) + TEST_FILES restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-1.toml + restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/BlobGranuleRestartCycle-1.toml - restarting/from_7.3.0/BlobGranuleRestartCycle-2.toml) + TEST_FILES restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-1.toml + restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/BlobGranuleRestartLarge-1.toml - restarting/from_7.3.0/BlobGranuleRestartLarge-2.toml) + TEST_FILES restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-1.toml + restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/SnapCycleRestart-1.txt - restarting/from_7.3.0/SnapCycleRestart-2.txt) + TEST_FILES restarting/from_7.3.0_until_7.3.50/SnapTestRestart-1.toml + restarting/from_7.3.0_until_7.3.50/SnapTestRestart-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/SnapTestAttrition-1.txt - restarting/from_7.3.0/SnapTestAttrition-2.txt) + TEST_FILES restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-1.toml + restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.0_until_7.3.50/CycleTestRestart-1.toml + restarting/from_7.3.0_until_7.3.50/CycleTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-1.toml + restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.49_until_7.3.50/ClientMetricRestart-1.toml + restarting/from_7.3.49_until_7.3.50/ClientMetricRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/ConfigureTestRestart-1.toml + restarting/from_7.3.50/ConfigureTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/ConfigureStorageMigrationTestRestart-1.toml + restarting/from_7.3.50/ConfigureStorageMigrationTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/DrUpgradeRestart-1.toml + restarting/from_7.3.50/DrUpgradeRestart-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/SnapTestSimpleRestart-1.txt - restarting/from_7.3.0/SnapTestSimpleRestart-2.txt) + TEST_FILES restarting/from_7.3.50/UpgradeAndBackupRestore-1.toml + restarting/from_7.3.50/UpgradeAndBackupRestore-2.toml) add_fdb_test( - TEST_FILES restarting/from_7.3.0/SnapTestRestart-1.txt - restarting/from_7.3.0/SnapTestRestart-2.txt) + TEST_FILES restarting/from_7.3.50/VersionVectorDisableRestart-1.toml + restarting/from_7.3.50/VersionVectorDisableRestart-2.toml IGNORE) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/VersionVectorEnableRestart-1.toml + restarting/from_7.3.50/VersionVectorEnableRestart-2.toml IGNORE) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/BlobGranuleRestartCycle-1.toml + restarting/from_7.3.50/BlobGranuleRestartCycle-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/BlobGranuleRestartLarge-1.toml + restarting/from_7.3.50/BlobGranuleRestartLarge-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/SnapCycleRestart-1.toml + restarting/from_7.3.50/SnapCycleRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/SnapTestAttrition-1.toml + restarting/from_7.3.50/SnapTestAttrition-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/SnapTestSimpleRestart-1.toml + restarting/from_7.3.50/SnapTestSimpleRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/SnapTestRestart-1.toml + restarting/from_7.3.50/SnapTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/ClientTransactionProfilingCorrectness-1.toml + restarting/from_7.3.50/ClientTransactionProfilingCorrectness-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/CycleTestRestart-1.toml + restarting/from_7.3.50/CycleTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/StorefrontTestRestart-1.toml + restarting/from_7.3.50/StorefrontTestRestart-2.toml) + add_fdb_test( + TEST_FILES restarting/from_7.3.50/ClientMetricRestart-1.toml + restarting/from_7.3.50/ClientMetricRestart-2.toml) add_fdb_test( TEST_FILES restarting/to_7.1.33_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml restarting/to_7.1.33_until_7.2.0/ConfigureStorageMigrationTestRestart-2.toml) add_fdb_test( TEST_FILES restarting/to_7.1.33_until_7.2.0/CycleTestRestart-1.toml restarting/to_7.1.33_until_7.2.0/CycleTestRestart-2.toml) - add_fdb_test( - TEST_FILES restarting/from_7.3.49/ClientMetricRestart-1.toml - restarting/from_7.3.49/ClientMetricRestart-2.toml) add_fdb_test( TEST_FILES restarting/to_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-1.toml restarting/to_7.2.0_until_7.3.0/ConfigureStorageMigrationTestRestart-2.toml IGNORE) diff --git a/tests/fast/AuthzSecurityWithBlobGranules.toml b/tests/fast/AuthzSecurityWithBlobGranules.toml index 8965eaa4730..0739cbb78b5 100644 --- a/tests/fast/AuthzSecurityWithBlobGranules.toml +++ b/tests/fast/AuthzSecurityWithBlobGranules.toml @@ -3,7 +3,7 @@ allowDefaultTenant = false tenantModes = ['optional', 'required'] blobGranulesEnabled = true # FIXME: re-enable rocks at some point -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[knobs]] audit_logging_enabled = false diff --git a/tests/fast/BlobRestoreBasic.toml b/tests/fast/BlobRestoreBasic.toml index d54590b746a..2415ff45766 100644 --- a/tests/fast/BlobRestoreBasic.toml +++ b/tests/fast/BlobRestoreBasic.toml @@ -4,7 +4,7 @@ blobGranulesEnabled = true extraDatabaseMode = 'Single' allowDefaultTenant = false disableTss = true -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[knobs]] bg_consistency_check_enabled = 0 diff --git a/tests/fast/BlobRestoreLarge.toml b/tests/fast/BlobRestoreLarge.toml index 43d7e4c1665..83a646cdf62 100644 --- a/tests/fast/BlobRestoreLarge.toml +++ b/tests/fast/BlobRestoreLarge.toml @@ -4,7 +4,7 @@ blobGranulesEnabled = true extraDatabaseMode = 'Single' allowDefaultTenant = false disableTss = true -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[knobs]] bg_consistency_check_enabled = 0 diff --git a/tests/fast/BlobRestoreToVersion.toml b/tests/fast/BlobRestoreToVersion.toml index 5e39422f3a8..9bccbe708c0 100644 --- a/tests/fast/BlobRestoreToVersion.toml +++ b/tests/fast/BlobRestoreToVersion.toml @@ -4,7 +4,7 @@ blobGranulesEnabled = true extraDatabaseMode = 'Single' allowDefaultTenant = false disableTss = true -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[knobs]] bg_consistency_check_enabled = 0 diff --git a/tests/fast/DataLossRecovery.toml b/tests/fast/DataLossRecovery.toml index a47ee131f47..ff64fc39b9e 100644 --- a/tests/fast/DataLossRecovery.toml +++ b/tests/fast/DataLossRecovery.toml @@ -1,7 +1,6 @@ [configuration] generateFearless = false config = 'triple' -storageEngineType = 0 processesPerMachine = 2 coordinators = 3 machineCount = 45 diff --git a/tests/restarting/from_5.0.0_until_6.3.0/StorefrontTestRestart-1.txt b/tests/restarting/from_5.0.0_until_6.3.0/StorefrontTestRestart-1.txt deleted file mode 100644 index b196a3bb5c5..00000000000 --- a/tests/restarting/from_5.0.0_until_6.3.0/StorefrontTestRestart-1.txt +++ /dev/null @@ -1,10 +0,0 @@ -testTitle=StorefrontTest -clearAfterTest=false -testName=Storefront -actorsPerClient=20 -transactionsPerSecond=200 -itemCount=20000 -maxOrderSize=4 -testName=SaveAndKill -restartInfoLocation=simfdb/restartInfo.ini -testDuration=10.0 diff --git a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-1.txt b/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-1.txt deleted file mode 100644 index c4ae7ad8e59..00000000000 --- a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-1.txt +++ /dev/null @@ -1,20 +0,0 @@ -extraDatabaseMode=Local - -testTitle=DrUpgrade -clearAfterTest=false -simBackupAgents=BackupToDB - - testName=Cycle - nodeCount=30000 - transactionsPerSecond=2500.0 - testDuration=30.0 - expectedRate=0 - - testName=BackupToDBUpgrade - backupAfter=10.0 - stopDifferentialAfter=50.0 - backupRangesCount=-1 - - testName=SaveAndKill - restartInfoLocation=simfdb/restartInfo.ini - testDuration=40.0 diff --git a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt b/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt deleted file mode 100644 index 366d8bc2a39..00000000000 --- a/tests/restarting/from_5.1.7_until_6.3.0/DrUpgradeRestart-2.txt +++ /dev/null @@ -1,18 +0,0 @@ -extraDatabaseMode=Local - -testTitle=DrUpgrade -runSetup=false -clearAfterTest=false -simBackupAgents=BackupToDB -waitForQuiescenceBegin=false - - testName=Cycle - nodeCount=30000 - transactionsPerSecond=2500.0 - testDuration=30.0 - expectedRate=0 - - testName=BackupToDBUpgrade - backupAfter=10.0 - backupRangesCount=-1 - stopDifferentialAfter=70.0 diff --git a/tests/restarting/from_6.3.13/StorefrontTestRestart-2.txt b/tests/restarting/from_6.3.13/StorefrontTestRestart-2.txt deleted file mode 100644 index 6a2a859714e..00000000000 --- a/tests/restarting/from_6.3.13/StorefrontTestRestart-2.txt +++ /dev/null @@ -1,7 +0,0 @@ -testTitle=StorefrontTest -runSetup=false -testName=Storefront -actorsPerClient=20 -transactionsPerSecond=200 -itemCount=20000 -maxOrderSize=4 diff --git a/tests/restarting/from_6.3.13/ClientTransactionProfilingCorrectness-1.txt b/tests/restarting/from_6.3.13_until_7.3.0/ClientTransactionProfilingCorrectness-1.txt similarity index 100% rename from tests/restarting/from_6.3.13/ClientTransactionProfilingCorrectness-1.txt rename to tests/restarting/from_6.3.13_until_7.3.0/ClientTransactionProfilingCorrectness-1.txt diff --git a/tests/restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-2.txt b/tests/restarting/from_6.3.13_until_7.3.0/ClientTransactionProfilingCorrectness-2.txt similarity index 100% rename from tests/restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-2.txt rename to tests/restarting/from_6.3.13_until_7.3.0/ClientTransactionProfilingCorrectness-2.txt diff --git a/tests/restarting/from_6.3.13/CycleTestRestart-1.txt b/tests/restarting/from_6.3.13_until_7.3.0/CycleTestRestart-1.txt similarity index 100% rename from tests/restarting/from_6.3.13/CycleTestRestart-1.txt rename to tests/restarting/from_6.3.13_until_7.3.0/CycleTestRestart-1.txt diff --git a/tests/restarting/from_5.0.0_until_6.3.0/CycleTestRestart-2.txt b/tests/restarting/from_6.3.13_until_7.3.0/CycleTestRestart-2.txt similarity index 100% rename from tests/restarting/from_5.0.0_until_6.3.0/CycleTestRestart-2.txt rename to tests/restarting/from_6.3.13_until_7.3.0/CycleTestRestart-2.txt diff --git a/tests/restarting/from_6.3.13/StorefrontTestRestart-1.txt b/tests/restarting/from_6.3.13_until_7.3.0/StorefrontTestRestart-1.txt similarity index 100% rename from tests/restarting/from_6.3.13/StorefrontTestRestart-1.txt rename to tests/restarting/from_6.3.13_until_7.3.0/StorefrontTestRestart-1.txt diff --git a/tests/restarting/from_5.0.0_until_6.3.0/StorefrontTestRestart-2.txt b/tests/restarting/from_6.3.13_until_7.3.0/StorefrontTestRestart-2.txt similarity index 100% rename from tests/restarting/from_5.0.0_until_6.3.0/StorefrontTestRestart-2.txt rename to tests/restarting/from_6.3.13_until_7.3.0/StorefrontTestRestart-2.txt diff --git a/tests/restarting/from_7.3.0/SnapCycleRestart-1.txt b/tests/restarting/from_7.3.0/SnapCycleRestart-1.txt deleted file mode 100644 index c98df2ffa2d..00000000000 --- a/tests/restarting/from_7.3.0/SnapCycleRestart-1.txt +++ /dev/null @@ -1,24 +0,0 @@ -storageEngineExcludeTypes=4,5 -logAntiQuorum=0 - -;Take snap and do cycle test -testTitle=SnapCyclePre -clearAfterTest=false - - testName=Cycle - transactionsPerSecond=2500.0 - nodeCount=2500 - testDuration=10.0 - expectedRate=0 - - testName=SnapTest - numSnaps=1 - maxSnapDelay=10.0 - testID=1 - -;save and shutdown -testTitle=SnapCycleShutdown - testName=SaveAndKill - restartInfoLocation=simfdb/restartInfo.ini - testDuration=10.0 - isRestoring=1 diff --git a/tests/restarting/from_7.3.0/SnapCycleRestart-2.txt b/tests/restarting/from_7.3.0/SnapCycleRestart-2.txt deleted file mode 100644 index 26680de04a8..00000000000 --- a/tests/restarting/from_7.3.0/SnapCycleRestart-2.txt +++ /dev/null @@ -1,13 +0,0 @@ -storageEngineExcludeTypes=4,5 -buggify=off - -testTitle=SnapCycleRestore -runSetup=false - - ;Post snap restore test - testName=Cycle - transactionsPerSecond=2500.0 - nodeCount=2500 - testDuration=10.0 - expectedRate=0 - enableDD=true diff --git a/tests/restarting/from_7.3.0/SnapTestAttrition-1.txt b/tests/restarting/from_7.3.0/SnapTestAttrition-1.txt deleted file mode 100644 index 709d62c505e..00000000000 --- a/tests/restarting/from_7.3.0/SnapTestAttrition-1.txt +++ /dev/null @@ -1,51 +0,0 @@ -storageEngineExcludeTypes=4,5 -logAntiQuorum=0 - -;write 1000 Keys ending with even numbers -testTitle=SnapTestPre -clearAfterTest=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=3.0 - testID=0 - -;Take snap and do read/write -testTitle=SnapTestTakeSnap -clearAfterTest=false - - testName=ReadWrite - testDuration=10.0 - transactionsPerSecond=10000 - writesPerTransactionA=0 - readsPerTransactionA=10 - writesPerTransactionB=10 - readsPerTransactionB=1 - alpha=0.5 - nodeCount=100000 - valueBytes=16 - discardEdgeMeasurements=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=10.0 - testID=1 - - testName=Attrition - testDuration=10.0 - -;write 1000 Keys ending with odd numbers -testTitle=SnapTestPost -clearAfterTest=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=25.0 - testID=2 - -; save and shutdown -testTitle=SnapSimpleShutdown - testName=SaveAndKill - restartInfoLocation=simfdb/restartInfo.ini - testDuration=10.0 - isRestoring=1 diff --git a/tests/restarting/from_7.3.0/SnapTestAttrition-2.txt b/tests/restarting/from_7.3.0/SnapTestAttrition-2.txt deleted file mode 100644 index c113a2e3043..00000000000 --- a/tests/restarting/from_7.3.0/SnapTestAttrition-2.txt +++ /dev/null @@ -1,12 +0,0 @@ -storageEngineExcludeTypes=4,5 - -buggify=off - -; verify all keys are even numbered -testTitle=SnapTestVerify - testName=SnapTest - numSnaps=1 - maxSnapDelay=3.0 - testID=3 - restartInfoLocation=simfdb/restartInfo.ini - enableDD=true diff --git a/tests/restarting/from_7.3.0/SnapTestRestart-1.txt b/tests/restarting/from_7.3.0/SnapTestRestart-1.txt deleted file mode 100644 index 3013a0fa0ae..00000000000 --- a/tests/restarting/from_7.3.0/SnapTestRestart-1.txt +++ /dev/null @@ -1,48 +0,0 @@ -storageEngineExcludeTypes=4,5 -logAntiQuorum=0 - -;write 1000 Keys ending with even numbers -testTitle=SnapTestPre -clearAfterTest=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=3.0 - testID=0 - -;Take snap and do read/write -testTitle=SnapTestTakeSnap -clearAfterTest=false - - testName=ReadWrite - testDuration=10.0 - transactionsPerSecond=10000 - writesPerTransactionA=0 - readsPerTransactionA=10 - writesPerTransactionB=10 - readsPerTransactionB=1 - alpha=0.5 - nodeCount=100000 - valueBytes=16 - discardEdgeMeasurements=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=10.0 - testID=1 - -;write 1000 Keys ending with odd numbers -testTitle=SnapTestPost -clearAfterTest=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=25.0 - testID=2 - -;save and shutdown -testTitle=SnapTestShutdown - testName=SaveAndKill - restartInfoLocation=simfdb/restartInfo.ini - testDuration=10.0 - isRestoring=1 diff --git a/tests/restarting/from_7.3.0/SnapTestRestart-2.txt b/tests/restarting/from_7.3.0/SnapTestRestart-2.txt deleted file mode 100644 index f01065eb73d..00000000000 --- a/tests/restarting/from_7.3.0/SnapTestRestart-2.txt +++ /dev/null @@ -1,11 +0,0 @@ -storageEngineExcludeTypes=4,5 - -buggify=off - -; verify all keys are even numbered -testTitle=SnapTestVerify - testName=SnapTest - numSnaps=1 - maxSnapDelay=3.0 - testID=3 - enableDD=true diff --git a/tests/restarting/from_7.3.0/SnapTestSimpleRestart-1.txt b/tests/restarting/from_7.3.0/SnapTestSimpleRestart-1.txt deleted file mode 100644 index 2068ee98ca8..00000000000 --- a/tests/restarting/from_7.3.0/SnapTestSimpleRestart-1.txt +++ /dev/null @@ -1,43 +0,0 @@ -storageEngineExcludeTypes=4,5 -logAntiQuorum=0 - -;write 1000 Keys ending with even number -testTitle=SnapSimplePre -clearAfterTest=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=30.0 - testID=0 - -;take snap -testTitle=SnapSimpleTakeSnap -clearAfterTest=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=5.0 - testID=1 - -;write 1000 Keys ending with odd number -testTitle=SnapSimplePost -clearAfterTest=false - - testName=SnapTest - numSnaps=1 - maxSnapDelay=3.0 - testID=2 - -;snapCreate with binary path that is not whitelisted -testTitle=SnapCreateNotWhitelistedBinaryPath - testName=SnapTest - numSnaps=1 - maxSnapDelay=3.0 - testID=4 - -;save and shutdown -testTitle=SnapSimpleShutdown - testName=SaveAndKill - restartInfoLocation=simfdb/restartInfo.ini - testDuration=10.0 - isRestoring=1 diff --git a/tests/restarting/from_7.3.0/SnapTestSimpleRestart-2.txt b/tests/restarting/from_7.3.0/SnapTestSimpleRestart-2.txt deleted file mode 100644 index c7cc7f4da1a..00000000000 --- a/tests/restarting/from_7.3.0/SnapTestSimpleRestart-2.txt +++ /dev/null @@ -1,11 +0,0 @@ -storageEngineExcludeTypes=4,5 - -buggify=off - -; verify all keys are even numbered -testTitle=SnapSimpleVerify -testName=SnapTest - numSnaps=1 - maxSnapDelay=3.0 - testID=3 - enableDD=true diff --git a/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-1.toml new file mode 100644 index 00000000000..020c1b5ec8e --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-1.toml @@ -0,0 +1,67 @@ +# Blob Granules are only upgrade-able as of snowflake/release-71.2.3 and release + +[configuration] +testClass = "BlobGranuleRestart" +blobGranulesEnabled = true +allowDefaultTenant = false +injectTargetedSSRestart = true +injectSSDelay = true +# FIXME: re-enable rocks at some point +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +enable_blob_granule_file_logical_size=false +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle = 'BlobGranuleRestartCycle' +clearAfterTest=false + + [[test.workload]] + testName = 'Cycle' + transactionsPerSecond = 500.0 + nodeCount = 2500 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 30.0 + # don't delete state after test + clearAndMergeCheck = false + doForcePurge = false + initAtEnd = false + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 30.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 30.0 + testDuration = 30.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 30.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 30.0 + + [[test.workload]] + testName = 'BlobFailureInjection' + testDuration = 30.0 + + [[test.workload]] + testName='SaveAndKill' + restartInfoLocation='simfdb/restartInfo.ini' + testDuration=30.0 diff --git a/tests/restarting/from_7.3.0/BlobGranuleRestartCycle-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-2.toml similarity index 97% rename from tests/restarting/from_7.3.0/BlobGranuleRestartCycle-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-2.toml index b94ba0d1315..b2c13e52209 100644 --- a/tests/restarting/from_7.3.0/BlobGranuleRestartCycle-2.toml +++ b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartCycle-2.toml @@ -7,7 +7,7 @@ allowDefaultTenant = false injectTargetedSSRestart = true injectSSDelay = true # FIXME: re-enable rocks at some point -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[test]] testTitle = 'BlobGranuleRestartCycle' diff --git a/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-1.toml new file mode 100644 index 00000000000..d645b15c41f --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-1.toml @@ -0,0 +1,75 @@ +# Blob Granules are only upgrade-able as of snowflake/release-71.2.3 and release + +[configuration] +testClass = "BlobGranuleRestart" +blobGranulesEnabled = true +allowDefaultTenant = false +injectTargetedSSRestart = true +injectSSDelay = true +# FIXME: re-enable rocks at some point +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +enable_blob_granule_file_logical_size=false +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle = 'BlobGranuleRestartLarge' +clearAfterTest=false + + [[test.workload]] + testName = 'ReadWrite' + testDuration = 60.0 + transactionsPerSecond = 200 + writesPerTransactionA = 5 + readsPerTransactionA = 1 + writesPerTransactionB = 10 + readsPerTransactionB = 1 + alpha = 0.5 + nodeCount = 2000000 + valueBytes = 128 + discardEdgeMeasurements = false + warmingDelay = 10.0 + setup = false + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 60.0 + # don't delete state after test + clearAndMergeCheck = false + doForcePurge = false + initAtEnd = false + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 60.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 60.0 + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'BlobFailureInjection' + testDuration = 60.0 + + [[test.workload]] + testName='SaveAndKill' + restartInfoLocation='simfdb/restartInfo.ini' + testDuration=60.0 diff --git a/tests/restarting/from_7.3.0/BlobGranuleRestartLarge-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-2.toml similarity index 97% rename from tests/restarting/from_7.3.0/BlobGranuleRestartLarge-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-2.toml index 1dd539f0911..b1c45c4df08 100644 --- a/tests/restarting/from_7.3.0/BlobGranuleRestartLarge-2.toml +++ b/tests/restarting/from_7.3.0_until_7.3.50/BlobGranuleRestartLarge-2.toml @@ -7,7 +7,7 @@ allowDefaultTenant = false injectTargetedSSRestart = true injectSSDelay = true # FIXME: re-enable rocks at some point -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[test]] testTitle = 'BlobGranuleRestartLarge' diff --git a/tests/restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-1.toml new file mode 100644 index 00000000000..3c722caf616 --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-1.toml @@ -0,0 +1,41 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle="ClientTransactionProfilingCorrectness" +clearAfterTest=false +runSetup=true +timeout=2100 + + [[test.workload]] + testName="ApiCorrectness" + numKeys=5000 + onlyLowerCase=true + shortKeysRatio=0.5 + minShortKeyLength=1 + maxShortKeyLength=3 + minLongKeyLength=1 + maxLongKeyLength=128 + minValueLength=1 + maxValueLength=1000 + numGets=1000 + numGetRanges=100 + numGetRangeSelectors=100 + numGetKeys=100 + numClears=100 + numClearRanges=10 + maxTransactionBytes=500000 + randomTestDuration=30 + + [[test.workload]] + testName="ClientTransactionProfileCorrectness" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=60 diff --git a/tests/restarting/from_6.3.13/ClientTransactionProfilingCorrectness-2.txt b/tests/restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-2.toml similarity index 70% rename from tests/restarting/from_6.3.13/ClientTransactionProfilingCorrectness-2.txt rename to tests/restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-2.toml index 8e799db151c..1ed1755d886 100644 --- a/tests/restarting/from_6.3.13/ClientTransactionProfilingCorrectness-2.txt +++ b/tests/restarting/from_7.3.0_until_7.3.50/ClientTransactionProfilingCorrectness-2.toml @@ -1,9 +1,11 @@ -testTitle=ClientTransactionProfilingCorrectness +[[test]] +testTitle="ClientTransactionProfilingCorrectness" clearAfterTest=true timeout=2100 runSetup=true - testName=ApiCorrectness + [[test.workload]] + testName="ApiCorrectness" numKeys=5000 onlyLowerCase=true shortKeysRatio=0.5 @@ -22,5 +24,6 @@ runSetup=true maxTransactionBytes=500000 randomTestDuration=60 - testName=ClientTransactionProfileCorrectness + [[test.workload]] + testName="ClientTransactionProfileCorrectness" diff --git a/tests/restarting/from_7.3.0_until_7.3.50/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/ConfigureStorageMigrationTestRestart-1.toml new file mode 100644 index 00000000000..f0502b6f623 --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/ConfigureStorageMigrationTestRestart-1.toml @@ -0,0 +1,33 @@ +[configuration] +extraMachineCountDC = 2 +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle = 'CloggedConfigureDatabaseTest' +clearAfterTest = false + + [[test.workload]] + testName = 'ConfigureDatabase' + testDuration = 30.0 + allowTestStorageMigration = true + allowDescriptorChange = false + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 30.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 30.0 + scale = 0.1 + clogginess = 2.0 + + [[test.workload]] + testName='SaveAndKill' + restartInfoLocation='simfdb/restartInfo.ini' + testDuration=30.0 \ No newline at end of file diff --git a/tests/restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/ConfigureStorageMigrationTestRestart-2.toml similarity index 100% rename from tests/restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/ConfigureStorageMigrationTestRestart-2.toml diff --git a/tests/restarting/from_7.3.0_until_7.3.50/ConfigureTestRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/ConfigureTestRestart-1.toml new file mode 100644 index 00000000000..c603893a91c --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/ConfigureTestRestart-1.toml @@ -0,0 +1,31 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle='CloggedConfigureDatabaseTest' +clearAfterTest=false + + [[test.workload]] + testName='ConfigureDatabase' + testDuration=30.0 + allowDescriptorChange=false + + [[test.workload]] + testName='RandomClogging' + testDuration=30.0 + + [[test.workload]] + testName='RandomClogging' + testDuration=30.0 + scale=0.1 + clogginess=2.0 + + [[test.workload]] + testName='SaveAndKill' + restartInfoLocation='simfdb/restartInfo.ini' + testDuration=30.0 diff --git a/tests/restarting/from_7.3.0/ConfigureTestRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/ConfigureTestRestart-2.toml similarity index 100% rename from tests/restarting/from_7.3.0/ConfigureTestRestart-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/ConfigureTestRestart-2.toml diff --git a/tests/restarting/from_7.3.0_until_7.3.50/CycleTestRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/CycleTestRestart-1.toml new file mode 100644 index 00000000000..08126de1425 --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/CycleTestRestart-1.toml @@ -0,0 +1,46 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle="Clogged" +clearAfterTest=false + + [[test.workload]] + testName="Cycle" + transactionsPerSecond=500.0 + nodeCount=2500 + testDuration=10.0 + expectedRate=0 + + [[test.workload]] + testName="RandomClogging" + testDuration=10.0 + + [[test.workload]] + testName="Rollback" + meanDelay=10.0 + testDuration=10.0 + + [[test.workload]] + testName="Attrition" + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName="Attrition" + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 diff --git a/tests/restarting/from_6.3.13/CycleTestRestart-2.txt b/tests/restarting/from_7.3.0_until_7.3.50/CycleTestRestart-2.toml similarity index 56% rename from tests/restarting/from_6.3.13/CycleTestRestart-2.txt rename to tests/restarting/from_7.3.0_until_7.3.50/CycleTestRestart-2.toml index 7eb6c64beab..7caffb3afcf 100644 --- a/tests/restarting/from_6.3.13/CycleTestRestart-2.txt +++ b/tests/restarting/from_7.3.0_until_7.3.50/CycleTestRestart-2.toml @@ -1,26 +1,32 @@ -testTitle=Clogged +[[test]] +testTitle="Clogged" runSetup=false - testName=Cycle + [[test.workload]] + testName="Cycle" transactionsPerSecond=2500.0 nodeCount=2500 testDuration=10.0 expectedRate=0 - testName=RandomClogging + [[test.workload]] + testName="RandomClogging" testDuration=10.0 - testName=Rollback + [[test.workload]] + testName="Rollback" meanDelay=10.0 testDuration=10.0 - testName=Attrition + [[test.workload]] + testName="Attrition" machinesToKill=10 machinesToLeave=3 reboot=true testDuration=10.0 - testName=Attrition + [[test.workload]] + testName="Attrition" machinesToKill=10 machinesToLeave=3 reboot=true diff --git a/tests/restarting/from_7.3.0_until_7.3.50/DrUpgradeRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/DrUpgradeRestart-1.toml new file mode 100644 index 00000000000..fcf8864426d --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/DrUpgradeRestart-1.toml @@ -0,0 +1,34 @@ +[configuration] +extraDatabaseMode = "Local" +# In 7.2, DR is not supported in required tenant mode +allowDefaultTenant = false +encryptModes = ['disabled'] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [4, 5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle = "DrUpgrade" +clearAfterTest = false +simBackupAgents = "BackupToDB" + + [[test.workload]] + testName = "Cycle" + nodeCount = 30000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = "BackupToDBUpgrade" + backupAfter = 10.0 + stopDifferentialAfter = 50.0 + backupRangesCount = -1 + + [[test.workload]] + testName = "SaveAndKill" + restartInfoLocation = "simfdb/restartInfo.ini" + testDuration = 40.0 diff --git a/tests/restarting/from_7.3.0/DrUpgradeRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/DrUpgradeRestart-2.toml similarity index 100% rename from tests/restarting/from_7.3.0/DrUpgradeRestart-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/DrUpgradeRestart-2.toml diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-1.toml new file mode 100644 index 00000000000..6c80d0fda9d --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-1.toml @@ -0,0 +1,36 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +# Take snap and do cycle test +testTitle="SnapCyclePre" +clearAfterTest=false + + [[test.workload]] + testName="Cycle" + transactionsPerSecond=2500.0 + nodeCount=2500 + testDuration=10.0 + expectedRate=0 + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=10.0 + testID=1 + +[[test]] +# save and shutdown +testTitle="SnapCycleShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-2.toml new file mode 100644 index 00000000000..77be982536f --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapCycleRestart-2.toml @@ -0,0 +1,16 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +testTitle="SnapCycleRestore" +runSetup=false + + [[test.workload]] + # Post snap restore test + testName="Cycle" + transactionsPerSecond=2500.0 + nodeCount=2500 + testDuration=10.0 + expectedRate=0 + enableDD=true diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-1.toml new file mode 100644 index 00000000000..ef16ff776d5 --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-1.toml @@ -0,0 +1,68 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +# write 1000 Keys ending with even numbers +testTitle="SnapTestPre" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=0 + +[[test]] +# Take snap and do read/write +testTitle="SnapTestTakeSnap" +clearAfterTest=false + + [[test.workload]] + testName="ReadWrite" + testDuration=10.0 + transactionsPerSecond=10000 + writesPerTransactionA=0 + readsPerTransactionA=10 + writesPerTransactionB=10 + readsPerTransactionB=1 + alpha=0.5 + nodeCount=100000 + valueBytes=16 + discardEdgeMeasurements=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=10.0 + testID=1 + + [[test.workload]] + testName="Attrition" + testDuration=10.0 + +[[test]] +# write 1000 Keys ending with odd numbers +testTitle="SnapTestPost" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=25.0 + testID=2 + +[[test]] +# save and shutdown +testTitle="SnapSimpleShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-2.toml new file mode 100644 index 00000000000..5ac004965ce --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestAttrition-2.toml @@ -0,0 +1,15 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +# verify all keys are even numbered +testTitle="SnapTestVerify" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=3 + restartInfoLocation="simfdb/restartInfo.ini" + enableDD=true diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapTestRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestRestart-1.toml new file mode 100644 index 00000000000..9ba9306a40e --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestRestart-1.toml @@ -0,0 +1,64 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +# write 1000 Keys ending with even numbers +testTitle="SnapTestPre" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=0 + +[[test]] +# Take snap and do read/write +testTitle="SnapTestTakeSnap" +clearAfterTest=false + + [[test.workload]] + testName="ReadWrite" + testDuration=10.0 + transactionsPerSecond=10000 + writesPerTransactionA=0 + readsPerTransactionA=10 + writesPerTransactionB=10 + readsPerTransactionB=1 + alpha=0.5 + nodeCount=100000 + valueBytes=16 + discardEdgeMeasurements=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=10.0 + testID=1 + +[[test]] +# write 1000 Keys ending with odd numbers +testTitle="SnapTestPost" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=25.0 + testID=2 + +[[test]] +# save and shutdown +testTitle="SnapTestShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapTestRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestRestart-2.toml new file mode 100644 index 00000000000..121fd5b24bd --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestRestart-2.toml @@ -0,0 +1,14 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +# verify all keys are even numbered +testTitle="SnapTestVerify" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=3 + enableDD=true diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-1.toml new file mode 100644 index 00000000000..f00ad6271a3 --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-1.toml @@ -0,0 +1,61 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +# write 1000 Keys ending with even number +testTitle="SnapSimplePre" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=30.0 + testID=0 + +[[test]] +# take snap +testTitle="SnapSimpleTakeSnap" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=5.0 + testID=1 + +[[test]] +# write 1000 Keys ending with odd number +testTitle="SnapSimplePost" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=2 + +[[test]] +# snapCreate with binary path that is not whitelisted +testTitle="SnapCreateNotWhitelistedBinaryPath" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=4 + +[[test]] +# save and shutdown +testTitle="SnapSimpleShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-2.toml new file mode 100644 index 00000000000..6706fc48b8e --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/SnapTestSimpleRestart-2.toml @@ -0,0 +1,14 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +# verify all keys are even numbered +testTitle="SnapSimpleVerify" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=3 + enableDD=true diff --git a/tests/restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-1.toml new file mode 100644 index 00000000000..2e7fc8720ac --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-1.toml @@ -0,0 +1,23 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle="StorefrontTest" +clearAfterTest=false + + [[test.workload]] + testName="Storefront" + actorsPerClient=20 + transactionsPerSecond=200 + itemCount=20000 + maxOrderSize=4 + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 diff --git a/tests/restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-2.toml new file mode 100644 index 00000000000..c25a3252c32 --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/StorefrontTestRestart-2.toml @@ -0,0 +1,10 @@ +[[test]] +testTitle="StorefrontTest" +runSetup=false + + [[test.workload]] + testName="Storefront" + actorsPerClient=20 + transactionsPerSecond=200 + itemCount=20000 + maxOrderSize=4 diff --git a/tests/restarting/from_7.3.0_until_7.3.50/UpgradeAndBackupRestore-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/UpgradeAndBackupRestore-1.toml new file mode 100644 index 00000000000..79fb477eebd --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/UpgradeAndBackupRestore-1.toml @@ -0,0 +1,72 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle = 'SubmitBackup' +simBackupAgents= 'BackupToFile' +clearAfterTest = false +runConsistencyCheck=false +disabledFailureInjectionWorkloads = 'Attrition' + + [[test.workload]] + testName = 'SubmitBackup' + delayFor = 0 + stopWhenDone = false + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 30.0 + +[[test]] +testTitle = 'FirstCycleTest' +clearAfterTest=false +runConsistencyCheck = false +disabledFailureInjectionWorkloads = 'Attrition' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 30000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = 'BeforeRestart' + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 90.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 90.0 + testDuration = 90.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 90.0 + + [[test.workload]] + testName='Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 90.0 + +[[test]] +testTitle = 'SaveDatabase' +clearAfterTest = false + + [[test.workload]] + testName = 'SaveAndKill' + restartInfoLocation = 'simfdb/restartInfo.ini' + testDuration=30.0 diff --git a/tests/restarting/from_7.3.0/UpgradeAndBackupRestore-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/UpgradeAndBackupRestore-2.toml similarity index 100% rename from tests/restarting/from_7.3.0/UpgradeAndBackupRestore-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/UpgradeAndBackupRestore-2.toml diff --git a/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorDisableRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorDisableRestart-1.toml new file mode 100644 index 00000000000..a7ccff9a527 --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorDisableRestart-1.toml @@ -0,0 +1,66 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +enable_version_vector = true +enable_version_vector_tlog_unicast = true +proxy_use_resolver_private_mutations = true +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle='VersionVectorDowngrade' +clearAfterTest=false + + [[test.workload]] + testName='Cycle' + transactionsPerSecond=2500.0 + nodeCount=1000 + testDuration=30.0 + expectedRate=0 + keyPrefix = 'cycle' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = '!' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = 'ZZZ' + + [[test.workload]] + testName='RandomClogging' + testDuration=10.0 + + [[test.workload]] + testName='Rollback' + meanDelay=10.0 + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName='SaveAndKill' + restartInfoLocation='simfdb/restartInfo.ini' + testDuration=60.0 diff --git a/tests/restarting/from_7.3.0/VersionVectorDisableRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorDisableRestart-2.toml similarity index 100% rename from tests/restarting/from_7.3.0/VersionVectorDisableRestart-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/VersionVectorDisableRestart-2.toml diff --git a/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorEnableRestart-1.toml b/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorEnableRestart-1.toml new file mode 100644 index 00000000000..f5bd18433de --- /dev/null +++ b/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorEnableRestart-1.toml @@ -0,0 +1,65 @@ +[configuration] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +enable_version_vector = false +enable_version_vector_tlog_unicast = false +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle='VersionVectorUpgrade' +clearAfterTest=false + + [[test.workload]] + testName='Cycle' + transactionsPerSecond=2500.0 + nodeCount=1000 + testDuration=30.0 + expectedRate=0 + keyPrefix = 'cycle' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = '!' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = 'ZZZ' + + [[test.workload]] + testName='RandomClogging' + testDuration=10.0 + + [[test.workload]] + testName='Rollback' + meanDelay=10.0 + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName='SaveAndKill' + restartInfoLocation='simfdb/restartInfo.ini' + testDuration=60.0 diff --git a/tests/restarting/from_7.3.0/VersionVectorEnableRestart-2.toml b/tests/restarting/from_7.3.0_until_7.3.50/VersionVectorEnableRestart-2.toml similarity index 100% rename from tests/restarting/from_7.3.0/VersionVectorEnableRestart-2.toml rename to tests/restarting/from_7.3.0_until_7.3.50/VersionVectorEnableRestart-2.toml diff --git a/tests/restarting/from_7.3.49_until_7.3.50/ClientMetricRestart-1.toml b/tests/restarting/from_7.3.49_until_7.3.50/ClientMetricRestart-1.toml new file mode 100644 index 00000000000..7ce53472dad --- /dev/null +++ b/tests/restarting/from_7.3.49_until_7.3.50/ClientMetricRestart-1.toml @@ -0,0 +1,24 @@ +[configuration] +tenantModes = ['disabled'] +# For ShardedRocksDB, an very old 7.3 binary is not compatible to the new binary +storageEngineExcludeTypes = [5] + +[[knobs]] +# For ShardEncodeLocationMetadata, an very old 7.3 binary is not compatible to the new binary +shard_encode_location_metadata = false + +[[test]] +testTitle='ClientMetricRestartTest' +clearAfterTest=false +waitForQuiescenceEnd=false + + [[test.workload]] + testName='ClientMetric' + toSet=true + samplingProbability=0.01 + testDuration=500.0 + + [[test.workload]] + testName='SaveAndKill' + restartInfoLocation='simfdb/restartInfo.ini' + testDuration=500.0 \ No newline at end of file diff --git a/tests/restarting/from_7.3.49/ClientMetricRestart-2.toml b/tests/restarting/from_7.3.49_until_7.3.50/ClientMetricRestart-2.toml similarity index 100% rename from tests/restarting/from_7.3.49/ClientMetricRestart-2.toml rename to tests/restarting/from_7.3.49_until_7.3.50/ClientMetricRestart-2.toml diff --git a/tests/restarting/from_7.3.0/BlobGranuleRestartCycle-1.toml b/tests/restarting/from_7.3.50/BlobGranuleRestartCycle-1.toml similarity index 97% rename from tests/restarting/from_7.3.0/BlobGranuleRestartCycle-1.toml rename to tests/restarting/from_7.3.50/BlobGranuleRestartCycle-1.toml index a06262654fa..ad178bb71f5 100644 --- a/tests/restarting/from_7.3.0/BlobGranuleRestartCycle-1.toml +++ b/tests/restarting/from_7.3.50/BlobGranuleRestartCycle-1.toml @@ -7,7 +7,7 @@ allowDefaultTenant = false injectTargetedSSRestart = true injectSSDelay = true # FIXME: re-enable rocks at some point -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[knobs]] enable_blob_granule_file_logical_size=false diff --git a/tests/restarting/from_7.3.50/BlobGranuleRestartCycle-2.toml b/tests/restarting/from_7.3.50/BlobGranuleRestartCycle-2.toml new file mode 100644 index 00000000000..b2c13e52209 --- /dev/null +++ b/tests/restarting/from_7.3.50/BlobGranuleRestartCycle-2.toml @@ -0,0 +1,55 @@ +# Blob Granules are only upgrade-able as of snowflake/release-71.2.3 and release-7.2 + +[configuration] +testClass = "BlobGranuleRestart" +blobGranulesEnabled = true +allowDefaultTenant = false +injectTargetedSSRestart = true +injectSSDelay = true +# FIXME: re-enable rocks at some point +storageEngineExcludeTypes = [5] + +[[test]] +testTitle = 'BlobGranuleRestartCycle' +clearAfterTest=false +runSetup=false + + [[test.workload]] + testName = 'Cycle' + transactionsPerSecond = 2500.0 + nodeCount = 2500 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 30.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 30.0 + testDuration = 30.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 30.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 30.0 + + [[test.workload]] + testName = 'BlobFailureInjection' + testDuration = 30.0 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 30.0 + # cycle does its own workload checking, don't want clear racing with its checking + clearAndMergeCheck = false diff --git a/tests/restarting/from_7.3.0/BlobGranuleRestartLarge-1.toml b/tests/restarting/from_7.3.50/BlobGranuleRestartLarge-1.toml similarity index 97% rename from tests/restarting/from_7.3.0/BlobGranuleRestartLarge-1.toml rename to tests/restarting/from_7.3.50/BlobGranuleRestartLarge-1.toml index 49d2b3cee18..4f2f0b4d362 100644 --- a/tests/restarting/from_7.3.0/BlobGranuleRestartLarge-1.toml +++ b/tests/restarting/from_7.3.50/BlobGranuleRestartLarge-1.toml @@ -7,7 +7,7 @@ allowDefaultTenant = false injectTargetedSSRestart = true injectSSDelay = true # FIXME: re-enable rocks at some point -storageEngineExcludeTypes = [4, 5] +storageEngineExcludeTypes = [5] [[knobs]] enable_blob_granule_file_logical_size=false diff --git a/tests/restarting/from_7.3.50/BlobGranuleRestartLarge-2.toml b/tests/restarting/from_7.3.50/BlobGranuleRestartLarge-2.toml new file mode 100644 index 00000000000..b1c45c4df08 --- /dev/null +++ b/tests/restarting/from_7.3.50/BlobGranuleRestartLarge-2.toml @@ -0,0 +1,61 @@ +# Blob Granules are only upgrade-able as of snowflake/release-71.2.3 and release-7.2 + +[configuration] +testClass = "BlobGranuleRestart" +blobGranulesEnabled = true +allowDefaultTenant = false +injectTargetedSSRestart = true +injectSSDelay = true +# FIXME: re-enable rocks at some point +storageEngineExcludeTypes = [5] + +[[test]] +testTitle = 'BlobGranuleRestartLarge' +clearAfterTest=false +runSetup=false + + [[test.workload]] + testName = 'ReadWrite' + testDuration = 60.0 + transactionsPerSecond = 200 + writesPerTransactionA = 5 + readsPerTransactionA = 1 + writesPerTransactionB = 10 + readsPerTransactionB = 1 + alpha = 0.5 + nodeCount = 2000000 + valueBytes = 128 + discardEdgeMeasurements = false + warmingDelay = 10.0 + setup = false + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 60.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 60.0 + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 60.0 + + [[test.workload]] + testName = 'BlobFailureInjection' + testDuration = 60.0 + + [[test.workload]] + testName = 'BlobGranuleVerifier' + testDuration = 60.0 diff --git a/tests/restarting/from_7.3.49/ClientMetricRestart-1.toml b/tests/restarting/from_7.3.50/ClientMetricRestart-1.toml similarity index 100% rename from tests/restarting/from_7.3.49/ClientMetricRestart-1.toml rename to tests/restarting/from_7.3.50/ClientMetricRestart-1.toml diff --git a/tests/restarting/from_7.3.50/ClientMetricRestart-2.toml b/tests/restarting/from_7.3.50/ClientMetricRestart-2.toml new file mode 100644 index 00000000000..4a81d94b890 --- /dev/null +++ b/tests/restarting/from_7.3.50/ClientMetricRestart-2.toml @@ -0,0 +1,10 @@ +[[test]] +testTitle='ClientMetricRestartTest' +clearAfterTest=false +waitForQuiescenceEnd=false +runConsistencyCheck=false + + [[test.workload]] + testName='ClientMetric' + toSet=false + testDuration=500.0 \ No newline at end of file diff --git a/tests/restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-1.txt b/tests/restarting/from_7.3.50/ClientTransactionProfilingCorrectness-1.toml similarity index 61% rename from tests/restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-1.txt rename to tests/restarting/from_7.3.50/ClientTransactionProfilingCorrectness-1.toml index 963051506f4..47dc6b7f928 100644 --- a/tests/restarting/from_5.2.0_until_6.3.0/ClientTransactionProfilingCorrectness-1.txt +++ b/tests/restarting/from_7.3.50/ClientTransactionProfilingCorrectness-1.toml @@ -1,9 +1,11 @@ -testTitle=ClientTransactionProfilingCorrectness +[[test]] +testTitle="ClientTransactionProfilingCorrectness" clearAfterTest=false runSetup=true timeout=2100 - testName=ApiCorrectness + [[test.workload]] + testName="ApiCorrectness" numKeys=5000 onlyLowerCase=true shortKeysRatio=0.5 @@ -22,8 +24,10 @@ timeout=2100 maxTransactionBytes=500000 randomTestDuration=30 - testName=ClientTransactionProfileCorrectness + [[test.workload]] + testName="ClientTransactionProfileCorrectness" - testName=SaveAndKill - restartInfoLocation=simfdb/restartInfo.ini + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" testDuration=60 diff --git a/tests/restarting/from_7.3.50/ClientTransactionProfilingCorrectness-2.toml b/tests/restarting/from_7.3.50/ClientTransactionProfilingCorrectness-2.toml new file mode 100644 index 00000000000..1ed1755d886 --- /dev/null +++ b/tests/restarting/from_7.3.50/ClientTransactionProfilingCorrectness-2.toml @@ -0,0 +1,29 @@ +[[test]] +testTitle="ClientTransactionProfilingCorrectness" +clearAfterTest=true +timeout=2100 +runSetup=true + + [[test.workload]] + testName="ApiCorrectness" + numKeys=5000 + onlyLowerCase=true + shortKeysRatio=0.5 + minShortKeyLength=1 + maxShortKeyLength=3 + minLongKeyLength=1 + maxLongKeyLength=128 + minValueLength=1 + maxValueLength=1000 + numGets=1000 + numGetRanges=100 + numGetRangeSelectors=100 + numGetKeys=100 + numClears=100 + numClearRanges=10 + maxTransactionBytes=500000 + randomTestDuration=60 + + [[test.workload]] + testName="ClientTransactionProfileCorrectness" + diff --git a/tests/restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/from_7.3.50/ConfigureStorageMigrationTestRestart-1.toml similarity index 100% rename from tests/restarting/from_7.3.0/ConfigureStorageMigrationTestRestart-1.toml rename to tests/restarting/from_7.3.50/ConfigureStorageMigrationTestRestart-1.toml diff --git a/tests/restarting/from_7.3.50/ConfigureStorageMigrationTestRestart-2.toml b/tests/restarting/from_7.3.50/ConfigureStorageMigrationTestRestart-2.toml new file mode 100644 index 00000000000..e9ae14f0c3e --- /dev/null +++ b/tests/restarting/from_7.3.50/ConfigureStorageMigrationTestRestart-2.toml @@ -0,0 +1,23 @@ +[configuration] +extraMachineCountDC = 2 + +[[test]] +testTitle = 'CloggedConfigureDatabaseTest' +runSetup=false +waitForQuiescenceBegin=false + + [[test.workload]] + testName = 'ConfigureDatabase' + testDuration = 300.0 + waitStoreTypeCheck = true + storageMigrationCompatibleConf = true + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 300.0 + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 300.0 + scale = 0.1 + clogginess = 2.0 \ No newline at end of file diff --git a/tests/restarting/from_7.3.0/ConfigureTestRestart-1.toml b/tests/restarting/from_7.3.50/ConfigureTestRestart-1.toml similarity index 100% rename from tests/restarting/from_7.3.0/ConfigureTestRestart-1.toml rename to tests/restarting/from_7.3.50/ConfigureTestRestart-1.toml diff --git a/tests/restarting/from_7.3.50/ConfigureTestRestart-2.toml b/tests/restarting/from_7.3.50/ConfigureTestRestart-2.toml new file mode 100644 index 00000000000..96f1ed9ea99 --- /dev/null +++ b/tests/restarting/from_7.3.50/ConfigureTestRestart-2.toml @@ -0,0 +1,21 @@ +[configuration] +randomlyRenameZoneId=true + +[[test]] +testTitle='CloggedConfigureDatabaseTest' +runSetup=false + + [[test.workload]] + testName='ConfigureDatabase' + testDuration=300.0 + allowDescriptorChange=false + + [[test.workload]] + testName='RandomClogging' + testDuration=300.0 + + [[test.workload]] + testName='RandomClogging' + testDuration=300.0 + scale=0.1 + clogginess=2.0 diff --git a/tests/restarting/from_5.0.0_until_6.3.0/CycleTestRestart-1.txt b/tests/restarting/from_7.3.50/CycleTestRestart-1.toml similarity index 50% rename from tests/restarting/from_5.0.0_until_6.3.0/CycleTestRestart-1.txt rename to tests/restarting/from_7.3.50/CycleTestRestart-1.toml index 8f7cacda930..1379bfa04d5 100644 --- a/tests/restarting/from_5.0.0_until_6.3.0/CycleTestRestart-1.txt +++ b/tests/restarting/from_7.3.50/CycleTestRestart-1.toml @@ -1,31 +1,38 @@ -testTitle=Clogged +[[test]] +testTitle="Clogged" clearAfterTest=false - testName=Cycle + [[test.workload]] + testName="Cycle" transactionsPerSecond=500.0 nodeCount=2500 testDuration=10.0 expectedRate=0 - testName=RandomClogging + [[test.workload]] + testName="RandomClogging" testDuration=10.0 - testName=Rollback + [[test.workload]] + testName="Rollback" meanDelay=10.0 testDuration=10.0 - testName=Attrition + [[test.workload]] + testName="Attrition" machinesToKill=10 machinesToLeave=3 reboot=true testDuration=10.0 - testName=Attrition + [[test.workload]] + testName="Attrition" machinesToKill=10 machinesToLeave=3 reboot=true testDuration=10.0 - testName=SaveAndKill - restartInfoLocation=simfdb/restartInfo.ini + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" testDuration=10.0 diff --git a/tests/restarting/from_7.3.50/CycleTestRestart-2.toml b/tests/restarting/from_7.3.50/CycleTestRestart-2.toml new file mode 100644 index 00000000000..7caffb3afcf --- /dev/null +++ b/tests/restarting/from_7.3.50/CycleTestRestart-2.toml @@ -0,0 +1,33 @@ +[[test]] +testTitle="Clogged" +runSetup=false + + [[test.workload]] + testName="Cycle" + transactionsPerSecond=2500.0 + nodeCount=2500 + testDuration=10.0 + expectedRate=0 + + [[test.workload]] + testName="RandomClogging" + testDuration=10.0 + + [[test.workload]] + testName="Rollback" + meanDelay=10.0 + testDuration=10.0 + + [[test.workload]] + testName="Attrition" + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName="Attrition" + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 diff --git a/tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml b/tests/restarting/from_7.3.50/DrUpgradeRestart-1.toml similarity index 94% rename from tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml rename to tests/restarting/from_7.3.50/DrUpgradeRestart-1.toml index e157ef85b97..2a85cdb3a0e 100644 --- a/tests/restarting/from_7.3.0/DrUpgradeRestart-1.toml +++ b/tests/restarting/from_7.3.50/DrUpgradeRestart-1.toml @@ -3,7 +3,7 @@ extraDatabaseMode = "Local" # In 7.2, DR is not supported in required tenant mode allowDefaultTenant = false encryptModes = ['disabled'] -storageEngineExcludeTypes = [4,5] +storageEngineExcludeTypes = [4, 5] [[test]] testTitle = "DrUpgrade" diff --git a/tests/restarting/from_7.3.50/DrUpgradeRestart-2.toml b/tests/restarting/from_7.3.50/DrUpgradeRestart-2.toml new file mode 100644 index 00000000000..9a883ee2f46 --- /dev/null +++ b/tests/restarting/from_7.3.50/DrUpgradeRestart-2.toml @@ -0,0 +1,22 @@ +[configuration] +extraDatabaseMode = "Local" + +[[test]] +testTitle = "DrUpgrade" +runSetup = false +clearAfterTest = false +simBackupAgents = "BackupToDB" +waitForQuiescenceBegin = false + + [[test.workload]] + testName = "Cycle" + nodeCount = 30000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + + [[test.workload]] + testName = "BackupToDBUpgrade" + backupAfter = 10.0 + backupRangesCount = -1 + stopDifferentialAfter = 70.0 diff --git a/tests/restarting/from_7.3.50/SnapCycleRestart-1.toml b/tests/restarting/from_7.3.50/SnapCycleRestart-1.toml new file mode 100644 index 00000000000..a4468bf3677 --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapCycleRestart-1.toml @@ -0,0 +1,31 @@ +[configuration] +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[test]] +# Take snap and do cycle test +testTitle="SnapCyclePre" +clearAfterTest=false + + [[test.workload]] + testName="Cycle" + transactionsPerSecond=2500.0 + nodeCount=2500 + testDuration=10.0 + expectedRate=0 + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=10.0 + testID=1 + +[[test]] +# save and shutdown +testTitle="SnapCycleShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.50/SnapCycleRestart-2.toml b/tests/restarting/from_7.3.50/SnapCycleRestart-2.toml new file mode 100644 index 00000000000..77be982536f --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapCycleRestart-2.toml @@ -0,0 +1,16 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +testTitle="SnapCycleRestore" +runSetup=false + + [[test.workload]] + # Post snap restore test + testName="Cycle" + transactionsPerSecond=2500.0 + nodeCount=2500 + testDuration=10.0 + expectedRate=0 + enableDD=true diff --git a/tests/restarting/from_7.3.50/SnapTestAttrition-1.toml b/tests/restarting/from_7.3.50/SnapTestAttrition-1.toml new file mode 100644 index 00000000000..3935cc871c8 --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapTestAttrition-1.toml @@ -0,0 +1,63 @@ +[configuration] +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[test]] +# write 1000 Keys ending with even numbers +testTitle="SnapTestPre" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=0 + +[[test]] +# Take snap and do read/write +testTitle="SnapTestTakeSnap" +clearAfterTest=false + + [[test.workload]] + testName="ReadWrite" + testDuration=10.0 + transactionsPerSecond=10000 + writesPerTransactionA=0 + readsPerTransactionA=10 + writesPerTransactionB=10 + readsPerTransactionB=1 + alpha=0.5 + nodeCount=100000 + valueBytes=16 + discardEdgeMeasurements=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=10.0 + testID=1 + + [[test.workload]] + testName="Attrition" + testDuration=10.0 + +[[test]] +# write 1000 Keys ending with odd numbers +testTitle="SnapTestPost" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=25.0 + testID=2 + +[[test]] +# save and shutdown +testTitle="SnapSimpleShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.50/SnapTestAttrition-2.toml b/tests/restarting/from_7.3.50/SnapTestAttrition-2.toml new file mode 100644 index 00000000000..5ac004965ce --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapTestAttrition-2.toml @@ -0,0 +1,15 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +# verify all keys are even numbered +testTitle="SnapTestVerify" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=3 + restartInfoLocation="simfdb/restartInfo.ini" + enableDD=true diff --git a/tests/restarting/from_7.3.50/SnapTestRestart-1.toml b/tests/restarting/from_7.3.50/SnapTestRestart-1.toml new file mode 100644 index 00000000000..337a7dd9a22 --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapTestRestart-1.toml @@ -0,0 +1,59 @@ +[configuration] +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[test]] +# write 1000 Keys ending with even numbers +testTitle="SnapTestPre" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=0 + +[[test]] +# Take snap and do read/write +testTitle="SnapTestTakeSnap" +clearAfterTest=false + + [[test.workload]] + testName="ReadWrite" + testDuration=10.0 + transactionsPerSecond=10000 + writesPerTransactionA=0 + readsPerTransactionA=10 + writesPerTransactionB=10 + readsPerTransactionB=1 + alpha=0.5 + nodeCount=100000 + valueBytes=16 + discardEdgeMeasurements=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=10.0 + testID=1 + +[[test]] +# write 1000 Keys ending with odd numbers +testTitle="SnapTestPost" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=25.0 + testID=2 + +[[test]] +# save and shutdown +testTitle="SnapTestShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.50/SnapTestRestart-2.toml b/tests/restarting/from_7.3.50/SnapTestRestart-2.toml new file mode 100644 index 00000000000..121fd5b24bd --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapTestRestart-2.toml @@ -0,0 +1,14 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +# verify all keys are even numbered +testTitle="SnapTestVerify" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=3 + enableDD=true diff --git a/tests/restarting/from_7.3.50/SnapTestSimpleRestart-1.toml b/tests/restarting/from_7.3.50/SnapTestSimpleRestart-1.toml new file mode 100644 index 00000000000..470a7d03a30 --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapTestSimpleRestart-1.toml @@ -0,0 +1,56 @@ +[configuration] +storageEngineExcludeTypes = [4, 5] +logAntiQuorum=0 + +[[test]] +# write 1000 Keys ending with even number +testTitle="SnapSimplePre" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=30.0 + testID=0 + +[[test]] +# take snap +testTitle="SnapSimpleTakeSnap" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=5.0 + testID=1 + +[[test]] +# write 1000 Keys ending with odd number +testTitle="SnapSimplePost" +clearAfterTest=false + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=2 + +[[test]] +# snapCreate with binary path that is not whitelisted +testTitle="SnapCreateNotWhitelistedBinaryPath" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=4 + +[[test]] +# save and shutdown +testTitle="SnapSimpleShutdown" + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 + isRestoring=1 diff --git a/tests/restarting/from_7.3.50/SnapTestSimpleRestart-2.toml b/tests/restarting/from_7.3.50/SnapTestSimpleRestart-2.toml new file mode 100644 index 00000000000..6706fc48b8e --- /dev/null +++ b/tests/restarting/from_7.3.50/SnapTestSimpleRestart-2.toml @@ -0,0 +1,14 @@ +[configuration] +storageEngineExcludeTypes=[4,5] +buggify=false + +[[test]] +# verify all keys are even numbered +testTitle="SnapSimpleVerify" + + [[test.workload]] + testName="SnapTest" + numSnaps=1 + maxSnapDelay=3.0 + testID=3 + enableDD=true diff --git a/tests/restarting/from_7.3.50/StorefrontTestRestart-1.toml b/tests/restarting/from_7.3.50/StorefrontTestRestart-1.toml new file mode 100644 index 00000000000..c597691eac8 --- /dev/null +++ b/tests/restarting/from_7.3.50/StorefrontTestRestart-1.toml @@ -0,0 +1,15 @@ +[[test]] +testTitle="StorefrontTest" +clearAfterTest=false + + [[test.workload]] + testName="Storefront" + actorsPerClient=20 + transactionsPerSecond=200 + itemCount=20000 + maxOrderSize=4 + + [[test.workload]] + testName="SaveAndKill" + restartInfoLocation="simfdb/restartInfo.ini" + testDuration=10.0 diff --git a/tests/restarting/from_7.3.50/StorefrontTestRestart-2.toml b/tests/restarting/from_7.3.50/StorefrontTestRestart-2.toml new file mode 100644 index 00000000000..c25a3252c32 --- /dev/null +++ b/tests/restarting/from_7.3.50/StorefrontTestRestart-2.toml @@ -0,0 +1,10 @@ +[[test]] +testTitle="StorefrontTest" +runSetup=false + + [[test.workload]] + testName="Storefront" + actorsPerClient=20 + transactionsPerSecond=200 + itemCount=20000 + maxOrderSize=4 diff --git a/tests/restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml b/tests/restarting/from_7.3.50/UpgradeAndBackupRestore-1.toml similarity index 97% rename from tests/restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml rename to tests/restarting/from_7.3.50/UpgradeAndBackupRestore-1.toml index 4b5253946b9..8c602d7cfd3 100644 --- a/tests/restarting/from_7.3.0/UpgradeAndBackupRestore-1.toml +++ b/tests/restarting/from_7.3.50/UpgradeAndBackupRestore-1.toml @@ -1,5 +1,5 @@ [configuration] -storageEngineExcludeTypes=[3,5] +storageEngineExcludeTypes = [5] [[test]] testTitle = 'SubmitBackup' diff --git a/tests/restarting/from_7.3.50/UpgradeAndBackupRestore-2.toml b/tests/restarting/from_7.3.50/UpgradeAndBackupRestore-2.toml new file mode 100644 index 00000000000..b5ac855d52a --- /dev/null +++ b/tests/restarting/from_7.3.50/UpgradeAndBackupRestore-2.toml @@ -0,0 +1,61 @@ +[[test]] +testTitle = 'SecondCycleTest' +simBackupAgents = 'BackupToFile' +clearAfterTest=false +runConsistencyCheck=false + + [[test.workload]] + testName = 'Cycle' + nodeCount = 30000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = 'AfterRestart' + + [[test.workload]] + testName = 'RandomClogging' + testDuration = 90.0 + + [[test.workload]] + testName = 'Rollback' + meanDelay = 90.0 + testDuration = 90.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 90.0 + + [[test.workload]] + testName = 'Attrition' + machinesToKill = 10 + machinesToLeave = 3 + reboot = true + testDuration = 90.0 + +[[test]] +testTitle= 'RestoreBackup' +simBackupAgents = 'BackupToFile' +clearAfterTest=false + + [[test.workload]] + testName = 'RestoreBackup' + tag = 'default' + +[[test]] +testTitle = 'CheckCycles' +checkOnly=true + + [[test.workload]] + testName = 'Cycle' + nodeCount=30000 + keyPrefix = 'AfterRestart' + expectedRate=0 + + [[test.workload]] + testName = 'Cycle' + nodeCount = 30000 + keyPrefix= 'BeforeRestart' + expectedRate = 0 diff --git a/tests/restarting/from_7.3.0/VersionVectorDisableRestart-1.toml b/tests/restarting/from_7.3.50/VersionVectorDisableRestart-1.toml similarity index 100% rename from tests/restarting/from_7.3.0/VersionVectorDisableRestart-1.toml rename to tests/restarting/from_7.3.50/VersionVectorDisableRestart-1.toml diff --git a/tests/restarting/from_7.3.50/VersionVectorDisableRestart-2.toml b/tests/restarting/from_7.3.50/VersionVectorDisableRestart-2.toml new file mode 100644 index 00000000000..5c178993a37 --- /dev/null +++ b/tests/restarting/from_7.3.50/VersionVectorDisableRestart-2.toml @@ -0,0 +1,54 @@ +[[knobs]] +enable_version_vector = false +enable_version_vector_tlog_unicast = false + +[[test]] +testTitle='VersionVectorDowngrade' +runSetup=false + + [[test.workload]] + testName='Cycle' + transactionsPerSecond=2500.0 + nodeCount=1000 + testDuration=30.0 + expectedRate=0 + keyPrefix = 'cycle' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = '!' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = 'ZZZ' + + [[test.workload]] + testName='RandomClogging' + testDuration=10.0 + + [[test.workload]] + testName='Rollback' + meanDelay=10.0 + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=60.0 diff --git a/tests/restarting/from_7.3.0/VersionVectorEnableRestart-1.toml b/tests/restarting/from_7.3.50/VersionVectorEnableRestart-1.toml similarity index 100% rename from tests/restarting/from_7.3.0/VersionVectorEnableRestart-1.toml rename to tests/restarting/from_7.3.50/VersionVectorEnableRestart-1.toml diff --git a/tests/restarting/from_7.3.50/VersionVectorEnableRestart-2.toml b/tests/restarting/from_7.3.50/VersionVectorEnableRestart-2.toml new file mode 100644 index 00000000000..c1b4bc1e3ef --- /dev/null +++ b/tests/restarting/from_7.3.50/VersionVectorEnableRestart-2.toml @@ -0,0 +1,55 @@ +[[knobs]] +enable_version_vector = true +enable_version_vector_tlog_unicast = true +proxy_use_resolver_private_mutations = true + +[[test]] +testTitle='VersionVectorUpgrade' +runSetup=false + + [[test.workload]] + testName='Cycle' + transactionsPerSecond=2500.0 + nodeCount=1000 + testDuration=30.0 + expectedRate=0 + keyPrefix = 'cycle' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = '!' + + [[test.workload]] + testName = 'Cycle' + nodeCount = 1000 + transactionsPerSecond = 2500.0 + testDuration = 30.0 + expectedRate = 0 + keyPrefix = 'ZZZ' + + [[test.workload]] + testName='RandomClogging' + testDuration=10.0 + + [[test.workload]] + testName='Rollback' + meanDelay=10.0 + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=10.0 + + [[test.workload]] + testName='Attrition' + machinesToKill=10 + machinesToLeave=3 + reboot=true + testDuration=60.0 diff --git a/tests/restarting/to_7.1.33_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/to_7.1.33_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml index f4c15caec32..d24db85f121 100644 --- a/tests/restarting/to_7.1.33_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml +++ b/tests/restarting/to_7.1.33_until_7.2.0/ConfigureStorageMigrationTestRestart-1.toml @@ -2,7 +2,7 @@ extraMachineCountDC = 2 maxTLogVersion=6 disableHostname=true -storageEngineExcludeTypes=[3, 4, 5] +storageEngineExcludeTypes=[3, 5] tenantModes=['disabled'] encryptModes=['disabled'] simHTTPServerEnabled=false diff --git a/tests/restarting/to_7.3.0/ConfigureStorageMigrationTestRestart-1.toml b/tests/restarting/to_7.3.0/ConfigureStorageMigrationTestRestart-1.toml index b00389af57d..d0dbf3bb33c 100644 --- a/tests/restarting/to_7.3.0/ConfigureStorageMigrationTestRestart-1.toml +++ b/tests/restarting/to_7.3.0/ConfigureStorageMigrationTestRestart-1.toml @@ -2,7 +2,6 @@ extraMachineCountDC = 2 maxTLogVersion=6 disableHostname=true -storageEngineExcludeTypes=[4] tenantModes=['disabled'] [[test]]