From c81e957fa6dd65ffe318e8dffcd3fc828495472f Mon Sep 17 00:00:00 2001 From: Zhe Wang Date: Wed, 18 Sep 2024 15:58:43 -0700 Subject: [PATCH 1/3] [Release-7.3] TeamRedundant and TeamUnhealthy data moves choose best destination with probability (#11668) * team redundant and unhealthy data moves can choose best dest with probability * nits * nits * enable wantTrueBestIfMoveout * fix getteam stuck --- fdbclient/ServerKnobs.cpp | 3 + fdbclient/include/fdbclient/ServerKnobs.h | 5 ++ fdbserver/DDRelocationQueue.actor.cpp | 80 ++++++++++++------- fdbserver/DDTeamCollection.actor.cpp | 13 +-- .../include/fdbserver/DataDistributionTeam.h | 16 ++-- 5 files changed, 77 insertions(+), 40 deletions(-) diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 81db2a85264..af3b6a09482 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -177,6 +177,9 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( ENABLE_REPLICA_CONSISTENCY_CHECK_ON_DATA_MOVEMENT, true ); init( CONSISTENCY_CHECK_REQUIRED_REPLICAS, 1 ); + + init( PROBABILITY_TEAM_REDUNDANT_DATAMOVE_CHOOSE_TRUE_BEST_DEST, 0.0 ); if (isSimulated) PROBABILITY_TEAM_REDUNDANT_DATAMOVE_CHOOSE_TRUE_BEST_DEST = deterministicRandom()->random01(); + init( PROBABILITY_TEAM_UNHEALTHY_DATAMOVE_CHOOSE_TRUE_BEST_DEST, 0.0 ); if (isSimulated) PROBABILITY_TEAM_UNHEALTHY_DATAMOVE_CHOOSE_TRUE_BEST_DEST = deterministicRandom()->random01(); // Data distribution init( AVAILABLE_SPACE_PIVOT_RATIO, 0.5 ); diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index 9c79dc48624..a0d7bddc4c5 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -201,6 +201,11 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl return; } +// With probability, set wantTrueBestIfMoveout true for teamUnhealthy data moves and teamRedundant data moves only. +// This flag takes effect in getTeam. When the flag is set true, DD always getBestTeam for teamRedundant data moves and +// do getBestTeam for a teamRedundant data move if the data move decides to move data out of a SS. +bool getWantTrueBestIfMoveout(int priority) { + if (priority == SERVER_KNOBS->PRIORITY_TEAM_UNHEALTHY) { + return deterministicRandom()->random01() < + SERVER_KNOBS->PROBABILITY_TEAM_UNHEALTHY_DATAMOVE_CHOOSE_TRUE_BEST_DEST; + } else if (priority == SERVER_KNOBS->PRIORITY_TEAM_REDUNDANT) { + return deterministicRandom()->random01() < + SERVER_KNOBS->PROBABILITY_TEAM_REDUNDANT_DATAMOVE_CHOOSE_TRUE_BEST_DEST; + } else { + return false; + } +} + // This actor relocates the specified keys to a good place. // The inFlightActor key range map stores the actor for each RelocateData ACTOR Future dataDistributionRelocator(DDQueue* self, @@ -1453,6 +1468,7 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, state double startTime = now(); state std::vector destIds; state WantTrueBest wantTrueBest(isValleyFillerPriority(rd.priority)); + state WantTrueBestIfMoveout wantTrueBestIfMoveout(getWantTrueBestIfMoveout(rd.priority)); state uint64_t debugID = deterministicRandom()->randomUInt64(); state bool enableShardMove = SERVER_KNOBS->SHARD_ENCODE_LOCATION_METADATA && SERVER_KNOBS->ENABLE_DD_PHYSICAL_SHARD; state bool doBulkLoading = rd.bulkLoadTask.present(); @@ -1467,6 +1483,7 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, .detail("KeyBegin", rd.keys.begin) .detail("KeyEnd", rd.keys.end) .detail("Priority", rd.priority) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) .detail("SuppressedEventCount", self->suppressIntervals); if (relocateShardInterval.severity != SevDebug) { @@ -1614,6 +1631,7 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, req.completeSources = rd.completeSources; req.storageQueueAware = SERVER_KNOBS->ENABLE_STORAGE_QUEUE_AWARE_TEAM_SELECTION; req.findTeamForBulkLoad = doBulkLoading; + req.wantTrueBestIfMoveout = wantTrueBestIfMoveout; if (enableShardMove && tciIndex == 1) { ASSERT(physicalShardIDCandidate != UID().first() && @@ -1622,8 +1640,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, self->physicalShardCollection->tryGetAvailableRemoteTeamWith( physicalShardIDCandidate, metrics, debugID); if (!remoteTeamWithPhysicalShard.second) { - // Physical shard with `physicalShardIDCandidate` is not available. Retry selecting new - // dst physical shard. + // Physical shard with `physicalShardIDCandidate` is not available. Retry selecting + // new dst physical shard. self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++; foundTeams = false; break; @@ -1636,8 +1654,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, } } - // bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have any - // server that hosts the relocateData. This is possible, for example, in a fearless + // bestTeam.second = false if the bestTeam in the teamCollection (in the DC) does not have + // any server that hosts the relocateData. This is possible, for example, in a fearless // configuration when the remote DC is just brought up. Future>, bool>> fbestTeam = brokenPromiseToNever(self->teamCollections[tciIndex].getTeam.getReply(req)); @@ -1680,11 +1698,11 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, if (enableShardMove) { if (tciIndex == 1 && !forceToUseNewPhysicalShard) { // critical to the correctness of team selection by PhysicalShardCollection - // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a primary - // team Thus, tryGetAvailableRemoteTeamWith() may select an almost full remote team In - // this case, we must re-select a remote team We set foundTeams = false to avoid - // finishing team selection Then, forceToUseNewPhysicalShard is set, which enforce to - // use getTeam to select a remote team + // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a + // primary team Thus, tryGetAvailableRemoteTeamWith() may select an almost full + // remote team In this case, we must re-select a remote team We set foundTeams = + // false to avoid finishing team selection Then, forceToUseNewPhysicalShard is set, + // which enforce to use getTeam to select a remote team bool minAvailableSpaceRatio = bestTeam.first.get()->getMinAvailableSpaceRatio(true); if (minAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) { self->retryFindDstReasonCount[DDQueue::RetryFindDstReason::RemoteTeamIsFull]++; @@ -1693,11 +1711,11 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, } // critical to the correctness of team selection by PhysicalShardCollection - // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a primary - // team Thus, tryGetAvailableRemoteTeamWith() may select an unhealthy remote team In - // this case, we must re-select a remote team We set foundTeams = false to avoid - // finishing team selection Then, forceToUseNewPhysicalShard is set, which enforce to - // use getTeam to select a remote team + // tryGetAvailableRemoteTeamWith() enforce to select a remote team paired with a + // primary team Thus, tryGetAvailableRemoteTeamWith() may select an unhealthy remote + // team In this case, we must re-select a remote team We set foundTeams = false to + // avoid finishing team selection Then, forceToUseNewPhysicalShard is set, which + // enforce to use getTeam to select a remote team if (!bestTeam.first.get()->isHealthy()) { self->retryFindDstReasonCount [DDQueue::RetryFindDstReason::RemoteTeamIsNotHealthy]++; @@ -1707,11 +1725,11 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, } bestTeams.emplace_back(bestTeam.first.get(), true); - // Always set bestTeams[i].second = true to disable optimization in data move between DCs - // for the correctness of PhysicalShardCollection - // Currently, enabling the optimization will break the invariant of PhysicalShardCollection - // Invariant: once a physical shard is created with a specific set of SSes, this SS set will - // never get changed. + // Always set bestTeams[i].second = true to disable optimization in data move between + // DCs for the correctness of PhysicalShardCollection Currently, enabling the + // optimization will break the invariant of PhysicalShardCollection Invariant: once a + // physical shard is created with a specific set of SSes, this SS set will never get + // changed. if (tciIndex == 0) { ASSERT(foundTeams); @@ -1731,9 +1749,9 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, self->retryFindDstReasonCount [DDQueue::RetryFindDstReason::NoAvailablePhysicalShard]++; if (wantTrueBest) { - // Next retry will likely get the same team, and we know that we can't reuse - // any existing physical shard in this team. So force to create new physical - // shard. + // Next retry will likely get the same team, and we know that we can't + // reuse any existing physical shard in this team. So force to create + // new physical shard. forceToUseNewPhysicalShard = true; } foundTeams = false; @@ -1784,6 +1802,9 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, TraceEvent(destOverloadedCount > 50 ? SevInfo : SevDebug, "DestSSBusy", distributorId) .suppressFor(1.0) .detail("TraceID", rd.randomId) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) + .detail("IsRestore", rd.isRestore()) + .detail("Priority", rd.priority) .detail("StuckCount", stuckCount) .detail("DestOverloadedCount", destOverloadedCount) .detail("TeamCollectionId", tciIndex) @@ -1802,6 +1823,9 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, TraceEvent(stuckCount > 50 ? SevWarnAlways : SevWarn, "BestTeamStuck", distributorId) .suppressFor(1.0) .detail("TraceID", rd.randomId) + .detail("WantTrueBestIfMoveout", wantTrueBestIfMoveout) + .detail("IsRestore", rd.isRestore()) + .detail("Priority", rd.priority) .detail("StuckCount", stuckCount) .detail("DestOverloadedCount", destOverloadedCount) .detail("TeamCollectionId", tciIndex) @@ -2141,8 +2165,8 @@ ACTOR Future dataDistributionRelocator(DDQueue* self, auto serverIds = bestTeams[i].first->getServerIDs(); selectedTeams.push_back(ShardsAffectedByTeamFailure::Team(serverIds, i == 0)); } - // The update of PhysicalShardToTeams, PhysicalShardInstances, keyRangePhysicalShardIDMap should - // be atomic + // The update of PhysicalShardToTeams, PhysicalShardInstances, keyRangePhysicalShardIDMap + // should be atomic self->physicalShardCollection->updatePhysicalShardCollection( rd.keys, rd.isRestore(), selectedTeams, rd.dataMoveId.first(), metrics, debugID); } @@ -2242,8 +2266,8 @@ inline double getWorstCpu(const HealthMetrics& metrics, const std::vector& return cpu; } -// Move the shard with the top K highest read density of sourceTeam's to destTeam if sourceTeam has much more read load -// than destTeam +// Move the shard with the top K highest read density of sourceTeam's to destTeam if sourceTeam has much more read +// load than destTeam ACTOR Future rebalanceReadLoad(DDQueue* self, DataMovementReason moveReason, Reference sourceTeam, @@ -2785,8 +2809,8 @@ struct DDQueueImpl { } } } catch (Error& e) { - if (e.code() != error_code_broken_promise && // FIXME: Get rid of these broken_promise errors every time we - // are killed by the master dying + if (e.code() != error_code_broken_promise && // FIXME: Get rid of these broken_promise errors every time + // we are killed by the master dying e.code() != error_code_movekeys_conflict && e.code() != error_code_data_move_cancelled && e.code() != error_code_data_move_dest_team_not_found) TraceEvent(SevError, "DataDistributionQueueError", self->distributorId).error(e); diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp index 74371df81cf..1657d1c90e6 100644 --- a/fdbserver/DDTeamCollection.actor.cpp +++ b/fdbserver/DDTeamCollection.actor.cpp @@ -535,7 +535,7 @@ class DDTeamCollectionImpl { if (req.storageQueueAware) { storageQueueThreshold = calculateTeamStorageQueueThreshold(self->teams); } - if (req.teamSelect == TeamSelect::WANT_TRUE_BEST) { + if (req.teamSelect == TeamSelect::WANT_TRUE_BEST || req.wantTrueBestIfMoveout) { ASSERT(!bestOption.present()); if (SERVER_KNOBS->ENFORCE_SHARD_COUNT_PER_TEAM && req.preferWithinShardLimit) { bestOption = getBestTeam(self, @@ -657,11 +657,14 @@ class DDTeamCollectionImpl { // self->traceAllInfo(true); } - if (req.storageQueueAware && !bestOption.present()) { + if (!bestOption.present() && (req.storageQueueAware || req.wantTrueBestIfMoveout)) { + // re-run getTeam without storageQueueAware and wantTrueBestIfMoveout req.storageQueueAware = false; - TraceEvent(SevWarn, "StorageQueueAwareGetTeamFailed", self->distributorId) - .detail("Reason", "bestOption not present"); - wait(getTeam(self, req)); // re-run getTeam without storageQueueAware + req.wantTrueBestIfMoveout = false; + TraceEvent(SevWarn, "GetTeamRetry", self->distributorId) + .detail("OldStorageQueueAware", req.storageQueueAware) + .detail("OldWantTrueBestIfMoveout", req.wantTrueBestIfMoveout); + wait(getTeam(self, req)); } else { req.reply.send(std::make_pair(bestOption, foundSrc)); } diff --git a/fdbserver/include/fdbserver/DataDistributionTeam.h b/fdbserver/include/fdbserver/DataDistributionTeam.h index bff3844fd9b..a360a8ab674 100644 --- a/fdbserver/include/fdbserver/DataDistributionTeam.h +++ b/fdbserver/include/fdbserver/DataDistributionTeam.h @@ -98,6 +98,7 @@ struct IDataDistributionTeam { FDB_BOOLEAN_PARAM(WantNewServers); FDB_BOOLEAN_PARAM(WantTrueBest); +FDB_BOOLEAN_PARAM(WantTrueBestIfMoveout); FDB_BOOLEAN_PARAM(PreferLowerDiskUtil); FDB_BOOLEAN_PARAM(TeamMustHaveShards); FDB_BOOLEAN_PARAM(ForReadBalance); @@ -146,7 +147,8 @@ struct GetTeamRequest { bool findTeamByServers; bool findTeamForBulkLoad; Optional keys; - bool storageQueueAware; + bool storageQueueAware = false; + bool wantTrueBestIfMoveout = false; // completeSources have all shards in the key range being considered for movement, src have at least 1 shard in the // key range for movement. From the point of set, completeSources is the Intersection set of several , @@ -173,14 +175,14 @@ struct GetTeamRequest { teamMustHaveShards(teamMustHaveShards), forReadBalance(forReadBalance), preferLowerReadUtil(preferLowerReadUtil), preferWithinShardLimit(preferWithinShardLimit), inflightPenalty(inflightPenalty), findTeamByServers(FindTeamByServers::False), - findTeamForBulkLoad(FindTeamForBulkLoad::False), keys(keys) {} + findTeamForBulkLoad(FindTeamForBulkLoad::False), keys(keys), wantTrueBestIfMoveout(false) {} GetTeamRequest(std::vector servers) : teamSelect(TeamSelect::WANT_COMPLETE_SRCS), storageQueueAware(false), preferLowerDiskUtil(PreferLowerDiskUtil::False), teamMustHaveShards(TeamMustHaveShards::False), forReadBalance(ForReadBalance::False), preferLowerReadUtil(PreferLowerReadUtil::False), preferWithinShardLimit(PreferWithinShardLimit::False), inflightPenalty(1.0), findTeamByServers(FindTeamByServers::True), findTeamForBulkLoad(FindTeamForBulkLoad::False), - src(std::move(servers)) {} + src(std::move(servers)), wantTrueBestIfMoveout(false) {} // return true if a.score < b.score [[nodiscard]] bool lessCompare(TeamRef a, TeamRef b, int64_t aLoadBytes, int64_t bLoadBytes) const { @@ -195,10 +197,10 @@ struct GetTeamRequest { std::stringstream ss; ss << "TeamSelect:" << teamSelect.toString() << " StorageQueueAware:" << storageQueueAware - << " PreferLowerDiskUtil:" << preferLowerDiskUtil << " PreferLowerReadUtil:" << preferLowerReadUtil - << " PreferWithinShardLimit:" << preferWithinShardLimit << " teamMustHaveShards:" << teamMustHaveShards - << " forReadBalance:" << forReadBalance << " inflightPenalty:" << inflightPenalty - << " findTeamByServers:" << findTeamByServers << ";"; + << " WantTrueBestIfMoveout:" << wantTrueBestIfMoveout << " PreferLowerDiskUtil:" << preferLowerDiskUtil + << " PreferLowerReadUtil:" << preferLowerReadUtil << " PreferWithinShardLimit:" << preferWithinShardLimit + << " teamMustHaveShards:" << teamMustHaveShards << " forReadBalance:" << forReadBalance + << " inflightPenalty:" << inflightPenalty << " findTeamByServers:" << findTeamByServers << ";"; ss << "CompleteSources:"; for (const auto& cs : completeSources) { ss << cs.toString() << ","; From 6ac0baf4ea2d51dcbf5a99c00e53fdb0003b5e76 Mon Sep 17 00:00:00 2001 From: Zhe Wang Date: Wed, 18 Sep 2024 18:19:48 -0700 Subject: [PATCH 2/3] [Release-7.3] Delay team remover when space pivot is low (#11665) --- fdbclient/ServerKnobs.cpp | 1 + fdbclient/include/fdbclient/ServerKnobs.h | 5 +++++ fdbserver/DDTeamCollection.actor.cpp | 23 +++++++++++++++++++++++ 3 files changed, 29 insertions(+) diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index af3b6a09482..2739f18214c 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -380,6 +380,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC, 2.0 ); if( randomize && BUGGIFY ) DD_BULKLOAD_SCHEDULE_MIN_INTERVAL_SEC = deterministicRandom()->random01() * 10 + 1; // TeamRemover + init( TR_LOW_SPACE_PIVOT_DELAY_SEC, 0 ); if (isSimulated) TR_LOW_SPACE_PIVOT_DELAY_SEC = deterministicRandom()->randomInt(0, 3); init( TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER, false ); if( randomize && BUGGIFY ) TR_FLAG_DISABLE_MACHINE_TEAM_REMOVER = deterministicRandom()->random01() < 0.1 ? true : false; // false by default. disable the consistency check when it's true init( TR_REMOVE_MACHINE_TEAM_DELAY, 60.0 ); if( randomize && BUGGIFY ) TR_REMOVE_MACHINE_TEAM_DELAY = deterministicRandom()->random01() * 60.0; init( TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS, true ); if( randomize && BUGGIFY ) TR_FLAG_REMOVE_MT_WITH_MOST_TEAMS = deterministicRandom()->random01() < 0.1 ? true : false; diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index a0d7bddc4c5..355c15885de 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -353,6 +353,11 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImplteams.size()) + .detail("Servers", self->server_info.size()) + .detail("HealthyServers", serverCount) .detail("DesiredTeams", desiredTeams) .detail("MaxTeams", maxTeams) .detail("StorageTeamSize", self->configuration.storageTeamSize) @@ -1979,6 +1982,16 @@ class DDTeamCollectionImpl { // tracker) and remove bad team (cancel the team tracker). wait(self->badTeamRemover); + if (SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC > 0 && + self->teamPivots.pivotAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) { + TraceEvent(SevWarn, "MachineTeamRemoverDelayedForLowSpacePivot", self->distributorId) + .detail("IsPrimary", self->primary) + .detail("CurrentSpacePivot", self->teamPivots.pivotAvailableSpaceRatio) + .detail("TargetSpacePivot", SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO); + wait(delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC)); + continue; + } + state int healthyMachineCount = self->calculateHealthyMachineCount(); // Check if all machines are healthy, if not, we wait for 1 second and loop back. // Eventually, all machines will become healthy. @@ -2107,6 +2120,16 @@ class DDTeamCollectionImpl { // adding the bad team (add the team tracker) and remove bad team (cancel the team tracker). wait(self->badTeamRemover); + if (SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC > 0 && + self->teamPivots.pivotAvailableSpaceRatio < SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO) { + TraceEvent(SevWarn, "ServerTeamRemoverDelayedForLowSpacePivot", self->distributorId) + .detail("IsPrimary", self->primary) + .detail("CurrentSpacePivot", self->teamPivots.pivotAvailableSpaceRatio) + .detail("TargetSpacePivot", SERVER_KNOBS->TARGET_AVAILABLE_SPACE_RATIO); + wait(delay(SERVER_KNOBS->TR_LOW_SPACE_PIVOT_DELAY_SEC)); + continue; + } + // From this point, all server teams should be healthy, because we wait above // until processingUnhealthy is done, and all machines are healthy int desiredServerTeams = SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * self->server_info.size(); From 2237199385d858a1d6d954c80b84089b89149ed3 Mon Sep 17 00:00:00 2001 From: Zhe Wang Date: Thu, 26 Sep 2024 10:16:45 -0700 Subject: [PATCH 3/3] [Release-7.3] Validate ServerTeam count per server in simulation (#11678) * validate server team count in simulation * change naming (not relevant to the PR title) * address comments and add a new trace event BuildTeamsLastBuildTeamsFailed triggered when buildTeam failed --- fdbclient/ServerKnobs.cpp | 1 + fdbclient/include/fdbclient/ServerKnobs.h | 2 ++ fdbserver/DDShardTracker.actor.cpp | 12 +++++------ fdbserver/DDTeamCollection.actor.cpp | 25 +++++++++++++++++++---- 4 files changed, 30 insertions(+), 10 deletions(-) diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 2739f18214c..d33cfb95c53 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -371,6 +371,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( DD_MAXIMUM_LARGE_TEAM_CLEANUP, 10000 ); if( randomize && BUGGIFY ) DD_MAXIMUM_LARGE_TEAM_CLEANUP = 10; init( DD_LARGE_TEAM_DELAY, 60.0 ); init( DD_FIX_WRONG_REPLICAS_DELAY, 60.0 ); + init (DD_VALIDATE_SERVER_TEAM_COUNT_AFTER_BUILD_TEAM, false ); if (isSimulated) DD_VALIDATE_SERVER_TEAM_COUNT_AFTER_BUILD_TEAM = true; // BulkLoading init( BULKLOAD_FILE_BYTES_MAX, 1*1024*1024*1024 ); // 1GB diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index 355c15885de..0a7587d3b00 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -351,6 +351,8 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl shardUsableRegions(DataDistributionTracker::SafeAccessor self double expectedCompletionSeconds = self()->shards->size() * 1.0 / SERVER_KNOBS->DD_SHARD_USABLE_REGION_CHECK_RATE; double delayTime = deterministicRandom()->random01() * expectedCompletionSeconds; wait(delayJittered(delayTime)); - auto [destTeams, srcTeams] = self()->shardsAffectedByTeamFailure->getTeamsForFirstShard(keys); - if (destTeams.size() < self()->usableRegions) { + auto [newTeam, previousTeam] = self()->shardsAffectedByTeamFailure->getTeamsForFirstShard(keys); + if (newTeam.size() < self()->usableRegions) { TraceEvent(SevWarn, "ShardUsableRegionMismatch", self()->distributorId) .suppressFor(5.0) - .detail("DestTeamSize", destTeams.size()) - .detail("SrcTeamSize", srcTeams.size()) - .detail("DestServers", describe(destTeams)) - .detail("SrcServers", describe(srcTeams)) + .detail("NewTeamSize", newTeam.size()) + .detail("PreviousTeamSize", previousTeam.size()) + .detail("NewServers", describe(newTeam)) + .detail("PreviousServers", describe(previousTeam)) .detail("UsableRegion", self()->usableRegions) .detail("Shard", keys); RelocateShard rs(keys, DataMovementReason::POPULATE_REGION, RelocateReason::OTHER); diff --git a/fdbserver/DDTeamCollection.actor.cpp b/fdbserver/DDTeamCollection.actor.cpp index 73ac90aeb60..a80a5dbd8a8 100644 --- a/fdbserver/DDTeamCollection.actor.cpp +++ b/fdbserver/DDTeamCollection.actor.cpp @@ -943,7 +943,8 @@ class DDTeamCollectionImpl { // If there are too few machines to even build teams or there are too few represented datacenters, can't // build any team. self->lastBuildTeamsFailed = true; - TraceEvent(SevWarnAlways, "BuildTeamsNotEnoughUniqueMachines", self->distributorId) + TraceEvent(SevWarnAlways, "BuildTeamsLastBuildTeamsFailed", self->distributorId) + .detail("Reason", "Do not have enough unique machines") .detail("Primary", self->primary) .detail("UniqueMachines", uniqueMachines) .detail("Replication", self->configuration.storageTeamSize); @@ -4526,7 +4527,8 @@ bool DDTeamCollection::isValidLocality(Reference storagePoli void DDTeamCollection::evaluateTeamQuality() const { int teamCount = teams.size(), serverCount = allServers.size(); double teamsPerServer = (double)teamCount * configuration.storageTeamSize / serverCount; - + const int targetTeamNumPerServer = + (SERVER_KNOBS->DESIRED_TEAMS_PER_SERVER * (configuration.storageTeamSize + 1)) / 2; ASSERT_EQ(serverCount, server_info.size()); int minTeams = std::numeric_limits::max(); @@ -4542,6 +4544,16 @@ void DDTeamCollection::evaluateTeamQuality() const { varTeams += (stc - teamsPerServer) * (stc - teamsPerServer); // Use zoneId as server's machine id machineTeams[info->getLastKnownInterface().locality.zoneId()] += stc; + // Check invariant: if latest buildTeam succeeds, then each server must have at least + // targetTeamNumPerServer serverTeams + // lastBuildTeamsFailed is set only when (1) machine count is less than configured team size; + // (2) Not find any server team candidates when creating server team; (3) failed to add machine team + if (SERVER_KNOBS->DD_VALIDATE_SERVER_TEAM_COUNT_AFTER_BUILD_TEAM && !lastBuildTeamsFailed && + stc < targetTeamNumPerServer) { + TraceEvent(SevError, "NewAddServerNotMatchTargetSTCount", distributorId) + .detail("CurrentServerTeams", stc) + .detail("TargetServerTeams", targetTeamNumPerServer); + } } } varTeams /= teamsPerServer * teamsPerServer; @@ -5165,9 +5177,9 @@ int DDTeamCollection::addBestMachineTeams(int machineTeamsToBuild) { // When too many teams exist in simulation, traceAllInfo will buffer too many trace logs before // trace has a chance to flush its buffer, which causes assertion failure. traceAllInfo(!g_network->isSimulated()); - TraceEvent(SevWarn, "DataDistributionBuildTeams", distributorId) + TraceEvent(SevWarn, "BuildTeamsLastBuildTeamsFailed", distributorId) .detail("Primary", primary) - .detail("Reason", "Unable to make desired machine Teams") + .detail("Reason", "Unable to make desired machineTeams") .detail("Hint", "Check TraceAllInfo event"); lastBuildTeamsFailed = true; break; @@ -5577,6 +5589,11 @@ int DDTeamCollection::addTeamsBestOf(int teamsToBuild, int desiredTeams, int max if (bestServerTeam.size() != configuration.storageTeamSize) { // Not find any team and will unlikely find a team lastBuildTeamsFailed = true; + TraceEvent(SevWarn, "BuildTeamsLastBuildTeamsFailed", distributorId) + .detail("Reason", "Unable to find any valid serverTeam") + .detail("Primary", primary) + .detail("BestServerTeam", describe(bestServerTeam)) + .detail("ConfigStorageTeamSize", configuration.storageTeamSize); break; }