Skip to content

Commit d51ccf7

Browse files
authored
HDDS-13016. Add a getAllNodeCount() method to NodeManager. (apache#8445)
1 parent 8b27fb9 commit d51ccf7

18 files changed

Lines changed: 79 additions & 94 deletions

File tree

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -421,7 +421,7 @@ private static Pipeline getFromProtobufSetCreationTimestamp(HddsProtos.Pipeline
421421
.build();
422422
}
423423

424-
public Pipeline copyWithNodesInOrder(List<DatanodeDetails> nodes) {
424+
public Pipeline copyWithNodesInOrder(List<? extends DatanodeDetails> nodes) {
425425
return toBuilder().setNodesInOrder(nodes).build();
426426
}
427427

@@ -611,7 +611,7 @@ public Builder setNodeOrder(List<Integer> orders) {
611611
return this;
612612
}
613613

614-
public Builder setNodesInOrder(List<DatanodeDetails> nodes) {
614+
public Builder setNodesInOrder(List<? extends DatanodeDetails> nodes) {
615615
this.nodesInOrder = new LinkedList<>(nodes);
616616
return this;
617617
}

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeDecommissionManager.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ private synchronized boolean checkIfDecommissionPossible(List<DatanodeDetails> d
423423
}
424424
int reqNodes = cif.getReplicationConfig().getRequiredNodes();
425425
if ((inServiceTotal - numDecom) < reqNodes) {
426-
int unHealthyTotal = nodeManager.getAllNodes().size() - inServiceTotal;
426+
final int unHealthyTotal = nodeManager.getAllNodeCount() - inServiceTotal;
427427
String errorMsg = "Insufficient nodes. Tried to decommission " + dns.size() +
428428
" nodes out of " + inServiceTotal + " IN-SERVICE HEALTHY and " + unHealthyTotal +
429429
" not IN-SERVICE or not HEALTHY nodes. Cannot decommission as a minimum of " + reqNodes +
@@ -591,7 +591,7 @@ private synchronized boolean checkIfMaintenancePossible(List<DatanodeDetails> dn
591591
minInService = maintenanceReplicaMinimum;
592592
}
593593
if ((inServiceTotal - numMaintenance) < minInService) {
594-
int unHealthyTotal = nodeManager.getAllNodes().size() - inServiceTotal;
594+
final int unHealthyTotal = nodeManager.getAllNodeCount() - inServiceTotal;
595595
String errorMsg = "Insufficient nodes. Tried to start maintenance for " + dns.size() +
596596
" nodes out of " + inServiceTotal + " IN-SERVICE HEALTHY and " + unHealthyTotal +
597597
" not IN-SERVICE or not HEALTHY nodes. Cannot enter maintenance mode as a minimum of " + minInService +

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManager.java

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -136,11 +136,14 @@ int getNodeCount(
136136
NodeOperationalState opState, NodeState health);
137137

138138
/**
139-
* Get all datanodes known to SCM.
140-
*
141-
* @return List of DatanodeDetails known to SCM.
139+
* @return all datanodes known to SCM.
142140
*/
143-
List<DatanodeDetails> getAllNodes();
141+
List<? extends DatanodeDetails> getAllNodes();
142+
143+
/** @return the number of datanodes. */
144+
default int getAllNodeCount() {
145+
return getAllNodes().size();
146+
}
144147

145148
/**
146149
* Returns the aggregated node stats.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeStateManager.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -544,6 +544,10 @@ public List<DatanodeInfo> getAllNodes() {
544544
return nodeStateMap.getAllDatanodeInfos();
545545
}
546546

547+
int getAllNodeCount() {
548+
return nodeStateMap.getNodeCount();
549+
}
550+
547551
/**
548552
* Sets the operational state of the given node. Intended to be called when
549553
* a node is being decommissioned etc.

hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java

Lines changed: 14 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@
3636
import java.util.Collections;
3737
import java.util.HashMap;
3838
import java.util.HashSet;
39-
import java.util.LinkedList;
4039
import java.util.List;
4140
import java.util.Map;
4241
import java.util.Objects;
@@ -252,15 +251,14 @@ public List<DatanodeDetails> getNodes(
252251
.map(node -> (DatanodeDetails)node).collect(Collectors.toList());
253252
}
254253

255-
/**
256-
* Returns all datanodes that are known to SCM.
257-
*
258-
* @return List of DatanodeDetails
259-
*/
260254
@Override
261-
public List<DatanodeDetails> getAllNodes() {
262-
return nodeStateManager.getAllNodes().stream()
263-
.map(node -> (DatanodeDetails) node).collect(Collectors.toList());
255+
public List<DatanodeInfo> getAllNodes() {
256+
return nodeStateManager.getAllNodes();
257+
}
258+
259+
@Override
260+
public int getAllNodeCount() {
261+
return nodeStateManager.getAllNodeCount();
264262
}
265263

266264
/**
@@ -449,9 +447,9 @@ public RegisteredCommand register(
449447
LOG.info("Updated datanode to: {}", dn);
450448
scmNodeEventPublisher.fireEvent(SCMEvents.NODE_ADDRESS_UPDATE, dn);
451449
} else if (isVersionChange(oldNode.getVersion(), datanodeDetails.getVersion())) {
452-
LOG.info("Update the version for registered datanode = {}, " +
450+
LOG.info("Update the version for registered datanode {}, " +
453451
"oldVersion = {}, newVersion = {}.",
454-
datanodeDetails.getUuid(), oldNode.getVersion(), datanodeDetails.getVersion());
452+
datanodeDetails, oldNode.getVersion(), datanodeDetails.getVersion());
455453
nodeStateManager.updateNode(datanodeDetails, layoutInfo);
456454
}
457455
} catch (NodeNotFoundException e) {
@@ -1725,29 +1723,16 @@ public DatanodeInfo getNode(DatanodeID id) {
17251723
*/
17261724
@Override
17271725
public List<DatanodeDetails> getNodesByAddress(String address) {
1728-
List<DatanodeDetails> allNodes = getAllNodes();
1729-
List<DatanodeDetails> results = new LinkedList<>();
17301726
if (Strings.isNullOrEmpty(address)) {
1731-
LOG.warn("address is null");
1732-
return results;
1727+
return Collections.emptyList();
17331728
}
17341729
Set<DatanodeID> datanodeIDS = dnsToDnIdMap.get(address);
17351730
if (datanodeIDS == null) {
1736-
LOG.debug("Cannot find node for address {}", address);
1737-
return results;
1731+
return Collections.emptyList();
17381732
}
1739-
1740-
datanodeIDS.forEach(datanodeID -> {
1741-
try {
1742-
List<DatanodeDetails> datanodeDetails = allNodes.stream().
1743-
filter(node -> node.getID().equals(datanodeID)).
1744-
collect(Collectors.toList());
1745-
results.addAll(datanodeDetails);
1746-
} catch (Exception e) {
1747-
LOG.warn("Error find node for DataNode ID {}", datanodeID);
1748-
}
1749-
});
1750-
return results;
1733+
return datanodeIDS.stream()
1734+
.map(this::getNode)
1735+
.collect(Collectors.toList());
17511736
}
17521737

17531738
/**

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/TestSCMCommonPlacementPolicy.java

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@
6868
*/
6969
public class TestSCMCommonPlacementPolicy {
7070

71-
private NodeManager nodeManager;
71+
private MockNodeManager nodeManager;
7272
private OzoneConfiguration conf;
7373

7474
@BeforeEach
@@ -535,7 +535,7 @@ private static class DummyPlacementPolicy extends SCMCommonPlacementPolicy {
535535
DummyPlacementPolicy(NodeManager nodeManager, ConfigurationSource conf,
536536
int rackCnt) {
537537
this(nodeManager, conf,
538-
IntStream.range(0, nodeManager.getAllNodes().size()).boxed()
538+
IntStream.range(0, nodeManager.getAllNodeCount()).boxed()
539539
.collect(Collectors.toMap(Function.identity(),
540540
idx -> idx % rackCnt)), rackCnt);
541541
}
@@ -552,7 +552,7 @@ private static class DummyPlacementPolicy extends SCMCommonPlacementPolicy {
552552
this.rackCnt = rackCnt;
553553
this.racks = IntStream.range(0, rackCnt)
554554
.mapToObj(i -> mock(Node.class)).collect(Collectors.toList());
555-
List<DatanodeDetails> datanodeDetails = nodeManager.getAllNodes();
555+
final List<? extends DatanodeDetails> datanodeDetails = nodeManager.getAllNodes();
556556
rackMap = datanodeRackMap.entrySet().stream()
557557
.collect(Collectors.toMap(
558558
entry -> datanodeDetails.get(entry.getKey()),

hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1828,7 +1828,7 @@ public void testScmRegisterNodeWith4LayerNetworkTopology()
18281828
assertEquals(nodeCount, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
18291829
assertEquals(nodeCount, clusterMap.getNumOfLeafNode(""));
18301830
assertEquals(4, clusterMap.getMaxLevel());
1831-
List<DatanodeDetails> nodeList = nodeManager.getAllNodes();
1831+
final List<DatanodeInfo> nodeList = nodeManager.getAllNodes();
18321832
nodeList.forEach(node -> assertTrue(
18331833
node.getNetworkLocation().startsWith("/rack1/ng")));
18341834
nodeList.forEach(node -> assertNotNull(node.getParent()));
@@ -1872,7 +1872,7 @@ void testScmRegisterNodeWithNetworkTopology(boolean useHostname)
18721872
nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
18731873
assertEquals(nodeCount, clusterMap.getNumOfLeafNode(""));
18741874
assertEquals(3, clusterMap.getMaxLevel());
1875-
List<DatanodeDetails> nodeList = nodeManager.getAllNodes();
1875+
final List<DatanodeInfo> nodeList = nodeManager.getAllNodes();
18761876
nodeList.forEach(node ->
18771877
assertEquals("/rack1", node.getNetworkLocation()));
18781878

@@ -2019,7 +2019,7 @@ public void testScmRegisterNodeWithUpdatedIpAndHostname()
20192019
nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
20202020
assertEquals(1, clusterMap.getNumOfLeafNode(""));
20212021
assertEquals(4, clusterMap.getMaxLevel());
2022-
List<DatanodeDetails> nodeList = nodeManager.getAllNodes();
2022+
final List<DatanodeInfo> nodeList = nodeManager.getAllNodes();
20232023
assertEquals(1, nodeList.size());
20242024

20252025
DatanodeDetails returnedNode = nodeList.get(0);
@@ -2039,7 +2039,7 @@ public void testScmRegisterNodeWithUpdatedIpAndHostname()
20392039
assertEquals(1, nodeManager.getNodeCount(NodeStatus.inServiceHealthy()));
20402040
assertEquals(1, clusterMap.getNumOfLeafNode(""));
20412041
assertEquals(4, clusterMap.getMaxLevel());
2042-
List<DatanodeDetails> updatedNodeList = nodeManager.getAllNodes();
2042+
final List<DatanodeInfo> updatedNodeList = nodeManager.getAllNodes();
20432043
assertEquals(1, updatedNodeList.size());
20442044

20452045
DatanodeDetails returnedUpdatedNode = updatedNodeList.get(0);

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestContainerOperations.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,7 @@ public void testDatanodeUsageInfoCompatibility() throws IOException {
188188

189189
@Test
190190
public void testDatanodeUsageInfoContainerCount() throws Exception {
191-
List<DatanodeDetails> dnList = cluster().getStorageContainerManager()
191+
List<? extends DatanodeDetails> dnList = cluster().getStorageContainerManager()
192192
.getScmNodeManager()
193193
.getAllNodes();
194194

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -630,7 +630,7 @@ public void testScmInfo(@TempDir Path tempDir) throws Exception {
630630
*/
631631
private void testScmProcessDatanodeHeartbeat(MiniOzoneCluster cluster) {
632632
NodeManager nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
633-
List<DatanodeDetails> allNodes = nodeManager.getAllNodes();
633+
List<? extends DatanodeDetails> allNodes = nodeManager.getAllNodes();
634634
assertEquals(cluster.getHddsDatanodes().size(), allNodes.size());
635635

636636
for (DatanodeDetails node : allNodes) {

hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/node/TestDecommissionAndMaintenance.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -316,7 +316,7 @@ public void testInsufficientNodesCannotBeDecommissioned()
316316
// Generate some data on the empty cluster to create some containers
317317
generateData(20, "key", ratisRepConfig);
318318

319-
final List<DatanodeDetails> toDecommission = nm.getAllNodes();
319+
final List<? extends DatanodeDetails> toDecommission = nm.getAllNodes();
320320

321321
// trying to decommission 5 nodes should leave the cluster with 2 nodes,
322322
// which is not sufficient for RATIS.THREE replication. It should not be allowed.
@@ -706,7 +706,7 @@ public void testInsufficientNodesCannotBePutInMaintenance()
706706
throws Exception {
707707
// Generate some data on the empty cluster to create some containers
708708
generateData(20, "key", ratisRepConfig);
709-
final List<DatanodeDetails> toMaintenance = nm.getAllNodes();
709+
final List<? extends DatanodeDetails> toMaintenance = nm.getAllNodes();
710710

711711
// trying to move 6 nodes to maintenance should leave the cluster with 1 node,
712712
// which is not sufficient for RATIS.THREE replication (3 - maintenanceReplicaMinimum = 2).

0 commit comments

Comments
 (0)