Skip to content

Commit a3c9c0e

Browse files
adoroszlaiadoroszlai
authored
HDDS-12443. Intermittent failure in TestContainerBalancerSubCommand (#7989)
1 parent 83fd8d7 commit a3c9c0e

3 files changed

Lines changed: 78 additions & 142 deletions

File tree

‎hadoop-hdds/tools/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,11 @@
140140
<type>test-jar</type>
141141
<scope>test</scope>
142142
</dependency>
143+
<dependency>
144+
<groupId>org.apache.ozone</groupId>
145+
<artifactId>hdds-test-utils</artifactId>
146+
<scope>test</scope>
147+
</dependency>
143148
</dependencies>
144149

145150
<build>

‎hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ContainerBalancerStatusSubcommand.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.time.LocalDateTime;
2727
import java.time.OffsetDateTime;
2828
import java.time.ZoneId;
29+
import java.time.format.DateTimeFormatter;
2930
import java.util.List;
3031
import java.util.stream.Collectors;
3132
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
@@ -69,7 +70,9 @@ public void execute(ScmClient scmClient) throws IOException {
6970
System.out.println("ContainerBalancer is Running.");
7071

7172
if (verbose) {
72-
System.out.printf("Started at: %s %s%n", dateTime.toLocalDate(), dateTime.toLocalTime());
73+
System.out.printf("Started at: %s %s%n",
74+
dateTime.toLocalDate().format(DateTimeFormatter.ISO_LOCAL_DATE),
75+
dateTime.toLocalTime().format(DateTimeFormatter.ISO_LOCAL_TIME));
7376
Duration balancingDuration = Duration.between(startedAtInstant, OffsetDateTime.now());
7477
System.out.printf("Balancing duration: %s%n%n", getPrettyDuration(balancingDuration));
7578
System.out.println(getConfigurationPrettyString(balancerStatusInfo.getConfiguration()));

‎hadoop-hdds/tools/src/test/java/org/apache/hadoop/hdds/scm/cli/datanode/TestContainerBalancerSubCommand.java‎

Lines changed: 69 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -18,19 +18,13 @@
1818
package org.apache.hadoop.hdds.scm.cli.datanode;
1919

2020
import static org.apache.hadoop.ozone.OzoneConsts.GB;
21-
import static org.junit.jupiter.api.Assertions.assertFalse;
22-
import static org.junit.jupiter.api.Assertions.assertTrue;
21+
import static org.assertj.core.api.Assertions.assertThat;
2322
import static org.mockito.Mockito.mock;
2423
import static org.mockito.Mockito.when;
2524

26-
import java.io.ByteArrayOutputStream;
2725
import java.io.IOException;
28-
import java.io.PrintStream;
29-
import java.io.UnsupportedEncodingException;
30-
import java.nio.charset.StandardCharsets;
3126
import java.time.OffsetDateTime;
3227
import java.util.Arrays;
33-
import java.util.regex.Matcher;
3428
import java.util.regex.Pattern;
3529
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos;
3630
import org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.ContainerBalancerStatusInfoProto;
@@ -40,6 +34,8 @@
4034
import org.apache.hadoop.hdds.scm.cli.ContainerBalancerStopSubcommand;
4135
import org.apache.hadoop.hdds.scm.client.ScmClient;
4236
import org.apache.hadoop.hdds.scm.container.balancer.ContainerBalancerConfiguration;
37+
import org.apache.hadoop.hdds.utils.IOUtils;
38+
import org.apache.ozone.test.GenericTestUtils;
4339
import org.junit.jupiter.api.AfterEach;
4440
import org.junit.jupiter.api.BeforeEach;
4541
import org.junit.jupiter.api.Test;
@@ -51,14 +47,44 @@
5147
*/
5248
class TestContainerBalancerSubCommand {
5349

54-
private static final String DEFAULT_ENCODING = StandardCharsets.UTF_8.name();
55-
private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
56-
private final ByteArrayOutputStream errContent = new ByteArrayOutputStream();
57-
private final PrintStream originalOut = System.out;
58-
private final PrintStream originalErr = System.err;
50+
private static final Pattern DURATION = Pattern.compile(
51+
"^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
52+
private static final Pattern FAILED_TO_START = Pattern.compile(
53+
"^Failed\\sto\\sstart\\sContainer\\sBalancer.");
54+
private static final Pattern IS_NOT_RUNNING = Pattern.compile(
55+
"^ContainerBalancer\\sis\\sNot\\sRunning.");
56+
private static final Pattern IS_RUNNING = Pattern.compile(
57+
"^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
58+
private static final Pattern STARTED_AT = Pattern.compile(
59+
"^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$", Pattern.MULTILINE);
60+
private static final Pattern STARTED_SUCCESSFULLY = Pattern.compile(
61+
"^Container\\sBalancer\\sstarted\\ssuccessfully.");
62+
private static final Pattern WAITING_TO_STOP = Pattern.compile(
63+
"^Sending\\sstop\\scommand.\\sWaiting\\sfor\\sContainer\\sBalancer\\sto\\sstop...\\n" +
64+
"Container\\sBalancer\\sstopped.");
65+
66+
private static final String BALANCER_CONFIG_OUTPUT = "Container Balancer Configuration values:\n" +
67+
"Key Value\n" +
68+
"Threshold 10.0\n" +
69+
"Max Datanodes to Involve per Iteration(percent) 20\n" +
70+
"Max Size to Move per Iteration 0GB\n" +
71+
"Max Size Entering Target per Iteration 26GB\n" +
72+
"Max Size Leaving Source per Iteration 26GB\n" +
73+
"Number of Iterations 3\n" +
74+
"Time Limit for Single Container's Movement 65min\n" +
75+
"Time Limit for Single Container's Replication 50min\n" +
76+
"Interval between each Iteration 0min\n" +
77+
"Whether to Enable Network Topology false\n" +
78+
"Whether to Trigger Refresh Datanode Usage Info false\n" +
79+
"Container IDs to Exclude from Balancing None\n" +
80+
"Datanodes Specified to be Balanced None\n" +
81+
"Datanodes Excluded from Balancing None";
82+
5983
private ContainerBalancerStopSubcommand stopCmd;
6084
private ContainerBalancerStartSubcommand startCmd;
6185
private ContainerBalancerStatusSubcommand statusCmd;
86+
private GenericTestUtils.PrintStreamCapturer out;
87+
private GenericTestUtils.PrintStreamCapturer err;
6288

6389
private static ContainerBalancerStatusInfoResponseProto getContainerBalancerStatusInfoResponseProto(
6490
ContainerBalancerConfiguration config) {
@@ -203,18 +229,17 @@ private static ContainerBalancerConfiguration getContainerBalancerConfiguration(
203229
}
204230

205231
@BeforeEach
206-
public void setup() throws UnsupportedEncodingException {
232+
void setup() {
207233
stopCmd = new ContainerBalancerStopSubcommand();
208234
startCmd = new ContainerBalancerStartSubcommand();
209235
statusCmd = new ContainerBalancerStatusSubcommand();
210-
System.setOut(new PrintStream(outContent, false, DEFAULT_ENCODING));
211-
System.setErr(new PrintStream(errContent, false, DEFAULT_ENCODING));
236+
out = GenericTestUtils.captureOut();
237+
err = GenericTestUtils.captureErr();
212238
}
213239

214240
@AfterEach
215-
public void tearDown() {
216-
System.setOut(originalOut);
217-
System.setErr(originalErr);
241+
void tearDown() {
242+
IOUtils.closeQuietly(out, err);
218243
}
219244

220245
@Test
@@ -230,30 +255,6 @@ void testContainerBalancerStatusInfoSubcommandRunningWithoutFlags()
230255
//test status is running
231256
when(scmClient.getContainerBalancerStatusInfo()).thenReturn(statusInfoResponseProto);
232257
statusCmd.execute(scmClient);
233-
Pattern p = Pattern.compile(
234-
"^ContainerBalancer\\sis\\sRunning.");
235-
String output = outContent.toString(DEFAULT_ENCODING);
236-
Matcher m = p.matcher(output);
237-
assertTrue(m.find());
238-
239-
String balancerConfigOutput =
240-
"Container Balancer Configuration values:\n" +
241-
"Key Value\n" +
242-
"Threshold 10.0\n" +
243-
"Max Datanodes to Involve per Iteration(percent) 20\n" +
244-
"Max Size to Move per Iteration 0GB\n" +
245-
"Max Size Entering Target per Iteration 26GB\n" +
246-
"Max Size Leaving Source per Iteration 26GB\n" +
247-
"Number of Iterations 3\n" +
248-
"Time Limit for Single Container's Movement 65min\n" +
249-
"Time Limit for Single Container's Replication 50min\n" +
250-
"Interval between each Iteration 0min\n" +
251-
"Whether to Enable Network Topology false\n" +
252-
"Whether to Trigger Refresh Datanode Usage Info false\n" +
253-
"Container IDs to Exclude from Balancing None\n" +
254-
"Datanodes Specified to be Balanced None\n" +
255-
"Datanodes Excluded from Balancing None";
256-
assertFalse(output.contains(balancerConfigOutput));
257258

258259
String currentIterationOutput =
259260
"Current iteration info:\n" +
@@ -273,9 +274,11 @@ void testContainerBalancerStatusInfoSubcommandRunningWithoutFlags()
273274
"Exited data from nodes \n" +
274275
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
275276
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
276-
assertFalse(output.contains(currentIterationOutput));
277277

278-
assertFalse(output.contains("Iteration history list:"));
278+
assertThat(out.get()).containsPattern(IS_RUNNING)
279+
.doesNotContain(BALANCER_CONFIG_OUTPUT)
280+
.doesNotContain(currentIterationOutput)
281+
.doesNotContain("Iteration history list:");
279282
}
280283

281284
@Test
@@ -293,42 +296,7 @@ void testContainerBalancerStatusInfoSubcommandVerboseHistory()
293296
CommandLine c = new CommandLine(statusCmd);
294297
c.parseArgs("--verbose", "--history");
295298
statusCmd.execute(scmClient);
296-
String output = outContent.toString(DEFAULT_ENCODING);
297-
Pattern p = Pattern.compile(
298-
"^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
299-
Matcher m = p.matcher(output);
300-
assertTrue(m.find());
301-
302-
p = Pattern.compile(
303-
"^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$", Pattern.MULTILINE);
304-
m = p.matcher(output);
305-
assertTrue(m.find());
306-
307-
p = Pattern.compile(
308-
"^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
309-
m = p.matcher(output);
310-
assertTrue(m.find());
311-
312-
String balancerConfigOutput =
313-
"Container Balancer Configuration values:\n" +
314-
"Key Value\n" +
315-
"Threshold 10.0\n" +
316-
"Max Datanodes to Involve per Iteration(percent) 20\n" +
317-
"Max Size to Move per Iteration 0GB\n" +
318-
"Max Size Entering Target per Iteration 26GB\n" +
319-
"Max Size Leaving Source per Iteration 26GB\n" +
320-
"Number of Iterations 3\n" +
321-
"Time Limit for Single Container's Movement 65min\n" +
322-
"Time Limit for Single Container's Replication 50min\n" +
323-
"Interval between each Iteration 0min\n" +
324-
"Whether to Enable Network Topology false\n" +
325-
"Whether to Trigger Refresh Datanode Usage Info false\n" +
326-
"Container IDs to Exclude from Balancing None\n" +
327-
"Datanodes Specified to be Balanced None\n" +
328-
"Datanodes Excluded from Balancing None";
329-
assertTrue(output.contains(balancerConfigOutput));
330-
331-
assertTrue(output.contains("Iteration history list:"));
299+
332300
String firstHistoryIterationOutput =
333301
"Key Value\n" +
334302
"Iteration number 3\n" +
@@ -346,7 +314,6 @@ void testContainerBalancerStatusInfoSubcommandVerboseHistory()
346314
"Exited data from nodes \n" +
347315
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
348316
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
349-
assertTrue(output.contains(firstHistoryIterationOutput));
350317

351318
String secondHistoryIterationOutput =
352319
"Key Value\n" +
@@ -365,7 +332,15 @@ void testContainerBalancerStatusInfoSubcommandVerboseHistory()
365332
"Exited data from nodes \n" +
366333
"b8b9c511-c30f-4933-8938-2f272e307070 -> 15 GB\n" +
367334
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 15 GB";
368-
assertTrue(output.contains(secondHistoryIterationOutput));
335+
336+
assertThat(out.get())
337+
.containsPattern(IS_RUNNING)
338+
.containsPattern(STARTED_AT)
339+
.containsPattern(DURATION)
340+
.contains(BALANCER_CONFIG_OUTPUT)
341+
.contains("Iteration history list:")
342+
.contains(firstHistoryIterationOutput)
343+
.contains(secondHistoryIterationOutput);
369344
}
370345

371346
@Test
@@ -383,40 +358,6 @@ void testContainerBalancerStatusInfoSubcommandVerbose()
383358
CommandLine c = new CommandLine(statusCmd);
384359
c.parseArgs("--verbose");
385360
statusCmd.execute(scmClient);
386-
String output = outContent.toString(DEFAULT_ENCODING);
387-
Pattern p = Pattern.compile(
388-
"^ContainerBalancer\\sis\\sRunning.$", Pattern.MULTILINE);
389-
Matcher m = p.matcher(output);
390-
assertTrue(m.find());
391-
392-
p = Pattern.compile(
393-
"^Started at: (\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})$", Pattern.MULTILINE);
394-
m = p.matcher(output);
395-
assertTrue(m.find());
396-
397-
p = Pattern.compile(
398-
"^Balancing duration: \\d{1}s$", Pattern.MULTILINE);
399-
m = p.matcher(output);
400-
assertTrue(m.find());
401-
402-
String balancerConfigOutput =
403-
"Container Balancer Configuration values:\n" +
404-
"Key Value\n" +
405-
"Threshold 10.0\n" +
406-
"Max Datanodes to Involve per Iteration(percent) 20\n" +
407-
"Max Size to Move per Iteration 0GB\n" +
408-
"Max Size Entering Target per Iteration 26GB\n" +
409-
"Max Size Leaving Source per Iteration 26GB\n" +
410-
"Number of Iterations 3\n" +
411-
"Time Limit for Single Container's Movement 65min\n" +
412-
"Time Limit for Single Container's Replication 50min\n" +
413-
"Interval between each Iteration 0min\n" +
414-
"Whether to Enable Network Topology false\n" +
415-
"Whether to Trigger Refresh Datanode Usage Info false\n" +
416-
"Container IDs to Exclude from Balancing None\n" +
417-
"Datanodes Specified to be Balanced None\n" +
418-
"Datanodes Excluded from Balancing None";
419-
assertTrue(output.contains(balancerConfigOutput));
420361

421362
String currentIterationOutput =
422363
"Current iteration info:\n" +
@@ -436,9 +377,14 @@ void testContainerBalancerStatusInfoSubcommandVerbose()
436377
"Exited data from nodes \n" +
437378
"b8b9c511-c30f-4933-8938-2f272e307070 -> 30 GB\n" +
438379
"7bd99815-47e7-4015-bc61-ca6ef6dfd130 -> 18 GB";
439-
assertTrue(output.contains(currentIterationOutput));
440380

441-
assertFalse(output.contains("Iteration history list:"));
381+
assertThat(out.get())
382+
.containsPattern(IS_RUNNING)
383+
.containsPattern(STARTED_AT)
384+
.containsPattern(DURATION)
385+
.contains(BALANCER_CONFIG_OUTPUT)
386+
.contains(currentIterationOutput)
387+
.doesNotContain("Iteration history list:");
442388
}
443389

444390
@Test
@@ -453,10 +399,7 @@ void testContainerBalancerStatusInfoSubcommandRunningOnStoppedBalancer()
453399
.build());
454400

455401
statusCmd.execute(scmClient);
456-
Pattern p = Pattern.compile(
457-
"^ContainerBalancer\\sis\\sNot\\sRunning.");
458-
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
459-
assertTrue(m.find());
402+
assertThat(out.get()).containsPattern(IS_NOT_RUNNING);
460403
}
461404

462405
@Test
@@ -471,23 +414,15 @@ void testContainerBalancerStatusSubcommandNotRunning()
471414

472415
statusCmd.execute(scmClient);
473416

474-
Pattern p = Pattern.compile(
475-
"^ContainerBalancer\\sis\\sNot\\sRunning.");
476-
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
477-
assertTrue(m.find());
417+
assertThat(out.get()).containsPattern(IS_NOT_RUNNING);
478418
}
479419

480420
@Test
481421
public void testContainerBalancerStopSubcommand() throws IOException {
482422
ScmClient scmClient = mock(ScmClient.class);
483423
stopCmd.execute(scmClient);
484424

485-
Pattern p = Pattern.compile("^Sending\\sstop\\scommand." +
486-
"\\sWaiting\\sfor\\sContainer\\sBalancer\\sto\\sstop...\\n" +
487-
"Container\\sBalancer\\sstopped.");
488-
489-
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
490-
assertTrue(m.find());
425+
assertThat(out.get()).containsPattern(WAITING_TO_STOP);
491426
}
492427

493428
@Test
@@ -503,10 +438,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsNotRunning()
503438
.build());
504439
startCmd.execute(scmClient);
505440

506-
Pattern p = Pattern.compile("^Container\\sBalancer\\sstarted" +
507-
"\\ssuccessfully.");
508-
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
509-
assertTrue(m.find());
441+
assertThat(out.get()).containsPattern(STARTED_SUCCESSFULLY);
510442
}
511443

512444
@Test
@@ -522,11 +454,7 @@ public void testContainerBalancerStartSubcommandWhenBalancerIsRunning()
522454
.build());
523455
startCmd.execute(scmClient);
524456

525-
Pattern p = Pattern.compile("^Failed\\sto\\sstart\\sContainer" +
526-
"\\sBalancer.");
527-
528-
Matcher m = p.matcher(outContent.toString(DEFAULT_ENCODING));
529-
assertTrue(m.find());
457+
assertThat(out.get()).containsPattern(FAILED_TO_START);
530458
}
531459

532460
}

0 commit comments

Comments
 (0)