Skip to content

Commit 613bd88

Browse files
willstrantonwillstranton
committed
Add strip_components to extract/download_and_extract http_archive
The `strip_components` attribute functions similar to tar --strip-components: > Strip NUMBER leading components from file names on extraction. This is an alternative to the existing `strip_prefix` attribute, which required knowing the exact prefix to be stripped. Only one of the two attributes (`strip_prefix`, `strip_components`) can be set at one time. Fixes #28879 RELNOTES[NEW]: Adds the `strip_components` attribute to `extract`/`download_and_extract`/`http_archive` to allow stripping of path components when extracting files.
1 parent a19391c commit 613bd88

15 files changed

Lines changed: 273 additions & 15 deletions

File tree

‎src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/CompressedFunction.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,9 @@
3030
* formats while all formats that collect multiple entries inside a single (potentially compressed)
3131
* archive are archiver formats. This class handles the former, compressor formats.
3232
*
33-
* <p>It ignores the {@link DecompressorDescriptor#prefix()} setting because compressed files cannot
34-
* contain directories.
33+
* <p>It ignores the {@link DecompressorDescriptor#prefix()} and {@link
34+
* DecompressorDescriptor#stripComponents()} setting because compressed files cannot contain
35+
* directories.
3536
*/
3637
public abstract class CompressedFunction implements Decompressor {
3738

‎src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/CompressedTarFunction.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ public Path decompress(DecompressorDescriptor descriptor)
108108
"Failed to extract %s, tarred paths cannot be absolute", strippedRelativePath));
109109
}
110110

111+
strippedRelativePath = strippedRelativePath.stripComponents(descriptor.stripComponents());
112+
if (strippedRelativePath == PathFragment.EMPTY_FRAGMENT) {
113+
continue;
114+
}
115+
111116
Path filePath = descriptor.destinationPath().getRelative(strippedRelativePath);
112117
if (!filePath.startsWith(descriptor.destinationPath())) {
113118
throw new IOException(

‎src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/DecompressorDescriptor.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ public record DecompressorDescriptor(
3333
Path archivePath,
3434
Path destinationPath,
3535
Optional<String> prefix,
36+
int stripComponents,
3637
ImmutableMap<String, String> renameFiles) {
3738
public DecompressorDescriptor {
3839
requireNonNull(context, "context");
@@ -45,6 +46,7 @@ public record DecompressorDescriptor(
4546
public static Builder builder() {
4647
return new AutoBuilder_DecompressorDescriptor_Builder()
4748
.setContext("")
49+
.setStripComponents(0)
4850
.setRenameFiles(ImmutableMap.of());
4951
}
5052

@@ -60,8 +62,18 @@ public abstract static class Builder {
6062

6163
public abstract Builder setPrefix(String prefix);
6264

65+
public abstract Builder setStripComponents(int stripComponents);
66+
6367
public abstract Builder setRenameFiles(Map<String, String> renameFiles);
6468

65-
public abstract DecompressorDescriptor build();
69+
public abstract DecompressorDescriptor autoBuild();
70+
71+
public DecompressorDescriptor build() {
72+
DecompressorDescriptor d = autoBuild();
73+
if (d.stripComponents != 0 && d.prefix().isPresent() && !d.prefix().get().isEmpty()) {
74+
throw new IllegalArgumentException("Only one of 'prefix' or 'stripComponents' can be set");
75+
}
76+
return d;
77+
}
6678
}
6779
}

‎src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressor.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,12 @@ public Path decompress(DecompressorDescriptor descriptor)
8686
if (entryPath.skip()) {
8787
continue;
8888
}
89-
extract7zEntry(sevenZFile, entry, destinationDirectory, entryPath.getPathFragment());
89+
PathFragment pathFragment =
90+
entryPath.getPathFragment().stripComponents(descriptor.stripComponents());
91+
if (pathFragment == PathFragment.EMPTY_FRAGMENT) {
92+
continue;
93+
}
94+
extract7zEntry(sevenZFile, entry, destinationDirectory, pathFragment);
9095
}
9196

9297
if (prefix.isPresent() && !foundPrefix) {

‎src/main/java/com/google/devtools/build/lib/bazel/repository/decompressor/ZipDecompressor.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,8 +95,13 @@ public Path decompress(DecompressorDescriptor descriptor)
9595
if (entryPath.skip()) {
9696
continue;
9797
}
98+
PathFragment pathFragment =
99+
entryPath.getPathFragment().stripComponents(descriptor.stripComponents());
100+
if (pathFragment == PathFragment.EMPTY_FRAGMENT) {
101+
continue;
102+
}
98103
extractZipEntry(
99-
reader, entry, destinationDirectory, entryPath.getPathFragment(), prefix, symlinks);
104+
reader, entry, destinationDirectory, pathFragment, prefix, symlinks);
100105
}
101106

102107
if (prefix.isPresent() && !foundPrefix) {

‎src/main/java/com/google/devtools/build/lib/bazel/repository/starlark/StarlarkBaseExternalContext.java‎

Lines changed: 41 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -977,7 +977,8 @@ field is for debugging purposes only and should not be relied upon as a stable A
977977
be used to strip it from extracted files.
978978
979979
<p>For compatibility, this parameter may also be used under the deprecated name
980-
<code>stripPrefix</code>.
980+
<code>stripPrefix</code>. Only one of <code>strip_prefix</code> or
981+
<code>strip_components</code> can be used.
981982
"""),
982983
@Param(
983984
name = "allow_fail",
@@ -1043,6 +1044,16 @@ field is for debugging purposes only and should not be relied upon as a stable A
10431044
positional = false,
10441045
named = true,
10451046
defaultValue = "''"),
1047+
@Param(
1048+
name = "strip_components",
1049+
positional = false,
1050+
named = true,
1051+
defaultValue = "0",
1052+
doc =
1053+
"""
1054+
Strip the given number leading components from file names on extraction. Only one of
1055+
<code>strip_components</code> or <code>strip_prefix</code> can be used.
1056+
"""),
10461057
})
10471058
public StructImpl downloadAndExtract(
10481059
Object url,
@@ -1057,9 +1068,12 @@ public StructImpl downloadAndExtract(
10571068
String integrity,
10581069
Dict<?, ?> renameFiles, // <String, String> expected
10591070
String oldStripPrefix,
1071+
StarlarkInt stripComponentsI,
10601072
StarlarkThread thread)
10611073
throws RepositoryFunctionException, InterruptedException, EvalException {
10621074
stripPrefix = renamedStripPrefix("download_and_extract", stripPrefix, oldStripPrefix);
1075+
int stripComponents = Starlark.toInt(stripComponentsI, "strip_components");
1076+
checkOneTypeOfStripping("download_and_extract", stripPrefix, stripComponents);
10631077
ImmutableMap<URI, Map<String, List<String>>> authHeaders =
10641078
getAuthHeaders(getAuthContents(authUnchecked, "auth"));
10651079

@@ -1162,6 +1176,7 @@ public StructImpl downloadAndExtract(
11621176
.setArchivePath(downloadedPath)
11631177
.setDestinationPath(outputPath.getPath())
11641178
.setPrefix(stripPrefix)
1179+
.setStripComponents(stripComponents)
11651180
.setRenameFiles(renameFilesMap)
11661181
.build(),
11671182
// Type does NOT need to be passed here, as the existing code renames the archive path to
@@ -1260,7 +1275,8 @@ private static void deleteTreeWithRetries(Path downloadDirectory)
12601275
used to strip it from extracted files.
12611276
12621277
<p>For compatibility, this parameter may also be used under the deprecated name
1263-
<code>stripPrefix</code>.
1278+
<code>stripPrefix</code>. Only one of <code>strip_prefix</code> or
1279+
<code>strip_components</code> can be set.
12641280
"""),
12651281
@Param(
12661282
name = "rename_files",
@@ -1291,6 +1307,16 @@ private static void deleteTreeWithRetries(Path downloadDirectory)
12911307
positional = false,
12921308
named = true,
12931309
defaultValue = "''"),
1310+
@Param(
1311+
name = "strip_components",
1312+
positional = false,
1313+
named = true,
1314+
defaultValue = "0",
1315+
doc =
1316+
"""
1317+
Strip the given number leading components from file names on extraction. Only one of
1318+
<code>strip_components</code> or <code>strip_prefix</code> can be set.
1319+
"""),
12941320
@Param(
12951321
name = "type",
12961322
defaultValue = "''",
@@ -1314,10 +1340,13 @@ public void extract(
13141340
Dict<?, ?> renameFiles, // <String, String> expected
13151341
String watchArchive,
13161342
String oldStripPrefix,
1343+
StarlarkInt stripComponentsI,
13171344
String type,
13181345
StarlarkThread thread)
13191346
throws RepositoryFunctionException, InterruptedException, EvalException {
13201347
stripPrefix = renamedStripPrefix("extract", stripPrefix, oldStripPrefix);
1348+
int stripComponents = Starlark.toInt(stripComponentsI, "strip_components");
1349+
checkOneTypeOfStripping("extract", stripPrefix, stripComponents);
13211350
StarlarkPath archivePath = getPath(archive);
13221351

13231352
if (!archivePath.exists()) {
@@ -1355,6 +1384,7 @@ public void extract(
13551384
.setArchivePath(archivePath.getPath())
13561385
.setDestinationPath(outputPath.getPath())
13571386
.setPrefix(stripPrefix)
1387+
.setStripComponents(stripComponents)
13581388
.setRenameFiles(renameFilesMap)
13591389
.build(),
13601390
Optional.ofNullable(type).filter(s -> !s.isBlank()));
@@ -1409,6 +1439,15 @@ private static String renamedStripPrefix(String method, String stripPrefix, Stri
14091439
method);
14101440
}
14111441

1442+
private static void checkOneTypeOfStripping(
1443+
String method, String stripPrefix, int stripComponents) throws EvalException {
1444+
if (!stripPrefix.isEmpty() && stripComponents > 0) {
1445+
throw Starlark.errorf(
1446+
"%s() got multiple strip values. Only one of 'strip_prefix' or 'strip_components' can be set",
1447+
method);
1448+
}
1449+
}
1450+
14121451
@StarlarkMethod(
14131452
name = "file",
14141453
doc = "Generates a file in the repository directory with the provided content.",

‎src/main/java/com/google/devtools/build/lib/vfs/PathFragment.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -668,6 +668,21 @@ private PathFragment subFragmentImpl(int beginIndex, int endIndex) {
668668
return makePathFragment(normalizedPath.substring(starti, endi), driveStrLength);
669669
}
670670

671+
/** Strip <code>numComponents</code> leading components from file names on extraction. */
672+
public PathFragment stripComponents(int numComponents) {
673+
if (numComponents == 0) {
674+
return this;
675+
}
676+
if (numComponents < 0) {
677+
throw new IllegalArgumentException(
678+
String.format("Invalid number of elements (%d)", numComponents));
679+
}
680+
if (numComponents >= this.segmentCount()) {
681+
return EMPTY_FRAGMENT;
682+
}
683+
return this.subFragment(numComponents);
684+
}
685+
671686
/**
672687
* Returns an {@link Iterable} that lazily yields the segments of this path.
673688
*

‎src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/CompressedTarFunctionTest.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,19 @@ public void testDecompressWithPrefix() throws Exception {
8080
archiveDescriptor.assertOutputFiles(outputDir, INNER_FOLDER_NAME);
8181
}
8282

83+
/**
84+
* Test decompressing a tar.gz file with hard link file and symbolic link file inside and
85+
* stripping the first component.
86+
*/
87+
@Test
88+
public void testDecompressWithStripComponents() throws Exception {
89+
DecompressorDescriptor.Builder descriptorBuilder =
90+
archiveDescriptor.createDescriptorBuilder().setStripComponents(1);
91+
Path outputDir = decompress(descriptorBuilder.build());
92+
93+
archiveDescriptor.assertOutputFiles(outputDir, INNER_FOLDER_NAME);
94+
}
95+
8396
/**
8497
* Test decompressing a tar.gz file, with some entries being renamed during the extraction
8598
* process.
@@ -116,6 +129,24 @@ public void testDecompressWithRenamedFilesAndPrefix() throws Exception {
116129
assertThat(innerDir.getRelative("renamedFile").exists()).isTrue();
117130
}
118131

132+
/** Test that entry renaming is applied prior to component stripping. */
133+
@Test
134+
public void testDecompressWithRenamedFilesAndStripComponents() throws Exception {
135+
String innerDirName = ROOT_FOLDER_NAME + "/" + INNER_FOLDER_NAME;
136+
137+
HashMap<String, String> renameFiles = new HashMap<>();
138+
renameFiles.put(innerDirName + "/hardLinkFile", innerDirName + "/renamedFile");
139+
DecompressorDescriptor.Builder descriptorBuilder =
140+
archiveDescriptor
141+
.createDescriptorBuilder()
142+
.setStripComponents(1)
143+
.setRenameFiles(renameFiles);
144+
Path outputDir = decompress(descriptorBuilder.build());
145+
146+
Path innerDir = outputDir.getRelative(INNER_FOLDER_NAME);
147+
assertThat(innerDir.getRelative("renamedFile").exists()).isTrue();
148+
}
149+
119150
private Path decompress(DecompressorDescriptor descriptor) throws Exception {
120151
return new CompressedTarFunction() {
121152
@Override

‎src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/SevenZDecompressorTest.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,19 @@ public void testDecompressWithPrefix() throws Exception {
9595
assertThat(files).contains(REGULAR_FILENAME);
9696
}
9797

98+
/** Test decompressing a .7z file and stripping components. */
99+
@Test
100+
public void testDecompressWithStripComponents() throws Exception {
101+
DecompressorDescriptor.Builder descriptorBuilder =
102+
archiveDescriptor().createDescriptorBuilder().setStripComponents(1);
103+
Path outputDir = decompress(descriptorBuilder.build());
104+
Path fileDir = outputDir.getRelative(INNER_FOLDER_NAME);
105+
106+
ImmutableList<String> files =
107+
fileDir.readdir(Symlinks.NOFOLLOW).stream().map(Dirent::getName).collect(toImmutableList());
108+
assertThat(files).contains(REGULAR_FILENAME);
109+
}
110+
98111
/** Test decompressing a .7z with entries being renamed during the extraction process. */
99112
@Test
100113
public void testDecompressWithRenamedFiles() throws Exception {
@@ -136,6 +149,35 @@ public void testDecompressWithRenamedFilesAndPrefix() throws Exception {
136149
assertThat(fileDir.getRelative("renamedFile").getFileSize()).isNotEqualTo(0);
137150
}
138151

152+
/** Test that entry renaming is applied prior to stripping components. */
153+
@Test
154+
public void testDecompressWithRenamedFilesAndStripComponents() throws Exception {
155+
String innerDirName = ROOT_FOLDER_NAME + "/" + INNER_FOLDER_NAME;
156+
157+
HashMap<String, String> renameFiles = new HashMap<>();
158+
renameFiles.put(innerDirName + "/" + REGULAR_FILENAME, innerDirName + "/renamedFile");
159+
DecompressorDescriptor.Builder descriptorBuilder =
160+
archiveDescriptor()
161+
.createDescriptorBuilder()
162+
.setStripComponents(1)
163+
.setRenameFiles(renameFiles);
164+
Path outputDir = decompress(descriptorBuilder.build());
165+
166+
Path fileDir = outputDir.getRelative(INNER_FOLDER_NAME);
167+
ImmutableList<String> files =
168+
fileDir.readdir(Symlinks.NOFOLLOW).stream().map(Dirent::getName).collect(toImmutableList());
169+
assertThat(files).contains("renamedFile");
170+
assertThat(fileDir.getRelative("renamedFile").getFileSize()).isNotEqualTo(0);
171+
}
172+
173+
/** Test decompressing a .7z file where everything is stripped */
174+
@Test
175+
public void testDecompressStripAllComponents() throws Exception {
176+
Path outputDir = decompress(archiveDescriptor().createDescriptorBuilder().setStripComponents(1_000).build());
177+
178+
assertThat(outputDir.exists()).isFalse();
179+
}
180+
139181
private File archiveDir;
140182
private File extractionDir;
141183

‎src/test/java/com/google/devtools/build/lib/bazel/repository/decompressor/ZipDecompressorTest.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ public void testDecompressWithoutPrefix() throws Exception {
8080
}
8181

8282
/**
83-
* Test decompressing a tar.gz file with hard link file and symbolic link file inside and
83+
* Test decompressing a zip file with hard link file and symbolic link file inside and
8484
* stripping a prefix
8585
*/
8686
@Test
@@ -93,6 +93,20 @@ public void testDecompressWithPrefix() throws Exception {
9393
archiveDescriptor.assertOutputFiles(outputDir, INNER_FOLDER_NAME);
9494
}
9595

96+
/**
97+
* Test decompressing a zip file with hard link file and symbolic link file inside and
98+
* stripping a component
99+
*/
100+
@Test
101+
public void testDecompressWithStripComponents() throws Exception {
102+
TestArchiveDescriptor archiveDescriptor = new TestArchiveDescriptor(ARCHIVE_NAME, "out", false);
103+
DecompressorDescriptor.Builder descriptorBuilder =
104+
archiveDescriptor.createDescriptorBuilder().setStripComponents(1);
105+
Path outputDir = decompress(descriptorBuilder.build());
106+
107+
archiveDescriptor.assertOutputFiles(outputDir, INNER_FOLDER_NAME);
108+
}
109+
96110
/**
97111
* Test decompressing a zip file, with some entries being renamed during the extraction process.
98112
*/
@@ -130,6 +144,25 @@ public void testDecompressWithRenamedFilesAndPrefix() throws Exception {
130144
assertThat(innerDir.getRelative("renamedFile").exists()).isTrue();
131145
}
132146

147+
/** Test that entry renaming is applied prior to stripping components. */
148+
@Test
149+
public void testDecompressWithRenamedFilesAndStripComponents() throws Exception {
150+
TestArchiveDescriptor archiveDescriptor = new TestArchiveDescriptor(ARCHIVE_NAME, "out", false);
151+
String innerDirName = ROOT_FOLDER_NAME + "/" + INNER_FOLDER_NAME;
152+
153+
HashMap<String, String> renameFiles = new HashMap<>();
154+
renameFiles.put(innerDirName + "/hardLinkFile", innerDirName + "/renamedFile");
155+
DecompressorDescriptor.Builder descriptorBuilder =
156+
archiveDescriptor
157+
.createDescriptorBuilder()
158+
.setStripComponents(1)
159+
.setRenameFiles(renameFiles);
160+
Path outputDir = decompress(descriptorBuilder.build());
161+
162+
Path innerDir = outputDir.getRelative(INNER_FOLDER_NAME);
163+
assertThat(innerDir.getRelative("renamedFile").exists()).isTrue();
164+
}
165+
133166
private Path decompress(DecompressorDescriptor descriptor) throws Exception {
134167
return ZipDecompressor.INSTANCE.decompress(descriptor);
135168
}

0 commit comments

Comments
 (0)