Skip to content

Fix treatment of "#" in S3Hook.parse_s3_url() #41796

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
eladkal merged 1 commit into from
Aug 29, 2024
Merged

Fix treatment of "#" in S3Hook.parse_s3_url() #41796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airflow/providers/amazon/aws/hooks/s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ def parse_s3_url(s3url: str) -> tuple[str, str]:
valid_s3_virtual_hosted_format = "https://bucket-name.s3.region-code.amazonaws.com/key-name"
format = s3url.split("//")
if re.match(r"s3[na]?:", format[0], re.IGNORECASE):
parsed_url = urlsplit(s3url)
parsed_url = urlsplit(s3url, allow_fragments=False)
if not parsed_url.netloc:
raise S3HookUriParseFailure(
"Please provide a bucket name using a valid format of the form: "
Expand Down
74 changes: 55 additions & 19 deletions tests/providers/amazon/aws/hooks/test_s3.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,25 +81,61 @@ def test_transfer_config_args_invalid(self, transfer_config_args):
with pytest.raises(TypeError, match="transfer_config_args expected dict, got .*"):
S3Hook(transfer_config_args=transfer_config_args)

def test_parse_s3_url(self):
parsed = S3Hook.parse_s3_url("s3://test/this/is/not/a-real-key.txt")
assert parsed == ("test", "this/is/not/a-real-key.txt"), "Incorrect parsing of the s3 url"

def test_parse_s3_url_s3a_style(self):
parsed = S3Hook.parse_s3_url("s3a://test/this/is/not/a-real-key.txt")
assert parsed == ("test", "this/is/not/a-real-key.txt"), "Incorrect parsing of the s3 url"

def test_parse_s3_url_s3n_style(self):
parsed = S3Hook.parse_s3_url("s3n://test/this/is/not/a-real-key.txt")
assert parsed == ("test", "this/is/not/a-real-key.txt"), "Incorrect parsing of the s3 url"

def test_parse_s3_url_path_style(self):
parsed = S3Hook.parse_s3_url("https://s3.us-west-2.amazonaws.com/DOC-EXAMPLE-BUCKET1/test.jpg")
assert parsed == ("DOC-EXAMPLE-BUCKET1", "test.jpg"), "Incorrect parsing of the s3 url"

def test_parse_s3_url_virtual_hosted_style(self):
parsed = S3Hook.parse_s3_url("https://DOC-EXAMPLE-BUCKET1.s3.us-west-2.amazonaws.com/test.png")
assert parsed == ("DOC-EXAMPLE-BUCKET1", "test.png"), "Incorrect parsing of the s3 url"
@pytest.mark.parametrize(
"url, expected",
[
pytest.param(
"s3://test/this/is/not/a-real-key.txt", ("test", "this/is/not/a-real-key.txt"), id="s3 style"
),
pytest.param(
"s3a://test/this/is/not/a-real-key.txt",
("test", "this/is/not/a-real-key.txt"),
id="s3a style",
),
pytest.param(
"s3n://test/this/is/not/a-real-key.txt",
("test", "this/is/not/a-real-key.txt"),
id="s3n style",
),
pytest.param(
"https://s3.us-west-2.amazonaws.com/DOC-EXAMPLE-BUCKET1/test.jpg",
("DOC-EXAMPLE-BUCKET1", "test.jpg"),
id="path style",
),
pytest.param(
"https://DOC-EXAMPLE-BUCKET1.s3.us-west-2.amazonaws.com/test.png",
("DOC-EXAMPLE-BUCKET1", "test.png"),
id="virtual hosted style",
),
pytest.param(
"s3://test/this/is/not/a-real-key #2.txt",
("test", "this/is/not/a-real-key #2.txt"),
id="s3 style with #",
),
pytest.param(
"s3a://test/this/is/not/a-real-key #2.txt",
("test", "this/is/not/a-real-key #2.txt"),
id="s3a style with #",
),
pytest.param(
"s3n://test/this/is/not/a-real-key #2.txt",
("test", "this/is/not/a-real-key #2.txt"),
id="s3n style with #",
),
pytest.param(
"https://s3.us-west-2.amazonaws.com/DOC-EXAMPLE-BUCKET1/test #2.jpg",
("DOC-EXAMPLE-BUCKET1", "test #2.jpg"),
id="path style with #",
),
pytest.param(
"https://DOC-EXAMPLE-BUCKET1.s3.us-west-2.amazonaws.com/test #2.png",
("DOC-EXAMPLE-BUCKET1", "test #2.png"),
id="virtual hosted style with #",
),
],
)
def test_parse_s3_url(self, url: str, expected: tuple[str, str]):
assert S3Hook.parse_s3_url(url) == expected, "Incorrect parsing of the s3 url"

def test_parse_invalid_s3_url_virtual_hosted_style(self):
with pytest.raises(
Expand Down