Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions moto/s3/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -200,11 +200,14 @@ def version_id(self) -> str:

@property
def value(self) -> bytes:
return copy.copy(self.get_value())

def get_value(self, start: int = 0, end: int | None = None) -> bytes:
"""Read a slice without materializing the rest of the object."""
with self.lock:
self._value_buffer.seek(0)
r = self._value_buffer.read()
r = copy.copy(r)
return r
start, stop, _ = slice(start, end).indices(self.size)
self._value_buffer.seek(start)
return self._value_buffer.read(max(0, stop - start))

@property
def arn(self) -> str:
Expand Down
21 changes: 15 additions & 6 deletions moto/s3/responses.py
Original file line number Diff line number Diff line change
Expand Up @@ -1794,21 +1794,28 @@ def _bucket_response_delete_keys(
def _handle_range_header(
self, request: Any, response_headers: dict[str, Any], response_content: Any
) -> TYPE_RESPONSE:
if request.method == "HEAD":
def read_content(start: int = 0, end: int | None = None) -> Any:
if isinstance(response_content, FakeKey):
return response_content.get_value(start, end)
return response_content[start:end]

if isinstance(response_content, FakeKey):
length = response_content.size
elif request.method == "HEAD":
length = int(response_headers["content-length"])
else:
length = len(response_content)
last = length - 1

_, rspec = request.headers.get("range").split("=")
if "," in rspec:
return 200, response_headers, response_content
return 200, response_headers, read_content()

try:
begin, end = [int(i) if i else None for i in rspec.split("-")]
except ValueError:
# if we can't parse the Range header, S3 just treat the request as a non-range request
return 200, response_headers, response_content
return 200, response_headers, read_content()

if (begin is None and end == 0) or (begin is not None and begin > last):
if request.method == "HEAD":
Expand All @@ -1824,19 +1831,19 @@ def _handle_range_header(
end = last
else:
# Treat as non-range request
return 200, response_headers, response_content
return 200, response_headers, read_content()

if begin > min(end, last):
# Treat as non-range request if after the logic is applied, the start of the range is greater than the end
return 200, response_headers, response_content
return 200, response_headers, read_content()

if begin or end < last:
# range requests do not return the checksum
for key in [h for h in response_headers if h.startswith("x-amz-checksum-")]:
del response_headers[key]

response_headers["content-range"] = f"bytes {begin}-{end}/{length}"
content = response_content[begin : end + 1]
content = read_content(begin, end + 1)
if request.method == "HEAD":
response_headers["content-length"] = str((end - begin) + 1)
else:
Expand Down Expand Up @@ -2094,6 +2101,8 @@ def get_object(self) -> TYPE_RESPONSE:
raise RangeNotSatisfiable
response_headers["content-range"] = f"bytes 0-{key.size - 1}/{key.size}"
return 206, response_headers, key.value
if self.headers.get("range"):
return self._handle_range_header(self.request, response_headers, key)
return 200, response_headers, key.value

def get_object_attributes(self) -> TYPE_RESPONSE:
Expand Down
45 changes: 44 additions & 1 deletion tests/test_s3/test_server.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import io
import uuid
from unittest import SkipTest
from unittest.mock import patch
from unittest.mock import call, patch
from urllib.parse import parse_qs, urlparse

import boto3
Expand All @@ -13,6 +13,8 @@
import moto.server as server
from moto import settings
from moto.moto_server.threaded_moto_server import ThreadedMotoServer
from moto.s3.models import s3_backends
from tests import DEFAULT_ACCOUNT_ID


class AuthenticatedClient(FlaskClient):
Expand All @@ -36,6 +38,47 @@ def test_s3_server_get():
assert b"ListAllMyBucketsResult" in res.data


@pytest.mark.parametrize("buffer_size", [256, 16384])
@pytest.mark.parametrize("versioned", [False, True])
def test_ranged_get_reads_only_requested_bytes(buffer_size, versioned):
client = authenticated_client()
bucket = str(uuid.uuid4())
client.put(f"/{bucket}")
if versioned:
client.put(
f"/{bucket}?versioning",
data="<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
)
data = bytes(range(256)) * 32
with patch(
"moto.s3.models.get_s3_default_key_buffer_size", return_value=buffer_size
):
created = client.put(f"/{bucket}/key", data=data)
version = created.headers.get("x-amz-version-id")
if versioned:
client.put(f"/{bucket}/key", data=b"new version")
key = s3_backends[DEFAULT_ACCOUNT_ID]["aws"].get_object(
bucket, "key", version_id=version
)
path = f"/{bucket}/key" + (f"?versionId={version}" if version else "")
for spec, start, end in [
("bytes=7-22", 7, 23),
("bytes=8180-", 8180, len(data)),
("bytes=-19", len(data) - 19, len(data)),
]:
with patch.object(
key._value_buffer, "read", wraps=key._value_buffer.read
) as read:
response = client.get(path, headers={"Range": spec})
assert response.status_code == 206
assert response.data == data[start:end]
assert response.headers["Content-Range"] == (
f"bytes {start}-{end - 1}/{len(data)}"
)
# Byte correctness alone would also pass after reading the whole key.
assert read.call_args_list == [call(end - start)]


@pytest.mark.parametrize("key_name", ["bar_baz", "bar+baz", "baz bar"])
def test_s3_server_bucket_create(key_name):
test_client = authenticated_client()
Expand Down