diff --git a/moto/s3/models.py b/moto/s3/models.py index a33ec611ca14..0fb61d64f87f 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -200,11 +200,14 @@ def version_id(self) -> str: @property def value(self) -> bytes: + return copy.copy(self.get_value()) + + def get_value(self, start: int = 0, end: int | None = None) -> bytes: + """Read a slice without materializing the rest of the object.""" with self.lock: - self._value_buffer.seek(0) - r = self._value_buffer.read() - r = copy.copy(r) - return r + start, stop, _ = slice(start, end).indices(self.size) + self._value_buffer.seek(start) + return self._value_buffer.read(max(0, stop - start)) @property def arn(self) -> str: diff --git a/moto/s3/responses.py b/moto/s3/responses.py index b80bc387ad5e..ee9dde8d1c0b 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -1794,7 +1794,14 @@ def _bucket_response_delete_keys( def _handle_range_header( self, request: Any, response_headers: dict[str, Any], response_content: Any ) -> TYPE_RESPONSE: - if request.method == "HEAD": + def read_content(start: int = 0, end: int | None = None) -> Any: + if isinstance(response_content, FakeKey): + return response_content.get_value(start, end) + return response_content[start:end] + + if isinstance(response_content, FakeKey): + length = response_content.size + elif request.method == "HEAD": length = int(response_headers["content-length"]) else: length = len(response_content) @@ -1802,13 +1809,13 @@ def _handle_range_header( _, rspec = request.headers.get("range").split("=") if "," in rspec: - return 200, response_headers, response_content + return 200, response_headers, read_content() try: begin, end = [int(i) if i else None for i in rspec.split("-")] except ValueError: # if we can't parse the Range header, S3 just treat the request as a non-range request - return 200, response_headers, response_content + return 200, response_headers, read_content() if (begin is None and end == 0) or (begin is not None and begin > last): if request.method == "HEAD": @@ -1824,11 +1831,11 @@ def _handle_range_header( end = last else: # Treat as non-range request - return 200, response_headers, response_content + return 200, response_headers, read_content() if begin > min(end, last): # Treat as non-range request if after the logic is applied, the start of the range is greater than the end - return 200, response_headers, response_content + return 200, response_headers, read_content() if begin or end < last: # range requests do not return the checksum @@ -1836,7 +1843,7 @@ def _handle_range_header( del response_headers[key] response_headers["content-range"] = f"bytes {begin}-{end}/{length}" - content = response_content[begin : end + 1] + content = read_content(begin, end + 1) if request.method == "HEAD": response_headers["content-length"] = str((end - begin) + 1) else: @@ -2094,6 +2101,8 @@ def get_object(self) -> TYPE_RESPONSE: raise RangeNotSatisfiable response_headers["content-range"] = f"bytes 0-{key.size - 1}/{key.size}" return 206, response_headers, key.value + if self.headers.get("range"): + return self._handle_range_header(self.request, response_headers, key) return 200, response_headers, key.value def get_object_attributes(self) -> TYPE_RESPONSE: diff --git a/tests/test_s3/test_server.py b/tests/test_s3/test_server.py index eb5e7655283f..bc637f10c183 100644 --- a/tests/test_s3/test_server.py +++ b/tests/test_s3/test_server.py @@ -1,7 +1,7 @@ import io import uuid from unittest import SkipTest -from unittest.mock import patch +from unittest.mock import call, patch from urllib.parse import parse_qs, urlparse import boto3 @@ -13,6 +13,8 @@ import moto.server as server from moto import settings from moto.moto_server.threaded_moto_server import ThreadedMotoServer +from moto.s3.models import s3_backends +from tests import DEFAULT_ACCOUNT_ID class AuthenticatedClient(FlaskClient): @@ -36,6 +38,47 @@ def test_s3_server_get(): assert b"ListAllMyBucketsResult" in res.data +@pytest.mark.parametrize("buffer_size", [256, 16384]) +@pytest.mark.parametrize("versioned", [False, True]) +def test_ranged_get_reads_only_requested_bytes(buffer_size, versioned): + client = authenticated_client() + bucket = str(uuid.uuid4()) + client.put(f"/{bucket}") + if versioned: + client.put( + f"/{bucket}?versioning", + data="Enabled", + ) + data = bytes(range(256)) * 32 + with patch( + "moto.s3.models.get_s3_default_key_buffer_size", return_value=buffer_size + ): + created = client.put(f"/{bucket}/key", data=data) + version = created.headers.get("x-amz-version-id") + if versioned: + client.put(f"/{bucket}/key", data=b"new version") + key = s3_backends[DEFAULT_ACCOUNT_ID]["aws"].get_object( + bucket, "key", version_id=version + ) + path = f"/{bucket}/key" + (f"?versionId={version}" if version else "") + for spec, start, end in [ + ("bytes=7-22", 7, 23), + ("bytes=8180-", 8180, len(data)), + ("bytes=-19", len(data) - 19, len(data)), + ]: + with patch.object( + key._value_buffer, "read", wraps=key._value_buffer.read + ) as read: + response = client.get(path, headers={"Range": spec}) + assert response.status_code == 206 + assert response.data == data[start:end] + assert response.headers["Content-Range"] == ( + f"bytes {start}-{end - 1}/{len(data)}" + ) + # Byte correctness alone would also pass after reading the whole key. + assert read.call_args_list == [call(end - start)] + + @pytest.mark.parametrize("key_name", ["bar_baz", "bar+baz", "baz bar"]) def test_s3_server_bucket_create(key_name): test_client = authenticated_client()