From 5a94364f5ad7aa8f8e79f0c962d2c07fec818ef0 Mon Sep 17 00:00:00 2001 From: s3inlc Date: Thu, 17 Sep 2026 11:50:37 +0200 Subject: [PATCH 01/11] adding local file and upload of cracker binaries --- ci/apiv2/test_agent_protocol.py | 82 ++- ci/apiv2/test_cracker.py | 504 +++++++++++++++++- ci/apiv2/test_taskwrapper.py | 4 +- ci/apiv2/utils.py | 82 ++- ci/phpunit/TestBase.php | 2 +- ci/phpunit/dba/AbstractModelFactoryTest.php | 2 +- ci/phpunit/dba/MassUpdateSetTest.php | 2 +- ci/phpunit/downloadapi/DownloadAppTest.php | 180 +++++++ .../openapi/crackerbinarytype.spec.json | 64 ++- ci/phpunit/inc/UtilTest.php | 148 ++++- .../inc/apiv2/openapi/SpecOverridesTest.php | 71 ++- .../inc/utils/CrackerBinaryUtilsTest.php | 2 +- ci/phpunit/inc/utils/CrackerUtilsTest.php | 314 ++++++++++- src/api/download.php | 18 + src/api/v2/index.php | 29 +- src/dba/models/CrackerBinary.php | 17 +- src/dba/models/CrackerBinaryFactory.php | 4 +- src/dba/models/generator.php | 4 +- src/inc/StartupConfig.php | 13 + src/inc/Util.php | 70 ++- .../agentapi/model/DownloadBinaryAction.php | 8 +- .../apiv2/auth/JwtAuthenticationFactory.php | 39 ++ src/inc/apiv2/common/AbstractHelperAPI.php | 112 +--- src/inc/apiv2/model/CrackerBinaryAPI.php | 76 +++ src/inc/apiv2/openapi/ModelApiPathBuilder.php | 6 + src/inc/apiv2/openapi/SpecOverrides.php | 85 ++- src/inc/defines/DDirectories.php | 11 +- .../CrackerBinaryDownloadHandler.php | 44 ++ src/inc/downloadapi/DownloadApp.php | 79 +++ .../downloadapi/DownloadAuthMiddleware.php | 44 ++ src/inc/downloadapi/DownloadRegistry.php | 19 + src/inc/startup/setup.json | 3 +- src/inc/startup/setup.php | 3 +- src/inc/utils/CrackerUtils.php | 265 ++++++++- src/inc/utils/DownloadUtils.php | 136 +++++ ...0917114300_cracker-binary-local-upload.sql | 6 + ...0917114300_cracker-binary-local-upload.sql | 3 + 37 files changed, 2356 insertions(+), 195 deletions(-) create mode 100644 ci/phpunit/downloadapi/DownloadAppTest.php create mode 100644 src/api/download.php create mode 100644 src/inc/apiv2/auth/JwtAuthenticationFactory.php create mode 100644 src/inc/downloadapi/CrackerBinaryDownloadHandler.php create mode 100644 src/inc/downloadapi/DownloadApp.php create mode 100644 src/inc/downloadapi/DownloadAuthMiddleware.php create mode 100644 src/inc/downloadapi/DownloadRegistry.php create mode 100644 src/inc/utils/DownloadUtils.php create mode 100644 src/migrations/mysql/20260917114300_cracker-binary-local-upload.sql create mode 100644 src/migrations/postgres/20260917114300_cracker-binary-local-upload.sql diff --git a/ci/apiv2/test_agent_protocol.py b/ci/apiv2/test_agent_protocol.py index e472ffbca..627d6dd52 100644 --- a/ci/apiv2/test_agent_protocol.py +++ b/ci/apiv2/test_agent_protocol.py @@ -16,12 +16,14 @@ import json import re import unittest +import urllib.parse import requests -from hashtopolis import Agent, Config, HealthCheck, Voucher +from hashtopolis import Agent, Config, Cracker, HealthCheck, Voucher from hashtopolis_agent import DummyAgent -from utils import BaseTest, do_create_agentassignent, do_create_dummy_agent, do_create_voucher, get_hashtopolis_uri +from utils import (BaseTest, SEVEN_ZIP_MAGIC, do_create_agentassignent, do_create_dummy_agent, + do_create_voucher, get_hashtopolis_uri) AGENT_ENDPOINT = '/api/server.php' @@ -32,6 +34,19 @@ def _uri(): return get_hashtopolis_uri() +def fetch_via_test_config(url, **kwargs): + """GET a server-generated url through the configured test server uri. + + The realworld dataset configures baseHost with a url that is only reachable + from outside the container, so server-generated absolute urls cannot be + fetched from within the tests. The authority is rewritten to the uri the + tests run against, path and query (which carry the agent token) are kept. + """ + parts = urllib.parse.urlparse(url) + base = urllib.parse.urlparse(_uri()) + return requests.get(base._replace(path=parts.path, query=parts.query).geturl(), **kwargs) + + def agent_request(payload): """POST a raw JSON payload to the agent API and return (status_code, body_text). @@ -535,6 +550,69 @@ def test_download_cracker_invalid_binary_version_id(self): assert_error_envelope(self, body, "downloadBinary") self.assertEqual(parse_envelope(body)['message'], "Invalid cracker binary type id!") + def test_download_cracker_local_binary(self): + """A locally stored cracker binary is served by the server itself: the + downloadBinary action returns the url of the download endpoint with the + requesting agent's token appended, so the archive can directly be fetched.""" + dummy = self._dummy() + content = SEVEN_ZIP_MAGIC + b'local-binary-download-test' + cracker = self.create_local_cracker(content=content, extra_payload={'version': '7.2.7'}) + + code, body = agent_request({ + "action": "downloadBinary", + "token": dummy.token, + "type": "cracker", + "binaryVersionId": cracker.id, + }) + self.assertEqual(code, 200) + resp = parse_envelope(body) + self.assertEqual(resp['response'], "SUCCESS") + url = resp['url'] + url_parts = urllib.parse.urlparse(url) + self.assertEqual(f'/api/download.php/crackerBinary/{cracker.id}', url_parts.path) + self.assertIn(f'token={dummy.token}', url_parts.query) + + # the agent can fetch the archive with the returned url + r = fetch_via_test_config(url) + self.assertEqual(200, r.status_code) + self.assertEqual(content, r.content) + + def test_download_cracker_local_binary_wrong_token_denied(self): + """The download url of a local binary only works with the agent token it + was issued for.""" + dummy = self._dummy() + cracker = self.create_local_cracker() + + code, body = agent_request({ + "action": "downloadBinary", + "token": dummy.token, + "type": "cracker", + "binaryVersionId": cracker.id, + }) + url = parse_envelope(body)['url'] + + r = fetch_via_test_config(url.replace(f'token={dummy.token}', 'token=wrong-token')) + self.assertEqual(401, r.status_code) + + def test_download_cracker_local_binary_external_unchanged(self): + """Cracker binaries referenced with an external url are answered with the + stored url, no token is appended.""" + dummy = self._dummy() + external_binaries = [c for c in Cracker.objects.filter() if not c.filename] + self.assertTrue(external_binaries, 'no externally referenced cracker binary found') + cracker = external_binaries[0] + + code, body = agent_request({ + "action": "downloadBinary", + "token": dummy.token, + "type": "cracker", + "binaryVersionId": cracker.id, + }) + resp = parse_envelope(body) + self.assertEqual(resp['response'], "SUCCESS") + self.assertEqual(cracker.downloadUrl, resp['url']) + self.assertNotIn('token=', resp['url']) + # --------------------------------------------------------------------------- # clientError diff --git a/ci/apiv2/test_cracker.py b/ci/apiv2/test_cracker.py index 91fe9e2c7..1fe83ef3b 100644 --- a/ci/apiv2/test_cracker.py +++ b/ci/apiv2/test_cracker.py @@ -1,5 +1,33 @@ -from hashtopolis import Cracker -from utils import BaseTest +import datetime +import glob +import io +import os +import threading +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + +import requests + +from hashtopolis import Cracker, CrackerType, FileImport, HashtopolisError +from utils import (BaseTest, SEVEN_ZIP_MAGIC, do_create_agent, do_create_local_cracker, + get_bearer_token, get_hashtopolis_uri) + + +CRACKERS_DIR = os.environ.get('HASHTOPOLIS_CRACKERS_PATH', '/usr/local/share/hashtopolis/crackers') +IMPORT_DIR = os.environ.get('HASHTOPOLIS_IMPORT_PATH', '/usr/local/share/hashtopolis/import') +APIV2 = get_hashtopolis_uri() + '/api/v2' + + +def archive_path(obj): + """Absolute path of the locally stored archive of a cracker binary.""" + return os.path.join(CRACKERS_DIR, f'{obj.id}_{obj.filename}') + + +def url_copy_path(obj): + """Absolute path of the local copy of a url-referenced cracker binary, + composed by the server from the cracker type and the version.""" + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + return os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-{obj.version}.7z') class CrackerTest(BaseTest): @@ -27,3 +55,475 @@ def test_expandables(self): model_obj = self.create_test_object() expandables = ['crackerBinaryType'] self._test_expandables(model_obj, expandables) + + +class TestCrackerUpload(BaseTest): + """Cracker binaries created by uploading a 7z archive instead of providing a url.""" + + def _assert_local_binary(self, obj, content, version): + # the archive filename is composed of the cracker type, the version and the extension + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + self.assertEqual(f'{cracker_type.typeName}-{version}.7z', obj.filename) + # the download url was generated automatically and points to the download endpoint + self.assertTrue(obj.downloadUrl.startswith('http')) + self.assertTrue(obj.downloadUrl.endswith(f'/api/download.php/crackerBinary/{obj.id}')) + # the archive is stored in the crackers directory + self.assertTrue(os.path.isfile(archive_path(obj)), 'archive is not stored in the crackers directory') + with open(archive_path(obj), 'rb') as f: + self.assertEqual(content, f.read()) + + def test_create_with_inline_source(self): + content = SEVEN_ZIP_MAGIC + b'inline-archive-content' + obj = self.create_local_cracker(content=content, extra_payload={'version': '7.2.7'}) + self._assert_local_binary(obj, content, '7.2.7') + + def test_create_with_import_source_chunked_upload(self): + """Arbitrary-size archives are chunk uploaded to the import directory + (the TUS protocol implementation) and then imported on creation.""" + content = SEVEN_ZIP_MAGIC + b'chunked-upload-archive-content' + import_name = f'cracker-upload-{datetime.datetime.now().isoformat()}.7z' + FileImport().do_upload(import_name, io.BytesIO(content)) + # the completed upload landed in the import directory + self.assertTrue(os.path.isfile(os.path.join(IMPORT_DIR, import_name))) + + obj = self.create_local_cracker(source_type='import', source_data=import_name, + extra_payload={'version': '7.2.7'}) + self._assert_local_binary(obj, content, '7.2.7') + # the archive was moved out of the import directory + self.assertFalse(os.path.isfile(os.path.join(IMPORT_DIR, import_name))) + + def test_create_with_url_source(self): + """The server fetches the archive itself from a http url.""" + content = SEVEN_ZIP_MAGIC + b'url-archive-content' + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', str(len(content))) + self.end_headers() + self.wfile.write(content) + + def log_message(self, format, *args): + pass + + server = HTTPServer(('127.0.0.1', 0), Handler) + threading.Thread(target=server.serve_forever).start() + try: + obj = self.create_local_cracker( + source_type='url', + source_data=f'http://127.0.0.1:{server.server_address[1]}/archive.7z', + extra_payload={'version': '7.2.7'}) + self._assert_local_binary(obj, content, '7.2.7') + finally: + server.shutdown() + + def test_create_invalid_archive_rejects(self): + """An archive which is not a 7z file is rejected, the import file is + restored and no binary is created.""" + version = f'9.9.9-{int(time.time())}' + import_name = f'cracker-invalid-{datetime.datetime.now().isoformat()}.txt' + FileImport().do_upload(import_name, io.BytesIO(b'not-a-7z-archive')) + + with self.assertRaises(HashtopolisError) as e: + do_create_local_cracker(source_type='import', source_data=import_name, + extra_payload={'version': version}) + self.assertEqual(400, e.exception.status_code) + + # the import file was put back to the import directory + self.assertTrue(os.path.isfile(os.path.join(IMPORT_DIR, import_name))) + # no binary was created + self.assertEqual([], list(Cracker.objects.filter(version=version))) + os.unlink(os.path.join(IMPORT_DIR, import_name)) + + def test_create_with_both_sources_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker( + extra_payload={'downloadUrl': 'https://example.org/files/cracker.7z'}, + delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_missing_source_data_rejects(self): + obj = Cracker(crackerBinaryTypeId=1, version='7.2.7', binaryName='cracker', + sourceType='inline') + with self.assertRaises(HashtopolisError) as e: + obj.save() + self.assertEqual(400, e.exception.status_code) + + def test_create_with_bogus_source_type_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_type='bogus', source_data='data', delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_invalid_base64_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_data='!!!no-base64!!!', delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_missing_import_file_rejects(self): + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_type='import', source_data='does-not-exist.7z', + delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_create_with_invalid_url_scheme_rejects(self): + """Only http and https urls can be fetched, no local files or stream wrappers.""" + with self.assertRaises(HashtopolisError) as e: + self.create_local_cracker(source_type='url', source_data='file:///etc/passwd', + delete=False) + self.assertEqual(400, e.exception.status_code) + + def test_patch_download_url_of_local_binary_rejects(self): + """The download url of a locally stored binary is owned by the server.""" + obj = self.create_local_cracker() + obj.downloadUrl = 'https://evil.example.org/cracker.7z' + with self.assertRaises(HashtopolisError) as e: + obj.save() + self.assertEqual(400, e.exception.status_code) + + # the url was not changed, but other attributes can still be patched + reloaded = Cracker.objects.get(pk=obj.id) + self.assertTrue(reloaded.downloadUrl.endswith(f'/api/download.php/crackerBinary/{obj.id}')) + reloaded.version = '8.0.0' + reloaded.save() + self.assertEqual('8.0.0', Cracker.objects.get(pk=obj.id).version) + + def test_patch_source_type_rejects(self): + """sourceType is only valid at creation, patching it is forbidden.""" + obj = self.create_local_cracker() + headers = {'Authorization': f'Bearer {get_bearer_token()}', + 'Content-Type': 'application/json'} + r = requests.patch( + f'{APIV2}/ui/crackers/{obj.id}', + headers=headers, + json={'data': {'type': 'crackerBinary', 'id': str(obj.id), + 'attributes': {'sourceType': 'url'}}}) + self.assertEqual(403, r.status_code) + + def test_delete_removes_archive(self): + obj = self.create_local_cracker(delete=False) + self.assertTrue(os.path.isfile(archive_path(obj))) + + obj.delete() + + self.assertFalse(os.path.isfile(archive_path(obj))) + # with valid authentication there is no such archive anymore + r = requests.get(f'{get_hashtopolis_uri()}/api/download.php/crackerBinary/{obj.id}', + headers={'Authorization': f'Bearer {get_bearer_token()}'}) + self.assertEqual(404, r.status_code) + + +class TestCrackerUrlCopy(BaseTest): + """Cracker binaries added with a download url: the server downloads a + local copy of the archive, so it has it for later analysis. The agents + still download the archive from the external url. If the download fails, + the create is rejected and nothing is added.""" + + def _serve_archive(self, content, status=200, content_length=None): + """Start a local http server answering every request with the same + response, returns the server and the url it is reachable at.""" + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(status) + if content is not None: + self.send_header('Content-Type', 'application/octet-stream') + self.send_header('Content-Length', + str(len(content) if content_length is None else content_length)) + self.end_headers() + self.wfile.write(content) + else: + self.end_headers() + + def log_message(self, format, *args): + pass + + server = HTTPServer(('127.0.0.1', 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server, f'http://127.0.0.1:{server.server_address[1]}/archive.7z' + + def _assert_rejected(self, url, expected_message, version): + """A create with a broken download url reports the failure on the + create and leaves no binary and no local copy of the archive behind.""" + with self.assertRaises(HashtopolisError) as e: + self.create_cracker(extra_payload={'downloadUrl': url, 'version': version}) + self.assertEqual(400, e.exception.status_code) + self.assertIn(expected_message, e.exception.title) + self.assertEqual([], list(Cracker.objects.filter(version=version))) + self.assertEqual([], glob.glob(os.path.join(CRACKERS_DIR, f'*{version}*'))) + + def test_create_with_download_url_stores_local_copy(self): + """The server downloads a local copy of the archive from the download + url, the url itself stays the external reference of the binary.""" + content = SEVEN_ZIP_MAGIC + b'url-copy-archive-content' + server, url = self._serve_archive(content) + try: + version = 'url-copy' + obj = self.create_cracker(extra_payload={'downloadUrl': url, 'version': version}, delete=False) + + # the binary still references the external url, it is not marked as locally stored + self.assertEqual(url, obj.downloadUrl) + self.assertIsNone(obj.filename) + + # the server downloaded a local copy of the archive + path = url_copy_path(obj) + self.assertTrue(os.path.isfile(path), 'local copy of the archive is missing') + with open(path, 'rb') as f: + self.assertEqual(content, f.read()) + + # deleting the binary removes the local copy + obj.delete() + self.assertFalse(os.path.isfile(path)) + finally: + server.shutdown() + + def test_create_with_unreachable_url_rejects(self): + """Nothing is listening on the url, the connection is refused and + the create is rejected.""" + self._assert_rejected('http://127.0.0.1:1/archive.7z', + 'Failed to download the archive from the download url', + f'unreach-{int(time.time())}') + + def test_create_with_not_found_url_rejects(self): + """The url answers with an http error status, the download fails and + the create is rejected.""" + server, url = self._serve_archive(None, status=404) + try: + self._assert_rejected(url, 'Failed to download the archive from the download url', + f'notfound-{int(time.time())}') + finally: + server.shutdown() + + def test_create_with_non_7z_archive_url_rejects(self): + """The url serves something else than a 7z archive, the create is + rejected and nothing is added.""" + server, url = self._serve_archive(b'not-a-7z-archive') + try: + self._assert_rejected(url, 'The archive at the download url is not a valid 7z archive!', + f'non7z-{int(time.time())}') + finally: + server.shutdown() + + def test_create_with_truncated_download_rejects(self): + """The url closes the connection before the announced content is + completely sent, the incomplete download is rejected.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'only-half-of-the-archive', + content_length=4096) + try: + self._assert_rejected(url, 'Download incomplete', + f'trunc-{int(time.time())}') + finally: + server.shutdown() + + def test_create_with_invalid_url_scheme_rejects(self): + """Only http and https urls are fetched by the server, no local files + or stream wrappers.""" + self._assert_rejected('file:///etc/passwd', + 'Only http and https download urls are supported!', + f'scheme-{int(time.time())}') + + def _assert_patch_rejected(self, obj, new_url, expected_message): + """Patching the download url to a broken url reports the failure on + the patch, the update is rolled back and the previously stored local + copy is left untouched.""" + old_url = obj.downloadUrl + path = url_copy_path(obj) + with open(path, 'rb') as f: + content_before = f.read() + + with self.assertRaises(HashtopolisError) as e: + obj.downloadUrl = new_url + obj.save() + self.assertEqual(400, e.exception.status_code) + self.assertIn(expected_message, e.exception.title) + + # the update was rolled back, the local copy is untouched + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual(old_url, reloaded.downloadUrl) + with open(path, 'rb') as f: + self.assertEqual(content_before, f.read()) + return reloaded + + def test_patch_download_url_refreshes_local_copy(self): + """Changing the download url re-downloads the local copy of the + archive from the new url, the binary itself keeps referencing the + new external url.""" + content_a = SEVEN_ZIP_MAGIC + b'archive-from-url-a' + server_a, url_a = self._serve_archive(content_a) + content_b = SEVEN_ZIP_MAGIC + b'archive-from-url-b' + server_b, url_b = self._serve_archive(content_b) + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url_a, 'version': 'url-patch'}, + delete=False) + path = url_copy_path(obj) + with open(path, 'rb') as f: + self.assertEqual(content_a, f.read()) + + obj.downloadUrl = url_b + obj.save() + + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual(url_b, reloaded.downloadUrl) + self.assertIsNone(reloaded.filename) + # the local copy was replaced with the archive from the new url + with open(path, 'rb') as f: + self.assertEqual(content_b, f.read()) + + obj.delete() + self.assertFalse(os.path.isfile(path)) + finally: + server_a.shutdown() + server_b.shutdown() + + def test_patch_download_url_with_version_stores_new_copy(self): + """Changing the download url together with the version stores the + refreshed local copy under the new archive filename and removes the + copy of the previous version.""" + content_a = SEVEN_ZIP_MAGIC + b'archive-old-version' + server_a, url_a = self._serve_archive(content_a) + content_b = SEVEN_ZIP_MAGIC + b'archive-new-version' + server_b, url_b = self._serve_archive(content_b) + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url_a, 'version': 'p-old'}, + delete=False) + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + old_path = os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-p-old.7z') + new_path = os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-p-new.7z') + self.assertTrue(os.path.isfile(old_path)) + + obj.downloadUrl = url_b + obj.version = 'p-new' + obj.save() + + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual('p-new', reloaded.version) + self.assertEqual(url_b, reloaded.downloadUrl) + with open(new_path, 'rb') as f: + self.assertEqual(content_b, f.read()) + self.assertFalse(os.path.isfile(old_path)) + + obj.delete() + self.assertFalse(os.path.isfile(new_path)) + finally: + server_a.shutdown() + server_b.shutdown() + + def test_patch_download_url_to_unreachable_url_rejects(self): + """Changing the download url to an url the server cannot download + from rejects the patch, the update is rolled back.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'url-patch-unreachable') + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url}, delete=False) + self._assert_patch_rejected(obj, 'http://127.0.0.1:1/archive.7z', + 'Failed to download the archive from the download url') + obj.delete() + finally: + server.shutdown() + + def test_patch_download_url_to_non_7z_archive_rejects(self): + """Changing the download url to an url which does not serve a 7z + archive rejects the patch and rolls the update back.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'url-patch-non-7z') + bad_server, bad_url = self._serve_archive(b'not-a-7z-archive') + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url}, delete=False) + self._assert_patch_rejected(obj, bad_url, + 'The archive at the download url is not a valid 7z archive!') + obj.delete() + finally: + server.shutdown() + bad_server.shutdown() + + def test_patch_download_url_invalid_scheme_rejects(self): + """The download url can only be changed to an http or https url.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'url-patch-scheme') + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url}, delete=False) + self._assert_patch_rejected(obj, 'file:///etc/passwd', + 'Only http and https download urls are supported!') + obj.delete() + finally: + server.shutdown() + + def test_patch_version_keeps_local_copy(self): + """The local copy tracks the download url: patching other attributes + like the version does not re-download it.""" + content = SEVEN_ZIP_MAGIC + b'url-copy-stays-on-version-patch' + server, url = self._serve_archive(content) + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url, 'version': 'keep-1'}, + delete=False) + cracker_type = CrackerType.objects.get(pk=obj.crackerBinaryTypeId) + path = os.path.join(CRACKERS_DIR, f'{obj.id}_{cracker_type.typeName}-keep-1.7z') + + obj.version = 'keep-2' + obj.save() + + reloaded = Cracker.objects.get(pk=obj.id) + self.assertEqual('keep-2', reloaded.version) + # the local copy is untouched, still under the filename of its version + self.assertTrue(os.path.isfile(path)) + with open(path, 'rb') as f: + self.assertEqual(content, f.read()) + + obj.delete() + self.assertFalse(os.path.isfile(path)) + finally: + server.shutdown() + + +class TestDownloadEndpoint(BaseTest): + """The download endpoint serving the locally stored cracker binary archives.""" + + def _download(self, obj, headers=None, params=None, kind='crackerBinary'): + return requests.get(f'{get_hashtopolis_uri()}/api/download.php/{kind}/{obj.id}', + headers=headers, params=params) + + def test_download_without_auth_rejected(self): + obj = self.create_local_cracker() + self.assertEqual(401, self._download(obj).status_code) + + def test_download_with_invalid_agent_token_rejected(self): + obj = self.create_local_cracker() + self.assertEqual(401, self._download(obj, params={'token': 'invalid-token'}).status_code) + + def test_download_with_invalid_bearer_rejected(self): + obj = self.create_local_cracker() + self.assertEqual(401, + self._download(obj, headers={'Authorization': 'Bearer invalid.jwt.token'}).status_code) + + def test_download_with_agent_token(self): + agent = do_create_agent() + self.delete_after_test(agent) + content = SEVEN_ZIP_MAGIC + b'download-endpoint-content' + obj = self.create_local_cracker(content=content) + + r = self._download(obj, params={'token': agent.token}) + self.assertEqual(200, r.status_code) + self.assertEqual(content, r.content) + self.assertEqual('application/x-7z-compressed', r.headers['Content-Type']) + self.assertEqual(f'attachment; filename="{obj.filename}"', r.headers['Content-Disposition']) + + def test_download_with_bearer_token(self): + content = SEVEN_ZIP_MAGIC + b'download-endpoint-content' + obj = self.create_local_cracker(content=content) + + r = self._download(obj, headers={'Authorization': f'Bearer {get_bearer_token()}'}) + self.assertEqual(200, r.status_code) + self.assertEqual(content, r.content) + + def test_download_range_request(self): + content = SEVEN_ZIP_MAGIC + b'download-endpoint-content' + obj = self.create_local_cracker(content=content) + + r = self._download(obj, headers={'Authorization': f'Bearer {get_bearer_token()}', + 'Range': 'bytes=0-5'}) + self.assertEqual(206, r.status_code) + self.assertEqual(content[:6], r.content) + self.assertEqual(f'bytes 0-5/{len(content)}', r.headers['Content-Range']) + + def test_download_unknown_kind_rejected(self): + agent = do_create_agent() + self.delete_after_test(agent) + obj = self.create_local_cracker() + r = self._download(obj, params={'token': agent.token}, kind='unknown') + self.assertEqual(404, r.status_code) diff --git a/ci/apiv2/test_taskwrapper.py b/ci/apiv2/test_taskwrapper.py index 5615c391c..653eb0829 100644 --- a/ci/apiv2/test_taskwrapper.py +++ b/ci/apiv2/test_taskwrapper.py @@ -1,6 +1,6 @@ from hashtopolis import Helper, HashtopolisError, TaskWrapper from hashtopolis import Cracker -from utils import BaseTest +from utils import BaseTest, get_cracker_archive_url class TaskWrapperTest(BaseTest): @@ -59,7 +59,7 @@ def test_helper_create_supertask_generic_cracker(self): cracker = Cracker( crackerBinaryTypeId=crackertype.id, version='1.2.3', - downloadUrl='https://example.org/generic-1.2.3.gz', + downloadUrl=get_cracker_archive_url(), binaryName='generic-x64') cracker.save() self.delete_after_test(cracker) diff --git a/ci/apiv2/utils.py b/ci/apiv2/utils.py index c41237094..4e399aeab 100644 --- a/ci/apiv2/utils.py +++ b/ci/apiv2/utils.py @@ -1,10 +1,13 @@ import abc +import base64 import datetime +from http.server import BaseHTTPRequestHandler, HTTPServer from io import BytesIO import json from pathlib import Path import requests import tempfile +import threading import time import unittest import zipfile @@ -51,6 +54,54 @@ def get_hashtopolis_uri(): return get_test_config()['hashtopolis_uri'] +def get_bearer_token(): + """Request an apiv2 JWT with the test credentials, for raw requests.""" + cfg = get_test_config() + r = requests.post(f"{cfg['hashtopolis_uri']}/api/v2/auth/token", + auth=(cfg['username'], cfg['password'])) + return json.loads(r.text)['token'] + + +# magic bytes every valid 7z archive starts with +SEVEN_ZIP_MAGIC = b'\x37\x7A\xBC\xAF\x27\x1C' + +# archive content served by the shared test server for download-url cracker +# creations, the server downloads a local copy of it on creation +CRACKER_URL_ARCHIVE_CONTENT = SEVEN_ZIP_MAGIC + b'cracker-download-url-archive' + + +class _CrackerArchiveHandler(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'application/x-7z-compressed') + self.send_header('Content-Length', str(len(CRACKER_URL_ARCHIVE_CONTENT))) + self.end_headers() + self.wfile.write(CRACKER_URL_ARCHIVE_CONTENT) + + def log_message(self, format, *args): + pass + + +_cracker_archive_server = None + + +def get_cracker_archive_url(): + """Url of a shared local http server which serves a valid 7z archive. + + Cracker binaries created with a download url need a url which really + serves an archive, the server downloads a local copy from it on creation. + The server is started lazily and kept alive for the whole test run, the + hashtopolis server can reach it because the tests run in the same + container/network namespace. + """ + global _cracker_archive_server + if _cracker_archive_server is None: + server = HTTPServer(('127.0.0.1', 0), _CrackerArchiveHandler) + threading.Thread(target=server.serve_forever, daemon=True).start() + _cracker_archive_server = server + return f'http://127.0.0.1:{_cracker_archive_server.server_address[1]}/cracker.7z' + + def _do_create_obj_from_file(model_class, file_prefix, extra_payload={}, **kwargs): file_id = kwargs.get('file_id') or '001' p = Path(__file__).parent.joinpath(f'testfiles/{model_class.__name__.lower()}/{file_prefix}_{file_id}.json') @@ -174,7 +225,33 @@ def do_create_accessgroup(**kwargs): def do_create_cracker(**kwargs): - return _do_create_obj_from_file(Cracker, 'create_cracker', **kwargs) + extra_payload = dict(kwargs.pop('extra_payload', None) or {}) + # the server downloads a local copy of the archive on creation, so the + # download url has to serve a real archive unless the test provides its own + extra_payload.setdefault('downloadUrl', get_cracker_archive_url()) + return _do_create_obj_from_file(Cracker, 'create_cracker', extra_payload, **kwargs) + + +def do_create_local_cracker(source_type='inline', source_data=None, + content=SEVEN_ZIP_MAGIC + b'local-cracker-test', + extra_payload={}, **kwargs): + """Create a cracker binary by uploading an archive instead of providing a url. + + source_data defaults to the base64 encoded content for source_type 'inline'. + For 'import' it is the filename of a file in the import directory (e.g. one + uploaded with FileImport.do_upload), for 'url' a http/https url. + """ + if source_data is None: + source_data = base64.b64encode(content).decode() + payload = json.loads( + Path(__file__).parent.joinpath('testfiles/cracker/create_cracker_001.json').read_text('UTF-8')) + # the download url is generated by the server when the archive is uploaded + del payload['downloadUrl'] + final_payload = {**payload, **extra_payload, + 'sourceType': source_type, 'sourceData': source_data} + obj = Cracker(**final_payload) + obj.save() + return obj def do_create_crackertype(**kwargs): @@ -388,6 +465,9 @@ def create_agentassignment(self, **kwargs): def create_cracker(self, **kwargs): return self._create_test_object(do_create_cracker, **kwargs) + def create_local_cracker(self, **kwargs): + return self._create_test_object(do_create_local_cracker, **kwargs) + def create_crackertype(self, **kwargs): return self._create_test_object(do_create_crackertype, **kwargs) diff --git a/ci/phpunit/TestBase.php b/ci/phpunit/TestBase.php index 3a420cc8c..c7408e17e 100644 --- a/ci/phpunit/TestBase.php +++ b/ci/phpunit/TestBase.php @@ -213,7 +213,7 @@ protected function createCrackerBinaryType(): CrackerBinaryType { protected function createCrackerBinary(CrackerBinaryType $crackerBinaryType): CrackerBinary { $crackerBinary = $this->createDatabaseObject( Factory::getCrackerBinaryFactory(), - new CrackerBinary(null, $crackerBinaryType->getId(), '1.0.' . uniqid(), 'https://example.invalid/' . uniqid(), 'binary_' . uniqid()) + new CrackerBinary(null, $crackerBinaryType->getId(), '1.0.' . uniqid(), 'https://example.invalid/' . uniqid(), 'binary_' . uniqid(), null) ); $this->assertTrue($crackerBinary instanceof CrackerBinary); return $crackerBinary; diff --git a/ci/phpunit/dba/AbstractModelFactoryTest.php b/ci/phpunit/dba/AbstractModelFactoryTest.php index 1b421fc2f..18adb7feb 100644 --- a/ci/phpunit/dba/AbstractModelFactoryTest.php +++ b/ci/phpunit/dba/AbstractModelFactoryTest.php @@ -1681,7 +1681,7 @@ private function setUpHealthCheck(): array { $crackerBinaryType = new CrackerBinaryType(null, '', 0); $crackerBinaryType = $this->createDatabaseObject(Factory::getCrackerBinaryTypeFactory(), $crackerBinaryType); - $crackerBinary = new CrackerBinary(null, $crackerBinaryType->getId(), '', '', ''); + $crackerBinary = new CrackerBinary(null, $crackerBinaryType->getId(), '', '', '', null); $crackerBinary = $this->createDatabaseObject(Factory::getCrackerBinaryFactory(), $crackerBinary); $healthCheck = new HealthCheck(null, 0, 0, 0, $hashType->getId(), $crackerBinary->getId(), 0, ''); diff --git a/ci/phpunit/dba/MassUpdateSetTest.php b/ci/phpunit/dba/MassUpdateSetTest.php index 66b7e92f8..5bbb199ca 100644 --- a/ci/phpunit/dba/MassUpdateSetTest.php +++ b/ci/phpunit/dba/MassUpdateSetTest.php @@ -159,7 +159,7 @@ public function testMassSingleUpdateWithMappedColumn(): void { $agent = $this->createDatabaseObject(Factory::getAgentFactory(), new Agent(null, '', '', 0, '', '', 0, 0, 0, '', '', 0, '', null, 0, '')); $hashType = $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, $prefix . '_ht', 0, 0)); $cbt = $this->createDatabaseObject(Factory::getCrackerBinaryTypeFactory(), new CrackerBinaryType(null, '', 0)); - $cb = $this->createDatabaseObject(Factory::getCrackerBinaryFactory(), new CrackerBinary(null, $cbt->getId(), '', '', '')); + $cb = $this->createDatabaseObject(Factory::getCrackerBinaryFactory(), new CrackerBinary(null, $cbt->getId(), '', '', '', null)); $healthCheck = $this->createDatabaseObject(Factory::getHealthCheckFactory(), new HealthCheck(null, 0, 0, 0, $hashType->getId(), $cb->getId(), 0, '')); $hca1 = $this->createDatabaseObject(Factory::getHealthCheckAgentFactory(), new HealthCheckAgent(null, $healthCheck->getId(), $agent->getId(), 0, 0, 0, 0, 100, '')); diff --git a/ci/phpunit/downloadapi/DownloadAppTest.php b/ci/phpunit/downloadapi/DownloadAppTest.php new file mode 100644 index 000000000..357685285 --- /dev/null +++ b/ci/phpunit/downloadapi/DownloadAppTest.php @@ -0,0 +1,180 @@ +type = $this->createDatabaseObject( + Factory::getCrackerBinaryTypeFactory(), + new CrackerBinaryType(null, 'download-test-type', 1) + ); + $this->externalBinary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://example.com/hc.7z', 'testcracker', null) + ); + + // create a locally stored binary through the import source + $this->agentToken = 'dl-test-' . uniqid(); + $this->createDatabaseObject( + Factory::getAgentFactory(), + new Agent(null, 'download-test-agent', '', 0, '', '', 0, 0, 0, $this->agentToken, '', 0, '', null, 0, '') + ); + + $importName = 'download-test-' . uniqid() . '.7z'; + $this->archiveContent = self::SEVEN_ZIP_MAGIC . 'download-test-content'; + file_put_contents(self::getImportPath() . $importName, $this->archiveContent); + $this->localBinary = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $importName); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $this->localBinary); + + if (isset($_SERVER['HTTP_RANGE'])) { + $this->savedHttpRange = true; + $this->savedHttpRangeValue = $_SERVER['HTTP_RANGE']; + } + } + + #[Override] + protected function tearDown(): void { + // remove the archive in case a test failed before it could clean up + $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); + if (file_exists($archive)) { + unlink($archive); + } + if ($this->savedHttpRange) { + $_SERVER['HTTP_RANGE'] = $this->savedHttpRangeValue; + } + else { + unset($_SERVER['HTTP_RANGE']); + } + parent::tearDown(); + } + + private static function getImportPath(): string { + return Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal() . '/'; + } + + private function runDownloadRequest(string $uriWithQuery, array $headers = []): ResponseInterface { + $request = (new ServerRequestFactory())->createServerRequest('GET', $uriWithQuery); + foreach ($headers as $name => $value) { + $request = $request->withHeader($name, $value); + } + return DownloadApp::create()->handle($request); + } + + private function localBinaryUri(): string { + return '/api/download.php/crackerBinary/' . $this->localBinary->getId(); + } + + // A request without any authentication is rejected with 401. + public function testNoAuthenticationIsRejected(): void { + $response = $this->runDownloadRequest($this->localBinaryUri()); + $this->assertEquals(401, $response->getStatusCode()); + $this->assertEquals('No access!', (string)$response->getBody()); + } + + // A request with an invalid agent token is rejected with 401. + public function testInvalidAgentTokenIsRejected(): void { + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=invalid-token'); + $this->assertEquals(401, $response->getStatusCode()); + $this->assertEquals('No access!', (string)$response->getBody()); + } + + // A request with an invalid JWT is rejected with 401. + public function testInvalidBearerTokenIsRejected(): void { + $response = $this->runDownloadRequest($this->localBinaryUri(), ['Authorization' => 'Bearer invalid.jwt.value']); + $this->assertEquals(401, $response->getStatusCode()); + } + + // A valid agent token allows to download the archive of a locally stored + // binary, including the download headers. + public function testAgentTokenCanDownloadArchive(): void { + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $this->assertEquals(200, $response->getStatusCode()); + $this->assertEquals($this->archiveContent, (string)$response->getBody()); + $this->assertEquals('application/x-7z-compressed', $response->getHeaderLine('Content-Type')); + $this->assertEquals( + 'attachment; filename="' . $this->localBinary->getFilename() . '"', + $response->getHeaderLine('Content-Disposition') + ); + $this->assertEquals(strlen($this->archiveContent), (int)$response->getHeaderLine('Content-Length')); + } + + // An unknown download kind is rejected with 404. + public function testUnknownKindIsRejected(): void { + $response = $this->runDownloadRequest('/api/download.php/unknown/' . $this->localBinary->getId() . '?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + $this->assertEquals('Unknown download kind!', (string)$response->getBody()); + } + + // A non existing binary id is rejected with 404. + public function testUnknownBinaryIdIsRejected(): void { + $response = $this->runDownloadRequest('/api/download.php/crackerBinary/99999999?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + } + + // Binaries which are not locally stored have no archive to download. + public function testExternalBinaryHasNoArchive(): void { + $response = $this->runDownloadRequest('/api/download.php/crackerBinary/' . $this->externalBinary->getId() . '?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + $this->assertEquals('No such cracker binary archive!', (string)$response->getBody()); + } + + // When the archive is not present on the server anymore, the download + // results in 404. + public function testMissingArchiveFileIsRejected(): void { + $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); + unlink($archive); + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $this->assertEquals(404, $response->getStatusCode()); + $this->assertEquals('The archive of this cracker binary is not present on the server!', (string)$response->getBody()); + } + + // A range request is answered with partial content. + public function testRangeRequestReturnsPartialContent(): void { + $_SERVER['HTTP_RANGE'] = 'bytes=0-5'; + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $this->assertEquals(206, $response->getStatusCode()); + $this->assertEquals(substr($this->archiveContent, 0, 6), (string)$response->getBody()); + $this->assertEquals('bytes 0-5/' . strlen($this->archiveContent), $response->getHeaderLine('Content-Range')); + } + + // A request with a matching ETag is answered with not modified. + public function testMatchingEtagReturnsNotModified(): void { + $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); + $etag = md5(filemtime($archive) . strlen($this->archiveContent)); + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken, ['If-None-Match' => $etag]); + $this->assertEquals(304, $response->getStatusCode()); + } +} diff --git a/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json b/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json index 37cd39695..e87ef913a 100644 --- a/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json +++ b/ci/phpunit/fixtures/openapi/crackerbinarytype.spec.json @@ -1847,7 +1847,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -1857,10 +1858,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -2217,7 +2229,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -2227,10 +2240,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -2587,7 +2611,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -2597,10 +2622,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -3009,7 +3045,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -3019,10 +3056,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } diff --git a/ci/phpunit/inc/UtilTest.php b/ci/phpunit/inc/UtilTest.php index 0a33763a5..93f8bc6c7 100644 --- a/ci/phpunit/inc/UtilTest.php +++ b/ci/phpunit/inc/UtilTest.php @@ -3,23 +3,48 @@ namespace Hashtopolis\inc; use Exception; +use Override; use Hashtopolis\dba\Factory; use Hashtopolis\dba\models\StoredValue; use Hashtopolis\TestBase; -use Hashtopolis\dba\AbstractModelFactory; -use Hashtopolis\dba\models\ConfigSection; -use Hashtopolis\dba\models\Config; -use Hashtopolis\dba\models\ApiGroup; -use Hashtopolis\dba\models\AgentBinary; -use Hashtopolis\dba\models\CrackerBinary; -use Hashtopolis\dba\models\CrackerBinaryType; -use Hashtopolis\dba\models\Preprocessor; -use Hashtopolis\dba\models\RightGroup; -use Hashtopolis\dba\models\HashType; +use Hashtopolis\inc\defines\DConfig; require_once(dirname(__FILE__) . '/../TestBase.php'); final class UtilTest extends TestBase { + private string|false $savedBackendUrl = false; + private array $savedServer = []; + + #[Override] + protected function setUp(): void { + parent::setUp(); + $this->savedBackendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + $this->savedServer = [ + 'HTTP_HOST' => $_SERVER['HTTP_HOST'] ?? null, + 'SERVER_PORT' => $_SERVER['SERVER_PORT'] ?? null, + 'HTTPS' => $_SERVER['HTTPS'] ?? null, + ]; + } + + #[Override] + protected function tearDown(): void { + if ($this->savedBackendUrl === false) { + putenv('HASHTOPOLIS_BACKEND_URL'); + } + else { + putenv('HASHTOPOLIS_BACKEND_URL=' . $this->savedBackendUrl); + } + foreach ($this->savedServer as $key => $value) { + if ($value === null) { + unset($_SERVER[$key]); + } + else { + $_SERVER[$key] = $value; + } + } + parent::tearDown(); + } + /** * extractFileExtension returns empty string when no dot is present. */ @@ -1034,4 +1059,107 @@ public function testCheckOrCreateInitialObjectAllSetupConfigs(): void { $this->assertNotNull($obj, "Config entry $id should exist"); } } + + /** + * Expected fallback result when HASHTOPOLIS_BACKEND_URL is not usable: + * the server URL derived from the current request plus the configured base URL. + * + * @throws Exception + */ + private function expectedFallbackUrl(): string { + return rtrim(Util::buildServerUrl() . SConfig::getInstance()->getVal(DConfig::BASE_URL), '/'); + } + + /** + * buildBackendBaseUrl takes scheme, host and port from HASHTOPOLIS_BACKEND_URL + * and strips any path it may contain. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnv(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://localhost:8080/api/v2'); + $this->assertEquals('http://localhost:8080', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl handles a HASHTOPOLIS_BACKEND_URL without path and port. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvNoPath(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://hashtopolis.example.com'); + $this->assertEquals('http://hashtopolis.example.com', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl keeps https scheme and non-default ports. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvHttpsPort(): void { + putenv('HASHTOPOLIS_BACKEND_URL=https://hashtopolis.example.com:8443/hashtopolis/api/v2'); + $this->assertEquals('https://hashtopolis.example.com:8443', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl keeps IPv6 hosts from HASHTOPOLIS_BACKEND_URL. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvIpv6(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://[::1]:8080/api/v2'); + $this->assertEquals('http://[::1]:8080', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl strips a trailing slash of the env value. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromEnvTrailingSlash(): void { + putenv('HASHTOPOLIS_BACKEND_URL=http://localhost:8080/'); + $this->assertEquals('http://localhost:8080', Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl falls back to the server URL for a malformed env value + * (missing scheme), instead of returning a broken URL. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlEnvMalformedFallsBack(): void { + $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; + $_SERVER['SERVER_PORT'] = '1234'; + unset($_SERVER['HTTPS']); + putenv('HASHTOPOLIS_BACKEND_URL=localhost:8080'); + $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); + $this->assertEquals('http://fallbackhost:1234' . SConfig::getInstance()->getVal(DConfig::BASE_URL), Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl falls back to the server URL when HASHTOPOLIS_BACKEND_URL is unset. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlEnvUnsetFallsBack(): void { + $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; + $_SERVER['SERVER_PORT'] = '1234'; + unset($_SERVER['HTTPS']); + putenv('HASHTOPOLIS_BACKEND_URL'); + $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); + $this->assertEquals('http://fallbackhost:1234' . SConfig::getInstance()->getVal(DConfig::BASE_URL), Util::buildBackendBaseUrl()); + } + + /** + * buildBackendBaseUrl falls back to the server URL for an empty env value. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlEnvEmptyFallsBack(): void { + $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; + $_SERVER['SERVER_PORT'] = '1234'; + unset($_SERVER['HTTPS']); + putenv('HASHTOPOLIS_BACKEND_URL='); + $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); + } } diff --git a/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php b/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php index 56f7bec0c..87874bf1c 100644 --- a/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php +++ b/ci/phpunit/inc/apiv2/openapi/SpecOverridesTest.php @@ -142,12 +142,79 @@ public function testTheDefaultsCoverEveryNonPublicUserAttribute(): void { } /** - * Only User carries 'public' features, so nothing else is corrected. + * Only User carries 'public' features, so it is the only model whose + * responses arrive attribute-filtered and need optional corrections. + * CrackerBinary only carries creation documentation. */ - public function testUserIsTheOnlyModelWithDefaults(): void { + public function testTheDefaultsContainTheExpectedModels(): void { $this->assertTrue(SpecOverrides::defaults()->has('User')); + $this->assertTrue(SpecOverrides::defaults()->has('CrackerBinary')); foreach (['Agent', 'Task', 'Hashlist', 'Config', 'ApiToken'] as $model) { $this->assertFalse(SpecOverrides::defaults()->has($model), $model); } } + + public function testAttributeDescriptionsAreAdded(): void { + $overrides = new SpecOverrides([ + 'Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => [ + 'email' => 'The email address', + 'state' => 'On or off', + ]], + ]); + $result = $overrides->apply('Foo', $this->attributesSchema()); + + $this->assertSame('The email address', $result['properties']['email']['description']); + $this->assertSame('On or off', $result['properties']['state']['description']); + /* purely additive, the required list and the types stay as they were */ + $this->assertSame(["name", "email", "isValid", "state"], $result['required']); + $this->assertSame(["type" => "string"], ['type' => $result['properties']['email']['type']]); + } + + /** + * Unlike the optional/nullable corrections, a description for an attribute + * which is not part of the schema shape is simply skipped: the same + * description set is applied to responses and requests, and creation-only + * form fields for example are not part of a response. + */ + public function testAttributeDescriptionsSkipAbsentProperties(): void { + $overrides = new SpecOverrides([ + 'Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => ['sourceType' => 'Upload source']], + ]); + $result = $overrides->apply('Foo', $this->attributesSchema()); + + $this->assertSame($this->attributesSchema(), $result); + } + + /** + * applyDescriptions() serves the create request schema: descriptions are + * added, but the response oriented optional/nullable corrections must not + * be applied to it. + */ + public function testApplyDescriptionsIgnoresTheOtherCorrections(): void { + $overrides = new SpecOverrides([ + 'Foo' => [ + SpecOverrides::OPTIONAL_ATTRIBUTES => ['email'], + SpecOverrides::NULLABLE_ATTRIBUTES => ['isValid'], + SpecOverrides::ATTRIBUTE_DESCRIPTIONS => ['email' => 'The email address'], + ], + ]); + $result = $overrides->applyDescriptions('Foo', $this->attributesSchema()); + + $this->assertSame('The email address', $result['properties']['email']['description']); + $this->assertSame(["name", "email", "isValid", "state"], $result['required']); + $this->assertSame('string', $result['properties']['email']['type']); + $this->assertSame('boolean', $result['properties']['isValid']['type']); + } + + public function testDescriptionsMustMapToNonEmptyStrings(): void { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("must map attribute names to non-empty descriptions"); + new SpecOverrides(['Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => ['email' => '']]]); + } + + public function testDescriptionsMustNameAttributes(): void { + $this->expectException(InvalidArgumentException::class); + $this->expectExceptionMessage("must map attribute names to non-empty descriptions"); + new SpecOverrides(['Foo' => [SpecOverrides::ATTRIBUTE_DESCRIPTIONS => [42 => 'desc']]]); + } } diff --git a/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php b/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php index db7cb5b70..d835092bf 100644 --- a/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php +++ b/ci/phpunit/inc/utils/CrackerBinaryUtilsTest.php @@ -36,7 +36,7 @@ protected function setUp(): void { private function addBinary(string $version): AbstractModel { return $this->createDatabaseObject( Factory::getCrackerBinaryFactory(), - new CrackerBinary(null, $this->type->getId(), $version, 'http://example.com', 'testcracker') + new CrackerBinary(null, $this->type->getId(), $version, 'http://example.com', 'testcracker', null) ); } diff --git a/ci/phpunit/inc/utils/CrackerUtilsTest.php b/ci/phpunit/inc/utils/CrackerUtilsTest.php index c2bc6497d..06d513719 100644 --- a/ci/phpunit/inc/utils/CrackerUtilsTest.php +++ b/ci/phpunit/inc/utils/CrackerUtilsTest.php @@ -4,8 +4,11 @@ use Hashtopolis\dba\AbstractModel; use Hashtopolis\dba\Factory; +use Hashtopolis\dba\QueryFilter; use Hashtopolis\dba\models\CrackerBinary; use Hashtopolis\dba\models\CrackerBinaryType; +use Hashtopolis\inc\Util; +use Hashtopolis\inc\defines\DDirectories; use Hashtopolis\inc\apiv2\error\HttpConflict; use Hashtopolis\inc\apiv2\error\HttpError; use Hashtopolis\inc\HTException; @@ -36,7 +39,7 @@ protected function setUp(): void { ); $this->binary = $this->createDatabaseObject( Factory::getCrackerBinaryFactory(), - new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://example.com', 'testcracker') + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://example.com', 'testcracker', null) ); } @@ -89,11 +92,310 @@ public function testCreateBinaryEmptyVersionThrowsHttpError(): void { CrackerUtils::createBinary('', 'testcracker', 'http://example.com', $this->type->getId()); } - // Verifies the full happy path: createBinary() creates and returns a new - // CrackerBinary when all fields are valid. - public function testCreateBinaryValidInputCreatesBinary(): void { - $b = CrackerUtils::createBinary('9.9.9', 'newcracker', 'http://example.com/dl', $this->type->getId()); + // Verifies that createBinary() rejects a download url the server cannot + // fetch, and that the failed creation leaves no binary and no partial + // archive behind. + public function testCreateBinaryUnreachableUrlRollsBack(): void { + $countBefore = $this->countBinariesOfType($this->type->getId()); + try { + CrackerUtils::createBinary('9.9.9', 'testcracker', 'http://127.0.0.1:1/cracker.7z', $this->type->getId()); + $this->fail('Expected HttpError for an unreachable download url'); + } + catch (HttpError $e) { + $this->assertStringContainsString('Failed to download the archive from the download url', $e->getMessage()); + } + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + $this->assertEmpty(glob(CrackerUtils::getCrackersPath() . '*_test-crackerutils-type-9.9.9.7z')); + } + + // Verifies that createBinary() only accepts http and https download urls, + // so the server cannot be pointed at local files or stream wrappers. + public function testCreateBinaryInvalidSchemeThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinary('9.9.9', 'testcracker', 'file:///etc/passwd', $this->type->getId()); + } + + private const SEVEN_ZIP_MAGIC = "\x37\x7A\xBC\xAF\x27\x1C"; + + private function getImportPath(): string { + return Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal() . '/'; + } + + private function countBinariesOfType(int $typeId): int { + $qF = new QueryFilter(CrackerBinary::CRACKER_BINARY_TYPE_ID, $typeId, '='); + return sizeof(Factory::getCrackerBinaryFactory()->filter([Factory::FILTER => $qF])); + } + + // Verifies the full happy path: createBinaryFromUpload() with sourceType 'import' + // moves the archive from the import directory to the crackers directory, composes + // the server-side filename and sets the download url to the download endpoint. + public function testCreateBinaryFromUploadImportSource(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + $content = self::SEVEN_ZIP_MAGIC . 'test-content'; + file_put_contents($this->getImportPath() . $name, $content); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + $this->assertEquals('test-crackerutils-type-7.2.7.7z', $b->getFilename()); + $this->assertEquals( + Util::buildBackendBaseUrl() . '/api/download.php/crackerBinary/' . $b->getId(), + $b->getDownloadUrl() + ); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + $this->assertEquals($content, file_get_contents($archive)); + $this->assertFileDoesNotExist($this->getImportPath() . $name); + + unlink($archive); + } + + // Verifies the full happy path: createBinaryFromUpload() with sourceType 'inline' + // stores the base64 decoded archive in the crackers directory. + public function testCreateBinaryFromUploadInlineSource(): void { + $content = self::SEVEN_ZIP_MAGIC . 'inline-content'; + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'inline', base64_encode($content)); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + $this->assertEquals('test-crackerutils-type-7.2.7.7z', $b->getFilename()); + $this->assertStringEndsWith('/api/download.php/crackerBinary/' . $b->getId(), $b->getDownloadUrl()); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + $this->assertEquals($content, file_get_contents($archive)); + + unlink($archive); + } + + // Verifies that the composed archive filename sanitizes all characters which are + // problematic in file names. + public function testCreateBinaryFromUploadSanitizesFilename(): void { + $type = $this->createDatabaseObject( + Factory::getCrackerBinaryTypeFactory(), + new CrackerBinaryType(null, 'weird cracker name!', 1) + ); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $type->getId(), 'inline', base64_encode(self::SEVEN_ZIP_MAGIC)); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + $this->assertEquals('weird-cracker-name--7.2.7.7z', $b->getFilename()); + unlink(CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename()); + } + + // Verifies that createBinaryFromUpload() rejects an unsupported sourceType. + public function testCreateBinaryFromUploadInvalidSourceTypeThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'bogus', 'data'); + } + + // Verifies that createBinaryFromUpload() rejects an empty version. + public function testCreateBinaryFromUploadEmptyVersionThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('', 'testcracker', $this->type->getId(), 'inline', base64_encode(self::SEVEN_ZIP_MAGIC)); + } + + // Verifies that createBinaryFromUpload() rejects missing sourceData. + public function testCreateBinaryFromUploadEmptySourceDataThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'inline', ''); + } + + // Verifies that createBinaryFromUpload() rejects sourceData which is not valid base64. + public function testCreateBinaryFromUploadInvalidBase64ThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'inline', '!!!no-base64!!!'); + } + + // Verifies that createBinaryFromUpload() only allows http and https urls, so no + // local files or stream wrappers can be fetched by the server. + public function testCreateBinaryFromUploadUrlSchemeThrowsHttpError(): void { + $this->expectException(HttpError::class); + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'url', 'file:///etc/passwd'); + } + + // Verifies that a non-7z archive is rejected and the import file is restored and + // no leftover binary or archive remains. + public function testCreateBinaryFromUploadImportNot7zRollsBack(): void { + $name = 'test-archive-' . uniqid() . '.txt'; + file_put_contents($this->getImportPath() . $name, 'not-a-7z-archive'); + $countBefore = $this->countBinariesOfType($this->type->getId()); + + try { + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->fail('Expected HttpError for a non-7z archive'); + } + catch (HttpError $e) { + // expected + } + + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + $this->assertFileExists($this->getImportPath() . $name); + $this->assertEmpty(glob(CrackerUtils::getCrackersPath() . '*_test-crackerutils-type-7.2.7.7z')); + + unlink($this->getImportPath() . $name); + } + + // Verifies that a missing import file results in an error and no leftover binary. + public function testCreateBinaryFromUploadImportFileMissingRollsBack(): void { + $countBefore = $this->countBinariesOfType($this->type->getId()); + + try { + CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', 'does-not-exist-' . uniqid() . '.7z'); + $this->fail('Expected HttpError for a missing import file'); + } + catch (HttpError $e) { + // expected + } + + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + } + + // Verifies that deleteBinary() removes the locally stored archive of the binary. + public function testDeleteBinaryRemovesLocalArchive(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'to-be-deleted'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); - $this->assertSame('9.9.9', $b->getVersion()); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + + CrackerUtils::deleteBinary($b->getId()); + + $this->assertFileDoesNotExist($archive); + $this->assertNull(Factory::getCrackerBinaryFactory()->get($b->getId())); + } + + // Verifies that deleteBinary() also removes the local copy which was + // downloaded from the download url of an externally referenced binary. + public function testDeleteBinaryRemovesDownloadedLocalCopy(): void { + $binary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '3.0.0', 'http://example.com/cracker.7z', 'testcracker', null) + ); + $copy = CrackerUtils::getCrackersPath() . $binary->getId() . '_test-crackerutils-type-3.0.0.7z'; + file_put_contents($copy, self::SEVEN_ZIP_MAGIC . 'downloaded-local-copy'); + $this->assertFileExists($copy); + + CrackerUtils::deleteBinary($binary->getId()); + + $this->assertFileDoesNotExist($copy); + } + + // Verifies that deleteBinaryType() removes the archives of all locally stored + // binaries of the type. + public function testDeleteBinaryTypeRemovesLocalArchives(): void { + $type = $this->createDatabaseObject( + Factory::getCrackerBinaryTypeFactory(), + new CrackerBinaryType(null, 'type2-' . uniqid(), 1) + ); + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'to-be-deleted'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + $archive = CrackerUtils::getCrackersPath() . $b->getId() . '_' . $b->getFilename(); + $this->assertFileExists($archive); + + CrackerUtils::deleteBinaryType($type->getId()); + + $this->assertFileDoesNotExist($archive); + $this->assertEquals(0, $this->countBinariesOfType($type->getId())); + } + + // Verifies that the download url of a locally stored binary cannot be changed. + public function testUpdateBinaryRejectsUrlChangeForLocalBinary(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'local'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + try { + CrackerUtils::updateBinary('8.0.0', 'testcracker', 'http://other.example.com/hc.7z', $b->getId()); + $this->fail('Expected HTException when changing the url of a local binary'); + } + catch (HTException $e) { + $this->assertStringContainsString('locally stored', $e->getMessage()); + } + + $reloaded = Factory::getCrackerBinaryFactory()->get($b->getId()); + $this->assertEquals($b->getDownloadUrl(), $reloaded->getDownloadUrl()); + CrackerUtils::deleteBinary($b->getId()); + } + + // Verifies that a locally stored binary can still be updated as long as the + // download url is not changed. + public function testUpdateBinaryAllowsUnchangedUrlForLocalBinary(): void { + $name = 'test-archive-' . uniqid() . '.7z'; + file_put_contents($this->getImportPath() . $name, self::SEVEN_ZIP_MAGIC . 'local'); + $b = CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'import', $name); + $this->registerDatabaseObject(Factory::getCrackerBinaryFactory(), $b); + + CrackerUtils::updateBinary('8.0.0', 'testcracker', $b->getDownloadUrl(), $b->getId()); + + $reloaded = Factory::getCrackerBinaryFactory()->get($b->getId()); + $this->assertEquals('8.0.0', $reloaded->getVersion()); + $this->assertEquals($b->getDownloadUrl(), $reloaded->getDownloadUrl()); + CrackerUtils::deleteBinary($b->getId()); + } + + // Verifies that changing the download url of an externally referenced binary + // re-downloads the local copy from it, and that a failed download rolls the + // update back so nothing is changed and the previous copy is kept. + public function testUpdateBinaryUrlChangeRollsBackOnFailedDownload(): void { + $binary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://127.0.0.1:1/original.7z', 'testcracker', null) + ); + $copy = CrackerUtils::getCrackersPath() . $binary->getId() . '_test-crackerutils-type-1.0.0.7z'; + file_put_contents($copy, self::SEVEN_ZIP_MAGIC . 'previous-local-copy'); + + try { + CrackerUtils::updateBinary('2.0.0', 'newcracker', 'http://127.0.0.1:1/changed.7z', $binary->getId()); + $this->fail('Expected HttpError for a failed download from the changed url'); + } + catch (HttpError $e) { + $this->assertStringContainsString('Failed to download the archive from the download url', $e->getMessage()); + } + + // the update was rolled back + $reloaded = Factory::getCrackerBinaryFactory()->get($binary->getId()); + $this->assertEquals('1.0.0', $reloaded->getVersion()); + $this->assertEquals('http://127.0.0.1:1/original.7z', $reloaded->getDownloadUrl()); + $this->assertEquals('testcracker', $reloaded->getBinaryName()); + // the previous local copy is still there, unchanged + $this->assertFileExists($copy); + $this->assertEquals(self::SEVEN_ZIP_MAGIC . 'previous-local-copy', file_get_contents($copy)); + } + + // Verifies that a changed download url of a url-referenced binary has to be + // an http or https url. + public function testUpdateBinaryUrlChangeInvalidSchemeThrowsHttpError(): void { + try { + CrackerUtils::updateBinary('2.0.0', 'testcracker', 'file:///etc/passwd', $this->binary->getId()); + $this->fail('Expected HttpError for an invalid download url scheme'); + } + catch (HttpError $e) { + $this->assertStringContainsString('Only http and https download urls are supported!', $e->getMessage()); + } + // nothing was changed + $reloaded = Factory::getCrackerBinaryFactory()->get($this->binary->getId()); + $this->assertEquals('http://example.com', $reloaded->getDownloadUrl()); + } + + // Verifies that updating a binary without changing its download url does + // not re-download the local copy, the copy tracks the download url. + public function testUpdateBinaryUnchangedUrlKeepsLocalCopy(): void { + $binary = $this->createDatabaseObject( + Factory::getCrackerBinaryFactory(), + new CrackerBinary(null, $this->type->getId(), '1.0.0', 'http://127.0.0.1:1/unreachable.7z', 'testcracker', null) + ); + $copy = CrackerUtils::getCrackersPath() . $binary->getId() . '_test-crackerutils-type-1.0.0.7z'; + file_put_contents($copy, self::SEVEN_ZIP_MAGIC . 'kept-local-copy'); + + // the url is not changed, so no download happens even though it is unreachable + CrackerUtils::updateBinary('2.0.0', 'newcracker', 'http://127.0.0.1:1/unreachable.7z', $binary->getId()); + + $reloaded = Factory::getCrackerBinaryFactory()->get($binary->getId()); + $this->assertEquals('2.0.0', $reloaded->getVersion()); + $this->assertEquals('newcracker', $reloaded->getBinaryName()); + // the local copy is untouched, still under the filename of its version + $this->assertFileExists($copy); + $this->assertEquals(self::SEVEN_ZIP_MAGIC . 'kept-local-copy', file_get_contents($copy)); } } diff --git a/src/api/download.php b/src/api/download.php new file mode 100644 index 000000000..66a86d44e --- /dev/null +++ b/src/api/download.php @@ -0,0 +1,18 @@ +run(); diff --git a/src/api/v2/index.php b/src/api/v2/index.php index 055215965..5d6809848 100644 --- a/src/api/v2/index.php +++ b/src/api/v2/index.php @@ -17,6 +17,7 @@ // }); use Hashtopolis\inc\apiv2\auth\HashtopolisAuthenticator; +use Hashtopolis\inc\apiv2\auth\JwtAuthenticationFactory; use Hashtopolis\inc\apiv2\auth\JWTBeforeHandler; use Hashtopolis\inc\apiv2\common\ClassMapper; use Hashtopolis\inc\apiv2\error\ErrorHandler; @@ -34,19 +35,12 @@ use Tuupola\Middleware\HttpBasicAuthentication; -use JimTools\JwtAuth\Decoder\FirebaseDecoder; -use JimTools\JwtAuth\Middleware\JwtAuthentication; -use JimTools\JwtAuth\Options; -use JimTools\JwtAuth\Secret; use JimTools\JwtAuth\Exceptions\AuthorizationException; use Middlewares\DeflateEncoder; use Psr\Http\Message\ServerRequestInterface as Request; -use JimTools\JwtAuth\Rules\RequestMethodRule; -use JimTools\JwtAuth\Rules\RequestPathRule; - require_once(__DIR__ . "/../../../vendor/autoload.php"); require_once(__DIR__ . "/../../inc/startup/include.php"); @@ -74,21 +68,12 @@ /* API token validation */ $container->set("JwtAuthentication", function (ContainerInterface $container) { - $decoder = new FirebaseDecoder( - new Secret(StartupConfig::getInstance()->getPepper(0), 'HS256', hash("sha256", StartupConfig::getInstance()->getPepper(0))) - ); - - $options = new Options( - isSecure: false, - attribute: null, - before: new JWTBeforeHandler - ); - - $rules = [ - new RequestPathRule(ignore: ["/api/v2/auth/token", "/api/v2/auth/oauth-token", "/api/v2/helper/resetUserPassword", "/api/v2/openapi.json"]), - new RequestMethodRule(ignore: ["OPTIONS"]) - ]; - return new JwtAuthentication($options, $decoder, $rules); + return JwtAuthenticationFactory::create([ + "/api/v2/auth/token", + "/api/v2/auth/oauth-token", + "/api/v2/helper/resetUserPassword", + "/api/v2/openapi.json" + ]); }); /* diff --git a/src/dba/models/CrackerBinary.php b/src/dba/models/CrackerBinary.php index 337a12331..55c01cfb5 100644 --- a/src/dba/models/CrackerBinary.php +++ b/src/dba/models/CrackerBinary.php @@ -10,13 +10,15 @@ class CrackerBinary extends AbstractModel { private ?string $version; private ?string $downloadUrl; private ?string $binaryName; + private ?string $filename; - function __construct(?int $crackerBinaryId, ?int $crackerBinaryTypeId, ?string $version, ?string $downloadUrl, ?string $binaryName) { + function __construct(?int $crackerBinaryId, ?int $crackerBinaryTypeId, ?string $version, ?string $downloadUrl, ?string $binaryName, ?string $filename) { $this->crackerBinaryId = $crackerBinaryId; $this->crackerBinaryTypeId = $crackerBinaryTypeId; $this->version = $version; $this->downloadUrl = $downloadUrl; $this->binaryName = $binaryName; + $this->filename = $filename; } function getKeyValueDict(): array { @@ -26,6 +28,7 @@ function getKeyValueDict(): array { $dict['version'] = $this->version; $dict['downloadUrl'] = $this->downloadUrl; $dict['binaryName'] = $this->binaryName; + $dict['filename'] = $this->filename; return $dict; } @@ -35,8 +38,9 @@ static function getFeatures(): array { $dict['crackerBinaryId'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => True, "protected" => True, "private" => False, "alias" => "crackerBinaryId", "public" => False, "dba_mapping" => False]; $dict['crackerBinaryTypeId'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "crackerBinaryTypeId", "public" => False, "dba_mapping" => False]; $dict['version'] = ['read_only' => False, "type" => "str(20)", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "version", "public" => False, "dba_mapping" => False]; - $dict['downloadUrl'] = ['read_only' => False, "type" => "str(150)", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "downloadUrl", "public" => False, "dba_mapping" => False]; + $dict['downloadUrl'] = ['read_only' => False, "type" => "str(255)", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => False, "private" => False, "alias" => "downloadUrl", "public" => False, "dba_mapping" => False]; $dict['binaryName'] = ['read_only' => False, "type" => "str(50)", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => False, "private" => False, "alias" => "binaryName", "public" => False, "dba_mapping" => False]; + $dict['filename'] = ['read_only' => True, "type" => "str(100)", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => True, "private" => False, "alias" => "filename", "public" => False, "dba_mapping" => False]; return $dict; } @@ -97,11 +101,20 @@ function setBinaryName(?string $binaryName): void { $this->binaryName = $binaryName; } + function getFilename(): ?string { + return $this->filename; + } + + function setFilename(?string $filename): void { + $this->filename = $filename; + } + const CRACKER_BINARY_ID = "crackerBinaryId"; const CRACKER_BINARY_TYPE_ID = "crackerBinaryTypeId"; const VERSION = "version"; const DOWNLOAD_URL = "downloadUrl"; const BINARY_NAME = "binaryName"; + const FILENAME = "filename"; const PERM_CREATE = "permCrackerBinaryCreate"; const PERM_READ = "permCrackerBinaryRead"; diff --git a/src/dba/models/CrackerBinaryFactory.php b/src/dba/models/CrackerBinaryFactory.php index 72b7f4878..459d29922 100644 --- a/src/dba/models/CrackerBinaryFactory.php +++ b/src/dba/models/CrackerBinaryFactory.php @@ -32,7 +32,7 @@ function getCacheValidTime(): int { * @return CrackerBinary */ function getNullObject(): CrackerBinary { - return new CrackerBinary(-1, null, null, null, null); + return new CrackerBinary(-1, null, null, null, null, null); } /** @@ -45,6 +45,6 @@ function createObjectFromDict(array $dict): CrackerBinary { $conv[strtolower($key)] = $val; } $dict = $conv; - return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname']); + return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname'], $dict['filename']); } } diff --git a/src/dba/models/generator.php b/src/dba/models/generator.php index ef58253aa..8103c4095 100644 --- a/src/dba/models/generator.php +++ b/src/dba/models/generator.php @@ -284,8 +284,10 @@ ['name' => 'crackerBinaryId', 'read_only' => True, 'type' => 'int', 'protected' => True], ['name' => 'crackerBinaryTypeId', 'read_only' => True, 'type' => 'int', 'relation' => 'CrackerBinaryType'], ['name' => 'version', 'read_only' => False, 'type' => 'str(20)'], - ['name' => 'downloadUrl', 'read_only' => False, 'type' => 'str(150)'], + ['name' => 'downloadUrl', 'read_only' => False, 'null' => True, 'type' => 'str(255)'], ['name' => 'binaryName', 'read_only' => False, 'type' => 'str(50)'], + // archive filename of a server-hosted binary; NULL means the binary is downloaded from downloadUrl + ['name' => 'filename', 'read_only' => True, 'null' => True, 'type' => 'str(100)', 'protected' => True], ], ]; $CONF['CrackerBinaryType'] = [ diff --git a/src/inc/StartupConfig.php b/src/inc/StartupConfig.php index 25b108214..1a2ccd273 100644 --- a/src/inc/StartupConfig.php +++ b/src/inc/StartupConfig.php @@ -21,6 +21,7 @@ class StartupConfig { private const DIRECTORY_LOG = "log"; private const DIRECTORY_CONFIG = "config"; private const DIRECTORY_TUS = "tus"; + private const DIRECTORY_CRACKERS = "crackers"; private const DB_PROPERTY_TYPE = "type"; private const DB_PROPERTY_USER = "user"; @@ -55,6 +56,7 @@ public function __construct() { "log" => "/usr/local/share/hashtopolis/log", "config" => "/usr/local/share/hashtopolis/config", "tus" => "/var/tmp/tus/", + "crackers" => "/usr/local/share/hashtopolis/crackers", ]; $this->db_properties = [ @@ -134,6 +136,9 @@ private function loadEnv(): void { if (getenv('HASHTOPOLIS_TUS_PATH') !== false) { $this->directories[self::DIRECTORY_TUS] = getenv('HASHTOPOLIS_TUS_PATH'); } + if (getenv('HASHTOPOLIS_CRACKERS_PATH') !== false) { + $this->directories[self::DIRECTORY_CRACKERS] = getenv('HASHTOPOLIS_CRACKERS_PATH'); + } } /** @@ -159,10 +164,14 @@ private function loadLegacyConfig(): void { "log" => dirname(__FILE__) . "/../log/", "config" => dirname(__FILE__) . "/../config/", "tus" => "/var/tmp/tus/", + "crackers" => dirname(__FILE__) . "/../crackers/", ]; } else { $this->directories = $DIRECTORIES; + if (!array_key_exists(self::DIRECTORY_CRACKERS, $this->directories)) { + $this->directories[self::DIRECTORY_CRACKERS] = dirname(__FILE__) . "/../crackers/"; + } } // extract old database settings format @@ -203,6 +212,10 @@ public function getDirectoryTus(): string { return $this->directories[self::DIRECTORY_TUS]; } + public function getDirectoryCrackers(): string { + return $this->directories[self::DIRECTORY_CRACKERS]; + } + public function getDatabaseType(): string { return $this->db_properties[self::DB_PROPERTY_TYPE]; } diff --git a/src/inc/Util.php b/src/inc/Util.php index 0b360a91c..0caac2634 100755 --- a/src/inc/Util.php +++ b/src/inc/Util.php @@ -1194,11 +1194,23 @@ public static function uploadFile(string $target, string $type, array|string $so break; case "url": - $furl = fopen($sourcedata, "rb"); + error_clear_last(); + $furl = @fopen($sourcedata, "rb"); if (!$furl) { $msg = "Could not open url at source data!"; + $lastError = error_get_last(); + if ($lastError !== null) { + $msg .= " (" . $lastError['message'] . ")"; + } } else { + $contentLength = null; + $meta = stream_get_meta_data($furl); + foreach ($meta['wrapper_data'] ?? [] as $header) { + if (preg_match('/^Content-Length:\s*(\d+)\s*$/i', $header, $matches)) { + $contentLength = (int)$matches[1]; + } + } $fileLocation = fopen($target, "w"); if (!$fileLocation) { $msg = "Could not open target file!"; @@ -1206,18 +1218,41 @@ public static function uploadFile(string $target, string $type, array|string $so else { $buffersize = 131072; $last_logged = time(); + $failed = false; + $received = 0; while (!feof($furl)) { - if (!$data = fread($furl, $buffersize)) { + $data = fread($furl, $buffersize); + if ($data === false) { $msg = "READ ERROR on download"; + $failed = true; break; } - fwrite($fileLocation, $data); + if ($data === '') { + break; + } + if (fwrite($fileLocation, $data) !== strlen($data)) { + $msg = "Failed to write downloaded data to the target file!"; + $failed = true; + break; + } + $received += strlen($data); if ($last_logged < time() - 10) { $last_logged = time(); } } fclose($fileLocation); - $success = true; + if (!$failed && $contentLength !== null && $received !== $contentLength) { + $msg = "Download incomplete, received " . $received . " of " . $contentLength . " bytes!"; + $failed = true; + } + if ($failed) { + if (file_exists($target)) { + unlink($target); + } + } + else { + $success = true; + } } fclose($furl); } @@ -1268,6 +1303,33 @@ public static function buildServerUrl(): string { return $protocol . $hostname . $port; } + /** + * Determines the base URL of the backend as it is reachable from the outside. + * If HASHTOPOLIS_BACKEND_URL is set in the environment, scheme, host and port are + * taken from it and any path is stripped (e.g. "http://localhost:8080/api/v2" + * results in "http://localhost:8080"). If the variable is not set or malformed, + * this falls back to the server URL derived from the current request together + * with the configured base URL, respecting the baseHost config override. + * Used to generate agent reachable URLs for locally hosted files, e.g. the + * download URL of an uploaded cracker binary archive. + * @return string backend base url without trailing slash + * @throws Exception + */ + public static function buildBackendBaseUrl(): string { + $backendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + if ($backendUrl !== false && strlen($backendUrl) > 0) { + $parts = parse_url($backendUrl); + if ($parts !== false && isset($parts['scheme'], $parts['host'])) { + $url = $parts['scheme'] . '://' . $parts['host']; + if (isset($parts['port'])) { + $url .= ':' . $parts['port']; + } + return rtrim($url, '/'); + } + } + return rtrim(Util::buildServerUrl() . SConfig::getInstance()->getVal(DConfig::BASE_URL), '/'); + } + /** * Round to a specific amount of decimal points * @param $num Number diff --git a/src/inc/agentapi/model/DownloadBinaryAction.php b/src/inc/agentapi/model/DownloadBinaryAction.php index 7026effe6..884b3f4b2 100644 --- a/src/inc/agentapi/model/DownloadBinaryAction.php +++ b/src/inc/agentapi/model/DownloadBinaryAction.php @@ -69,8 +69,14 @@ public function __invoke(Request $request, Response $response): ResponseInterfac $crackerBinaryType = Factory::getCrackerBinaryTypeFactory()->get($crackerBinary->getCrackerBinaryTypeId()); DServerLog::log(DServerLog::TRACE, 'Agent ' . $agent->getId() . ' downloaded cracker binary ' . $crackerBinary->getId()); $ext = Util::getFileExtension($agent->getOs()); + $url = $crackerBinary->getDownloadUrl(); + // locally stored binaries are downloaded from this server, the download + // endpoint requires the token of the requesting agent as authentication + if ($crackerBinary->getFilename() !== null) { + $url .= '?token=' . $agent->getToken(); + } return $this->success($response, PActions::DOWNLOAD_BINARY, [ - PResponseBinaryDownload::URL => $crackerBinary->getDownloadUrl(), + PResponseBinaryDownload::URL => $url, PResponseBinaryDownload::NAME => $crackerBinaryType->getTypeName(), PResponseBinaryDownload::EXECUTABLE => $crackerBinary->getBinaryName() . $ext, ]); diff --git a/src/inc/apiv2/auth/JwtAuthenticationFactory.php b/src/inc/apiv2/auth/JwtAuthenticationFactory.php new file mode 100644 index 000000000..92d9776e3 --- /dev/null +++ b/src/inc/apiv2/auth/JwtAuthenticationFactory.php @@ -0,0 +1,39 @@ +getPepper(0), 'HS256', hash("sha256", StartupConfig::getInstance()->getPepper(0))) + ); + + $options = new Options( + isSecure: false, + attribute: null, + before: new JWTBeforeHandler + ); + + $rules = [ + new RequestPathRule(ignore: $ignorePaths), + new RequestMethodRule(ignore: ["OPTIONS"]) + ]; + return new JwtAuthentication($options, $decoder, $rules); + } +} diff --git a/src/inc/apiv2/common/AbstractHelperAPI.php b/src/inc/apiv2/common/AbstractHelperAPI.php index ba72c8131..8c9a24870 100644 --- a/src/inc/apiv2/common/AbstractHelperAPI.php +++ b/src/inc/apiv2/common/AbstractHelperAPI.php @@ -15,7 +15,7 @@ use Psr\Http\Message\ServerRequestInterface as Request; use Slim\App; use Slim\Exception\HttpForbiddenException; -use Hashtopolis\inc\Util; +use Hashtopolis\inc\utils\DownloadUtils; abstract class AbstractHelperAPI extends AbstractBaseAPI { abstract public function actionPost(array $data): AbstractModel|array|null; @@ -140,118 +140,16 @@ static public function register(App $app): void { } /** - * Handles HTTP range requests for partial content delivery + * Streams the given file as a download response, handling ETag based caching + * and partial content (range) requests. * - * This method processes the `Range` header from the HTTP request - * to determine the start and end byte positions for the response, - * ensuring the range is valid and updates the file pointer accordingly. - * - * @param int &$start A reference to the starting byte of the range. This value will be updated. - * @param int &$end A reference to the ending byte of the range. This value will be updated. - * @param int $size The total size of the content in bytes. - * @param resource $fp A file pointer resource to seek to the correct position for the range. - * @return bool Returns `true` if the range request is valid and successfully processed, or `false` otherwise. - * - * @throws InvalidArgumentException If the `Range` header is malformed. - * - * @note This function assumes the presence of the `HTTP_RANGE` header in the `$_SERVER` superglobal. - */ - protected function handleRangeRequest(int &$start, int &$end, int $size, $fp): bool { - $c_end = $end; - - list(, $range) = explode('=', $_SERVER['HTTP_RANGE'], 2); - - if (str_contains($range, ',')) { - return false; - } - if ($range == '-') { - $c_start = $size - (int)substr($range, 1); - } - else { - $range = explode('-', $range); - $c_start = (int)$range[0]; - if ((isset($range[1]) && is_numeric($range[1]))) { - $c_end = (int)$range[1]; - } - else { - $c_end = $size; - } - } - if ($c_end > $end) { - $c_end = $end; - } - if ($c_start > $c_end || $c_start > $size - 1 || $c_end >= $size) { - return false; - } - $start = $c_start; - $end = $c_end; - fseek($fp, $start); - return true; - } - - /** * @param Request $request * @param Response $response - * @param string $filename + * @param string $filename Absolute path of the file to stream * @return Response * @throws HttpForbiddenException */ protected function startDownload(Request $request, Response $response, string $filename): Response { - $size = Util::filesize($filename); - $lastModified = filemtime($filename); - - $etag = md5($lastModified . $size); - $ifNoneMatch = $request->getHeaderLine('If-None-Match'); - if ($ifNoneMatch === $etag) { - return $response->withStatus(304); - } - - $exp = explode(".", $filename); - if ($exp[sizeof($exp) - 1] == '7z') { - $contentType = "application/x-7z-compressed"; - } - else { - $contentType = "application/force-download"; - } - $fp = @fopen($filename, "rb"); - - if (!$fp) { - throw new HttpForbiddenException($request, "Can't open the file"); - } - - $start = 0; // Start byte - $end = $size - 1; // End byte - - $status = 200; - if (isset($_SERVER['HTTP_RANGE'])) { - if (!$this->handleRangeRequest($start, $end, $size, $fp)) { - fclose($fp); - return $response->withStatus(416) - ->withHeader("Content-Range", "bytes $start-$end/$size"); - } - else { - $status = 206; - } - } - - $length = $end - $start + 1; //content-length - $buffer = 1024 * 100; - $stream = $response->getBody(); - while (!feof($fp) && ($p = ftell($fp)) <= $end) { - if ($p + $buffer > $end) { - $buffer = $end - $p + 1; - } - $stream->write(fread($fp, $buffer)); - } - fclose($fp); - - return $response->withStatus($status) - ->withHeader("Content-Type", $contentType) - ->withHeader("Content-Description", $filename) - ->withHeader("Content-Disposition", "attachment; filename=\"" . $filename . "\"") - ->withHeader("Accept-Ranges", "Byte") - ->withHeader("Content-Range", "bytes $start-$end/$size") - ->withHeader("Content-Length", $length) - ->withHeader("ETag", $etag); + return DownloadUtils::startDownload($request, $response, $filename); } } diff --git a/src/inc/apiv2/model/CrackerBinaryAPI.php b/src/inc/apiv2/model/CrackerBinaryAPI.php index bdd14e11a..c97d8f178 100644 --- a/src/inc/apiv2/model/CrackerBinaryAPI.php +++ b/src/inc/apiv2/model/CrackerBinaryAPI.php @@ -10,6 +10,8 @@ use Hashtopolis\dba\models\Task; use Hashtopolis\inc\apiv2\common\AbstractModelAPI; use Hashtopolis\inc\apiv2\error\HttpError; +use Hashtopolis\inc\apiv2\error\HttpForbidden; +use Hashtopolis\inc\apiv2\error\ResourceNotFoundError; use Hashtopolis\inc\HTException; @@ -24,6 +26,23 @@ public static function getBaseUri(): string { public static function getDBAclass(): string { return CrackerBinary::class; } + + /** + * Extra fields which are valid for creation of object. With one of the source + * fields given, the archive of the cracker binary is uploaded to the server + * instead of referencing it with an external download url. + */ + public function getFormFields(): array { + return [ + "sourceType" => ['type' => 'str', 'null' => True, + 'choices' => [ + "inline" => "Archive provided as base64 data in sourceData", + "import" => "Archive taken from the import directory, sourceData is the filename", + "url" => "Archive fetched from an http(s) url given in sourceData" + ]], + "sourceData" => ['type' => 'str', 'null' => True] + ]; + } public static function getToOneRelationships(): array { return [ @@ -52,6 +71,25 @@ public static function getToManyRelationships(): array { * @throws HTException */ protected function createObject(array $data): int { + if (isset($data["sourceType"])) { + if (isset($data[CrackerBinary::DOWNLOAD_URL])) { + throw new HttpError("downloadUrl cannot be provided when the archive is uploaded with sourceType!"); + } + if (!isset($data["sourceData"])) { + throw new HttpError("sourceData is required when sourceType is provided!"); + } + $binary = CrackerUtils::createBinaryFromUpload( + $data[CrackerBinary::VERSION], + $data[CrackerBinary::BINARY_NAME], + $data[CrackerBinary::CRACKER_BINARY_TYPE_ID], + $data["sourceType"], + $data["sourceData"] + ); + return $binary->getId(); + } + if (!isset($data[CrackerBinary::DOWNLOAD_URL]) || strlen($data[CrackerBinary::DOWNLOAD_URL]) == 0) { + throw new HttpError("Please provide all information!"); + } $binary = CrackerUtils::createBinary( $data[CrackerBinary::VERSION], $data[CrackerBinary::BINARY_NAME], @@ -68,4 +106,42 @@ protected function createObject(array $data): int { protected function deleteObject(AbstractModel $object): void { CrackerUtils::deleteBinary($object->getId()); } + + /** + * The download url of locally stored binaries is owned by the server, so it + * cannot be overwritten with a patch. When the download url of a binary which + * is referenced by an external url is changed, the server downloads a new + * local copy of the archive from it; if that download fails, the update is + * rolled back so nothing is changed. + * + * @param int $objectId + * @param array $data + * @throws HttpError + * @throws HttpForbidden + * @throws ResourceNotFoundError + * @throws HTException + */ + protected function updateObject(int $objectId, array $data): void { + $binary = CrackerUtils::getBinary($objectId); + if (array_key_exists(CrackerBinary::DOWNLOAD_URL, $data)) { + if ($binary->getFilename() !== null) { + throw new HttpError("The download url of a locally stored cracker binary cannot be changed!"); + } + } + $refreshLocalCopy = $binary->getFilename() === null + && ($data[CrackerBinary::DOWNLOAD_URL] ?? null) !== null + && $data[CrackerBinary::DOWNLOAD_URL] != $binary->getDownloadUrl(); + if ($refreshLocalCopy) { + CrackerUtils::validateDownloadUrl($data[CrackerBinary::DOWNLOAD_URL]); + } + $previousValues = [ + CrackerBinary::VERSION => $binary->getVersion(), + CrackerBinary::DOWNLOAD_URL => $binary->getDownloadUrl(), + CrackerBinary::BINARY_NAME => $binary->getBinaryName() + ]; + parent::updateObject($objectId, $data); + if ($refreshLocalCopy) { + CrackerUtils::refreshLocalCopy($objectId, $previousValues); + } + } } diff --git a/src/inc/apiv2/openapi/ModelApiPathBuilder.php b/src/inc/apiv2/openapi/ModelApiPathBuilder.php index 4d174d913..60dcfcc56 100644 --- a/src/inc/apiv2/openapi/ModelApiPathBuilder.php +++ b/src/inc/apiv2/openapi/ModelApiPathBuilder.php @@ -188,6 +188,12 @@ public function addRoute(RouteTarget $target, AbstractModelAPI $api, ContainerIn array_filter($createFeatures, fn($f) => !$f['null']) )); $properties_create = $this->jsonApiFragments->buildPatchPost($this->typeMapper->makeProperties($createFeatures), $typeName, null, $requiredCreateAttributes); + /* Descriptions document the creation, the response corrections do not + apply to a request schema */ + $properties_create["data"]["properties"]["attributes"] = $this->overrides->applyDescriptions( + $name, + $properties_create["data"]["properties"]["attributes"] + ); $properties_patch = $this->jsonApiFragments->buildPatchPost($this->typeMapper->makeProperties($class->getPatchValidFeatures(), true), $typeName); $components[$name . "Create"] = diff --git a/src/inc/apiv2/openapi/SpecOverrides.php b/src/inc/apiv2/openapi/SpecOverrides.php index f817f9748..ccce41901 100644 --- a/src/inc/apiv2/openapi/SpecOverrides.php +++ b/src/inc/apiv2/openapi/SpecOverrides.php @@ -16,14 +16,20 @@ * while the model still declares its column NOT NULL, so the derived schema * demands it and a permission-filtered response fails validation. * - * Two independent corrections are available per model, both keyed by the - * attribute alias as it appears in the response: + * Three independent corrections are available per model, the first two keyed + * by the attribute alias as it appears in the response: * * - OPTIONAL_ATTRIBUTES: the attribute may be missing, so it is dropped from * the "required" list of the attributes object. This is what permission * filtering does: the key is not there at all. * - NULLABLE_ATTRIBUTES: the attribute may be null, so "null" joins its type * (or a "null" branch its oneOf). The key stays required. + * - ATTRIBUTE_DESCRIPTIONS: a map of attribute alias to description text. + * Purely additive documentation, applied to the response schemas and, via + * applyDescriptions(), to the create request schema. Properties the schema + * does not carry are skipped, so one description set can serve the + * different shapes a model's attributes take (responses do not contain + * creation-only form fields, for example). * * Both the openapi.json route and ci/tools/generate-openapi.php build with * defaults(), so the served spec and the committed one agree. @@ -35,15 +41,19 @@ class SpecOverrides { /** Attributes a response may send as null; "null" joins their type. */ public const NULLABLE_ATTRIBUTES = 'nullableAttributes'; - private const KEYS = [self::OPTIONAL_ATTRIBUTES, self::NULLABLE_ATTRIBUTES]; + /** Attribute descriptions; a map of attribute alias to description text. */ + public const ATTRIBUTE_DESCRIPTIONS = 'attributeDescriptions'; - /** @var array>> */ + private const KEYS = [self::OPTIONAL_ATTRIBUTES, self::NULLABLE_ATTRIBUTES, self::ATTRIBUTE_DESCRIPTIONS]; + + /** @var array|array>> */ private array $byModel = []; /** - * @param array>> $overrides model name + * @param array|array>> $overrides model name * (as the component schemas spell it, e.g. "User") to a map of - * OPTIONAL_ATTRIBUTES and/or NULLABLE_ATTRIBUTES to attribute aliases + * OPTIONAL_ATTRIBUTES and/or NULLABLE_ATTRIBUTES to attribute aliases, + * or of ATTRIBUTE_DESCRIPTIONS to attribute alias => description */ public function __construct(array $overrides = []) { foreach ($overrides as $model => $entry) { @@ -63,12 +73,24 @@ public function __construct(array $overrides = []) { if (!is_array($aliases)) { throw new InvalidArgumentException("Spec override '$key' of '$model' must be a list of attribute names"); } - foreach ($aliases as $alias) { - if (!is_string($alias) || $alias === '') { - throw new InvalidArgumentException("Spec override '$key' of '$model' must only name attributes"); + if ($key === self::ATTRIBUTE_DESCRIPTIONS) { + foreach ($aliases as $alias => $description) { + if (!is_string($alias) || $alias === '' || !is_string($description) || $description === '') { + throw new InvalidArgumentException( + "Spec override '$key' of '$model' must map attribute names to non-empty descriptions" + ); + } + } + $normalized[$key] = $aliases; + } + else { + foreach ($aliases as $alias) { + if (!is_string($alias) || $alias === '') { + throw new InvalidArgumentException("Spec override '$key' of '$model' must only name attributes"); + } } + $normalized[$key] = array_values(array_unique($aliases)); } - $normalized[$key] = array_values(array_unique($aliases)); } $this->byModel[$model] = $normalized; } @@ -95,6 +117,14 @@ public static function defaults(): self { 'globalPermissionGroupId', ], ], + 'CrackerBinary' => [ + self::ATTRIBUTE_DESCRIPTIONS => [ + 'downloadUrl' => 'External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards.', + 'filename' => 'Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided.', + 'sourceType' => 'Source the 7z archive is uploaded from: inline (base64 archive data in sourceData), import (filename of a file in the import directory as sourceData) or url (http/https url in sourceData, fetched by the server). Mutually exclusive with downloadUrl.', + 'sourceData' => 'Source of the archive upload, depending on sourceType: base64 encoded archive data, filename of a file in the import directory or a http/https url to fetch the archive from.', + ], + ], ]); } @@ -121,6 +151,11 @@ public function apply(string $model, array $schema): array { $properties = $schema['properties'] ?? []; foreach ($entry as $key => $aliases) { + if ($key === self::ATTRIBUTE_DESCRIPTIONS) { + // descriptions are additive documentation, properties which are not + // part of this particular schema shape are simply skipped + continue; + } foreach ($aliases as $alias) { if (!array_key_exists($alias, $properties)) { throw new InvalidArgumentException( @@ -139,6 +174,36 @@ public function apply(string $model, array $schema): array { $schema['properties'][$alias] = $this->makeNullable($schema['properties'][$alias]); } + return $this->applyDescriptionEntries($entry[self::ATTRIBUTE_DESCRIPTIONS] ?? [], $schema); + } + + /** + * Apply only the attribute descriptions of one model to a request attributes + * schema (create), where the response oriented optional/nullable corrections + * must not be applied. Properties the schema does not carry are skipped. + * + * The schema is returned unchanged when nothing is configured for the model. + * + * @param string $model model name as the component schemas spell it + * @param array $schema the "attributes" object, with "required" and "properties" + */ + public function applyDescriptions(string $model, array $schema): array { + if (!$this->has($model)) { + return $schema; + } + $descriptions = $this->byModel[$model][self::ATTRIBUTE_DESCRIPTIONS] ?? []; + return $this->applyDescriptionEntries($descriptions, $schema); + } + + /** + * @param array $descriptions attribute alias to description text + */ + private function applyDescriptionEntries(array $descriptions, array $schema): array { + foreach ($descriptions as $alias => $description) { + if (array_key_exists($alias, $schema['properties'] ?? [])) { + $schema['properties'][$alias]['description'] = $description; + } + } return $schema; } diff --git a/src/inc/defines/DDirectories.php b/src/inc/defines/DDirectories.php index cc0fbeefa..8557bc6e5 100644 --- a/src/inc/defines/DDirectories.php +++ b/src/inc/defines/DDirectories.php @@ -3,9 +3,10 @@ namespace Hashtopolis\inc\defines; class DDirectories { - const FILES = "directory_files"; - const IMPORT = "directory_import"; - const LOG = "directory_log"; - const CONFIG = "directory_config"; - const TUS = "directory_tus"; + const FILES = "directory_files"; + const IMPORT = "directory_import"; + const LOG = "directory_log"; + const CONFIG = "directory_config"; + const TUS = "directory_tus"; + const CRACKERS = "directory_crackers"; } \ No newline at end of file diff --git a/src/inc/downloadapi/CrackerBinaryDownloadHandler.php b/src/inc/downloadapi/CrackerBinaryDownloadHandler.php new file mode 100644 index 000000000..72fb0c03f --- /dev/null +++ b/src/inc/downloadapi/CrackerBinaryDownloadHandler.php @@ -0,0 +1,44 @@ +get((int)$args['id']); + if ($binary === null || $binary->getFilename() === null) { + return CrackerBinaryDownloadHandler::notFound($response, 'No such cracker binary archive!'); + } + $path = CrackerUtils::getCrackersPath() . $binary->getId() . '_' . $binary->getFilename(); + if (!file_exists($path)) { + return CrackerBinaryDownloadHandler::notFound($response, 'The archive of this cracker binary is not present on the server!'); + } + + $agent = $request->getAttribute(AgentAction::AGENT_ATTRIBUTE); + if ($agent instanceof Agent) { + DServerLog::log(DServerLog::TRACE, 'Agent ' . $agent->getId() . ' downloaded the archive of cracker binary ' . $binary->getId()); + } + else { + DServerLog::log(DServerLog::TRACE, 'User ' . ($request->getAttribute('userId') ?? 'unknown') . + ' downloaded the archive of cracker binary ' . $binary->getId()); + } + return DownloadUtils::startDownload($request, $response, $path, $binary->getFilename()); + } + + private static function notFound(Response $response, string $message): Response { + $response->getBody()->write($message); + return $response->withStatus(404)->withHeader('Content-Type', 'text/plain'); + } +} diff --git a/src/inc/downloadapi/DownloadApp.php b/src/inc/downloadapi/DownloadApp.php new file mode 100644 index 000000000..f2798c6b1 --- /dev/null +++ b/src/inc/downloadapi/DownloadApp.php @@ -0,0 +1,79 @@ +add(new DownloadAuthMiddleware()); + + $errorMiddleware = $app->addErrorMiddleware(true, true, true); + $errorMiddleware->setDefaultErrorHandler( + function ( + Request $request, + Throwable $exception, + bool $displayErrorDetails, + bool $logErrors, + bool $logErrorDetails, + ): ResponseInterface { + error_log("DownloadApi: " . $exception->getMessage()); + $response = new Response(500); + $response->getBody()->write('Internal server error'); + return $response; + } + ); + $errorMiddleware->setErrorHandler(AuthorizationException::class, function ( + Request $request, + Throwable $exception, + bool $displayErrorDetails, + bool $logErrors, + bool $logErrorDetails, + ): ResponseInterface { + $response = new Response(401); + $response->getBody()->write('No access!'); + return $response; + }); + $errorMiddleware->setErrorHandler(HttpNotFoundException::class, function ( + Request $request, + Throwable $exception, + bool $displayErrorDetails, + bool $logErrors, + bool $logErrorDetails, + ): ResponseInterface { + $response = new Response(404); + $response->getBody()->write('Not found'); + return $response; + }); + $app->addRoutingMiddleware(); + + $app->get('/api/download.php/{kind}/{id}', function (Request $request, Response $response, array $args): ResponseInterface { + $handlerClass = DownloadRegistry::getHandler($args['kind']); + if ($handlerClass === null) { + $response->getBody()->write('Unknown download kind!'); + return $response->withStatus(404)->withHeader('Content-Type', 'text/plain'); + } + /** @var CrackerBinaryDownloadHandler $handler */ + $handler = new $handlerClass(); + return $handler($request, $response, $args); + }); + + return $app; + } +} diff --git a/src/inc/downloadapi/DownloadAuthMiddleware.php b/src/inc/downloadapi/DownloadAuthMiddleware.php new file mode 100644 index 000000000..22c19fe5b --- /dev/null +++ b/src/inc/downloadapi/DownloadAuthMiddleware.php @@ -0,0 +1,44 @@ +hasHeader('Authorization')) { + // apiv2 JWT authentication, attaches the userId, scope and aud attributes + return JwtAuthenticationFactory::create([])->process($request, $handler); + } + + $token = $request->getQueryParams()['token'] ?? null; + if (is_string($token) && strlen($token) > 0) { + $qF = new QueryFilter(Agent::TOKEN, $token, '='); + $agent = Factory::getAgentFactory()->filter([Factory::FILTER => $qF], true); + if ($agent !== null) { + return $handler->handle($request->withAttribute(AgentAction::AGENT_ATTRIBUTE, $agent)); + } + } + + $response = new Response(401); + $response->getBody()->write('No access!'); + return $response; + } +} diff --git a/src/inc/downloadapi/DownloadRegistry.php b/src/inc/downloadapi/DownloadRegistry.php new file mode 100644 index 000000000..ced811f42 --- /dev/null +++ b/src/inc/downloadapi/DownloadRegistry.php @@ -0,0 +1,19 @@ + CrackerBinaryDownloadHandler::class, + ]; + + public static function getHandler(string $kind): ?string { + return self::HANDLERS[$kind] ?? null; + } +} diff --git a/src/inc/startup/setup.json b/src/inc/startup/setup.json index 76fb51d6a..9fc9d5326 100644 --- a/src/inc/startup/setup.json +++ b/src/inc/startup/setup.json @@ -393,7 +393,8 @@ "crackerBinaryTypeId" : 1, "version" : "7.1.2", "downloadUrl" : "https://hashcat.net/files/hashcat-7.1.2.7z", - "binaryName" : "hashcat" + "binaryName" : "hashcat", + "filename" : null } ], "CrackerBinaryType" : [ diff --git a/src/inc/startup/setup.php b/src/inc/startup/setup.php index 2ed12da5f..0b8e84167 100755 --- a/src/inc/startup/setup.php +++ b/src/inc/startup/setup.php @@ -212,4 +212,5 @@ Util::checkDataDirectory(DDirectories::IMPORT, StartupConfig::getInstance()->getDirectoryImport()); Util::checkDataDirectory(DDirectories::LOG, StartupConfig::getInstance()->getDirectoryLog()); Util::checkDataDirectory(DDirectories::CONFIG, StartupConfig::getInstance()->getDirectoryConfig()); -Util::checkDataDirectory(DDirectories::TUS, StartupConfig::getInstance()->getDirectoryTus()); \ No newline at end of file +Util::checkDataDirectory(DDirectories::TUS, StartupConfig::getInstance()->getDirectoryTus()); +Util::checkDataDirectory(DDirectories::CRACKERS, StartupConfig::getInstance()->getDirectoryCrackers()); \ No newline at end of file diff --git a/src/inc/utils/CrackerUtils.php b/src/inc/utils/CrackerUtils.php index a4cfc7f7a..1412897a2 100644 --- a/src/inc/utils/CrackerUtils.php +++ b/src/inc/utils/CrackerUtils.php @@ -10,6 +10,7 @@ use Hashtopolis\dba\ContainFilter; use Hashtopolis\dba\Factory; use Hashtopolis\dba\models\Pretask; +use Hashtopolis\inc\defines\DDirectories; use Hashtopolis\inc\apiv2\error\HttpConflict; use Hashtopolis\inc\apiv2\error\HttpError; use Hashtopolis\inc\HTException; @@ -55,6 +56,12 @@ public static function createBinaryType(string $typeName): CrackerBinaryType { } /** + * Creates a new cracker binary which is referenced by an external download url. + * The server downloads a local copy of the archive into the crackers directory, + * so it has it available for later analysis. The agents still download the + * archive from the external download url. If the download fails or the archive + * is not a valid 7z archive, nothing is added. + * * @param string $version * @param string $name * @param string $url @@ -69,8 +76,231 @@ public static function createBinary(string $version, string $name, string $url, if (strlen($version) == 0 || strlen($name) == 0 || strlen($url) == 0) { throw new HttpError("Please provide all information!"); } - $binary = new CrackerBinary(null, $binaryType->getId(), $version, $url, $name); - return Factory::getCrackerBinaryFactory()->save($binary); + CrackerUtils::validateDownloadUrl($url); + // create the entry first, the id is needed for the filename of the local copy + $binary = Factory::getCrackerBinaryFactory()->save( + new CrackerBinary(null, $binaryType->getId(), $version, $url, $name, null) + ); + try { + CrackerUtils::storeLocalCopy($binary); + } + catch (HttpError $e) { + Factory::getCrackerBinaryFactory()->delete($binary); + throw $e; + } + return $binary; + } + + /** + * Creates a new cracker binary from an uploaded 7z archive. The archive is stored in + * the crackers directory and the downloadUrl is set to the download endpoint of this + * server, so it can directly be used by the agents to download the binary. + * + * @param string $version + * @param string $name + * @param int $binaryTypeId + * @param string $sourceType choices inline, import, url + * @param string $sourceData base64 data, filename in the import directory or download url + * @return CrackerBinary + * @throws HttpError + * @throws HTException + * @throws Exception + */ + public static function createBinaryFromUpload(string $version, string $name, int $binaryTypeId, string $sourceType, string $sourceData): CrackerBinary { + $binaryType = CrackerUtils::getBinaryType($binaryTypeId); + if (strlen($version) == 0 || strlen($name) == 0 || strlen($sourceData) == 0) { + throw new HttpError("Please provide all information!"); + } + + // determine the source of the archive and validate it + switch ($sourceType) { + case "inline": + $archiveData = base64_decode($sourceData, true); + if ($archiveData === false) { + throw new HttpError("sourceData not valid base64 encoding"); + } + $uploadType = "paste"; + $uploadData = $archiveData; + break; + case "import": + $realname = str_replace(" ", "_", htmlentities(basename($sourceData), ENT_QUOTES, "UTF-8")); + if ($sourceData != $realname) { + throw new HttpError("sourceData is invalid filename suggestion '$realname'"); + } + $uploadType = "import"; + $uploadData = $sourceData; + break; + case "url": + $scheme = parse_url($sourceData, PHP_URL_SCHEME); + if ($scheme != "http" && $scheme != "https") { + throw new HttpError("Only http and https URLs are supported as sourceData!"); + } + $uploadType = "url"; + $uploadData = $sourceData; + break; + default: + throw new HttpError("sourceType value '" . $sourceType . "' is not supported (choices inline, import, url"); + } + + $filename = CrackerUtils::buildArchiveFilename($binaryType, $version); + + // create the entry first with a placeholder download url, the final one + // contains the id and can only be set once it is known + $binary = Factory::getCrackerBinaryFactory()->save( + new CrackerBinary(null, $binaryType->getId(), $version, "", $name, null) + ); + + $target = CrackerUtils::getCrackersPath() . $binary->getId() . '_' . $filename; + [$success, $msg] = Util::uploadFile($target, $uploadType, $uploadData); + if (!$success) { + Factory::getCrackerBinaryFactory()->delete($binary); + throw new HttpError("Failed to store the archive: " . $msg); + } + + if (!CrackerUtils::isSevenZipArchive($target)) { + // in case the archive was imported, put the file back to the import directory + if ($sourceType == "import") { + rename($target, CrackerUtils::getImportPath() . $sourceData); + } + else { + unlink($target); + } + Factory::getCrackerBinaryFactory()->delete($binary); + throw new HttpError("The provided archive is not a valid 7z archive!"); + } + + return Factory::getCrackerBinaryFactory()->mset($binary, [ + CrackerBinary::DOWNLOAD_URL => Util::buildBackendBaseUrl() . '/api/download.php/crackerBinary/' . $binary->getId(), + CrackerBinary::FILENAME => $filename + ]); + } + + /** + * @throws Exception + */ + public static function getCrackersPath(): string { + return rtrim(Factory::getStoredValueFactory()->get(DDirectories::CRACKERS)->getVal(), '/') . '/'; + } + + /** + * @throws Exception + */ + private static function getImportPath(): string { + return rtrim(Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal(), '/') . '/'; + } + + /** + * Composed server-side archive filename for a locally stored cracker binary, + * the '.7z' extension is enforced by construction. + */ + private static function buildArchiveFilename(CrackerBinaryType $binaryType, string $version): string { + $sanitized = preg_replace('/[^A-Za-z0-9._-]/', '-', $binaryType->getTypeName() . '-' . $version) ?? ''; + return $sanitized . '.7z'; + } + + private static function isSevenZipArchive(string $path): bool { + $magic = "\x37\x7A\xBC\xAF\x27\x1C"; + $fp = @fopen($path, "rb"); + if ($fp === false) { + return false; + } + $header = fread($fp, strlen($magic)); + fclose($fp); + return $header === $magic; + } + + /** + * Validates that the server is allowed to fetch the given url as the + * download url of a cracker binary archive. + * + * @param string $url + * @throws HttpError + */ + public static function validateDownloadUrl(string $url): void { + $scheme = parse_url($url, PHP_URL_SCHEME); + if ($scheme != "http" && $scheme != "https") { + throw new HttpError("Only http and https download urls are supported!"); + } + } + + /** + * Downloads a local copy of the archive of a url-referenced cracker binary + * from its download url into the crackers directory. The archive is + * downloaded to a temporary file first, so a failed download cannot destroy + * a previously stored local copy, and only moved into place after it was + * validated as a 7z archive. Local copies of previous versions or urls of + * the binary are removed. + * + * @param CrackerBinary $binary the binary to download the archive for + * @throws HttpError + * @throws HTException + * @throws Exception + */ + private static function storeLocalCopy(CrackerBinary $binary): void { + $binaryType = CrackerUtils::getBinaryType($binary->getCrackerBinaryTypeId()); + $crackersPath = CrackerUtils::getCrackersPath(); + $filename = CrackerUtils::buildArchiveFilename($binaryType, $binary->getVersion()); + $target = $crackersPath . $binary->getId() . '_' . $filename; + $temporary = $target . '.part'; + if (file_exists($temporary)) { + unlink($temporary); + } + [$success, $msg] = Util::uploadFile($temporary, "url", $binary->getDownloadUrl()); + if (!$success) { + if (file_exists($temporary)) { + unlink($temporary); + } + throw new HttpError("Failed to download the archive from the download url: " . $msg); + } + if (!CrackerUtils::isSevenZipArchive($temporary)) { + unlink($temporary); + throw new HttpError("The archive at the download url is not a valid 7z archive!"); + } + rename($temporary, $target); + // remove local copies of previous versions or urls of this binary + foreach (glob($crackersPath . $binary->getId() . '_*') ?: [] as $path) { + if ($path != $target) { + unlink($path); + } + } + } + + /** + * Refreshes the local copy of the archive of a url-referenced cracker binary + * from its download url, to be called after the binary was updated in the + * database. If the download fails, the update is rolled back by restoring + * the given previous values before the error is rethrown, so nothing of + * the update remains. + * + * @param int $binaryId + * @param array $previousValues previous values of the updated fields, keyed by the CrackerBinary feature constants + * @throws HttpError + * @throws HTException + * @throws Exception + */ + public static function refreshLocalCopy(int $binaryId, array $previousValues): void { + $binary = CrackerUtils::getBinary($binaryId); + try { + CrackerUtils::storeLocalCopy($binary); + } + catch (HttpError $e) { + Factory::getCrackerBinaryFactory()->mset($binary, $previousValues); + throw $e; + } + } + + /** + * Removes the locally stored archive and any downloaded local copy of a + * cracker binary, all of them are prefixed with the id of the binary. + * + * @throws Exception + */ + private static function deleteLocalArchive(CrackerBinary $binary): void { + foreach (glob(CrackerUtils::getCrackersPath() . $binary->getId() . '_*') ?: [] as $path) { + if (file_exists($path)) { + unlink($path); + } + } } /** @@ -85,6 +315,8 @@ public static function deleteBinary(int $binaryId): void { if (sizeof($check) > 0) { throw new HTException("There are tasks which use this binary!"); } + // remove a locally stored archive if there is one + CrackerUtils::deleteLocalArchive($binary); Factory::getCrackerBinaryFactory()->delete($binary); } @@ -114,17 +346,28 @@ public static function deleteBinaryType(int $binaryTypeId): void { throw new HTException("There are pretasks which use this cracker type!"); } + // remove the archives of locally stored binaries + foreach ($binaries as $binary) { + CrackerUtils::deleteLocalArchive($binary); + } + // delete Factory::getCrackerBinaryFactory()->massDeletion([Factory::FILTER => $qF]); Factory::getCrackerBinaryTypeFactory()->delete($binaryType); } /** + * Updates a cracker binary. When the download url of a binary which is + * referenced by an external url is changed, the server downloads a new local + * copy of the archive from it; if that download fails, the update is rolled + * back so nothing is changed. + * * @param string $version * @param string $name * @param string $url * @param int $binaryId * @return CrackerBinaryType + * @throws HttpError * @throws HTException * @throws Exception */ @@ -133,12 +376,30 @@ public static function updateBinary(string $version, string $name, string $url, if (strlen($version) == 0 || strlen($name) == 0 || strlen($url) == 0) { throw new HTException("Please provide all information!"); } + // locally stored binaries are downloaded from this server, so the url is owned by the server + if ($binary->getFilename() !== null && $url != $binary->getDownloadUrl()) { + throw new HTException("The download url of a locally stored cracker binary cannot be changed!"); + } + // a changed download url of a url-referenced binary requires the server to + // download a new local copy of the archive from it + $refreshLocalCopy = $binary->getFilename() === null && $url != $binary->getDownloadUrl(); + if ($refreshLocalCopy) { + CrackerUtils::validateDownloadUrl($url); + } + $previousValues = [ + CrackerBinary::VERSION => $binary->getVersion(), + CrackerBinary::DOWNLOAD_URL => $binary->getDownloadUrl(), + CrackerBinary::BINARY_NAME => $binary->getBinaryName() + ]; $binary = Factory::getCrackerBinaryFactory()->mset($binary, [ CrackerBinary::BINARY_NAME => htmlentities($name, ENT_QUOTES, "UTF-8"), CrackerBinary::DOWNLOAD_URL => $url, CrackerBinary::VERSION => $version ] ); + if ($refreshLocalCopy) { + CrackerUtils::refreshLocalCopy($binary->getId(), $previousValues); + } return Factory::getCrackerBinaryTypeFactory()->get($binary->getCrackerBinaryTypeId()); } diff --git a/src/inc/utils/DownloadUtils.php b/src/inc/utils/DownloadUtils.php new file mode 100644 index 000000000..1717a3972 --- /dev/null +++ b/src/inc/utils/DownloadUtils.php @@ -0,0 +1,136 @@ + $end) { + $c_end = $end; + } + if ($c_start > $c_end || $c_start > $size - 1 || $c_end >= $size) { + return false; + } + $start = $c_start; + $end = $c_end; + fseek($fp, $start); + return true; + } + + /** + * Streams the given file as a download response, handling ETag based + * caching ('If-None-Match') and partial content ('Range') requests. + * + * @param Request $request + * @param Response $response + * @param string $path Absolute path of the file to stream + * @param string|null $displayName Filename to announce to the client, defaults to the base name of the path + * @return Response + * @throws HttpForbiddenException + */ + public static function startDownload(Request $request, Response $response, string $path, ?string $displayName = null): Response { + if ($displayName === null) { + $displayName = basename($path); + } + $size = Util::filesize($path); + $lastModified = filemtime($path); + + $etag = md5($lastModified . $size); + $ifNoneMatch = $request->getHeaderLine('If-None-Match'); + if ($ifNoneMatch === $etag) { + return $response->withStatus(304); + } + + $exp = explode(".", $path); + if ($exp[sizeof($exp) - 1] == '7z') { + $contentType = "application/x-7z-compressed"; + } + else { + $contentType = "application/force-download"; + } + $fp = @fopen($path, "rb"); + + if (!$fp) { + throw new HttpForbiddenException($request, "Can't open the file"); + } + + $start = 0; // Start byte + $end = $size - 1; // End byte + + $status = 200; + if (isset($_SERVER['HTTP_RANGE'])) { + if (!DownloadUtils::handleRangeRequest($start, $end, $size, $fp)) { + fclose($fp); + return $response->withStatus(416) + ->withHeader("Content-Range", "bytes $start-$end/$size"); + } + else { + $status = 206; + } + } + + $length = $end - $start + 1; //content-length + $buffer = 1024 * 100; + $stream = $response->getBody(); + while (!feof($fp) && ($p = ftell($fp)) <= $end) { + if ($p + $buffer > $end) { + $buffer = $end - $p + 1; + } + $stream->write(fread($fp, $buffer)); + } + fclose($fp); + + return $response->withStatus($status) + ->withHeader("Content-Type", $contentType) + ->withHeader("Content-Description", $displayName) + ->withHeader("Content-Disposition", "attachment; filename=\"" . $displayName . "\"") + ->withHeader("Accept-Ranges", "Byte") + ->withHeader("Content-Range", "bytes $start-$end/$size") + ->withHeader("Content-Length", $length) + ->withHeader("ETag", $etag); + } +} diff --git a/src/migrations/mysql/20260917114300_cracker-binary-local-upload.sql b/src/migrations/mysql/20260917114300_cracker-binary-local-upload.sql new file mode 100644 index 000000000..73540f147 --- /dev/null +++ b/src/migrations/mysql/20260917114300_cracker-binary-local-upload.sql @@ -0,0 +1,6 @@ +-- Cracker binaries can be hosted on the server itself: 'filename' stores the archive +-- filename of the locally stored .7z archive, NULL means the binary is downloaded from downloadUrl. +ALTER TABLE CrackerBinary ADD COLUMN filename VARCHAR(100) NULL AFTER binaryName; + +-- downloadUrl can now point to the hashtopolis server download endpoint, so more space is needed +ALTER TABLE CrackerBinary MODIFY downloadUrl VARCHAR(255) NOT NULL; diff --git a/src/migrations/postgres/20260917114300_cracker-binary-local-upload.sql b/src/migrations/postgres/20260917114300_cracker-binary-local-upload.sql new file mode 100644 index 000000000..57a244fbd --- /dev/null +++ b/src/migrations/postgres/20260917114300_cracker-binary-local-upload.sql @@ -0,0 +1,3 @@ +-- Cracker binaries can be hosted on the server itself: 'filename' stores the archive +-- filename of the locally stored .7z archive, NULL means the binary is downloaded from downloadUrl. +ALTER TABLE CrackerBinary ADD COLUMN filename TEXT NULL; From 910b4b7632847138a7c6000bff28c2a4b719a0c3 Mon Sep 17 00:00:00 2001 From: s3inlc Date: Thu, 17 Sep 2026 12:14:31 +0200 Subject: [PATCH 02/11] updated openapi.json --- openapi.json | 239 +++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 211 insertions(+), 28 deletions(-) diff --git a/openapi.json b/openapi.json index 10cf5a133..e7314caa1 100644 --- a/openapi.json +++ b/openapi.json @@ -41445,6 +41445,36 @@ "attributes": { "type": "object", "properties": { + "sourceType": { + "oneOf": [ + { + "const": "inline", + "title": "Archive provided as base64 data in sourceData", + "type": "string" + }, + { + "const": "import", + "title": "Archive taken from the import directory, sourceData is the filename", + "type": "string" + }, + { + "const": "url", + "title": "Archive fetched from an http(s) url given in sourceData", + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Source the 7z archive is uploaded from: inline (base64 archive data in sourceData), import (filename of a file in the import directory as sourceData) or url (http/https url in sourceData, fetched by the server). Mutually exclusive with downloadUrl." + }, + "sourceData": { + "type": [ + "string", + "null" + ], + "description": "Source of the archive upload, depending on sourceType: base64 encoded archive data, filename of a file in the import directory or a http/https url to fetch the archive from." + }, "crackerBinaryTypeId": { "type": "integer" }, @@ -41452,7 +41482,11 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" @@ -41461,7 +41495,6 @@ "required": [ "crackerBinaryTypeId", "version", - "downloadUrl", "binaryName" ] } @@ -41493,7 +41526,10 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ] }, "version": { "type": "string" @@ -41535,7 +41571,10 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ] }, "version": { "type": "string" @@ -41640,7 +41679,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -41650,10 +41690,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } }, @@ -42014,7 +42065,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -42024,10 +42076,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } }, @@ -42439,7 +42502,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -42449,10 +42513,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } }, @@ -43214,7 +43289,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -43224,10 +43300,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -43584,7 +43671,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -43594,10 +43682,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -44006,7 +44105,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -44016,10 +44116,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -53410,7 +53521,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -53420,10 +53532,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -53864,7 +53987,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -53874,10 +53998,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -54370,7 +54505,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -54380,10 +54516,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -61871,7 +62018,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -61881,10 +62029,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -62834,7 +62993,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -62844,10 +63004,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } @@ -63849,7 +64020,8 @@ "crackerBinaryTypeId", "version", "downloadUrl", - "binaryName" + "binaryName", + "filename" ], "properties": { "crackerBinaryTypeId": { @@ -63859,10 +64031,21 @@ "type": "string" }, "downloadUrl": { - "type": "string" + "type": [ + "string", + "null" + ], + "description": "External http/https url where the agent downloads the binary archive from. The server keeps a local copy of the archive for later analysis: on creation it is downloaded from this url, and changing the url re-downloads it from the new url. The creation or change is rejected if that download fails or the archive is not a valid 7z file. Mutually exclusive with sourceType: when the archive is uploaded with sourceType, this url is set automatically to the download endpoint of this server and cannot be changed afterwards." }, "binaryName": { "type": "string" + }, + "filename": { + "type": [ + "string", + "null" + ], + "description": "Filename of the locally stored 7z archive, null when the binary is downloaded from the downloadUrl. Cannot be provided." } } } From e67eb089ed0c6167190358d2f498d52b7bbb1a3d Mon Sep 17 00:00:00 2001 From: s3inlc Date: Tue, 22 Sep 2026 14:38:13 +0200 Subject: [PATCH 03/11] added crackers binary path to dockerfile env --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index a39043ec0..5b04d143e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ ENV HASHTOPOLIS_IMPORT_PATH=${HASHTOPOLIS_PATH}/import ENV HASHTOPOLIS_LOG_PATH=${HASHTOPOLIS_PATH}/log ENV HASHTOPOLIS_CONFIG_PATH=${HASHTOPOLIS_PATH}/config ENV HASHTOPOLIS_BINARIES_PATH=${HASHTOPOLIS_PATH}/binaries +ENV HASHTOPOLIS_CRACKERS_PATH=${HASHTOPOLIS_PATH}/crackers ENV HASHTOPOLIS_TUS_PATH=/var/tmp/tus ENV HASHTOPOLIS_TEMP_UPLOADS_PATH=${HASHTOPOLIS_TUS_PATH}/uploads ENV HASHTOPOLIS_TEMP_META_PATH=${HASHTOPOLIS_TUS_PATH}/meta @@ -80,6 +81,7 @@ RUN mkdir -p \ ${HASHTOPOLIS_LOG_PATH} \ ${HASHTOPOLIS_CONFIG_PATH} \ ${HASHTOPOLIS_BINARIES_PATH} \ + ${HASHTOPOLIS_CRACKERS_PATH} \ ${HASHTOPOLIS_TUS_PATH} \ ${HASHTOPOLIS_TEMP_UPLOADS_PATH} \ ${HASHTOPOLIS_TEMP_META_PATH} \ From 2220b6c1fb141418bc2653f7c7d93f56396cdac6 Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 09:28:36 +0200 Subject: [PATCH 04/11] updated docker entrypoint to include new ENV variable already in this branch --- docker-entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index fca641e31..d6f7a8858 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -55,6 +55,7 @@ directories=( "${HASHTOPOLIS_LOG_PATH}" "${HASHTOPOLIS_IMPORT_PATH}" "${HASHTOPOLIS_BINARIES_PATH}" + "${HASHTOPOLIS_CRACKERS_PATH}" "${HASHTOPOLIS_TUS_PATH}" "${HASHTOPOLIS_TEMP_UPLOADS_PATH}" "${HASHTOPOLIS_TEMP_META_PATH}" From 50b38fd6745c808950f63313d3f7c77c0dffec4a Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 09:45:35 +0200 Subject: [PATCH 05/11] do not allow HOST address from client to be used and improved validation --- ci/phpunit/inc/UtilTest.php | 67 ++++++++++++----------- ci/phpunit/inc/utils/CrackerUtilsTest.php | 47 ++++++++++++++++ src/inc/Util.php | 45 ++++++++++----- src/inc/utils/CrackerUtils.php | 6 +- 4 files changed, 119 insertions(+), 46 deletions(-) diff --git a/ci/phpunit/inc/UtilTest.php b/ci/phpunit/inc/UtilTest.php index 93f8bc6c7..5c6df4cff 100644 --- a/ci/phpunit/inc/UtilTest.php +++ b/ci/phpunit/inc/UtilTest.php @@ -13,12 +13,14 @@ final class UtilTest extends TestBase { private string|false $savedBackendUrl = false; + private mixed $savedBaseHost = null; private array $savedServer = []; #[Override] protected function setUp(): void { parent::setUp(); $this->savedBackendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + $this->savedBaseHost = SConfig::getInstance()->getVal(DConfig::BASE_HOST); $this->savedServer = [ 'HTTP_HOST' => $_SERVER['HTTP_HOST'] ?? null, 'SERVER_PORT' => $_SERVER['SERVER_PORT'] ?? null, @@ -34,6 +36,7 @@ protected function tearDown(): void { else { putenv('HASHTOPOLIS_BACKEND_URL=' . $this->savedBackendUrl); } + SConfig::getInstance()->addValue(DConfig::BASE_HOST, $this->savedBaseHost); foreach ($this->savedServer as $key => $value) { if ($value === null) { unset($_SERVER[$key]); @@ -1060,16 +1063,6 @@ public function testCheckOrCreateInitialObjectAllSetupConfigs(): void { } } - /** - * Expected fallback result when HASHTOPOLIS_BACKEND_URL is not usable: - * the server URL derived from the current request plus the configured base URL. - * - * @throws Exception - */ - private function expectedFallbackUrl(): string { - return rtrim(Util::buildServerUrl() . SConfig::getInstance()->getVal(DConfig::BASE_URL), '/'); - } - /** * buildBackendBaseUrl takes scheme, host and port from HASHTOPOLIS_BACKEND_URL * and strips any path it may contain. @@ -1077,6 +1070,7 @@ private function expectedFallbackUrl(): string { * @throws Exception */ public function testBuildBackendBaseUrlFromEnv(): void { + $_SERVER['HTTP_HOST'] = 'attacker.example'; putenv('HASHTOPOLIS_BACKEND_URL=http://localhost:8080/api/v2'); $this->assertEquals('http://localhost:8080', Util::buildBackendBaseUrl()); } @@ -1122,44 +1116,53 @@ public function testBuildBackendBaseUrlFromEnvTrailingSlash(): void { } /** - * buildBackendBaseUrl falls back to the server URL for a malformed env value - * (missing scheme), instead of returning a broken URL. + * A malformed environment value is rejected instead of falling back to another + * host and hiding the configuration error. * * @throws Exception */ - public function testBuildBackendBaseUrlEnvMalformedFallsBack(): void { - $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; - $_SERVER['SERVER_PORT'] = '1234'; - unset($_SERVER['HTTPS']); + public function testBuildBackendBaseUrlEnvMalformedThrows(): void { + SConfig::getInstance()->addValue(DConfig::BASE_HOST, 'https://configured.example'); putenv('HASHTOPOLIS_BACKEND_URL=localhost:8080'); - $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); - $this->assertEquals('http://fallbackhost:1234' . SConfig::getInstance()->getVal(DConfig::BASE_URL), Util::buildBackendBaseUrl()); + $this->expectException(Exception::class); + Util::buildBackendBaseUrl(); + } + + /** + * buildBackendBaseUrl uses the explicitly configured baseHost for installations + * which do not provide HASHTOPOLIS_BACKEND_URL. + * + * @throws Exception + */ + public function testBuildBackendBaseUrlFromConfiguredBaseHost(): void { + $_SERVER['HTTP_HOST'] = 'attacker.example'; + SConfig::getInstance()->addValue(DConfig::BASE_HOST, 'https://configured.example:8443'); + putenv('HASHTOPOLIS_BACKEND_URL'); + $expected = 'https://configured.example:8443/' . ltrim((string)SConfig::getInstance()->getVal(DConfig::BASE_URL), '/'); + $this->assertEquals(rtrim($expected, '/'), Util::buildBackendBaseUrl()); } /** - * buildBackendBaseUrl falls back to the server URL when HASHTOPOLIS_BACKEND_URL is unset. + * An unconfigured backend URL fails closed and never uses the request Host header. * * @throws Exception */ - public function testBuildBackendBaseUrlEnvUnsetFallsBack(): void { - $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; - $_SERVER['SERVER_PORT'] = '1234'; - unset($_SERVER['HTTPS']); + public function testBuildBackendBaseUrlUnconfiguredDoesNotUseRequestHost(): void { + $_SERVER['HTTP_HOST'] = 'attacker.example'; + SConfig::getInstance()->addValue(DConfig::BASE_HOST, ''); putenv('HASHTOPOLIS_BACKEND_URL'); - $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); - $this->assertEquals('http://fallbackhost:1234' . SConfig::getInstance()->getVal(DConfig::BASE_URL), Util::buildBackendBaseUrl()); + $this->expectException(Exception::class); + Util::buildBackendBaseUrl(); } /** - * buildBackendBaseUrl falls back to the server URL for an empty env value. + * Only HTTP and HTTPS backend URLs can be persisted for agents. * * @throws Exception */ - public function testBuildBackendBaseUrlEnvEmptyFallsBack(): void { - $_SERVER['HTTP_HOST'] = 'fallbackhost:1234'; - $_SERVER['SERVER_PORT'] = '1234'; - unset($_SERVER['HTTPS']); - putenv('HASHTOPOLIS_BACKEND_URL='); - $this->assertEquals($this->expectedFallbackUrl(), Util::buildBackendBaseUrl()); + public function testBuildBackendBaseUrlRejectsInvalidScheme(): void { + putenv('HASHTOPOLIS_BACKEND_URL=ftp://configured.example/api/v2'); + $this->expectException(Exception::class); + Util::buildBackendBaseUrl(); } } diff --git a/ci/phpunit/inc/utils/CrackerUtilsTest.php b/ci/phpunit/inc/utils/CrackerUtilsTest.php index 06d513719..0b19d33be 100644 --- a/ci/phpunit/inc/utils/CrackerUtilsTest.php +++ b/ci/phpunit/inc/utils/CrackerUtilsTest.php @@ -14,6 +14,7 @@ use Hashtopolis\inc\HTException; use Hashtopolis\inc\utils\CrackerUtils; use Hashtopolis\TestBase; +use Override; require_once(dirname(__FILE__) . '/../../TestBase.php'); require_once(dirname(__FILE__) . '/../../../../src/inc/startup/include.php'); @@ -27,12 +28,15 @@ final class CrackerUtilsTest extends TestBase { private ?AbstractModel $type = null; private ?AbstractModel $binary = null; + private string|false $savedBackendUrl = false; // Creates a CrackerBinaryType and one CrackerBinary before each test. // These records provide valid IDs for the "happy path" tests and a known // duplicate name for the conflict test. protected function setUp(): void { parent::setUp(); + $this->savedBackendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + putenv('HASHTOPOLIS_BACKEND_URL=http://localhost:8080/api/v2'); $this->type = $this->createDatabaseObject( Factory::getCrackerBinaryTypeFactory(), new CrackerBinaryType(null, 'test-crackerutils-type', 1) @@ -43,6 +47,21 @@ protected function setUp(): void { ); } + #[Override] + protected function tearDown(): void { + try { + parent::tearDown(); + } + finally { + if ($this->savedBackendUrl === false) { + putenv('HASHTOPOLIS_BACKEND_URL'); + } + else { + putenv('HASHTOPOLIS_BACKEND_URL=' . $this->savedBackendUrl); + } + } + } + // Verifies that getBinary() throws HTException when the ID does not match // any row — the caller must handle the "binary not found" case. public function testGetBinaryInvalidIdThrowsHTException(): void { @@ -165,6 +184,34 @@ public function testCreateBinaryFromUploadInlineSource(): void { unlink($archive); } + // A missing or invalid public backend URL is detected before the archive or + // database record is stored, and a request Host header cannot replace it. + public function testCreateBinaryFromUploadInvalidBackendUrlHasNoSideEffects(): void { + $savedBackendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + $countBefore = $this->countBinariesOfType($this->type->getId()); + $_SERVER['HTTP_HOST'] = 'attacker.example'; + putenv('HASHTOPOLIS_BACKEND_URL=ftp://configured.example/api/v2'); + try { + try { + CrackerUtils::createBinaryFromUpload('8.8.8', 'testcracker', $this->type->getId(), 'inline', base64_encode(self::SEVEN_ZIP_MAGIC)); + $this->fail('Expected invalid backend URL configuration to reject the upload'); + } + catch (\Exception $e) { + $this->assertStringContainsString('configured backend URL is invalid', $e->getMessage()); + } + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + $this->assertEmpty(glob(CrackerUtils::getCrackersPath() . '*_test-crackerutils-type-8.8.8.7z')); + } + finally { + if ($savedBackendUrl === false) { + putenv('HASHTOPOLIS_BACKEND_URL'); + } + else { + putenv('HASHTOPOLIS_BACKEND_URL=' . $savedBackendUrl); + } + } + } + // Verifies that the composed archive filename sanitizes all characters which are // problematic in file names. public function testCreateBinaryFromUploadSanitizesFilename(): void { diff --git a/src/inc/Util.php b/src/inc/Util.php index 0caac2634..37c334616 100755 --- a/src/inc/Util.php +++ b/src/inc/Util.php @@ -1307,9 +1307,9 @@ public static function buildServerUrl(): string { * Determines the base URL of the backend as it is reachable from the outside. * If HASHTOPOLIS_BACKEND_URL is set in the environment, scheme, host and port are * taken from it and any path is stripped (e.g. "http://localhost:8080/api/v2" - * results in "http://localhost:8080"). If the variable is not set or malformed, - * this falls back to the server URL derived from the current request together - * with the configured base URL, respecting the baseHost config override. + * results in "http://localhost:8080"). Otherwise, the explicitly configured + * baseHost and baseUrl are used. Request headers are never used because this URL + * may be persisted and later receive agent credentials. * Used to generate agent reachable URLs for locally hosted files, e.g. the * download URL of an uploaded cracker binary archive. * @return string backend base url without trailing slash @@ -1317,17 +1317,36 @@ public static function buildServerUrl(): string { */ public static function buildBackendBaseUrl(): string { $backendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); - if ($backendUrl !== false && strlen($backendUrl) > 0) { - $parts = parse_url($backendUrl); - if ($parts !== false && isset($parts['scheme'], $parts['host'])) { - $url = $parts['scheme'] . '://' . $parts['host']; - if (isset($parts['port'])) { - $url .= ':' . $parts['port']; - } - return rtrim($url, '/'); - } + $fromEnvironment = $backendUrl !== false && strlen($backendUrl) > 0; + if (!$fromEnvironment) { + $backendUrl = SConfig::getInstance()->getVal(DConfig::BASE_HOST); + } + if (!is_string($backendUrl) || strlen($backendUrl) == 0) { + throw new Exception("A valid HASHTOPOLIS_BACKEND_URL or baseHost must be configured"); + } + if ($backendUrl !== trim($backendUrl) || preg_match('/[\x00-\x20\x7f]/', $backendUrl)) { + throw new Exception("The configured backend URL is invalid"); + } + $parts = parse_url($backendUrl); + if ($parts === false || !isset($parts['scheme'], $parts['host']) + || !in_array(strtolower($parts['scheme']), ['http', 'https'], true) + || isset($parts['user']) || isset($parts['pass']) + || isset($parts['query']) || isset($parts['fragment']) + || (!$fromEnvironment && isset($parts['path']) && $parts['path'] !== '' && $parts['path'] !== '/')) { + throw new Exception("The configured backend URL is invalid"); + } + $url = strtolower($parts['scheme']) . '://' . $parts['host']; + if (isset($parts['port'])) { + $url .= ':' . $parts['port']; + } + if (filter_var($url, FILTER_VALIDATE_URL) === false) { + throw new Exception("The configured backend URL is invalid"); + } + if (!$fromEnvironment) { + $baseUrl = SConfig::getInstance()->getVal(DConfig::BASE_URL); + $url .= '/' . ltrim((string)$baseUrl, '/'); } - return rtrim(Util::buildServerUrl() . SConfig::getInstance()->getVal(DConfig::BASE_URL), '/'); + return rtrim($url, '/'); } /** diff --git a/src/inc/utils/CrackerUtils.php b/src/inc/utils/CrackerUtils.php index 1412897a2..78b859ab1 100644 --- a/src/inc/utils/CrackerUtils.php +++ b/src/inc/utils/CrackerUtils.php @@ -141,6 +141,10 @@ public static function createBinaryFromUpload(string $version, string $name, int default: throw new HttpError("sourceType value '" . $sourceType . "' is not supported (choices inline, import, url"); } + + // Resolve this before storing anything so invalid server configuration cannot + // leave a partial archive or database record behind. + $backendBaseUrl = Util::buildBackendBaseUrl(); $filename = CrackerUtils::buildArchiveFilename($binaryType, $version); @@ -170,7 +174,7 @@ public static function createBinaryFromUpload(string $version, string $name, int } return Factory::getCrackerBinaryFactory()->mset($binary, [ - CrackerBinary::DOWNLOAD_URL => Util::buildBackendBaseUrl() . '/api/download.php/crackerBinary/' . $binary->getId(), + CrackerBinary::DOWNLOAD_URL => $backendBaseUrl . '/api/download.php/crackerBinary/' . $binary->getId(), CrackerBinary::FILENAME => $filename ]); } From 37d62c9ac501381fe4070bc89cf24dcc31bd41f2 Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 10:02:35 +0200 Subject: [PATCH 06/11] added check to avoid memory issues on inline upload handling --- ci/phpunit/inc/utils/CrackerUtilsTest.php | 25 +++++++++++++++ src/inc/utils/CrackerUtils.php | 37 ++++++++++++++++++++--- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/ci/phpunit/inc/utils/CrackerUtilsTest.php b/ci/phpunit/inc/utils/CrackerUtilsTest.php index 0b19d33be..893a8af1b 100644 --- a/ci/phpunit/inc/utils/CrackerUtilsTest.php +++ b/ci/phpunit/inc/utils/CrackerUtilsTest.php @@ -250,6 +250,31 @@ public function testCreateBinaryFromUploadInvalidBase64ThrowsHttpError(): void { CrackerUtils::createBinaryFromUpload('7.2.7', 'testcracker', $this->type->getId(), 'inline', '!!!no-base64!!!'); } + // Verifies that an inline archive which cannot be decoded with sufficient + // memory headroom is rejected before a database row or file is created. + public function testCreateBinaryFromUploadInlineTooLargeHasNoSideEffects(): void { + $memoryLimit = ini_get('memory_limit'); + $countBefore = $this->countBinariesOfType($this->type->getId()); + $testLimit = memory_get_usage(true) + 16 * 1024 * 1024; + try { + if (ini_set('memory_limit', (string)$testLimit) === false) { + $this->markTestSkipped('memory_limit cannot be changed in this environment'); + } + try { + CrackerUtils::createBinaryFromUpload('8.8.8', 'testcracker', $this->type->getId(), 'inline', base64_encode(self::SEVEN_ZIP_MAGIC)); + $this->fail('Expected the inline archive to exceed the safe memory allowance'); + } + catch (HttpError $e) { + $this->assertStringContainsString('inline archive is too large', $e->getMessage()); + } + $this->assertEquals($countBefore, $this->countBinariesOfType($this->type->getId())); + $this->assertEmpty(glob(CrackerUtils::getCrackersPath() . '*_test-crackerutils-type-8.8.8.7z')); + } + finally { + ini_set('memory_limit', (string)$memoryLimit); + } + } + // Verifies that createBinaryFromUpload() only allows http and https urls, so no // local files or stream wrappers can be fetched by the server. public function testCreateBinaryFromUploadUrlSchemeThrowsHttpError(): void { diff --git a/src/inc/utils/CrackerUtils.php b/src/inc/utils/CrackerUtils.php index 78b859ab1..39b2bb325 100644 --- a/src/inc/utils/CrackerUtils.php +++ b/src/inc/utils/CrackerUtils.php @@ -17,6 +17,8 @@ use Hashtopolis\inc\Util; class CrackerUtils { + private const INLINE_MEMORY_RESERVE = 16 * 1024 * 1024; + /** * @param CrackerBinaryType $cracker * @return CrackerBinary[] @@ -115,12 +117,8 @@ public static function createBinaryFromUpload(string $version, string $name, int // determine the source of the archive and validate it switch ($sourceType) { case "inline": - $archiveData = base64_decode($sourceData, true); - if ($archiveData === false) { - throw new HttpError("sourceData not valid base64 encoding"); - } $uploadType = "paste"; - $uploadData = $archiveData; + $uploadData = CrackerUtils::decodeInlineArchive($sourceData); break; case "import": $realname = str_replace(" ", "_", htmlentities(basename($sourceData), ENT_QUOTES, "UTF-8")); @@ -212,6 +210,35 @@ private static function isSevenZipArchive(string $path): bool { fclose($fp); return $header === $magic; } + + /** + * Decodes an inline archive while leaving sufficient PHP memory for the + * decoded data and subsequent request processing. + * + * @throws HttpError + */ + private static function decodeInlineArchive(string $sourceData): string { + $memoryLimit = ini_parse_quantity((string)ini_get('memory_limit')); + if ($memoryLimit != -1) { + $sourceLength = strlen($sourceData); + $decodedSize = intdiv($sourceLength, 4) * 3; + if ($sourceLength % 4 != 0) { + $decodedSize += 3; + } + + $reserve = max(self::INLINE_MEMORY_RESERVE, intdiv($memoryLimit, 4)); + $available = $memoryLimit - memory_get_usage(true) - $reserve; + if ($available <= 0 || $decodedSize > intdiv($available, 2)) { + throw new HttpError("The inline archive is too large for this server. Upload it through TUS and use sourceType 'import'."); + } + } + + $archiveData = base64_decode($sourceData, true); + if ($archiveData === false) { + throw new HttpError("sourceData not valid base64 encoding"); + } + return $archiveData; + } /** * Validates that the server is allowed to fetch the given url as the From dd410c2e6c905869e1b1411dd0a40253f899059c Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 10:04:41 +0200 Subject: [PATCH 07/11] check downloadUrl on cracker binary creation --- ci/apiv2/test_cracker.py | 22 ++++++++++++++++++++++ src/inc/apiv2/model/CrackerBinaryAPI.php | 3 +++ 2 files changed, 25 insertions(+) diff --git a/ci/apiv2/test_cracker.py b/ci/apiv2/test_cracker.py index 1fe83ef3b..d98df9c23 100644 --- a/ci/apiv2/test_cracker.py +++ b/ci/apiv2/test_cracker.py @@ -374,6 +374,28 @@ def test_patch_download_url_refreshes_local_copy(self): server_a.shutdown() server_b.shutdown() + def test_patch_download_url_to_null_rejects(self): + """A nullable create field cannot be set to null by PATCH.""" + server, url = self._serve_archive(SEVEN_ZIP_MAGIC + b'url-patch-null') + obj = None + try: + obj = self.create_cracker(extra_payload={'downloadUrl': url}, delete=False) + headers = {'Authorization': f'Bearer {get_bearer_token()}', + 'Content-Type': 'application/json'} + response = requests.patch( + f'{APIV2}/ui/crackers/{obj.id}', + headers=headers, + json={'data': {'type': 'crackerBinary', 'id': str(obj.id), + 'attributes': {'downloadUrl': None}}}) + + self.assertEqual(400, response.status_code) + self.assertIn('downloadUrl cannot be null', response.text) + self.assertEqual(url, Cracker.objects.get(pk=obj.id).downloadUrl) + finally: + if obj is not None: + obj.delete() + server.shutdown() + def test_patch_download_url_with_version_stores_new_copy(self): """Changing the download url together with the version stores the refreshed local copy under the new archive filename and removes the diff --git a/src/inc/apiv2/model/CrackerBinaryAPI.php b/src/inc/apiv2/model/CrackerBinaryAPI.php index c97d8f178..156ad12e4 100644 --- a/src/inc/apiv2/model/CrackerBinaryAPI.php +++ b/src/inc/apiv2/model/CrackerBinaryAPI.php @@ -124,6 +124,9 @@ protected function deleteObject(AbstractModel $object): void { protected function updateObject(int $objectId, array $data): void { $binary = CrackerUtils::getBinary($objectId); if (array_key_exists(CrackerBinary::DOWNLOAD_URL, $data)) { + if ($data[CrackerBinary::DOWNLOAD_URL] === null) { + throw new HttpError("downloadUrl cannot be null!"); + } if ($binary->getFilename() !== null) { throw new HttpError("The download url of a locally stored cracker binary cannot be changed!"); } From dad8c22fff2e75ab7f74bc373443cc72b3c474b8 Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 10:16:53 +0200 Subject: [PATCH 08/11] fixed to correct range requests --- ci/apiv2/test_file.py | 14 +++++++++ ci/phpunit/downloadapi/DownloadAppTest.php | 20 +++++++++++++ src/getFile.php | 33 ++-------------------- src/inc/utils/DownloadUtils.php | 20 +++++++++---- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/ci/apiv2/test_file.py b/ci/apiv2/test_file.py index 4f123c61b..9678b717c 100644 --- a/ci/apiv2/test_file.py +++ b/ci/apiv2/test_file.py @@ -56,6 +56,20 @@ def test_range_request_get_file(self): file_data = helper.get_file(file=model_obj, range="bytes=9-15") self.assertEqual(file_data, "123456\n") + def test_suffix_range_request_get_file(self): + model_obj = self.create_test_object() + + helper = Helper() + file_data = helper.get_file(file=model_obj, range="bytes=-9") + self.assertEqual(file_data, "princess\n") + + def test_open_ended_range_request_get_file(self): + model_obj = self.create_test_object() + + helper = Helper() + file_data = helper.get_file(file=model_obj, range="bytes=9-") + self.assertEqual(file_data, "123456\nprincess\n") + def test_bulk_delete(self): files = [self.create_test_object(delete=False) for i in range(5)] File.objects.delete_many(files) diff --git a/ci/phpunit/downloadapi/DownloadAppTest.php b/ci/phpunit/downloadapi/DownloadAppTest.php index 357685285..2f384fdb1 100644 --- a/ci/phpunit/downloadapi/DownloadAppTest.php +++ b/ci/phpunit/downloadapi/DownloadAppTest.php @@ -170,6 +170,26 @@ public function testRangeRequestReturnsPartialContent(): void { $this->assertEquals('bytes 0-5/' . strlen($this->archiveContent), $response->getHeaderLine('Content-Range')); } + // A suffix range returns the requested number of bytes from the end. + public function testSuffixRangeRequestReturnsFinalBytes(): void { + $_SERVER['HTTP_RANGE'] = 'bytes=-6'; + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $size = strlen($this->archiveContent); + $this->assertEquals(206, $response->getStatusCode()); + $this->assertEquals(substr($this->archiveContent, -6), (string)$response->getBody()); + $this->assertEquals('bytes ' . ($size - 6) . '-' . ($size - 1) . '/' . $size, $response->getHeaderLine('Content-Range')); + } + + // An open-ended range returns all bytes through the end of the file. + public function testOpenEndedRangeRequestReturnsRemainingBytes(): void { + $_SERVER['HTTP_RANGE'] = 'bytes=6-'; + $response = $this->runDownloadRequest($this->localBinaryUri() . '?token=' . $this->agentToken); + $size = strlen($this->archiveContent); + $this->assertEquals(206, $response->getStatusCode()); + $this->assertEquals(substr($this->archiveContent, 6), (string)$response->getBody()); + $this->assertEquals('bytes 6-' . ($size - 1) . '/' . $size, $response->getHeaderLine('Content-Range')); + } + // A request with a matching ETag is answered with not modified. public function testMatchingEtagReturnsNotModified(): void { $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); diff --git a/src/getFile.php b/src/getFile.php index fc3858d18..304e8969d 100644 --- a/src/getFile.php +++ b/src/getFile.php @@ -13,6 +13,7 @@ use Hashtopolis\inc\Util; use Hashtopolis\inc\utils\AccessControl; use Hashtopolis\inc\utils\AccessUtils; +use Hashtopolis\inc\utils\DownloadUtils; require_once(dirname(__FILE__) . "/inc/startup/load.php"); @@ -110,42 +111,12 @@ header("Content-Disposition: attachment; filename=\"" . $line->getFilename() . "\""); if (isset($_SERVER['HTTP_RANGE'])) { - - $c_start = $start; - $c_end = $end; - - list(, $range) = explode('=', $_SERVER['HTTP_RANGE'], 2); - - if (strpos($range, ',') !== false) { - header('HTTP/1.1 416 Requested Range Not Satisfiable'); - header("Content-Range: bytes $start-$end/$size"); - exit; - } - if ($range == '-') { - $c_start = $size - substr($range, 1); - } - else { - $range = explode('-', $range); - $c_start = $range[0]; - if ((isset($range[1]) && is_numeric($range[1]))) { - $c_end = $range[1]; - } - else { - $c_end = $size; - } - } - if ($c_end > $end) { - $c_end = $end; - } - if ($c_start > $c_end || $c_start > $size - 1 || $c_end >= $size) { + if (!DownloadUtils::handleRangeRequest($start, $end, $size, $fp)) { header('HTTP/1.1 416 Requested Range Not Satisfiable'); header("Content-Range: bytes $start-$end/$size"); exit; } - $start = $c_start; - $end = $c_end; $length = $end - $start + 1; - fseek($fp, $start); header('HTTP/1.1 206 Partial Content'); } diff --git a/src/inc/utils/DownloadUtils.php b/src/inc/utils/DownloadUtils.php index 1717a3972..af1f23b2d 100644 --- a/src/inc/utils/DownloadUtils.php +++ b/src/inc/utils/DownloadUtils.php @@ -36,17 +36,27 @@ public static function handleRangeRequest(int &$start, int &$end, int $size, $fp if (str_contains($range, ',')) { return false; } - if ($range == '-') { - $c_start = $size - (int)substr($range, 1); + $range = explode('-', $range, 2); + if ($range[0] === '') { + if (!isset($range[1]) || !ctype_digit($range[1]) || (int)$range[1] == 0) { + return false; + } + $c_start = max(0, $size - (int)$range[1]); + $c_end = $size - 1; } else { - $range = explode('-', $range); + if (!ctype_digit($range[0])) { + return false; + } $c_start = (int)$range[0]; - if ((isset($range[1]) && is_numeric($range[1]))) { + if (isset($range[1]) && $range[1] !== '' && ctype_digit($range[1])) { $c_end = (int)$range[1]; } + else if (isset($range[1]) && $range[1] === '') { + $c_end = $size - 1; + } else { - $c_end = $size; + return false; } } if ($c_end > $end) { From e789b6c2c5e197d7acde886a223ebed0c503b0d8 Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 10:19:42 +0200 Subject: [PATCH 09/11] added basic agents guideline for automated tests runs --- AGENTS.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..b056aedee --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,37 @@ +# Agent Guidelines + +## Testing With The Devcontainer + +Use the devcontainer Compose setup so tests run with the expected PHP extensions, +dependencies, directories, and database. Run these commands from the repository +root: + +```bash +docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml build +docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml run --rm --entrypoint composer hashtopolis-server-dev install --working-dir=/var/www/html +docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml up -d +``` + +If migrations or an incompatible test schema cause failures, recreate the test +database first, then start the stack again: + +```bash +docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml down --volumes --remove-orphans +``` + +Run PHP tests and static analysis inside `hashtopolis-server-dev`: + +```bash +docker exec -e HASHTOPOLIS_BACKEND_URL=http://localhost hashtopolis-server-dev vendor/bin/phpunit +docker exec hashtopolis-server-dev vendor/bin/phpstan analyse --no-progress +``` + +Pass a test file to PHPUnit for focused runs. API tests must run from +`/var/www/html/ci/apiv2` so their default configuration is discovered: + +```bash +docker exec -w /var/www/html/ci/apiv2 hashtopolis-server-dev python -m pytest test_file.py -q +``` + +The PostgreSQL devcontainer Compose file can be used in the same way when that +database backend is relevant. From 63ab3e82816cab78cef0694feb80834c166f68a9 Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 11:12:30 +0200 Subject: [PATCH 10/11] fixed failing tests --- ci/phpunit/downloadapi/DownloadAppTest.php | 40 +++++++++++++++------- src/dba/models/CrackerBinaryFactory.php | 2 +- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/ci/phpunit/downloadapi/DownloadAppTest.php b/ci/phpunit/downloadapi/DownloadAppTest.php index 2f384fdb1..48301d930 100644 --- a/ci/phpunit/downloadapi/DownloadAppTest.php +++ b/ci/phpunit/downloadapi/DownloadAppTest.php @@ -30,6 +30,7 @@ final class DownloadAppTest extends TestBase { private CrackerBinary $externalBinary; private string $agentToken; private string $archiveContent; + private string|false $savedBackendUrl = false; private bool $savedHttpRange = false; private string|false $savedHttpRangeValue = false; @@ -37,9 +38,14 @@ final class DownloadAppTest extends TestBase { protected function setUp(): void { parent::setUp(); + $this->savedBackendUrl = getenv('HASHTOPOLIS_BACKEND_URL'); + putenv('HASHTOPOLIS_BACKEND_URL=http://localhost/api/v2'); + + $suffix = uniqid(); + $this->type = $this->createDatabaseObject( Factory::getCrackerBinaryTypeFactory(), - new CrackerBinaryType(null, 'download-test-type', 1) + new CrackerBinaryType(null, 'dl-' . $suffix, 1) ); $this->externalBinary = $this->createDatabaseObject( Factory::getCrackerBinaryFactory(), @@ -50,7 +56,7 @@ protected function setUp(): void { $this->agentToken = 'dl-test-' . uniqid(); $this->createDatabaseObject( Factory::getAgentFactory(), - new Agent(null, 'download-test-agent', '', 0, '', '', 0, 0, 0, $this->agentToken, '', 0, '', null, 0, '') + new Agent(null, 'download-test-agent-' . $suffix, '', 0, '', '', 0, 0, 0, $this->agentToken, '', 0, '', null, 0, '') ); $importName = 'download-test-' . uniqid() . '.7z'; @@ -67,18 +73,28 @@ protected function setUp(): void { #[Override] protected function tearDown(): void { - // remove the archive in case a test failed before it could clean up - $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); - if (file_exists($archive)) { - unlink($archive); - } - if ($this->savedHttpRange) { - $_SERVER['HTTP_RANGE'] = $this->savedHttpRangeValue; + try { + // remove the archive in case a test failed before it could clean up + $archive = CrackerUtils::getCrackersPath() . $this->localBinary->getId() . '_' . $this->localBinary->getFilename(); + if (file_exists($archive)) { + unlink($archive); + } + parent::tearDown(); } - else { - unset($_SERVER['HTTP_RANGE']); + finally { + if ($this->savedBackendUrl === false) { + putenv('HASHTOPOLIS_BACKEND_URL'); + } + else { + putenv('HASHTOPOLIS_BACKEND_URL=' . $this->savedBackendUrl); + } + if ($this->savedHttpRange) { + $_SERVER['HTTP_RANGE'] = $this->savedHttpRangeValue; + } + else { + unset($_SERVER['HTTP_RANGE']); + } } - parent::tearDown(); } private static function getImportPath(): string { diff --git a/src/dba/models/CrackerBinaryFactory.php b/src/dba/models/CrackerBinaryFactory.php index 459d29922..3b7a975b3 100644 --- a/src/dba/models/CrackerBinaryFactory.php +++ b/src/dba/models/CrackerBinaryFactory.php @@ -45,6 +45,6 @@ function createObjectFromDict(array $dict): CrackerBinary { $conv[strtolower($key)] = $val; } $dict = $conv; - return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname'], $dict['filename']); + return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname'], $dict['filename'] ?? null); } } From 70d24f4a9990f10b6680c965a44b9007f9f1db7c Mon Sep 17 00:00:00 2001 From: s3inlc Date: Wed, 23 Sep 2026 11:38:54 +0200 Subject: [PATCH 11/11] fixed tests failing due to missing ENV, by default now localhost is in test and devcontainer composes --- .devcontainer/docker-compose.mysql.yml | 1 + .devcontainer/docker-compose.postgres.yml | 1 + .github/docker-compose.mysql.yml | 1 + .github/docker-compose.postgres.yml | 1 + ci/phpunit/inc/UtilTest.php | 4 +++- src/dba/models/CrackerBinaryFactory.php | 2 +- 6 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.devcontainer/docker-compose.mysql.yml b/.devcontainer/docker-compose.mysql.yml index bbeffdf61..71da6fb9b 100644 --- a/.devcontainer/docker-compose.mysql.yml +++ b/.devcontainer/docker-compose.mysql.yml @@ -14,6 +14,7 @@ services: HASHTOPOLIS_DB_HOST: hashtopolis-db-dev HASHTOPOLIS_DB_DATABASE: hashtopolis HASHTOPOLIS_APIV2_ENABLE: 1 + HASHTOPOLIS_BACKEND_URL: http://localhost depends_on: - hashtopolis-db-dev ports: diff --git a/.devcontainer/docker-compose.postgres.yml b/.devcontainer/docker-compose.postgres.yml index a0b2e13e9..1174f7db4 100644 --- a/.devcontainer/docker-compose.postgres.yml +++ b/.devcontainer/docker-compose.postgres.yml @@ -14,6 +14,7 @@ services: HASHTOPOLIS_DB_HOST: hashtopolis-db-dev HASHTOPOLIS_DB_DATABASE: hashtopolis HASHTOPOLIS_APIV2_ENABLE: 1 + HASHTOPOLIS_BACKEND_URL: http://localhost depends_on: - hashtopolis-db-dev ports: diff --git a/.github/docker-compose.mysql.yml b/.github/docker-compose.mysql.yml index babdc4121..845dcd46a 100644 --- a/.github/docker-compose.mysql.yml +++ b/.github/docker-compose.mysql.yml @@ -15,6 +15,7 @@ services: HASHTOPOLIS_DB_HOST: hashtopolis-db-dev HASHTOPOLIS_DB_DATABASE: hashtopolis HASHTOPOLIS_APIV2_ENABLE: 1 + HASHTOPOLIS_BACKEND_URL: http://localhost depends_on: - hashtopolis-db-dev ports: diff --git a/.github/docker-compose.postgres.yml b/.github/docker-compose.postgres.yml index d52312acf..4c910a0e8 100644 --- a/.github/docker-compose.postgres.yml +++ b/.github/docker-compose.postgres.yml @@ -15,6 +15,7 @@ services: HASHTOPOLIS_DB_HOST: hashtopolis-db-dev HASHTOPOLIS_DB_DATABASE: hashtopolis HASHTOPOLIS_APIV2_ENABLE: 1 + HASHTOPOLIS_BACKEND_URL: http://localhost depends_on: - hashtopolis-db-dev ports: diff --git a/ci/phpunit/inc/UtilTest.php b/ci/phpunit/inc/UtilTest.php index 5c6df4cff..28eb938a0 100644 --- a/ci/phpunit/inc/UtilTest.php +++ b/ci/phpunit/inc/UtilTest.php @@ -967,7 +967,8 @@ public function testCheckOrCreateInitialObjectCrackerBinary(): void { 'crackerBinaryTypeId' => $typeId, 'version' => '7.0.0', 'downloadUrl' => 'https://example.com/test.7z', - 'binaryName' => 'testHashcat' + 'binaryName' => 'testHashcat', + 'filename' => null ]; Util::checkOrCreateInitialObject(Factory::getCrackerBinaryFactory(), $data); $obj = Factory::getCrackerBinaryFactory()->get($id); @@ -976,6 +977,7 @@ public function testCheckOrCreateInitialObjectCrackerBinary(): void { $this->assertEquals('7.0.0', $obj->getVersion()); $this->assertEquals('https://example.com/test.7z', $obj->getDownloadUrl()); $this->assertEquals('testHashcat', $obj->getBinaryName()); + $this->assertNull($obj->getFilename()); Factory::getCrackerBinaryFactory()->delete($obj); Factory::getCrackerBinaryTypeFactory()->delete(Factory::getCrackerBinaryTypeFactory()->get($typeId)); diff --git a/src/dba/models/CrackerBinaryFactory.php b/src/dba/models/CrackerBinaryFactory.php index 3b7a975b3..459d29922 100644 --- a/src/dba/models/CrackerBinaryFactory.php +++ b/src/dba/models/CrackerBinaryFactory.php @@ -45,6 +45,6 @@ function createObjectFromDict(array $dict): CrackerBinary { $conv[strtolower($key)] = $val; } $dict = $conv; - return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname'], $dict['filename'] ?? null); + return new CrackerBinary($dict['crackerbinaryid'], $dict['crackerbinarytypeid'], $dict['version'], $dict['downloadurl'], $dict['binaryname'], $dict['filename']); } }