Allow uploading and storing cracker binaries on the server

.devcontainer/docker-compose.mysql.yml

@@ -14,6 +14,7 @@ services:
       HASHTOPOLIS_DB_HOST: hashtopolis-db-dev
       HASHTOPOLIS_DB_DATABASE: hashtopolis
       HASHTOPOLIS_APIV2_ENABLE: 1
+      HASHTOPOLIS_BACKEND_URL: http://localhost
     depends_on:
       - hashtopolis-db-dev
     ports:

.devcontainer/docker-compose.postgres.yml

@@ -14,6 +14,7 @@ services:
       HASHTOPOLIS_DB_HOST: hashtopolis-db-dev
       HASHTOPOLIS_DB_DATABASE: hashtopolis
       HASHTOPOLIS_APIV2_ENABLE: 1
+      HASHTOPOLIS_BACKEND_URL: http://localhost
     depends_on:
       - hashtopolis-db-dev
     ports:

.github/docker-compose.mysql.yml

@@ -15,6 +15,7 @@ services:
       HASHTOPOLIS_DB_HOST: hashtopolis-db-dev
       HASHTOPOLIS_DB_DATABASE: hashtopolis
       HASHTOPOLIS_APIV2_ENABLE: 1
+      HASHTOPOLIS_BACKEND_URL: http://localhost
     depends_on:
       - hashtopolis-db-dev
     ports:

.github/docker-compose.postgres.yml

@@ -15,6 +15,7 @@ services:
       HASHTOPOLIS_DB_HOST: hashtopolis-db-dev
       HASHTOPOLIS_DB_DATABASE: hashtopolis
       HASHTOPOLIS_APIV2_ENABLE: 1
+      HASHTOPOLIS_BACKEND_URL: http://localhost
     depends_on:
       - hashtopolis-db-dev
     ports:

AGENTS.md

@@ -0,0 +1,37 @@
+# Agent Guidelines
+
+## Testing With The Devcontainer
+
+Use the devcontainer Compose setup so tests run with the expected PHP extensions,
+dependencies, directories, and database. Run these commands from the repository
+root:
+
+```bash
+docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml build
+docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml run --rm --entrypoint composer hashtopolis-server-dev install --working-dir=/var/www/html
+docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml up -d
+```
+
+If migrations or an incompatible test schema cause failures, recreate the test
+database first, then start the stack again:
+
+```bash
+docker compose -p devcontainer -f .devcontainer/docker-compose.mysql.yml down --volumes --remove-orphans
+```
+
+Run PHP tests and static analysis inside `hashtopolis-server-dev`:
+
+```bash
+docker exec -e HASHTOPOLIS_BACKEND_URL=http://localhost hashtopolis-server-dev vendor/bin/phpunit
+docker exec hashtopolis-server-dev vendor/bin/phpstan analyse --no-progress
+```
+
+Pass a test file to PHPUnit for focused runs. API tests must run from
+`/var/www/html/ci/apiv2` so their default configuration is discovered:
+
+```bash
+docker exec -w /var/www/html/ci/apiv2 hashtopolis-server-dev python -m pytest test_file.py -q
+```
+
+The PostgreSQL devcontainer Compose file can be used in the same way when that
+database backend is relevant.

Dockerfile

@@ -30,6 +30,7 @@ ENV HASHTOPOLIS_IMPORT_PATH=${HASHTOPOLIS_PATH}/import
 ENV HASHTOPOLIS_LOG_PATH=${HASHTOPOLIS_PATH}/log
 ENV HASHTOPOLIS_CONFIG_PATH=${HASHTOPOLIS_PATH}/config
 ENV HASHTOPOLIS_BINARIES_PATH=${HASHTOPOLIS_PATH}/binaries
+ENV HASHTOPOLIS_CRACKERS_PATH=${HASHTOPOLIS_PATH}/crackers
 ENV HASHTOPOLIS_TUS_PATH=/var/tmp/tus
 ENV HASHTOPOLIS_TEMP_UPLOADS_PATH=${HASHTOPOLIS_TUS_PATH}/uploads
 ENV HASHTOPOLIS_TEMP_META_PATH=${HASHTOPOLIS_TUS_PATH}/meta
@@ -80,6 +81,7 @@ RUN mkdir -p \
     ${HASHTOPOLIS_LOG_PATH} \
     ${HASHTOPOLIS_CONFIG_PATH} \
     ${HASHTOPOLIS_BINARIES_PATH} \
+    ${HASHTOPOLIS_CRACKERS_PATH} \
     ${HASHTOPOLIS_TUS_PATH} \
     ${HASHTOPOLIS_TEMP_UPLOADS_PATH} \
     ${HASHTOPOLIS_TEMP_META_PATH} \

ci/apiv2/test_agent_protocol.py

@@ -16,12 +16,14 @@
 import json
 import re
 import unittest
+import urllib.parse
 
 import requests
 
-from hashtopolis import Agent, Config, HealthCheck, Voucher
+from hashtopolis import Agent, Config, Cracker, HealthCheck, Voucher
 from hashtopolis_agent import DummyAgent
-from utils import BaseTest, do_create_agentassignent, do_create_dummy_agent, do_create_voucher, get_hashtopolis_uri
+from utils import (BaseTest, SEVEN_ZIP_MAGIC, do_create_agentassignent, do_create_dummy_agent,
+                   do_create_voucher, get_hashtopolis_uri)
 
 
 AGENT_ENDPOINT = '/api/server.php'
@@ -32,6 +34,19 @@ def _uri():
     return get_hashtopolis_uri()
 
 
+def fetch_via_test_config(url, **kwargs):
+    """GET a server-generated url through the configured test server uri.
+
+    The realworld dataset configures baseHost with a url that is only reachable
+    from outside the container, so server-generated absolute urls cannot be
+    fetched from within the tests. The authority is rewritten to the uri the
+    tests run against, path and query (which carry the agent token) are kept.
+    """
+    parts = urllib.parse.urlparse(url)
+    base = urllib.parse.urlparse(_uri())
+    return requests.get(base._replace(path=parts.path, query=parts.query).geturl(), **kwargs)
+
+
 def agent_request(payload):
     """POST a raw JSON payload to the agent API and return (status_code, body_text).
 
@@ -535,6 +550,69 @@ def test_download_cracker_invalid_binary_version_id(self):
         assert_error_envelope(self, body, "downloadBinary")
         self.assertEqual(parse_envelope(body)['message'], "Invalid cracker binary type id!")
 
+    def test_download_cracker_local_binary(self):
+        """A locally stored cracker binary is served by the server itself: the
+        downloadBinary action returns the url of the download endpoint with the
+        requesting agent's token appended, so the archive can directly be fetched."""
+        dummy = self._dummy()
+        content = SEVEN_ZIP_MAGIC + b'local-binary-download-test'
+        cracker = self.create_local_cracker(content=content, extra_payload={'version': '7.2.7'})
+
+        code, body = agent_request({
+            "action": "downloadBinary",
+            "token": dummy.token,
+            "type": "cracker",
+            "binaryVersionId": cracker.id,
+        })
+        self.assertEqual(code, 200)
+        resp = parse_envelope(body)
+        self.assertEqual(resp['response'], "SUCCESS")
+        url = resp['url']
+        url_parts = urllib.parse.urlparse(url)
+        self.assertEqual(f'/api/download.php/crackerBinary/{cracker.id}', url_parts.path)
+        self.assertIn(f'token={dummy.token}', url_parts.query)
+
+        # the agent can fetch the archive with the returned url
+        r = fetch_via_test_config(url)
+        self.assertEqual(200, r.status_code)
+        self.assertEqual(content, r.content)
+
+    def test_download_cracker_local_binary_wrong_token_denied(self):
+        """The download url of a local binary only works with the agent token it
+        was issued for."""
+        dummy = self._dummy()
+        cracker = self.create_local_cracker()
+
+        code, body = agent_request({
+            "action": "downloadBinary",
+            "token": dummy.token,
+            "type": "cracker",
+            "binaryVersionId": cracker.id,
+        })
+        url = parse_envelope(body)['url']
+
+        r = fetch_via_test_config(url.replace(f'token={dummy.token}', 'token=wrong-token'))
+        self.assertEqual(401, r.status_code)
+
+    def test_download_cracker_local_binary_external_unchanged(self):
+        """Cracker binaries referenced with an external url are answered with the
+        stored url, no token is appended."""
+        dummy = self._dummy()
+        external_binaries = [c for c in Cracker.objects.filter() if not c.filename]
+        self.assertTrue(external_binaries, 'no externally referenced cracker binary found')
+        cracker = external_binaries[0]
+
+        code, body = agent_request({
+            "action": "downloadBinary",
+            "token": dummy.token,
+            "type": "cracker",
+            "binaryVersionId": cracker.id,
+        })
+        resp = parse_envelope(body)
+        self.assertEqual(resp['response'], "SUCCESS")
+        self.assertEqual(cracker.downloadUrl, resp['url'])
+        self.assertNotIn('token=', resp['url'])
+
 
 # ---------------------------------------------------------------------------
 # clientError