ADFA-5405: Let templates reference each other

app/src/main/java/com/itsaky/androidide/localWebServer/WebServer.kt

@@ -1,6 +1,5 @@
 package com.itsaky.androidide.localWebServer
 
-import android.database.Cursor
 import android.database.sqlite.SQLiteDatabase
 import android.net.TrafficStats
 import android.os.Environment.getExternalStorageDirectory
@@ -11,6 +10,7 @@ import com.itsaky.androidide.documentation.DocumentationContent
 import com.itsaky.androidide.documentation.DocumentationContentSource
 import com.itsaky.androidide.documentation.DocumentationLookup
 import com.itsaky.androidide.documentation.DocumentationRequestInterceptor
+import com.itsaky.androidide.documentation.TemplateRenderException
 import com.itsaky.androidide.utils.ContentTypeHeaders
 import com.itsaky.androidide.utils.DatabaseVersionResolver
 import org.slf4j.LoggerFactory
@@ -148,18 +148,6 @@ class WebServer(
 			.serializeNulls()
 			.create()
 
-	// -1 means "not fetched yet". Volatile because the WebView transport shares this server's
-	// process, and the interceptor's reads can run on WebView threads while the accept loop writes.
-	@Volatile
-	private var bookshelfTemplateId: Int = -1
-
-	private val cacheLock = Any()
-
-	// Which of the source's databases bookshelfTemplateId was filled from. The compiled templates
-	// themselves live in the source and are dropped by its own swap.
-	@Volatile
-	private var cachedDatabaseGeneration = 0L
-
 	// Long enough to stop a descriptor-exhaustion spin starving the connections whose closing would
 	// fix it; short enough to be invisible to a user, and never paid on a successful accept.
 	private val initialAcceptBackoffMs = 50L
@@ -547,31 +535,11 @@ class WebServer(
 			return sendError(writer, output, 501, "Not Implemented")
 		}
 
-		// serveRequest applies any pending sdcard debug-database swap via the content source.
+		// The content source applies a pending sdcard debug-database swap inside lookup()/withDatabase(),
+		// so a request reaching neither -- an unknown /pr/ target -- does not poll for one.
 		serveRequest(writer, output, path)
 	}
 
-	/**
-	 * Invalidates the cached bookshelf template identifier when the documentation database changes.
-	 */
-	private fun discardCachesIfDatabaseChanged() {
-		// Apply any pending swap first. The source swaps inside lookup()/withDatabase(), so checking
-		// the generation before those runs reads the generation from before the swap: on the very
-		// request that swaps, this would leave bookshelfTemplateId pointing at the previous
-		// database's template row -- rendering the old bookshelf, or 500ing if that id is absent.
-		contentSource.refreshDatabase()
-
-		if (contentSource.generation == cachedDatabaseGeneration) return
-
-		synchronized(cacheLock) {
-			val generation = contentSource.generation
-			if (generation == cachedDatabaseGeneration) return
-
-			bookshelfTemplateId = -1
-			cachedDatabaseGeneration = generation
-		}
-	}
-
 	/**
 	 * Serves a parsed request using the appropriate diagnostic endpoint or documentation content.
 	 *
@@ -584,8 +552,6 @@ class WebServer(
 		output: java.io.OutputStream,
 		path: String,
 	) {
-		discardCachesIfDatabaseChanged()
-
 		// Handle the special "pr" endpoint with highest priority
 		if (path.startsWith("pr/", false)) {
 			if (debugEnabled) log.debug("Found a pr/ path, '{}'.", path)
@@ -804,7 +770,26 @@ class WebServer(
 			outputStarted = realHandleBsEndpoint(writer, output) { outputStarted = true }
 		} catch (e: Exception) {
 			log.error("Error handling /pr/bs endpoint: {}", e.message)
-			sendError(writer, output, httpInternalServerError, "Internal Server Error 6", "Error generating bookshelf HTML.", outputStarted)
+			// The message is echoed ONLY for a template failure. That one names a template -- the
+			// bookshelf row itself, or anything it references -- and the name is the whole diagnostic
+			// (ADFA-5405). Everything else keeps the generic text, because this catch spans the whole
+			// of realHandleBsEndpoint: a SQLiteException carries SQL, and withDatabase's
+			// check(openIfNeeded()) carries the database's filesystem path. Any app on the device can
+			// GET this port, so echoing those was handing out internals for the sake of one
+			// diagnostic.
+			val detail =
+				when (e) {
+					is TemplateRenderException -> e.message ?: "Error generating bookshelf HTML."
+					else -> "Error generating bookshelf HTML."
+				}
+			sendError(
+				writer,
+				output,
+				httpInternalServerError,
+				"Internal Server Error 6",
+				detail,
+				outputStarted,
+			)
 		}
 
 		if (debugEnabled) log.debug("Leaving handleBsEndpoint().")
@@ -877,32 +862,23 @@ class WebServer(
 		val jsonText =
 			contentSource.withDatabase { database ->
 				try {
-					val json = bookshelfJson(database)
-					if (debugEnabled) log.debug("json content = '{}'.", String(json, Charsets.UTF_8))
-					if (debugEnabled) log.debug("before fetch bookshelf template ID = '{}'", bookshelfTemplateId)
-
-					// Have we already fetched the template
-					if (bookshelfTemplateId == -1) {
-						database.rawQuery("SELECT id FROM Templates WHERE name = 'bookshelf'", arrayOf()).use { cursor ->
-							if (!isCursorOneRow(cursor, writer, output)) {
-								return@withDatabase null
-							}
-
-							cursor.moveToFirst()
-							bookshelfTemplateId = cursor.getInt(0)
-							if (debugEnabled) log.debug("after the fetch bookshelf template ID = '{}'", bookshelfTemplateId)
-						}
+					bookshelfJson(database).also {
+						if (debugEnabled) log.debug("json content = '{}'.", String(it, Charsets.UTF_8))
 					}
-
-					json
 				} catch (e: Exception) {
-					log.error("Error processing request: {}", e.message)
-					sendError(writer, output, httpInternalServerError, "Internal Server Error", e.message ?: "")
+					log.error("Error building the bookshelf JSON: {}", e.message, e)
+					// The message stays in the log and out of the response. Everything reachable here is
+					// a database failure -- bookshelfJson is the SQL join -- so the message carries SQL
+					// text, or the database's filesystem path when withDatabase's check() is what threw.
+					// Any app on the device can GET this port. Narrowing handleBsEndpoint's own catch is
+					// not enough on its own: this one answers and returns null, so the outer catch never
+					// sees the exception at all.
+					sendError(writer, output, httpInternalServerError, "Internal Server Error", "Error generating bookshelf HTML.")
 					null
 				}
 			} ?: return false
 
-		val result = contentSource.renderTemplate(bookshelfTemplateId, jsonText, "/bookshelf")
+		val result = contentSource.renderNamedTemplate("bookshelf", jsonText, "/bookshelf")
 
 		if (debugEnabled) log.debug("Bookshelf result is '{}'.", String(result))
 
@@ -1070,22 +1046,6 @@ ORDER BY BC.category,
 		)
 	}
 
-	private fun isCursorOneRow(
-		cursor: Cursor,
-		writer: PrintWriter,
-		output: java.io.OutputStream,
-	): Boolean {
-		if (cursor.count == 1) {
-			return true
-		}
-		if (cursor.count == 0) {
-			sendError(writer, output, httpNotFound, "Corrupt database, no rows found, expected one.")
-		} else {
-			sendError(writer, output, httpInternalServerError, "Corrupt database - found ${cursor.count} rows when 1 was expected.")
-		}
-		return false
-	}
-
 	/**
 	 * Builds an HTML table of recent projects from the provided project database and writes it to the client.
 	 *

app/src/test/java/com/itsaky/androidide/localWebServer/WebServerTest.kt

@@ -363,15 +363,9 @@ class WebServerTest {
 		// The bookshelf join matches nothing: a cursor whose moveToNext() is immediately false.
 		every { db.rawQuery(match { it.contains("FROM Content AS C") }, any()) } returns
 			mockk<Cursor>(relaxed = true) { every { moveToNext() } returns false }
-		// The bookshelf template: its id lookup, then its body -- a Pebble expression over the JSON
-		// context, so the assertion proves the empty-shelf payload actually reached the render.
+		// The bookshelf template's body, fetched by name (ADFA-5405) -- a Pebble expression over the
+		// JSON context, so the assertion proves the empty-shelf payload actually reached the render.
 		every { db.rawQuery(match { it.contains("FROM Templates WHERE name") }, any()) } returns
-			mockk<Cursor>(relaxed = true) {
-				every { count } returns 1
-				every { moveToFirst() } returns true
-				every { getInt(0) } returns 7
-			}
-		every { db.rawQuery(match { it.contains("FROM Templates WHERE id") }, any()) } returns
 			mockk<Cursor>(relaxed = true) {
 				every { count } returns 1
 				every { moveToFirst() } returns true
@@ -392,6 +386,80 @@ class WebServerTest {
 		}
 	}
 
+	// ADFA-5405: a template the endpoint cannot resolve -- the bookshelf row, or anything it
+	// references -- is named by the loader's throw, and handleBsEndpoint has to pass that name on
+	// rather than replace it with its generic text. The name is the whole diagnostic.
+	@Test
+	fun `a bookshelf template that is not in the database answers 500 naming it`() {
+		val port = freePort()
+		val db = mockk<SQLiteDatabase>(relaxed = true)
+		every { SQLiteDatabase.openDatabase(any(), isNull(), any()) } returns db
+		every { db.rawQuery(match { it.contains("FROM Content AS C") }, any()) } returns
+			mockk<Cursor>(relaxed = true) { every { moveToNext() } returns false }
+		// No bookshelf row: a relaxed cursor's moveToFirst() is already false.
+		every { db.rawQuery(match { it.contains("FROM Templates WHERE name") }, any()) } returns mockk<Cursor>(relaxed = true)
+
+		val server = WebServer(testConfig(port))
+		val serverThread = Thread { server.start() }.apply { isDaemon = true }
+		serverThread.start()
+		try {
+			awaitPortBound(port)
+			val response = sendRawGetRequest(port, "/pr/bs")
+			assertTrue("Expected a 500 status line, got:\n$response", response.startsWith("HTTP/1.1 500"))
+			// The loader's own text, not just the template name: the generic fallback on the same
+			// sendError call is "Error generating bookshelf HTML.", which contains "bookshelf" too,
+			// so asserting on the name alone passes with or without the fix.
+			assertTrue(
+				"Expected the loader's diagnostic, got:\n$response",
+				response.contains("Template 'bookshelf' not found in the database"),
+			)
+			assertFalse("Expected no Pebble placeholder padding, got:\n$response", response.contains("(?:?)"))
+		} finally {
+			server.stop()
+			serverThread.join(2_000)
+		}
+	}
+
+	// The other half of the ADFA-5405 diagnostic: handleBsEndpoint's catch spans the whole handler,
+	// so echoing e.message put anything thrown in there into the response body -- a SQLiteException's
+	// SQL, or withDatabase's check() failure naming the database file. Any app on the device can GET
+	// this port. Only a template failure is echoed now; this pins that the rest is not.
+	@Test
+	fun `a failure that is not a template failure answers 500 without leaking internals`() {
+		val port = freePort()
+		val db = mockk<SQLiteDatabase>(relaxed = true)
+		every { SQLiteDatabase.openDatabase(any(), isNull(), any()) } returns db
+		// Thrown from inside the handler rather than from openDatabase, which would fail start()
+		// before the port is bound. The type matters more than the origin: this is the same
+		// IllegalStateException that withDatabase's check() raises, which used to be
+		// indistinguishable from a template diagnostic and so was echoed verbatim -- and its message
+		// carries both a filesystem path and SQL, the two things worth not sending.
+		every { db.rawQuery(match { it.contains("FROM Content AS C") }, any()) } throws
+			IllegalStateException(
+				"unable to open database file /data/user/0/com.itsaky.androidide/databases/documentation.db " +
+					"(while compiling: SELECT C.content FROM Content AS C JOIN ContentTypes)",
+			)
+
+		val server = WebServer(testConfig(port))
+		val serverThread = Thread { server.start() }.apply { isDaemon = true }
+		serverThread.start()
+		try {
+			awaitPortBound(port)
+			val response = sendRawGetRequest(port, "/pr/bs")
+			assertTrue("Expected a 500 status line, got:\n$response", response.startsWith("HTTP/1.1 500"))
+			assertTrue(
+				"Expected the generic text, got:\n$response",
+				response.contains("Error generating bookshelf HTML."),
+			)
+			assertFalse("Leaked a filesystem path:\n$response", response.contains("/data/user/0/"))
+			assertFalse("Leaked a database filename:\n$response", response.contains("documentation.db"))
+			assertFalse("Leaked SQL text:\n$response", response.contains("SELECT C.content"))
+		} finally {
+			server.stop()
+			serverThread.join(2_000)
+		}
+	}
+
 	// ADFA-5241: the two transports have to answer the same way about what a response says, and
 	// only a real response proves what this one sends. The decision itself lives in
 	// ContentTypeHeaders, shared with DocumentationRequestInterceptor.

common/src/main/java/com/itsaky/androidide/documentation/DatabaseTemplateLoader.kt

@@ -0,0 +1,87 @@
+/*
+ *  This file is part of Code on the Go.
+ *
+ *  AndroidIDE is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ *
+ *  AndroidIDE is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *   along with AndroidIDE.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package com.itsaky.androidide.documentation
+
+import android.database.sqlite.SQLiteDatabase
+import io.pebbletemplates.pebble.error.LoaderException
+import io.pebbletemplates.pebble.loader.Loader
+import java.io.Reader
+import java.io.StringReader
+
+/**
+ * Resolves Pebble template names against the `Templates` table, so a template can pull in another
+ * one with `extends`, `include`, `import` or `embed` (ADFA-5405).
+ *
+ * This replaces Pebble's `StringLoader`, which treats the name it is handed *as* the template body.
+ * That works for one self-contained template and silently breaks every cross-reference:
+ * `{% include "nav.peb" %}` asks the loader for "nav.peb", `StringLoader` hands back those eight
+ * characters as a template, and the page renders the literal text instead of the partial -- no
+ * exception, no log line.
+ *
+ * Names are `Templates.name` values, matched exactly: the table is a flat namespace with no
+ * directories, so there is no prefix, suffix or relative path to apply.
+ *
+ * @param database Supplies the database to read, or null when none is open. Called on every
+ * resolution rather than captured, because the source swaps the handle when a newer database
+ * appears; callers resolve under the read lock that a swap excludes, so the handle cannot change
+ * mid-render.
+ */
+internal class DatabaseTemplateLoader(
+	private val database: () -> SQLiteDatabase?,
+) : Loader<String> {
+	override fun getReader(name: String): Reader {
+		val database = database() ?: throw LoaderException(null, "No documentation database is open, for template '$name'")
+
+		return database.rawQuery(TEMPLATE_QUERY, arrayOf(name)).use { cursor ->
+			if (!cursor.moveToFirst()) {
+				throw LoaderException(null, "Template '$name' not found in the database")
+			}
+			StringReader(cursor.getBlob(0).toString(Charsets.UTF_8))
+		}
+	}
+
+	override fun resourceExists(name: String): Boolean {
+		val database = database() ?: return false
+
+		// Not TEMPLATE_QUERY: that copies the whole template blob into a CursorWindow to answer a
+		// boolean. Pebble reaches this only through the delegating and servlet loaders, neither of
+		// which is wired here, so the cost would be invisible -- which is the reason to get it right.
+		return database.rawQuery(EXISTS_QUERY, arrayOf(name)).use { it.moveToFirst() }
+	}
+
+	override fun createCacheKey(name: String): String = name
+
+	/** The table is a flat namespace, so a reference resolves to itself -- as with Pebble's own `MemoryLoader`. */
+	override fun resolveRelativePath(
+		relativePath: String,
+		anchorPath: String,
+	): String = relativePath
+
+	/** Template bodies are stored as UTF-8 blobs, so the engine's charset setting does not apply. */
+	override fun setCharset(charset: String) = Unit
+
+	/** Names are exact `Templates.name` values; decorating them would stop them matching. */
+	override fun setPrefix(prefix: String) = Unit
+
+	override fun setSuffix(suffix: String) = Unit
+
+	private companion object {
+		private const val TEMPLATE_QUERY = "SELECT content FROM Templates WHERE name = ?"
+		private const val EXISTS_QUERY = "SELECT 1 FROM Templates WHERE name = ? LIMIT 1"
+	}
+}