diff --git a/THIRD_PARTY_AI_MODELS.md b/THIRD_PARTY_AI_MODELS.md index 170651c51..0acf19303 100644 --- a/THIRD_PARTY_AI_MODELS.md +++ b/THIRD_PARTY_AI_MODELS.md @@ -20,7 +20,8 @@ the binary). Attribution + licenses for the models and their training data: > [`QtMeshEditor-facemesh-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-facemesh-onnx), > [`QtMeshEditor-faceblendshapes-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-faceblendshapes-onnx), > [`QtMeshEditor-blazepose-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-blazepose-onnx), -> [`QtMeshEditor-poselandmarks-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-poselandmarks-onnx) +> [`QtMeshEditor-poselandmarks-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-poselandmarks-onnx), +> [`QtMeshEditor-depthanything-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-depthanything-onnx) > (plus the in-house > [`QtMeshEditor-rmib-inbetween`](https://huggingface.co/fernandotonon/QtMeshEditor-rmib-inbetween), > [`QtMeshEditor-mesh-segmentation`](https://huggingface.co/fernandotonon/QtMeshEditor-mesh-segmentation), @@ -184,6 +185,44 @@ the binary). Attribution + licenses for the models and their training data: `QTMESH_REMBG_MODEL_BASE_URL` / `QSettings ai/rembgModelBaseUrl`; offline guard `QTMESH_REMBG_NO_DOWNLOAD`). Falls back to the raw image when unavailable. +## Depth-Anything-V2-Small — photo depth estimation (issue #1018, epic #818 C4) + +- **Model:** `depth-anything/Depth-Anything-V2-Small-hf` exported to ONNX + (`depth/da2_small.onnx`, ~99 MB), monocular relative-depth estimation from a + single photograph. +- **Licence: Apache-2.0 — the Small variant ONLY.** + **Depth-Anything-V2 Base and Large are `cc-by-nc-4.0` and must NEVER be + shipped** (non-commercial, fails the project's permissive-redistribution bar + the same way SF3D and Hunyuan3D did). This is not left to reviewer vigilance: + `scripts/export-depth-anything-onnx.py` queries the HF model API at export + time and **hard-fails** on any licence outside its allow-list, before + downloading a single weight. +- **Mirror repo:** + [`QtMeshEditor-depthanything-onnx`](https://huggingface.co/fernandotonon/QtMeshEditor-depthanything-onnx) + (standalone model card + I/O contract; the runtime downloads from the + aggregate repo under `depth/`). Refresh with + `scripts/sync-hf-model-repos.sh depthanything`. +- **Used by** `src/PhotoDepth.{h,cpp}`: CLI `qtmesh material --photo-depth + [-o out.png] [--depth-letterbox]`. Produces an 8-bit grayscale map with + **near = bright**, the same convention `MeshDepthRenderer` emits, so a photo + and a rendered mesh can feed `SDWorker::generateTextureControlled` + interchangeably. Downloads on first use to `AppData/ai_models/depth/` + (override `QTMESH_DEPTH_MODEL_BASE_URL` / `QSettings ai/depthModelBaseUrl`; + offline guard `QTMESH_DEPTH_NO_DOWNLOAD`). +- **Export contract:** input `pixel_values` float32 `[1,3,518,518]` + ImageNet-normalised; output `predicted_depth` float32 `[1,518,518]`, larger = + nearer. The spatial dims are **pinned, not dynamic**: a dynamic-axis export + traces cleanly but drifts from the traced resolution (measured 2.3e-02 + relative error at 462² and 4.9e-02 at 392², versus 2.7e-06 at 518²) because + the ViT position-embedding interpolation is only partly captured. The + reference preprocessor resizes every input to 518 anyway, so pinning costs + nothing and removes a silent-wrong-answer failure mode. +- **Parity-verified before upload:** the export script asserts torch-vs-ORT + agreement on both noise and a smooth structured image (measured 4.1e-06 and + 6.9e-06 max abs difference) and refuses to write a graph with any non-finite + output. This exists because the UniRig export (#1025) shipped a graph + emitting 100% NaN latents that went unnoticed for months. + ## PBRify_Remix — PBR map synthesis (issue #404) - Three SPAN models from https://github.com/Kim2091/PBRify_Remix — **CC0-1.0**, diff --git a/qml/TexturePropertiesPanel.qml b/qml/TexturePropertiesPanel.qml index 4afb90efd..935e7676b 100644 --- a/qml/TexturePropertiesPanel.qml +++ b/qml/TexturePropertiesPanel.qml @@ -278,6 +278,7 @@ GroupBox { // apply it via other tools even when in-app application of // the texture to certain materials isn't working yet). RowLayout { + id: depthRow Layout.fillWidth: true // #404: synthesize normal/roughness/height from the current diffuse. // Only shown on an ONNX build; disabled until a real texture is set. @@ -343,8 +344,44 @@ GroupBox { onClicked: { pbrStatus.text = "Cancelling…"; MaterialEditorQML.cancelUpscale() } } + // #1018: monocular depth from a PHOTO (Depth-Anything-V2-Small). + // Unlike the #403 mesh-depth path this needs no selected mesh — it + // estimates depth from any 2-D image, so a reference photo can + // condition the ControlNet texture generation below. Writes + // _depth.png next to the chosen photo. + property bool depthRunning: false + ThemedButton { + text: "Depth from photo…" + visible: MaterialEditorQML.aiPhotoDepthAvailable() && !parent.depthRunning + onClicked: { + const src = MaterialEditorQML.openFileDialog() + if (src && src.length > 0) { + parent.depthRunning = true + pbrStatus.text = "Estimating depth…" + MaterialEditorQML.generatePhotoDepth(src) + } + } + } + ThemedLabel { + text: "Estimating depth…" + visible: parent.depthRunning + elide: Text.ElideRight + } + Connections { target: MaterialEditorQML + // #1018 photo depth + function onPhotoDepthStarted() { pbrStatus.text = "Estimating depth…" } + function onPhotoDepthCompleted(depthPath) { + depthRow.depthRunning = false + // Show the file name, not the whole path — the panel is narrow. + const name = depthPath.split("/").pop() + pbrStatus.text = "Depth map written: " + name + } + function onPhotoDepthError(message) { + depthRow.depthRunning = false + pbrStatus.text = "Depth failed: " + message + } function onPbrSynthCompleted(result) { pbrStatus.text = result.fromCache ? "PBR maps ready (cached)." : "PBR maps generated." diff --git a/scripts/export-depth-anything-onnx.py b/scripts/export-depth-anything-onnx.py new file mode 100755 index 000000000..e6c74c78a --- /dev/null +++ b/scripts/export-depth-anything-onnx.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Export Depth-Anything-V2-Small to ONNX for QtMeshEditor (#1018, epic #818 C4). + +ONE-TIME, OFFLINE DEV TOOL — not shipped with the app, not run at runtime. +Mirrors scripts/export-pbrify-onnx.py / export-skintokens-onnx.py. + +LICENCE GATE (the reason this script refuses anything but Small): + Depth-Anything-V2-Small apache-2.0 -> redistributable, SHIPPED + Depth-Anything-V2-Base cc-by-nc-4.0 -> non-commercial, REJECTED + Depth-Anything-V2-Large cc-by-nc-4.0 -> non-commercial, REJECTED +Verified against the HF model API at export time. The project's bar is +permissive redistribution (Homebrew/Snap/WinGet/Docker), so the NC variants can +never ship no matter how much better they look. The script hard-fails on them +rather than leaving that to reviewer vigilance. + +Contract the C++ side (src/PhotoDepth.{h,cpp}) relies on: + input pixel_values float32 [1,3,H,W] ImageNet-normalised RGB + output predicted_depth float32 [1,H,W] RELATIVE inverse depth, + larger = NEARER +H and W are PINNED to 518x518 — only the batch axis is dynamic, and the C++ +runtime hard-codes the same size (PhotoDepth::kInputSize). A dynamic-axis export +traces cleanly but drifts away from the traced resolution (measured 2.3e-02 +relative error at 462 and 4.9e-02 at 392, versus 2.7e-06 at 518) because the ViT +position-embedding interpolation is only partly captured. The C++ side resizes +the source to this fixed shape and normalises the output to 0..255 itself, +matching MeshDepthRenderer's near=bright convention. +""" +import argparse +import json +import sys +import urllib.request + +REPO_SMALL = "depth-anything/Depth-Anything-V2-Small-hf" +# Must equal PhotoDepth::kInputSize in src/PhotoDepth.h. +RUNTIME_SIZE = 518 +ALLOWED_LICENSES = {"apache-2.0"} +PATCH = 14 + + +def check_license(repo: str) -> str: + """Refuse to export a model whose licence we cannot redistribute.""" + url = f"https://huggingface.co/api/models/{repo}" + with urllib.request.urlopen(url, timeout=60) as r: + meta = json.load(r) + lic = (meta.get("cardData") or {}).get("license") or "unknown" + if lic not in ALLOWED_LICENSES: + sys.exit( + f"REFUSING to export {repo}: license '{lic}' is not in the " + f"permissive allow-list {sorted(ALLOWED_LICENSES)}.\n" + "Depth-Anything-V2 Base/Large are cc-by-nc-4.0 and must never be " + "shipped (see THIRD_PARTY_AI_MODELS.md). Only the Small variant " + "clears the bar." + ) + return lic + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--repo", default=REPO_SMALL, + help="HF model id (must be permissively licensed)") + ap.add_argument("-o", "--out", default="da2_small.onnx") + ap.add_argument("--opset", type=int, default=18) + ap.add_argument("--size", type=int, default=RUNTIME_SIZE, + help=f"pinned input resolution; the C++ runtime " + f"(PhotoDepth::kInputSize) hard-codes {RUNTIME_SIZE}, " + f"so --allow-size-mismatch is required to change it") + ap.add_argument("--allow-size-mismatch", action="store_true", + help="export at a --size the shipped runtime cannot consume " + "(for experiments only; do NOT host the result)") + ap.add_argument("--tolerance", type=float, default=2e-3, + help="max abs torch-vs-ORT difference accepted") + args = ap.parse_args() + + if args.size % PATCH: + sys.exit(f"--size must be a multiple of {PATCH} (ViT patch size)") + # The graph is PINNED to --size, but PhotoDepth::estimate always builds a + # [1,3,518,518] tensor. A 392-pinned artifact hosted under the expected + # filename would fail EVERY inference with a dimension mismatch, so refuse + # by default rather than produce a parity-checked but unusable model. + if args.size != RUNTIME_SIZE and not args.allow_size_mismatch: + sys.exit( + f"--size {args.size} does not match the runtime's pinned " + f"{RUNTIME_SIZE} (PhotoDepth::kInputSize). The exported graph would " + f"reject every inference with a dimension mismatch. Pass " + f"--allow-size-mismatch only for experiments, and do not host the " + f"result; changing this for real means updating kInputSize too." + ) + + lic = check_license(args.repo) + print(f"[licence] {args.repo}: {lic} — OK") + + import numpy as np + import torch + from transformers import AutoModelForDepthEstimation + + print(f"[load] {args.repo}") + model = AutoModelForDepthEstimation.from_pretrained(args.repo) + model.eval() + + class Wrap(torch.nn.Module): + """Return the bare depth tensor, resized to follow the INPUT size. + + The HF head ends with + + nn.functional.interpolate( + ..., (int(patch_height * self.patch_size), int(...)), ...) + + and those `int()` casts turn the target size into a TorchScript + constant. The exported graph then declares dynamic height/width but + always emits the tracing resolution — measured: a 462x462 input still + produced a 518x518 depth map. Callers would silently get a depth map of + the wrong resolution for every size but the traced one. + + Re-interpolating here to `pixel_values.shape[-2:]` keeps the size as a + traced tensor shape rather than a baked constant, so the output really + does follow the input. At the tracing size this is a no-op resize. + """ + def __init__(self, m): + super().__init__() + self.m = m + + def forward(self, pixel_values): + depth = self.m(pixel_values=pixel_values).predicted_depth + if depth.dim() == 3: + depth = depth.unsqueeze(1) # [B,1,H,W] for interpolate + depth = torch.nn.functional.interpolate( + depth, size=pixel_values.shape[-2:], + mode="bilinear", align_corners=False) + return depth.squeeze(1) # back to [B,H,W] + + wrapped = Wrap(model).eval() + example = torch.randn(1, 3, args.size, args.size, dtype=torch.float32) + + print(f"[export] opset {args.opset} -> {args.out}") + torch.onnx.export( + wrapped, (example,), args.out, + input_names=["pixel_values"], output_names=["predicted_depth"], + # Batch stays dynamic; H/W are PINNED to the tracing size on purpose. + # A dynamic-axis export traces clean but drifts badly away from the + # traced resolution: the ViT position-embedding interpolation is only + # partly captured, measured at 2.3e-02 relative error at 462x462 and + # 4.9e-02 at 392x392 (vs 2.7e-06 at 518x518) on smooth images. Since + # the reference preprocessor resizes every input to 518x518 anyway + # (preprocessor_config.json: size 518, keep_aspect_ratio, + # ensure_multiple_of 14), a fixed shape costs nothing and removes a + # silent-wrong-answer failure mode. The C++ side letterboxes to 518. + dynamic_axes={"pixel_values": {0: "batch"}, + "predicted_depth": {0: "batch"}}, + opset_version=args.opset, do_constant_folding=True, dynamo=False, + ) + + # ---- PARITY: the step the UniRig export (#1025) skipped ---------------- + # That export shipped a graph emitting 100% NaN latents and nobody noticed + # until the feature was investigated months later. Never upload a graph + # whose output has not been compared against torch on real input. + import onnxruntime as ort + print("[parity] torch vs onnxruntime") + with torch.no_grad(): + ref = wrapped(example).cpu().numpy() + sess = ort.InferenceSession(args.out, providers=["CPUExecutionProvider"]) + got = sess.run(None, {"pixel_values": example.numpy()})[0] + + if got.shape != ref.shape: + sys.exit(f"shape mismatch: torch {ref.shape} vs ORT {got.shape}") + if not np.isfinite(got).all(): + bad = int((~np.isfinite(got)).sum()) + sys.exit(f"ONNX output has {bad}/{got.size} non-finite values — " + "the export is numerically broken (this is exactly the " + "UniRig #1025 failure; do NOT upload)") + err = float(np.abs(ref - got).max()) + rng = float(ref.max() - ref.min()) + print(f" shape {got.shape} max|diff| {err:.3e} (output range {rng:.3f})") + if err > args.tolerance: + sys.exit(f"parity FAILED: {err:.3e} > tolerance {args.tolerance:.3e}") + + # Contract check: the input shape must be PINNED. A graph that advertises + # dynamic H/W here would be accepted by the C++ side and then return + # quietly wrong depth at any other size (see the dynamic_axes note above). + decl = sess.get_inputs()[0].shape + if decl[2] != args.size or decl[3] != args.size: + sys.exit(f"expected pinned {args.size}x{args.size} input, got {decl} — " + "a dynamic-axis export drifts away from the traced size") + print(f" input shape pinned: {decl}") + + # Parity on a SMOOTH image too. White noise is the easiest case for a + # resize mismatch to hide in; a structured image is what the model will + # actually see. + yy, xx = np.mgrid[0:args.size, 0:args.size].astype(np.float32) / args.size + smooth = np.stack([np.sin(6 * xx) * np.cos(5 * yy), + np.cos(4 * xx + yy), xx * yy], 0)[None].astype(np.float32) + with torch.no_grad(): + ref3 = wrapped(torch.from_numpy(smooth)).cpu().numpy() + got3 = sess.run(None, {"pixel_values": smooth})[0] + if not np.isfinite(got3).all(): + sys.exit("smooth-image output is non-finite — do NOT upload") + err3 = float(np.abs(ref3 - got3).max()) + print(f"[parity] smooth image max|diff| {err3:.3e}") + if err3 > args.tolerance: + sys.exit(f"smooth-image parity FAILED: {err3:.3e}") + + print(f"[ok] {args.out} — parity verified, safe to host") + + +if __name__ == "__main__": + main() diff --git a/scripts/sync-hf-model-repos.sh b/scripts/sync-hf-model-repos.sh index a8d56bcee..16637b6b0 100755 --- a/scripts/sync-hf-model-repos.sh +++ b/scripts/sync-hf-model-repos.sh @@ -61,6 +61,7 @@ declare -A REPOS=( [faceblendshapes]=QtMeshEditor-faceblendshapes-onnx [blazepose]=QtMeshEditor-blazepose-onnx [poselandmarks]=QtMeshEditor-poselandmarks-onnx + [depthanything]=QtMeshEditor-depthanything-onnx ) declare -A FILES=( [pbrify]="1x-PBRify_NormalV3.onnx 1x-PBRify_RoughnessV2.onnx 1x-PBRify_Height.onnx" @@ -81,6 +82,7 @@ declare -A FILES=( [faceblendshapes]="mocap/face/face_blendshapes.onnx" [blazepose]="mocap/pose/pose_detector.onnx" [poselandmarks]="mocap/pose/pose_landmarks.onnx" + [depthanything]="depth/da2_small.onnx" ) sync_one() { diff --git a/src/CLIPipeline.cpp b/src/CLIPipeline.cpp index 2b1cd6ac6..fb81c4d6a 100644 --- a/src/CLIPipeline.cpp +++ b/src/CLIPipeline.cpp @@ -93,6 +93,7 @@ #include #include #include +#include #include #include #ifdef ENABLE_ONNX @@ -100,6 +101,11 @@ #include "PbrMapSynth.h" #include "TextureUpscaler.h" #endif +// PhotoDepth is ALWAYS included: unlike the headers above it ships non-ONNX +// stubs (isAvailable() -> false, estimate() -> a "rebuild with -DENABLE_ONNX" +// error), and cmdMaterialPhotoDepth calls them unconditionally so the default +// build can report the feature as unavailable instead of failing to compile. +#include "PhotoDepth.h" // RTShaderHelper is a core RTSS helper (no ONNX/SD/LLM dependency) used // unconditionally by the #406 describe-material path, so it must NOT sit inside // the ENABLE_ONNX guard above — Windows MinGW builds ONNX off and would @@ -5264,6 +5270,8 @@ int CLIPipeline::cmdMaterial(int argc, char* argv[]) QString describePrompt, llmModel; // #404 PBR map synthesis from a diffuse texture. QString pbrAlbedo; + QString photoDepthSrc; // #1018: photo -> monocular depth map + bool depthLetterbox = false; bool generatePbr = false; int pbrTileSize = 256; bool pbrNoNormal = false, pbrNoRoughness = false, pbrNoHeight = false; @@ -5306,6 +5314,11 @@ int CLIPipeline::cmdMaterial(int argc, char* argv[]) continue; } if (arg == "--generate-pbr") { generatePbr = true; continue; } + if (arg == "--photo-depth" && i + 1 < argc) { // #1018 + photoDepthSrc = QString(argv[++i]); + continue; + } + if (arg == "--depth-letterbox") { depthLetterbox = true; continue; } if (arg == "--upscale" && i + 1 < argc) { bool usOk = false; upscaleFactor = QString(argv[++i]).toInt(&usOk); @@ -5364,6 +5377,11 @@ int CLIPipeline::cmdMaterial(int argc, char* argv[]) // Depth-conditioned mesh-aware texture generation (issue #403). Distinct // enough (async SD worker, model loading, depth RTT) to live in its own // helper; presets continue below. + // #1018: monocular depth from a PHOTO (Depth-Anything-V2-Small). + if (!photoDepthSrc.isEmpty()) { + return cmdMaterialPhotoDepth(photoDepthSrc, outputPath, depthLetterbox); + } + // #405: Real-ESRGAN texture upscaling (--texture in, -o out). if (upscaleFactor != 0) { return cmdMaterialUpscale(pbrAlbedo, outputPath, upscaleFactor); @@ -6242,6 +6260,66 @@ int CLIPipeline::cmdMaterialGeneratePbr(const QString& albedoPath, #endif } +int CLIPipeline::cmdMaterialPhotoDepth(const QString& srcPath, + QString outputPath, bool letterbox) +{ + // #1018 (epic #818 C4): estimate a depth map from a PHOTO, for use as a + // ControlNet conditioning image. Distinct from `MeshDepthRenderer`, which + // renders depth from a mesh already in the scene; both emit the same + // near=bright grayscale so either can feed the SD texture path. + const QFileInfo fi(srcPath); + if (!fi.exists()) { + err() << "Error: " << srcPath << " not found." << Qt::endl; + return 1; + } + // Honour EXIF orientation: a phone JPEG stores portrait rotation in + // metadata only, and a plain QImage(path) reads the UNROTATED pixels — the + // model would then estimate a sideways scene and the PNG we write carries + // no metadata to compensate. + QImageReader reader(srcPath); + reader.setAutoTransform(true); + const QImage photo = reader.read(); + if (photo.isNull()) { + err() << "Error: could not read " << srcPath << " as an image (" + << reader.errorString() << ")." << Qt::endl; + return 1; + } + if (!PhotoDepth::isAvailable()) { + err() << "Error: photo depth needs an ONNX build " + "(rebuild with -DENABLE_ONNX)." << Qt::endl; + return 1; + } + const QString model = PhotoDepth::ensureModelBlocking(); + if (model.isEmpty()) { + err() << "Error: the Depth-Anything-V2-Small model is unavailable " + "(offline, or QTMESH_DEPTH_NO_DOWNLOAD is set)." << Qt::endl; + return 1; + } + + PhotoDepth::Options opts; + opts.letterbox = letterbox; + const auto r = PhotoDepth::estimate(photo, model, opts); + if (!r.ok) { + err() << "Error: " << r.error << Qt::endl; + return 1; + } + + if (outputPath.isEmpty()) + outputPath = QDir(fi.absolutePath()) + .filePath(fi.completeBaseName() + "_depth.png"); + if (!r.depth.save(outputPath)) { + err() << "Error: could not write " << outputPath << Qt::endl; + return 1; + } + cliWrite(QString("Wrote depth map: %1 (%2x%3, near = bright)\n") + .arg(QFileInfo(outputPath).fileName()) + .arg(r.depth.width()).arg(r.depth.height())); + SentryReporter::addBreadcrumb(QStringLiteral("ai.assist.photo_depth"), + QStringLiteral("photo_depth %1x%2") + .arg(photo.width()).arg(photo.height())); + return 0; +} + int CLIPipeline::cmdMaterialUpscale(const QString& srcPath, QString outputPath, int scale) { diff --git a/src/CLIPipeline.h b/src/CLIPipeline.h index 6dd8c084c..974cba2d5 100644 --- a/src/CLIPipeline.h +++ b/src/CLIPipeline.h @@ -130,6 +130,9 @@ class CLIPipeline { /// #405: Real-ESRGAN texture upscaling (2x/4x) via ONNX. Reads --texture, /// writes the upscaled PNG to -o (or _upscaled.png). Exit 1 when /// built without ENABLE_ONNX. + /// #1018: photo -> monocular depth map (Depth-Anything-V2-Small). + static int cmdMaterialPhotoDepth(const QString& srcPath, QString outputPath, + bool letterbox); static int cmdMaterialUpscale(const QString& srcPath, QString outputPath, int scale); /// #406: LLM-assisted material authoring. Headless equivalent of the diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d0f687f26..83b643964 100755 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -225,6 +225,7 @@ ImageTo3D/MeshGenController.cpp ImageTo3D/BackgroundRemover.cpp ImageTo3D/ImageCaptioner.cpp PbrMapSynth.cpp +PhotoDepth.cpp TextureUpscaler.cpp AIAssistManager.cpp MemoryEstimator.cpp @@ -389,6 +390,7 @@ UniRigPredictor.h MotionInbetween.h MotionLibrary.h MotionComposer.h +PhotoDepth.h MotionGenerator.h MeshSegmenter.h MeshDepthRenderer.h diff --git a/src/MCPServer.cpp b/src/MCPServer.cpp index f82430cbb..18983bc67 100644 --- a/src/MCPServer.cpp +++ b/src/MCPServer.cpp @@ -1,4 +1,6 @@ #include "MCPServer.h" +#include "PhotoDepth.h" +#include #include "MeshWeldOps.h" #include "commands/WeldVerticesCommand.h" #include "mainwindow.h" @@ -667,6 +669,7 @@ const QMap& MCPServer::toolHandlers() {QStringLiteral("generate_mesh_texture"), &MCPServer::toolGenerateMeshTexture}, {QStringLiteral("generate_pbr_maps"), &MCPServer::toolGeneratePbrMaps}, {QStringLiteral("upscale_texture"), &MCPServer::toolUpscaleTexture}, + {QStringLiteral("photo_depth"), &MCPServer::toolPhotoDepth}, {QStringLiteral("get_scene_info"), &MCPServer::toolGetSceneInfo}, {QStringLiteral("take_screenshot"), &MCPServer::toolTakeScreenshot}, {QStringLiteral("create_primitive"), &MCPServer::toolCreatePrimitive}, @@ -909,6 +912,7 @@ QJsonObject MCPServer::callTool(const QString &name, const QJsonObject &args) {QStringLiteral("generate_mesh_from_image"), QStringLiteral("image_to_3d")}, {QStringLiteral("generate_pbr_maps"), QStringLiteral("pbr_synth")}, {QStringLiteral("upscale_texture"), QStringLiteral("pbr_synth")}, + {QStringLiteral("photo_depth"), QStringLiteral("pbr_synth")}, {QStringLiteral("generate_normal_map"), QStringLiteral("pbr_synth")}, {QStringLiteral("pack_textures"), QStringLiteral("texture_atlas")}, {QStringLiteral("paint_bake"), QStringLiteral("texture_atlas")}, @@ -936,7 +940,7 @@ QJsonObject MCPServer::callTool(const QString &name, const QJsonObject &args) QStringLiteral("export_mesh"), QStringLiteral("auto_uv_unwrap"), QStringLiteral("uv_project"), QStringLiteral("uv_set_seams"), QStringLiteral("uv_unwrap_selection"), QStringLiteral("retopologize"), QStringLiteral("compute_skin_weights"), QStringLiteral("auto_rig"), QStringLiteral("generate_mesh_texture"), - QStringLiteral("generate_pbr_maps"), QStringLiteral("upscale_texture"), QStringLiteral("create_primitive"), + QStringLiteral("generate_pbr_maps"), QStringLiteral("upscale_texture"), QStringLiteral("photo_depth"), QStringLiteral("create_primitive"), QStringLiteral("animate"), QStringLiteral("add_keyframe"), QStringLiteral("remove_keyframe"), QStringLiteral("merge_animations"), QStringLiteral("resample_animation"), QStringLiteral("simplify_animation"), QStringLiteral("bake_animation_fps"), QStringLiteral("motion_in_between"), QStringLiteral("generate_motion"), @@ -3023,6 +3027,63 @@ QJsonObject MCPServer::toolUpscaleTexture(const QJsonObject &args) #endif } +QJsonObject MCPServer::toolPhotoDepth(const QJsonObject &args) +{ +#ifndef ENABLE_ONNX + Q_UNUSED(args); + return makeErrorResult( + "This build was compiled without photo depth estimation " + "(rebuild with -DENABLE_ONNX=ON)."); +#else + const QString srcPath = args.value("image_path").toString(); + if (srcPath.trimmed().isEmpty()) + return makeErrorResult("'image_path' is required."); + if (!QFileInfo::exists(srcPath)) + return makeErrorResult(QStringLiteral("image not found: %1").arg(srcPath)); + // Honour EXIF orientation (see the CLI path): QImage(path) reads the + // UNROTATED pixels, so a portrait phone JPEG would be estimated sideways. + QImageReader reader(srcPath); + reader.setAutoTransform(true); + const QImage photo = reader.read(); + if (photo.isNull()) + return makeErrorResult( + QStringLiteral("could not read %1 as an image (%2).") + .arg(srcPath, reader.errorString())); + + const QString model = PhotoDepth::ensureModelBlocking(); + if (model.isEmpty()) + return makeErrorResult( + "Depth model unavailable (offline, or QTMESH_DEPTH_NO_DOWNLOAD is set)."); + + PhotoDepth::Options opts; + opts.letterbox = args.value("letterbox").toBool(false); + const PhotoDepth::Result r = PhotoDepth::estimate(photo, model, opts); + if (!r.ok) + return makeErrorResult(QStringLiteral("depth estimation failed: %1") + .arg(r.error)); + + const QFileInfo fi(srcPath); + QString out = args.value("output_path").toString(); + if (out.trimmed().isEmpty()) + out = QDir(fi.absolutePath()) + .filePath(fi.completeBaseName() + QStringLiteral("_depth.png")); + if (!r.depth.save(out)) + return makeErrorResult(QStringLiteral("could not write %1").arg(out)); + + QJsonObject result = makeSuccessResult( + QStringLiteral("Estimated depth for '%1' (%2x%3, near = bright).") + .arg(fi.fileName()).arg(r.depth.width()).arg(r.depth.height())); + result["outputPath"] = out; + result["width"] = r.depth.width(); + result["height"] = r.depth.height(); + // Raw relative-depth range before normalisation — a near-zero span means + // the model found no structure, which a caller may want to act on. + result["rawMin"] = r.rawMin; + result["rawMax"] = r.rawMax; + return result; +#endif +} + QJsonObject MCPServer::toolGetSceneInfo(const QJsonObject &args) { Q_UNUSED(args); @@ -11198,6 +11259,36 @@ QJsonArray MCPServer::buildToolsList() QJsonArray{"texture_path"} ); } + + // photo_depth (#1018) + { + QJsonObject props; + props["image_path"] = QJsonObject{{"type", "string"}, + {"description", + "Path to the PHOTOGRAPH to estimate depth for. The map is written " + "next to it as _depth.png unless output_path is given."}}; + props["output_path"] = QJsonObject{{"type", "string"}, + {"description", "Optional explicit output path for the depth PNG."}}; + props["letterbox"] = QJsonObject{{"type", "boolean"}, + {"description", + "Preserve the source aspect ratio by padding into the model's " + "square input instead of stretching (default false, matching the " + "reference pipeline)."}}; + appendTool( + "photo_depth", + "Monocular depth estimation from a PHOTO (issue #1018) via " + "Depth-Anything-V2-Small (Apache-2.0; Base/Large are CC-BY-NC and " + "are never shipped). Returns an 8-bit grayscale map where NEAR is " + "BRIGHT — the same convention the mesh-depth renderer emits, so a " + "photo and a rendered mesh are interchangeable as ControlNet " + "conditioning images for generate_mesh_texture. This ESTIMATES " + "depth from a 2-D image; it does not render depth from a scene " + "mesh. The ~99 MB model downloads on first use. Reports the raw " + "relative-depth range, so a caller can detect a featureless result.", + props, + QJsonArray{"image_path"} + ); + } #endif // ENABLE_ONNX // retopologize diff --git a/src/MCPServer.h b/src/MCPServer.h index 8838e6418..c1818c5d2 100644 --- a/src/MCPServer.h +++ b/src/MCPServer.h @@ -198,6 +198,7 @@ private slots: /// Issue #405: Real-ESRGAN 2x/4x texture super-resolution. Writes /// _upscaled.png next to the source; model downloaded on first use. QJsonObject toolUpscaleTexture(const QJsonObject &args); + QJsonObject toolPhotoDepth(const QJsonObject &args); // #1018 QJsonObject toolGetSceneInfo(const QJsonObject &args); QJsonObject toolTakeScreenshot(const QJsonObject &args); QJsonObject toolCreatePrimitive(const QJsonObject &args); diff --git a/src/MaterialEditorQML.cpp b/src/MaterialEditorQML.cpp index 53d3f553a..97a0e8108 100644 --- a/src/MaterialEditorQML.cpp +++ b/src/MaterialEditorQML.cpp @@ -30,6 +30,8 @@ #endif #ifdef ENABLE_ONNX #include "AIAssistManager.h" +#include "PhotoDepth.h" +#include #include "TextureUpscaler.h" #include #include @@ -4364,6 +4366,79 @@ void MaterialEditorQML::generatePbrFromDiffuse() #endif } +bool MaterialEditorQML::aiPhotoDepthAvailable() const +{ + return PhotoDepth::isAvailable(); +} + +QString MaterialEditorQML::lastPhotoDepthPath() const +{ + return m_lastPhotoDepth; +} + +void MaterialEditorQML::generatePhotoDepth(const QString &photoPath) +{ +#ifndef ENABLE_ONNX + Q_UNUSED(photoPath); + emit photoDepthError(tr("Photo depth is not enabled. Rebuild with ENABLE_ONNX=ON.")); +#else + QString src = photoPath; + if (src.startsWith(QStringLiteral("file://"))) + src = QUrl(src).toLocalFile(); + if (src.isEmpty() || !QFileInfo::exists(src)) { + emit photoDepthError(tr("Pick a photo first.")); + return; + } + // Honour EXIF orientation (see CLIPipeline::cmdMaterialPhotoDepth): a + // portrait phone JPEG stores its rotation in metadata only, and a plain + // QImage(path) would hand the model a sideways scene. + QImageReader reader(src); + reader.setAutoTransform(true); + const QImage photo = reader.read(); + if (photo.isNull()) { + emit photoDepthError(tr("Could not read %1 as an image (%2).") + .arg(QFileInfo(src).fileName(), + reader.errorString())); + return; + } + + emit photoDepthStarted(); + // Resolve the model on THIS (GUI) thread: ensureModelBlocking spins a + // QEventLoop over ModelDownloader's queued signals, which needs a running + // event loop — the same constraint the #405 upscale path documents. + const QString model = PhotoDepth::ensureModelBlocking(); + if (model.isEmpty()) { + emit photoDepthError( + tr("Depth model unavailable (offline, or downloads disabled).")); + return; + } + + const QFileInfo fi(src); + const QString outPath = QDir(fi.absolutePath()) + .filePath(fi.completeBaseName() + QStringLiteral("_depth.png")); + + QPointer self(this); + // Inference is pure CPU and takes a second or two on a large photo — run it + // off the GUI thread and marshal the result back, as the upscale path does. + std::thread([self, photo, model, outPath]() { + const PhotoDepth::Result res = PhotoDepth::estimate(photo, model); + const bool ok = res.ok && !res.depth.isNull() + && res.depth.save(outPath, "PNG"); + const QString err = res.ok ? QString() : res.error; + QMetaObject::invokeMethod(qApp, [self, ok, err, outPath]() { + if (!self) return; + if (ok) { + self->m_lastPhotoDepth = outPath; + emit self->photoDepthCompleted(outPath); + } else { + emit self->photoDepthError( + err.isEmpty() ? tr("Could not write the depth map.") : err); + } + }, Qt::QueuedConnection); + }).detach(); +#endif +} + void MaterialEditorQML::upscaleCurrentTexture(int scale) { #ifndef ENABLE_ONNX diff --git a/src/MaterialEditorQML.h b/src/MaterialEditorQML.h index daaeece87..c6ae7e4c6 100644 --- a/src/MaterialEditorQML.h +++ b/src/MaterialEditorQML.h @@ -588,6 +588,21 @@ public slots: /// Request cancellation of an in-flight upscale (checked per tile). Q_INVOKABLE void cancelUpscale(); + // #1018: monocular depth from a PHOTO (Depth-Anything-V2-Small). + // Distinct from the mesh-rendered depth used by #403 — this estimates + // depth from a 2-D image, so a reference photo can condition the SD + // ControlNet path. Both emit the same near=bright grayscale. + // aiPhotoDepthAvailable() is true only on an ENABLE_ONNX build. + Q_INVOKABLE bool aiPhotoDepthAvailable() const; + /// Estimate depth for `photoPath` and write _depth.png beside it. + /// Runs on a worker thread (the ~99 MB model may download on first use); + /// emits photoDepthStarted / photoDepthCompleted(depthPath) / + /// photoDepthError. The resulting path is also remembered as the + /// ControlNet source for the next mesh-texture generation. + Q_INVOKABLE void generatePhotoDepth(const QString &photoPath); + /// Last depth map produced by generatePhotoDepth (empty when none). + Q_INVOKABLE QString lastPhotoDepthPath() const; + // Issue #403: mesh-aware texture generation. Same as // generateTextureFromPrompt but renders the selected entity's // depth map and conditions generation on it via a ControlNet @@ -738,6 +753,10 @@ public slots: void upscaleDownloading(); // first-run model fetch in progress void upscaleProgress(int tilesDone, int tilesTotal); void upscaleCompleted(const QString &outputPath); + // #1018 photo depth + void photoDepthStarted(); + void photoDepthCompleted(const QString &depthPath); + void photoDepthError(const QString &message); void upscaleError(const QString &error); /** Non-fatal informational message during generation (e.g. degraded mode); unlike sdGenerationError it must NOT abort or reset the in-flight run. */ @@ -832,6 +851,7 @@ public slots: // progress callback. shared_ptr so the detached worker keeps it alive even // if a new upscale starts; atomic for cross-thread access. std::shared_ptr m_upscaleCancel; + QString m_lastPhotoDepth; // #1018: last depth map produced double m_scrollAnimUSpeed = 0.0; double m_scrollAnimVSpeed = 0.0; diff --git a/src/PhotoDepth.cpp b/src/PhotoDepth.cpp new file mode 100644 index 000000000..faf563fa1 --- /dev/null +++ b/src/PhotoDepth.cpp @@ -0,0 +1,299 @@ +#include "PhotoDepth.h" + +#include "AppStorage.h" +#include "ModelDownloader.h" +#ifdef ENABLE_ONNX +#include "OnnxRuntimeSettings.h" +#include +#endif + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace { + +constexpr const char* kModelFile = "da2_small.onnx"; +constexpr const char* kDefaultModelBaseUrl = + "https://huggingface.co/fernandotonon/QtMeshEditor-models/resolve/main/depth/"; +constexpr const char* kBaseUrlSettingsKey = "ai/depthModelBaseUrl"; +constexpr const char* kModelLabel = "Depth-Anything-V2-Small depth model"; + +// ImageNet normalisation — the values the reference preprocessor uses +// (preprocessor_config.json: image_mean / image_std). +constexpr float kMean[3] = {0.485f, 0.456f, 0.406f}; +constexpr float kStd[3] = {0.229f, 0.224f, 0.225f}; + +QString modelDir() +{ + return QDir(AppStorage::aiModelsRoot()).filePath(QStringLiteral("depth/")); +} + +} // namespace + +namespace PhotoDepth { + +QString modelPath() +{ + return QDir(modelDir()).filePath(QString::fromLatin1(kModelFile)); +} + +bool modelPresent() { return QFileInfo::exists(modelPath()); } + +// --------------------------------------------------------------------------- +// Pure helpers — always compiled, so they are testable on any build +// --------------------------------------------------------------------------- + +std::vector toModelInput(const QImage& photo, bool letterbox) +{ + std::vector out(static_cast(kInputSize) * kInputSize * 3, 0.0f); + if (photo.isNull()) return out; + + QImage square(kInputSize, kInputSize, QImage::Format_RGB888); + square.fill(Qt::black); + if (letterbox) { + // Preserve aspect: scale the long edge to kInputSize and centre it. + const QImage fit = photo.scaled(kInputSize, kInputSize, Qt::KeepAspectRatio, + Qt::SmoothTransformation) + .convertToFormat(QImage::Format_RGB888); + QPainter p(&square); + p.drawImage((kInputSize - fit.width()) / 2, + (kInputSize - fit.height()) / 2, fit); + } else { + square = photo.scaled(kInputSize, kInputSize, Qt::IgnoreAspectRatio, + Qt::SmoothTransformation) + .convertToFormat(QImage::Format_RGB888); + } + + // CHW, ImageNet-normalised. + const size_t plane = static_cast(kInputSize) * kInputSize; + for (int y = 0; y < kInputSize; ++y) { + const uchar* line = square.constScanLine(y); + for (int x = 0; x < kInputSize; ++x) { + for (int c = 0; c < 3; ++c) { + const float v = static_cast(line[x * 3 + c]) / 255.0f; + out[c * plane + static_cast(y) * kInputSize + x] = + (v - kMean[c]) / kStd[c]; + } + } + } + return out; +} + +QImage toDepthImage(const std::vector& raw, int w, int h, + float* outMin, float* outMax) +{ + if (w <= 0 || h <= 0 + || raw.size() < static_cast(w) * static_cast(h)) + return {}; + + float mn = std::numeric_limits::max(); + float mx = -std::numeric_limits::max(); + for (size_t i = 0, n = static_cast(w) * h; i < n; ++i) { + const float v = raw[i]; + // Skip EVERY non-finite value, not just NaN. std::min/max do already + // shrug off NaN (they return their second argument when the comparison + // is false), but an isolated +inf/-inf compares normally and poisons + // the range: +inf collapses every finite pixel toward black, and -inf + // makes the normalisation inf/inf = NaN on the way to the 0..255 cast. + // The inference probe cannot catch this — it only rejects output where + // EVERY sampled value is non-finite. + if (!std::isfinite(v)) continue; + mn = std::min(mn, v); + mx = std::max(mx, v); + } + if (mn > mx) { mn = 0.0f; mx = 0.0f; } // all non-finite + if (outMin) *outMin = mn; + if (outMax) *outMax = mx; + + QImage img(w, h, QImage::Format_Grayscale8); + const float span = mx - mn; + for (int y = 0; y < h; ++y) { + uchar* line = img.scanLine(y); + for (int x = 0; x < w; ++x) { + const float v = raw[static_cast(y) * w + x]; + // The model emits INVERSE depth (larger = nearer), which is already + // the near=bright convention MeshDepthRenderer uses, so no flip. + // A constant field would divide by zero — emit mid-grey instead of + // a NaN-filled image. + const float t = (span > 1e-9f && std::isfinite(v)) + ? (v - mn) / span : 0.5f; + line[x] = static_cast( + std::clamp(t, 0.0f, 1.0f) * 255.0f + 0.5f); + } + } + return img; +} + +QImage cropLetterbox(const QImage& square, int srcW, int srcH) +{ + if (square.isNull() || srcW <= 0 || srcH <= 0) return square; + if (srcW == srcH) return square; + const int side = square.width(); + // Mirror toModelInput's fit: the long edge fills `side`. + const double s = static_cast(side) / std::max(srcW, srcH); + const int w = std::max(1, static_cast(std::lround(srcW * s))); + const int h = std::max(1, static_cast(std::lround(srcH * s))); + return square.copy((side - w) / 2, (side - h) / 2, w, h); +} + +#ifndef ENABLE_ONNX + +bool isAvailable() { return false; } +QString ensureModelBlocking() { return {}; } + +Result estimate(const QImage&, const QString&, const Options&) +{ + Result r; + r.error = QStringLiteral("Photo depth needs an ONNX build " + "(rebuild with -DENABLE_ONNX)."); + return r; +} + +#else // ENABLE_ONNX + +bool isAvailable() { return true; } + +QString ensureModelBlocking() +{ + const QString dst = modelPath(); + if (QFileInfo::exists(dst)) return dst; + if (!qEnvironmentVariableIsEmpty("QTMESH_DEPTH_NO_DOWNLOAD")) return {}; + + QString base; + { + QSettings s; + base = s.value(QString::fromLatin1(kBaseUrlSettingsKey)).toString(); + if (base.isEmpty()) { + const QByteArray env = qgetenv("QTMESH_DEPTH_MODEL_BASE_URL"); + base = env.isEmpty() ? QString::fromLatin1(kDefaultModelBaseUrl) + : QString::fromUtf8(env); + } + } + if (base.isEmpty()) return {}; + if (!base.endsWith('/')) base += '/'; + + auto* dl = ModelDownloader::instance(); + if (!dl) return {}; + + QDir().mkpath(QFileInfo(dst).absolutePath()); + const QString url = base + QString::fromLatin1(kModelFile); + QEventLoop loop; + bool ok = false, timedOut = false; + auto onDone = QObject::connect(dl, &ModelDownloader::downloadCompleted, &loop, + [&](const QString& name, const QString&) { + if (name == QString::fromLatin1(kModelLabel)) { ok = true; loop.quit(); } + }); + auto onErr = QObject::connect(dl, &ModelDownloader::downloadError, &loop, + [&](const QString& name, const QString&) { + if (name == QString::fromLatin1(kModelLabel)) { ok = false; loop.quit(); } + }); + QTimer timeout; + timeout.setSingleShot(true); + QObject::connect(&timeout, &QTimer::timeout, &loop, + [&]() { timedOut = true; loop.quit(); }); + timeout.start(600000); // 10 min — the model is ~99 MB + dl->startDownload(url, dst, QString::fromLatin1(kModelLabel)); + loop.exec(); + QObject::disconnect(onDone); + QObject::disconnect(onErr); + if (timedOut) dl->cancelDownload(); + return (ok && !timedOut && QFileInfo::exists(dst)) ? dst : QString(); +} + +Result estimate(const QImage& photo, const QString& modelFile, + const Options& opts) +{ + Result r; + if (photo.isNull()) { r.error = QStringLiteral("empty image"); return r; } + if (!QFileInfo::exists(modelFile)) { + r.error = QStringLiteral("Depth model not found at %1.").arg(modelFile); + return r; + } + + try { + Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "qtmesh_depth"); + Ort::SessionOptions so; + OnnxRuntimeSettings::configureSessionOptions(so); +#ifdef _WIN32 + Ort::Session session(env, modelFile.toStdWString().c_str(), so); +#else + Ort::Session session(env, modelFile.toStdString().c_str(), so); +#endif + Ort::AllocatorWithDefaultOptions alloc; + + std::vector input = toModelInput(photo, opts.letterbox); + const int64_t shape[4] = {1, 3, kInputSize, kInputSize}; + auto mem = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault); + Ort::Value in = Ort::Value::CreateTensor( + mem, input.data(), input.size(), shape, 4); + + auto inName = session.GetInputNameAllocated(0, alloc); + auto outName = session.GetOutputNameAllocated(0, alloc); + const char* inNames[] = {inName.get()}; + const char* outNames[] = {outName.get()}; + + auto outs = session.Run(Ort::RunOptions{nullptr}, + inNames, &in, 1, outNames, 1); + if (outs.empty() || !outs[0].IsTensor()) { + r.error = QStringLiteral("depth model produced no tensor output."); + return r; + } + auto info = outs[0].GetTensorTypeAndShapeInfo(); + const size_t n = info.GetElementCount(); + if (n < static_cast(kInputSize) * kInputSize) { + r.error = QStringLiteral("depth output too small (%1 values).").arg(n); + return r; + } + const float* data = outs[0].GetTensorData(); + std::vector raw(data, data + n); + + // Guard the #1025 failure mode: a numerically broken export returns + // finite-shaped, all-NaN output. Say so instead of emitting a + // mid-grey image that looks like a plausible flat depth map. + { + const size_t probe = std::min(n, 1024); + size_t bad = 0; + for (size_t i = 0; i < probe; ++i) + if (!std::isfinite(raw[i])) ++bad; + if (bad == probe) { + r.error = QStringLiteral( + "the depth model returned non-finite values for every " + "sampled output — the export is numerically broken."); + return r; + } + } + + QImage square = toDepthImage(raw, kInputSize, kInputSize, + &r.rawMin, &r.rawMax); + if (square.isNull()) { + r.error = QStringLiteral("failed to build the depth image."); + return r; + } + if (opts.letterbox) + square = cropLetterbox(square, photo.width(), photo.height()); + + const int outW = opts.outputSize > 0 ? opts.outputSize : photo.width(); + const int outH = opts.outputSize > 0 ? opts.outputSize : photo.height(); + r.depth = square.scaled(outW, outH, Qt::IgnoreAspectRatio, + Qt::SmoothTransformation); + r.ok = !r.depth.isNull(); + if (!r.ok) r.error = QStringLiteral("failed to resize the depth image."); + return r; + } catch (const Ort::Exception& e) { + r.error = QStringLiteral("depth inference failed: %1") + .arg(QString::fromUtf8(e.what())); + return r; + } +} + +#endif // ENABLE_ONNX + +} // namespace PhotoDepth diff --git a/src/PhotoDepth.h b/src/PhotoDepth.h new file mode 100644 index 000000000..de94ad5da --- /dev/null +++ b/src/PhotoDepth.h @@ -0,0 +1,98 @@ +#pragma once + +#include +#include + +/// #1018 (epic #818 Track C4): monocular depth estimation from a PHOTO via +/// Depth-Anything-V2-**Small**. +/// +/// Pure-data — no Ogre, no Qt singleton (the PbrMapSynth / TextureUpscaler +/// shape), so the geometry helpers unit-test without a GL context. ONNX +/// inference is compiled only under ENABLE_ONNX; the pre/post-processing is +/// always available and always tested. +/// +/// NOT to be confused with `MeshDepthRenderer`, which RENDERS a depth map of a +/// mesh already in the scene. This ESTIMATES depth from a 2-D photograph. +/// Both deliberately emit the same convention — 8-bit grayscale, **near = +/// bright** (MiDaS/ControlNet) — so either can feed +/// `SDWorker::generateTextureControlled` interchangeably. +/// +/// LICENCE: only the **Small** variant may ever ship. Small is Apache-2.0; +/// Base and Large are CC-BY-NC-4.0 and fail the project's permissive- +/// redistribution bar. `scripts/export-depth-anything-onnx.py` refuses to +/// export anything else. See THIRD_PARTY_AI_MODELS.md. +/// +/// MODEL CONTRACT (fixed by the export, asserted there against torch): +/// input pixel_values float32 [1,3,518,518] ImageNet-normalised RGB +/// output predicted_depth float32 [1,518,518] relative inverse depth, +/// larger = NEARER +/// The input size is **pinned**, not dynamic: a dynamic-axis export traces +/// clean but drifts from the traced resolution (measured 2.3e-02 relative error +/// at 462² vs 2.7e-06 at 518²) because the ViT position-embedding interpolation +/// is only partly captured. The reference preprocessor resizes everything to +/// 518 anyway, so pinning costs nothing and removes a silent-wrong-answer mode. +namespace PhotoDepth { + +/// The model's pinned input edge length (also the output edge length). +inline constexpr int kInputSize = 518; + +struct Options { + /// Output edge length. 0 = the SOURCE image's own size (the common case: + /// a control image should match the image it conditions). The model always + /// runs at kInputSize internally; this only controls the final resize. + int outputSize = 0; + /// Preserve the source aspect ratio by letterboxing into the square model + /// input instead of stretching. Off by default because ControlNet depth is + /// used as a whole-frame conditioning image, and the reference pipeline + /// stretches; enable when the aspect distortion matters more than coverage. + bool letterbox = false; +}; + +struct Result { + bool ok = false; + QString error; + /// Grayscale8, near = bright. Empty on failure. + QImage depth; + /// Raw model output range before normalisation — useful for diagnostics + /// (a constant map means the model saw nothing it understood). + float rawMin = 0.0f; + float rawMax = 0.0f; +}; + +/// Absolute path the model is cached at (AppData/ai_models/depth/da2_small.onnx). +QString modelPath(); +bool modelPresent(); +/// Download on first use (blocks via a local ModelDownloader event loop). +/// Returns the path, or empty when offline/disabled/failed. Honours +/// QTMESH_DEPTH_NO_DOWNLOAD and the base-URL override +/// QTMESH_DEPTH_MODEL_BASE_URL / QSettings ai/depthModelBaseUrl. +QString ensureModelBlocking(); + +/// True only when built with ENABLE_ONNX. +bool isAvailable(); + +/// Estimate depth for `photo` using the ONNX model at `modelPath`. +/// On any failure (no ONNX, missing model, inference error) returns ok=false +/// with a reason and an empty image — never a partially-valid map. +Result estimate(const QImage& photo, const QString& modelPath, + const Options& opts = {}); + +// ---- Pure helpers (always compiled; unit-tested without ONNX) ------------- + +/// ImageNet-normalise `photo` into the model's NCHW input buffer, resizing to +/// kInputSize. Returns kInputSize*kInputSize*3 floats in CHW order. +/// `letterbox` pads with black to preserve aspect instead of stretching. +std::vector toModelInput(const QImage& photo, bool letterbox); + +/// Normalise raw model output to Grayscale8 with **near = bright**, and report +/// the pre-normalisation range. A constant field maps to mid-grey rather than +/// dividing by zero. +QImage toDepthImage(const std::vector& raw, int w, int h, + float* outMin = nullptr, float* outMax = nullptr); + +/// Undo a letterbox: crop the padded region back out of a square depth map for +/// a source of the given aspect. Returns `square` unchanged when it was not +/// letterboxed (srcW/srcH already square) or on degenerate input. +QImage cropLetterbox(const QImage& square, int srcW, int srcH); + +} // namespace PhotoDepth diff --git a/src/PhotoDepth_test.cpp b/src/PhotoDepth_test.cpp new file mode 100644 index 000000000..b78bffabc --- /dev/null +++ b/src/PhotoDepth_test.cpp @@ -0,0 +1,193 @@ +#include + +#include "PhotoDepth.h" + +#include +#include + +#include +#include +#include + +// PhotoDepth's pre/post-processing is pure data (no Ogre, no GL, no ONNX +// session), so these run on every build. The ONNX inference itself needs the +// hosted model and is exercised manually; what is pinned here is the CONTRACT +// the model relies on — get the normalisation or the near/far convention wrong +// and the depth map is silently useless as a ControlNet input. + +TEST(PhotoDepth, ModelInputIsImageNetNormalisedCHW) +{ + QImage img(64, 32, QImage::Format_RGB888); + img.fill(QColor(255, 0, 0)); // pure red + + const auto in = PhotoDepth::toModelInput(img, /*letterbox=*/false); + ASSERT_EQ(in.size(), static_cast(PhotoDepth::kInputSize) + * PhotoDepth::kInputSize * 3); + + // Channel-major (CHW), not interleaved: the whole R plane comes first. + const size_t plane = static_cast(PhotoDepth::kInputSize) + * PhotoDepth::kInputSize; + // R = (1.0 - 0.485) / 0.229, G = (0 - 0.456) / 0.224, B = (0 - 0.406)/0.225 + EXPECT_NEAR(in[0], (1.0f - 0.485f) / 0.229f, 1e-3f); + EXPECT_NEAR(in[plane], (0.0f - 0.456f) / 0.224f, 1e-3f); + EXPECT_NEAR(in[2 * plane], (0.0f - 0.406f) / 0.225f, 1e-3f); +} + +TEST(PhotoDepth, ModelInputIsAlwaysThePinnedSize) +{ + // The export pins H/W to 518 (a dynamic-axis graph drifts badly away from + // the traced size), so ANY source must come out at exactly that size. + for (const QSize s : {QSize(16, 16), QSize(1024, 768), QSize(7, 913)}) { + QImage img(s, QImage::Format_RGB888); + img.fill(Qt::gray); + EXPECT_EQ(PhotoDepth::toModelInput(img, false).size(), + static_cast(PhotoDepth::kInputSize) + * PhotoDepth::kInputSize * 3) + << "source " << s.width() << "x" << s.height(); + } +} + +TEST(PhotoDepth, NullImageYieldsZeroedInputNotACrash) +{ + const auto in = PhotoDepth::toModelInput(QImage(), false); + EXPECT_EQ(in.size(), static_cast(PhotoDepth::kInputSize) + * PhotoDepth::kInputSize * 3); + for (float v : in) EXPECT_FLOAT_EQ(v, 0.0f); +} + +TEST(PhotoDepth, DepthImageKeepsNearBright) +{ + // The model emits INVERSE depth (larger = nearer) and MeshDepthRenderer + // uses near=bright, so the mapping must NOT invert. Flipping this produces + // a map that looks plausible but conditions ControlNet backwards. + std::vector raw = {0.0f, 1.0f, 2.0f, 3.0f}; + float mn = 0, mx = 0; + const QImage img = PhotoDepth::toDepthImage(raw, 2, 2, &mn, &mx); + ASSERT_FALSE(img.isNull()); + EXPECT_FLOAT_EQ(mn, 0.0f); + EXPECT_FLOAT_EQ(mx, 3.0f); + EXPECT_EQ(qGray(img.pixel(0, 0)), 0); // smallest value -> darkest + EXPECT_EQ(qGray(img.pixel(1, 1)), 255); // largest value -> brightest + EXPECT_GT(qGray(img.pixel(1, 0)), qGray(img.pixel(0, 0))); +} + +TEST(PhotoDepth, ConstantFieldBecomesMidGreyNotNaN) +{ + // A flat field would divide by zero when normalising. Emitting a NaN-filled + // image would then propagate silently into the texture pipeline. + std::vector raw(16, 2.5f); + const QImage img = PhotoDepth::toDepthImage(raw, 4, 4); + ASSERT_FALSE(img.isNull()); + for (int y = 0; y < 4; ++y) + for (int x = 0; x < 4; ++x) + EXPECT_EQ(qGray(img.pixel(x, y)), 128); +} + +TEST(PhotoDepth, NonFiniteValuesDoNotPoisonTheRange) +{ + // One stray NaN must not drag min/max to NaN and blank the whole map. + // NaN FIRST: std::min/max propagate a leading NaN through every later + // comparison, so this ordering is what actually exercises the guard. With + // NaN in the middle the comparisons happen to survive and the test passes + // even unguarded (verified by mutation). + std::vector raw = {std::numeric_limits::quiet_NaN(), + 0.0f, 1.0f, 2.0f}; + float mn = 0, mx = 0; + const QImage img = PhotoDepth::toDepthImage(raw, 2, 2, &mn, &mx); + ASSERT_FALSE(img.isNull()); + EXPECT_TRUE(std::isfinite(mn)); + EXPECT_TRUE(std::isfinite(mx)); + EXPECT_FLOAT_EQ(mn, 0.0f); + EXPECT_FLOAT_EQ(mx, 2.0f); + EXPECT_EQ(qGray(img.pixel(1, 1)), 255); // the real max still maps to white + // The NaN TEXEL must land on a defined value. Without the per-pixel guard + // the NaN flows into the 0..255 cast, which is undefined behaviour — in + // practice it lands on 0 on this toolchain, so assert the defined value the + // guard produces (mid-grey, the same neutral a flat field maps to) rather + // than merely "in range", which UB happens to satisfy. + EXPECT_EQ(qGray(img.pixel(0, 0)), 128) + << "a non-finite texel must map to the neutral mid-grey"; +} + +TEST(PhotoDepth, DepthImageRejectsUndersizedInput) +{ + std::vector raw = {0.0f, 1.0f}; + EXPECT_TRUE(PhotoDepth::toDepthImage(raw, 4, 4).isNull()); + EXPECT_TRUE(PhotoDepth::toDepthImage(raw, 0, 0).isNull()); +} + +TEST(PhotoDepth, CropLetterboxRecoversTheSourceAspect) +{ + // A 2:1 source letterboxed into a square must crop back to 2:1. + QImage square(100, 100, QImage::Format_Grayscale8); + square.fill(0); + const QImage cropped = PhotoDepth::cropLetterbox(square, 200, 100); + EXPECT_EQ(cropped.width(), 100); + EXPECT_EQ(cropped.height(), 50); + // A square source is returned untouched. + EXPECT_EQ(PhotoDepth::cropLetterbox(square, 100, 100).size(), square.size()); +} + +TEST(PhotoDepth, LetterboxPreservesAspectByPadding) +{ + // A wide image letterboxed into the square input must leave black bars, + // i.e. the top row is padding while the middle row carries image data. + QImage wide(200, 50, QImage::Format_RGB888); + wide.fill(QColor(255, 255, 255)); + const auto in = PhotoDepth::toModelInput(wide, /*letterbox=*/true); + const size_t W = PhotoDepth::kInputSize; + // Row 0 is padding -> normalised black, i.e. (0 - mean)/std (negative). + EXPECT_LT(in[0], 0.0f); + // The centre row is white -> (1 - mean)/std (positive). + EXPECT_GT(in[(W / 2) * W + W / 2], 0.0f); +} + +TEST(PhotoDepth, EstimateFailsCleanlyWithoutAModel) +{ + QImage img(32, 32, QImage::Format_RGB888); + img.fill(Qt::gray); + const auto r = PhotoDepth::estimate(img, QStringLiteral("/no/such/model.onnx")); + EXPECT_FALSE(r.ok); + EXPECT_FALSE(r.error.isEmpty()); + EXPECT_TRUE(r.depth.isNull()) << "a failed estimate must not return a map"; +} + +TEST(PhotoDepth, EstimateRejectsAnEmptyImage) +{ + const auto r = PhotoDepth::estimate(QImage(), PhotoDepth::modelPath()); + EXPECT_FALSE(r.ok); + EXPECT_FALSE(r.error.isEmpty()); +} + +TEST(PhotoDepth, ModelPathLivesUnderTheDepthCache) +{ + EXPECT_TRUE(PhotoDepth::modelPath().contains(QStringLiteral("ai_models"))); + EXPECT_TRUE(PhotoDepth::modelPath().contains(QStringLiteral("depth"))); + EXPECT_TRUE(PhotoDepth::modelPath().endsWith(QStringLiteral("da2_small.onnx"))); +} + +TEST(PhotoDepth, InfinitiesDoNotPoisonTheRange) +{ + // Distinct from the NaN case: std::min/max shrug off NaN (they return the + // second argument when a comparison is false), but +inf/-inf compare + // NORMALLY and would take over the range — +inf collapses every finite + // pixel toward black, -inf turns the normalisation into inf/inf = NaN on + // the way to the 0..255 cast. The inference probe cannot catch this: it + // only rejects output where EVERY sampled value is non-finite. + const float inf = std::numeric_limits::infinity(); + std::vector raw = {0.0f, inf, 1.0f, 2.0f}; + float mn = 0, mx = 0; + QImage img = PhotoDepth::toDepthImage(raw, 2, 2, &mn, &mx); + ASSERT_FALSE(img.isNull()); + EXPECT_TRUE(std::isfinite(mn)); + EXPECT_TRUE(std::isfinite(mx)); + EXPECT_FLOAT_EQ(mx, 2.0f) << "+inf must not become the range maximum"; + EXPECT_EQ(qGray(img.pixel(1, 1)), 255) << "the real maximum still maps to white"; + + std::vector raw2 = {0.0f, -inf, 1.0f, 2.0f}; + img = PhotoDepth::toDepthImage(raw2, 2, 2, &mn, &mx); + ASSERT_FALSE(img.isNull()); + EXPECT_TRUE(std::isfinite(mn)); + EXPECT_FLOAT_EQ(mn, 0.0f) << "-inf must not become the range minimum"; + EXPECT_EQ(qGray(img.pixel(0, 0)), 0); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index dd34bc008..e1e942848 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -261,6 +261,8 @@ if(BUILD_TESTS) ${CMAKE_CURRENT_SOURCE_DIR}/../src/MotionInbetween.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MotionLibrary.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MotionComposer.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/PhotoDepth.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../src/PhotoDepth_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MotionComposer_test.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MotionGenerator.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../src/MeshSegmenter.cpp