fix(rust): build Cargo workspace members into a shared target dir

aws_lambda_builders/workflows/rust_cargo/DESIGN.md

@@ -16,10 +16,16 @@ The general algorithm for preparing a rust executable for use on AWS Lambda is a
 
 It builds a binary in the standard cargo target directory. The binary's name is always `bootstrap`, and it's always located under `target/lambda/HANDLER_NAME/bootstrap`.
 
+For a Cargo workspace, the build targets the workspace's shared `target` directory rather than a `target` directory under each member. Because `sam build` invokes this workflow once per function, sharing a single target directory lets cargo compile common dependencies once instead of recompiling the whole dependency tree for every function. The shared directory and the member's binary name are both read from a single `cargo metadata` call. For a standalone (non-workspace) project the shared `target` directory is the project's own — unchanged from prior behavior. An explicit `CARGO_TARGET_DIR` in the environment always takes precedence.
+
 ### Copy and Rename executable
 
 It then copies the executable to the target directory honoring the provided runtime's [expectation on executable names](https://docs.aws.amazon.com/lambda/latest/dg/runtimes-custom.html).
 
+Because every workspace member now builds into the same `target/lambda` directory, that directory holds all of the workspace's binaries. When no handler (`artifact_executable_name`) is given, the copy step selects the binary named for the package whose manifest lives in the function's source directory (from the same `cargo metadata` call). It falls back to the previous single-directory heuristic when the binary name cannot be resolved.
+
+If two or more workspace members define a `bin` target with the same name (for example each package declaring a `bootstrap` bin), those binaries compile to the same path in the shared directory and overwrite each other. `sam build` still produces correct artifacts because it copies each function's binary out immediately after building it, but this relies on build ordering, so the workflow logs a warning recommending unique bin names per function.
+
 ## Notes
 
 Like the go builders, the workflow argument `options.artifact_executable_name`

aws_lambda_builders/workflows/rust_cargo/actions.py

@@ -104,7 +104,9 @@ class RustCopyAndRenameAction(BaseAction):
     DESCRIPTION = "Copy Rust executable, renaming if needed"
     PURPOSE = Purpose.COPY_SOURCE
 
-    def __init__(self, source_dir, artifacts_dir, handler=None, osutils=OSUtils()):
+    def __init__(
+        self, source_dir, artifacts_dir, handler=None, binaries=None, subprocess_cargo_lambda=None, osutils=OSUtils()
+    ):
         """
         Copy and rename Rust executable
 
@@ -119,21 +121,57 @@ def __init__(self, source_dir, artifacts_dir, handler=None, osutils=OSUtils()):
         handler : str, optional
             Handler name in `package.bin_name` or `bin_name` format
 
+        binaries : dict, optional
+            Resolved path dependencies, used to locate the `cargo` binary when
+            resolving the workspace target directory
+
+        subprocess_cargo_lambda : aws_lambda_builders.workflows.rust_cargo.cargo_lambda.SubprocessCargoLambda, optional
+            The Cargo Lambda process wrapper, used to resolve the same target
+            directory the build action compiled into
+
         osutils : aws_lambda_builders.workflows.rust_cargo.utils.OSUtils, optional
             Optional, External IO utils
         """
         self._source_dir = source_dir
         self._handler = handler
         self._artifacts_dir = artifacts_dir
+        self._binaries = binaries
+        self._subprocess_cargo_lambda = subprocess_cargo_lambda
         self._osutils = osutils
 
+    def _workspace_layout(self):
+        # Resolve the same shared target directory and binary name the build action
+        # used, from a single cached cargo metadata call. Returns None when the
+        # cargo wrapper is unavailable (e.g. in unit tests exercising the legacy path).
+        if self._subprocess_cargo_lambda and self._binaries and self._binaries.get("cargo"):
+            return self._subprocess_cargo_lambda.resolve_workspace_layout(
+                self._binaries["cargo"].binary_path, self._source_dir
+            )
+        return None
+
     def base_path(self):
+        # For a workspace member this is the workspace root's shared target/lambda; for
+        # a standalone project it is source_dir/target/lambda, matching the legacy path.
+        layout = self._workspace_layout()
+        if layout and layout.get("target_directory"):
+            return os.path.join(layout["target_directory"], "lambda")
         return os.path.join(self._source_dir, "target", "lambda")
 
     def binary_path(self):
         base = self.base_path()
-        if self._handler:
-            binary_path = os.path.join(base, self._handler, "bootstrap")
+
+        # An explicit handler (artifact_executable_name) always wins.
+        binary_name = self._handler
+        # Otherwise use the bin name cargo reported for this member. This is what lets
+        # the copy step pick the right binary now that every member shares one
+        # target/lambda directory holding all of the workspace's binaries.
+        if not binary_name:
+            layout = self._workspace_layout()
+            if layout:
+                binary_name = layout.get("binary_name")
+
+        if binary_name:
+            binary_path = os.path.join(base, binary_name, "bootstrap")
             LOG.debug("copying function binary from %s", binary_path)
             return binary_path
 

aws_lambda_builders/workflows/rust_cargo/cargo_lambda.py

@@ -3,6 +3,7 @@
 """
 
 import io
+import json
 import logging
 import os
 import shutil
@@ -39,6 +40,7 @@ def __init__(self, which, executable_search_paths=None, osutils=OSUtils()):
         self._which = which
         self._executable_search_paths = executable_search_paths
         self._osutils = osutils
+        self._workspace_layout_cache = {}
 
     def check_cargo_lambda_installation(self):
         """
@@ -69,6 +71,119 @@ def check_cargo_lambda_installation(self):
                 "https://www.cargo-lambda.info/guide/getting-started.html"
             )
 
+    def resolve_workspace_layout(self, cargo_path, source_dir):
+        """
+        Resolves the Cargo target directory and the binary produced for ``source_dir``.
+
+        A single ``cargo metadata`` call yields both:
+
+        - ``target_directory`` is the workspace's shared ``target`` directory.
+          For a Cargo workspace member this is the workspace root's ``target``,
+          which cargo shares across every member. Building each function into
+          that shared directory lets cargo reuse compiled dependencies across
+          the separate per-function builds that ``sam build`` runs, instead of
+          recompiling the whole dependency tree once per function. For a
+          standalone project it is the project's own ``target`` -- the location
+          used before this change.
+
+        - ``binary_name`` is the name of the ``bin`` target defined by the
+          package whose manifest lives in ``source_dir``. Because every member
+          now builds into the same ``target/lambda`` directory, the copy step
+          can no longer assume that directory holds a single binary; this name
+          tells it which one belongs to the function being built.
+
+        Results are cached per ``source_dir``.
+
+        Parameters
+        ----------
+        cargo_path : str
+            Path to the ``cargo`` binary.
+
+        source_dir : str
+            Path to the folder containing the function's source code.
+
+        Returns
+        -------
+        dict
+            ``{"target_directory": str or None, "binary_name": str or None}``.
+            Either value is ``None`` when it cannot be resolved, in which case
+            callers fall back to the previous behavior.
+        """
+
+        if source_dir in self._workspace_layout_cache:
+            return self._workspace_layout_cache[source_dir]
+
+        layout = {"target_directory": None, "binary_name": None}
+        command = [cargo_path, "metadata", "--no-deps", "--format-version", "1"]
+        LOG.debug("Resolving cargo workspace layout: %s", " ".join(command))
+        try:
+            process = self._osutils.popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source_dir)
+            out, err = process.communicate()
+            if process.returncode == 0:
+                metadata = json.loads(out.decode("utf-8"))
+                layout["target_directory"] = metadata.get("target_directory")
+                layout["binary_name"] = self._find_binary_name(metadata, source_dir)
+                self._warn_on_colliding_binaries(metadata)
+            else:
+                LOG.debug(
+                    "Could not resolve cargo workspace layout, falling back to previous behavior: %s",
+                    err.decode("utf-8", "replace").strip(),
+                )
+        except (OSError, ValueError, json.JSONDecodeError) as ex:
+            LOG.debug("Could not run cargo metadata, falling back to previous behavior: %s", ex)
+
+        self._workspace_layout_cache[source_dir] = layout
+        return layout
+
+    @staticmethod
+    def _find_binary_name(metadata, source_dir):
+        """
+        Finds the bin target name of the package whose manifest is in source_dir.
+        """
+        # cargo metadata emits absolute, symlink-resolved manifest paths, so resolve
+        # both operands the same way; os.path.normpath alone would never match a
+        # relative source_dir against cargo's absolute path.
+        member_manifest = os.path.realpath(os.path.join(source_dir, "Cargo.toml"))
+        for package in metadata.get("packages", []):
+            if os.path.realpath(package.get("manifest_path", "")) != member_manifest:
+                continue
+            bin_targets = [target["name"] for target in package.get("targets", []) if "bin" in target.get("kind", [])]
+            if len(bin_targets) == 1:
+                return bin_targets[0]
+            # A package with zero or several bins is ambiguous; let the copy step
+            # fall back to its directory-listing heuristic.
+            LOG.debug("Package %s does not have exactly one bin target: %s", package.get("name"), bin_targets)
+            return None
+        return None
+
+    @staticmethod
+    def _warn_on_colliding_binaries(metadata):
+        """
+        Warns when workspace members share a bin target name.
+
+        Since every member now builds into the same target/lambda directory,
+        two bins with the same name (e.g. several packages each defining a
+        `bootstrap` bin) compile to the same path and overwrite each other.
+        `sam build` still produces correct artifacts because it copies each
+        function's binary out immediately after building it, but the shared
+        output is fragile; unique bin names per function avoid it.
+        """
+        packages_by_bin = {}
+        for package in metadata.get("packages", []):
+            for target in package.get("targets", []):
+                if "bin" in target.get("kind", []):
+                    packages_by_bin.setdefault(target["name"], []).append(package.get("name"))
+
+        for bin_name, owners in packages_by_bin.items():
+            if len(owners) > 1:
+                LOG.warning(
+                    "Multiple workspace packages define a bin named '%s' (%s). They build to the same path in the "
+                    "shared target directory and overwrite each other; give each function a unique bin name to "
+                    "avoid relying on build ordering.",
+                    bin_name,
+                    ", ".join(sorted(owners)),
+                )
+
     def run(self, command, cwd):
         """
         Runs the build command.
@@ -101,16 +216,25 @@ def run(self, command, cwd):
                 os.environ["RUST_LOG"] = "debug"
             LOG.debug("RUST_LOG environment variable set to `%s`", os.environ.get("RUST_LOG"))
 
-        if not os.getenv("CARGO_TARGET_DIR"):
-            # This results in the "target" dir being created under the member dir of a cargo workspace
-            # This is for supporting sam build for a Cargo Workspace project
-            os.environ["CARGO_TARGET_DIR"] = "target"
+        cargo_env = dict(os.environ)
+        if not cargo_env.get("CARGO_TARGET_DIR"):
+            # Point every build at the workspace's shared target directory so cargo
+            # compiles dependencies once rather than once per function. For a standalone
+            # project this is the project's own target directory, matching the previous
+            # behavior. An explicit CARGO_TARGET_DIR in the environment is left untouched.
+            # The first element of command is the cargo binary path.
+            target_directory = self.resolve_workspace_layout(command[0], cwd)["target_directory"]
+            # Fall back to the relative "target" the workflow used before this change when
+            # metadata is unavailable, so the build still lands where the copy step (which
+            # falls back the same way) looks for it.
+            cargo_env["CARGO_TARGET_DIR"] = target_directory or "target"
 
         cargo_process = self._osutils.popen(
             command,
             stderr=subprocess.PIPE,
             stdout=subprocess.PIPE,
             cwd=cwd,
+            env=cargo_env,
         )
         stdout = ""
         # Create a buffer and use a thread to gather the stderr stream into the buffer

aws_lambda_builders/workflows/rust_cargo/workflow.py

@@ -51,7 +51,7 @@ def __init__(
                 handler,
                 flags,
             ),
-            RustCopyAndRenameAction(source_dir, artifacts_dir, handler),
+            RustCopyAndRenameAction(source_dir, artifacts_dir, handler, self.binaries, subprocess_cargo_lambda),
         ]
 
     def get_resolvers(self):

tests/integration/workflows/rust_cargo/test_rust_cargo.py

@@ -139,6 +139,35 @@ def test_builds_workspace_member(self):
         self.assertEqual(expected_files, output_files)
         self.assertIn("bar", os.path.join(source_dir, "bar", "target", "lambda"))
 
+    def test_builds_workspace_members_into_shared_target_dir(self):
+        # Building each member of a workspace, without an explicit handler, should place
+        # every binary under the single workspace-root target/lambda directory so cargo
+        # reuses compiled dependencies across the per-function builds. Each member's own
+        # binary must still be copied to its artifacts dir even though the shared
+        # target/lambda now holds every member's binary.
+        source_dir = os.path.join(self.TEST_DATA_FOLDER, "workspaces")
+        rm_target(source_dir)
+
+        member_artifacts = {}
+        for member in ("foo", "bar"):
+            artifacts_dir = tempfile.mkdtemp()
+            member_artifacts[member] = artifacts_dir
+            self.builder.build(
+                os.path.join(source_dir, member),
+                artifacts_dir,
+                self.scratch_dir,
+                os.path.join(source_dir, member, "Cargo.toml"),
+                runtime=self.runtime,
+            )
+
+        shared_lambda_dir = os.path.join(source_dir, "target", "lambda")
+        self.assertEqual({"foo", "bar"}, set(os.listdir(shared_lambda_dir)))
+        self.assertFalse(os.path.isdir(os.path.join(source_dir, "foo", "target")))
+        self.assertFalse(os.path.isdir(os.path.join(source_dir, "bar", "target")))
+        for member, artifacts_dir in member_artifacts.items():
+            self.assertEqual({"bootstrap"}, set(os.listdir(artifacts_dir)))
+            shutil.rmtree(artifacts_dir, ignore_errors=True)
+
     def test_builds_workspaces_project_with_package_option(self):
         source_dir = os.path.join(self.TEST_DATA_FOLDER, "workspaces")
         rm_target(source_dir)