Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,893 changes: 1,027 additions & 866 deletions pixi.lock

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ dependencies = [
"ninja==1.13.2; sys_platform != 'emscripten'",
"py==1.11.0",
"typer==0.15.1",
"platformdirs==4.10.0",
"pexpect==4.9.0; sys_platform != 'emscripten'",
"wasmtime==48.0.0; sys_platform != 'emscripten'",
"ziglang==0.13.0; sys_platform != 'emscripten'",
Expand Down
43 changes: 41 additions & 2 deletions spy/build/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ class BuildConfig:
warning_as_error: bool = False
gc: GCOption = "none"
static: bool = False
# Value passed as `-march=<march>` to the C compiler.
# None means "don't pass -march at all".
# Only meaningful for --target native[-static].
march: Optional[str] = None

def __post_init__(self) -> None:
if self.kind == "testlib" and self.target not in ("wasi", "emscripten"):
Expand Down Expand Up @@ -60,15 +64,32 @@ def __init__(self, config: BuildConfig):
else:
flags_target = config.target

if config.march is not None:
# Only asserted here as a last-resort check in case BuildConfig
# is constructed directly (e.g. from tests) rather than via the
# CLI, which already validates --march against --target.
assert config.target == "native", (
f"--march is only supported for --target native "
f"(including --static), got target={config.target!r}"
)

self.CC = get_cc(flags_target)
self.cflags += get_cflags(
flags_target, config.build_type, config.kind, config.warning_as_error
flags_target,
config.build_type,
config.kind,
config.warning_as_error,
config.march,
)
self.cflags += EXTRA_CFLAGS

self.ldflags += get_ldflags(flags_target, config.build_type)

libdir = get_libdir(flags_target, config.build_type, config.kind)
libdir = str(
self._ensure_libspy_built(
flags_target, config.build_type, config.kind, config.march
)
)
if config.target == "wasi" and config.kind == "testlib":
# WASM testlibs are used by tests: in this case we want to make sure to
# include the whole libspy.a, so that helper functions such as spy_str_alloc
Expand Down Expand Up @@ -188,6 +209,24 @@ def add_libgc(self, config: BuildConfig) -> None:
# 4. fallback, let's hope libgc is installed system-wide
self.ldflags += ["-lgc"]

@staticmethod
def _ensure_libspy_built(
target: str, build_type: BuildType, kind: OutputKind, march: Optional[str]
) -> py.path.local:
"""
Return the directory containing the libspy.a to link against.

Without --march it is the one built by `make -C spy/libspy`. With
--march, libspy has to be compiled for the same ISA as the module
(libspy is built with -flto): it is built on demand and cached, under
a name which includes a hash of everything which influences the build.
"""
if march is None:
return py.path.local(get_libdir(target, build_type, kind))
from spy.libspy.builder import ensure_libspy

return ensure_libspy(target, build_type, kind, march)

@staticmethod
def _build_bdwgc_static() -> None:
deps_dir = str(spy.libspy.DEPS)
Expand Down
79 changes: 78 additions & 1 deletion spy/build/flags.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,12 @@
"""

import argparse
import os
import re
import shlex
import subprocess
import sys
import tempfile
from os import getenv
from typing import Optional

Expand Down Expand Up @@ -129,6 +134,7 @@ def get_cflags(
build_type: BuildType,
output_kind: OutputKind = "exe",
warning_as_error: bool = False,
march: Optional[str] = None,
) -> list[str]:
_check_target(target)
_check_build_type(build_type)
Expand All @@ -138,13 +144,15 @@ def get_cflags(
else:
warning_flags = _WARNING_CFLAGS
include = ["-I", str(_INCLUDE)]
march_flags = [f"-march={validate_march(march)}"] if march is not None else []
return (
_BASE_CFLAGS
+ _TARGET_CFLAGS[target]
+ _BUILD_TYPE_CFLAGS[build_type]
+ _OUTPUT_KIND_CFLAGS[output_kind]
+ warning_flags
+ include
+ march_flags
)


Expand All @@ -154,6 +162,73 @@ def get_ldflags(target: str, build_type: BuildType) -> list[str]:
return _TARGET_LDFLAGS[target] + _BUILD_TYPE_LDFLAGS[build_type]


# A -march value ends up on a command line and in the name of a directory (see
# spy.libspy.builder.ensure_libspy). Rather than trying to quote/sanitize it, we
# accept only what real values look like: 'native', 'x86-64-v3', 'x86_64_v3',
# 'armv8.2-a+crc', 'znver4', ...
_MARCH_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9._+-]*")


def validate_march(march: str) -> str:
"""
Return `march` unchanged if it is syntactically a sane -march value, else
raise ValueError. This does NOT check that the compiler knows it: see
check_march for that.
"""
if not _MARCH_RE.fullmatch(march):
raise ValueError(
f"invalid --march value {march!r}: it must start with a letter or "
f"a digit, and contain only letters, digits and '.', '_', '+', '-'"
)
return march


def check_march(target: str, march: str) -> None:
"""
Check that `march` is a valid value for `-march=` for the C compiler of
`target`, by asking the compiler itself (it takes a few ms).

Raise ValueError with a short, user-friendly message if it is not: this is
nicer than letting the build fail with the raw compiler output (zig
prints the list of all the CPUs it knows).

Note that gcc/clang and zig don't agree on the names: e.g. 'x86-64-v3' is
fine for gcc but not for `zig cc`, which wants 'x86_64_v3'.
"""
validate_march(march)
_check_target(target)
cc = get_cc(target)
target_flags = [f for f in _TARGET_CFLAGS[target] if f.startswith("--target=")]
with tempfile.TemporaryDirectory() as tmpdir:
cmd = [
*shlex.split(cc),
*target_flags,
f"-march={march}",
"-x", "c", "-c", "-",
"-o", os.path.join(tmpdir, "probe.o"),
] # fmt: skip
try:
res = subprocess.run(cmd, input="", capture_output=True, text=True)
except FileNotFoundError:
# no compiler at all: the build will complain by itself
return
if res.returncode == 0:
return

# zig prints "info: available CPUs ..." before the actual error: look for it
lines = [line.strip() for line in res.stderr.splitlines() if line.strip()]
errors = [line for line in lines if "error" in line]
reason = errors[0] if errors else (lines[0] if lines else "unknown error")
msg = f"{march!r} is not a valid -march value for {cc!r}: {reason}"
if target == "native-static":
msg += (
"\n--static builds use zig cc, which has its own CPU names (e.g. "
"'x86_64_v3' instead of 'x86-64-v3'; list them with "
"`python -m ziglang targets`). 'native' works everywhere."
)
raise ValueError(msg)


def get_build_dirname(build_type: BuildType, output_kind: OutputKind = "exe") -> str:
"""
Name of the libspy build dir for the given flavor, e.g. 'debug' or
Expand All @@ -170,7 +245,9 @@ def get_build_dirname(build_type: BuildType, output_kind: OutputKind = "exe") ->


def get_libdir(
target: str, build_type: BuildType, output_kind: OutputKind = "exe"
target: str,
build_type: BuildType,
output_kind: OutputKind = "exe",
) -> str:
_check_target(target)
return str(_BUILD.join(target, get_build_dirname(build_type, output_kind)))
Expand Down
2 changes: 2 additions & 0 deletions spy/cli/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

from spy.cli.commands.astcompile import astcompile
from spy.cli.commands.build import build
from spy.cli.commands.cache import cache_app
from spy.cli.commands.cleanup import cleanup
from spy.cli.commands.colorize import colorize
from spy.cli.commands.execute import execute
Expand Down Expand Up @@ -39,3 +40,4 @@
app.spy_command(scopes, name="scopes")
app.spy_command(cleanup, name="cleanup")
app.spy_command(format, name="format")
app.add_typer(cache_app, name="cache")
24 changes: 24 additions & 0 deletions spy/cli/commands/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from spy.backend.c.cbackend import CBackend
from spy.build.build_info import BuildTarget
from spy.build.config import BuildConfig, GCOption, OutputKind
from spy.build.flags import check_march
from spy.cli._runners import init_vm, nullcontext, timer
from spy.cli.commands.shared_args import (
Base_Args,
Expand Down Expand Up @@ -109,6 +110,15 @@ class _build_mixin:
),
] = False

march: Annotated[
Optional[str],
Option(
"--march",
help="Pass -march={VALUE} to the C compiler, e.g. --march=native "
"(requires --target native).",
),
] = None


@dataclass
class Build_Args(
Expand Down Expand Up @@ -148,13 +158,27 @@ async def build(args: Build_Args) -> None:
if args.static and sys.platform == "darwin":
raise click.UsageError("--static is not supported on macOS")

if args.march is not None and args.target != "native":
raise click.UsageError(
f"--march can only be used with --target native, got --target={args.target}"
)

if args.march is not None:
# fail early (before generating any C and building libspy) and with a
# clear message: gcc/clang and zig (--static) don't accept the same names
try:
check_march("native-static" if args.static else "native", args.march)
except ValueError as e:
raise click.UsageError(str(e))

config = BuildConfig(
target=args.target,
kind=args.output_kind,
build_type="release" if args.release_mode else "debug",
warning_as_error=args.warning_as_error,
gc=gc,
static=args.static,
march=args.march,
)

cwd = py.path.local(".")
Expand Down
19 changes: 19 additions & 0 deletions spy/cli/commands/cache.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import typer

from spy.libspy.builder import cache_dir, clear_cache

cache_app = typer.Typer(
help="Manage the libspy variants which are built on demand",
no_args_is_help=True,
)


@cache_app.command("clear")
def clear() -> None:
"""Delete the libspy variants built on demand"""
root = cache_dir()
n = clear_cache()
if n == 0:
print(f"Nothing to remove in {root}")
else:
print(f"Removed {n} libspy build(s) from {root}")
Loading
Loading