diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bdc8b11..de5f8c8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,12 @@ conflated how the provider resolves the value with how the key hashes it. **Cross.** +- feat(cross): modules are now built during `--prepare` (staged into + `/modules/`) and during `--build` without `--app` (standalone + module build). Modules whose path references `${app_root}` are skipped with + an informational message when `--app` is absent. `build.app_path` is + deprecated; pass `--app` on the command line instead. + - feat(cross): `sdk_path` and `sdk_env_setup` may be written relative to the manifest that declares them, on the same rule `augment[].patches`, augment `path:` and `modules[].path` already follow. Absolute values are diff --git a/examples/cross/README.md b/examples/cross/README.md index 726eda3c..e218135b 100644 --- a/examples/cross/README.md +++ b/examples/cross/README.md @@ -176,8 +176,10 @@ cross: tree. Versioned outputs (`libhello.so.1.2.3` with `libhello.so` / `.so.1` symlinks) are staged whole — the real file plus every soname link — so both the bare name and any `DT_NEEDED` SONAME resolve at load time. -- Modules build (and stage) only when a runnable bundle is produced — i.e. with - `--app`. Declared without `--app`, they are skipped with a warning. +- Without `--app`, modules whose path does not reference `${app_root}` are + still built and staged into `/modules/`. Modules whose path + contains `${app_root}` require `--app` to resolve that variable and are + skipped with an informational message when `--app` is absent. - `build: cargo` cross-compiles a Rust crate: emb derives the Rust target from the GNU triple, synthesizes the cargo/cc-rs env from the profile (target- scoped `CC_` / `CARGO_TARGET__LINKER` / `CFLAGS_` / @@ -186,11 +188,16 @@ cross: `cargo` on `PATH`; targets the `arm-gnu` provider (the compiler is a real cross `gcc`). `features:` map to `--features`. -**Where modules ship.** They land in the app bundle's `lib/`, so the runnable -bundle (`--app`, `--tar`, `--deploy`) and the flatpak packager carry them -automatically. The deb/ipk/rpm/targz packagers package the *embedder binary* -plus `cross.package.files:` — not the app bundle — so a module reaches those -only if the bundle tree is listed under `files:` (the same as `libapp.so`). +**Where modules ship.** With `--app` they land in the app bundle's `lib/`, +so the runnable bundle (`--app`, `--tar`, `--deploy`) and the flatpak packager +carry them automatically. Without `--app` (standalone or `--prepare` build) +they are staged into `/modules/`. Nothing in a manifest reaches that +directory — the path variables are `${embedder_root}`, `${app_root}` and +`${runnable}`, none of which names the build root — so those artifacts are for +tooling outside emb, not for `files:` to pick up. +The deb/ipk/rpm/targz packagers package the *embedder binary* plus +`cross.package.files:` — not the app bundle — so a module reaches those only +if the bundle tree is listed under `files:` (the same as `libapp.so`). ## Local augment sources (`path:`) diff --git a/lib/src/commands/cross_command.dart b/lib/src/commands/cross_command.dart index 516f8706..79f41dc0 100644 --- a/lib/src/commands/cross_command.dart +++ b/lib/src/commands/cross_command.dart @@ -47,6 +47,7 @@ import 'package:emb_cli/src/host/host_info.dart'; import 'package:emb_cli/src/host/interactivity.dart'; import 'package:emb_cli/src/host/preflight.dart'; import 'package:emb_cli/src/json_output.dart'; +import 'package:emb_cli/src/manifest/build_config.dart'; import 'package:emb_cli/src/manifest/manifest_loader.dart'; import 'package:emb_cli/src/manifest/source_repo.dart'; import 'package:emb_cli/src/repo/git_repo.dart'; @@ -174,6 +175,7 @@ class CrossCommand extends Command { }) : _logger = logger, _environment = environment ?? Platform.environment, _host = host, + _loader = loader, _project = CrossProjectResolver(loader), _aotFactoryInjected = aotFactory, _bundleFactory = bundleFactory ?? BundleBuilder.new, @@ -470,6 +472,7 @@ class CrossCommand extends Command { final Logger _logger; final Map _environment; final HostInfo? _host; + final ManifestLoader _loader; final CrossProjectResolver _project; final AotBuilder Function(Workspace ws, HostInfo host)? _aotFactoryInjected; final BundleBuilder Function(Workspace ws) _bundleFactory; @@ -705,6 +708,26 @@ class CrossCommand extends Command { var resolved = resolveTarget(); if (resolved == null) return ExitCode.usage.code; + // `build.app_path` has no effect here: BuildConfig is read by `emb build`. + // Checked in both manifests this build read, because a `.emb/` project can + // declare a shared `build:` block in its base while the selected target + // comes from a sibling file — the two are the same path for a flat + // manifest, which the set collapses. + if (building) { + for (final path in { + selection.sourcePath, + project.nativeSourcePath, + }.whereType()) { + final raw = _loader.loadManifestFile(File(path))?.raw; + if (declaresAppPath(raw?['build'])) { + _logger.warn( + '$path: build.app_path is deprecated and has no effect; ' + 'pass --app on the command line instead.', + ); + } + } + } + if (building && args['flatpak'] == true && appDirArg == null && @@ -1053,6 +1076,22 @@ class CrossCommand extends Command { selectedBackends: selectedBackends, ); if (preparedEmbedder == null) return ExitCode.software.code; + if (target.modules.isNotEmpty) { + final libDir = Directory( + p.join(preparedEmbedder.buildRoot.path, 'modules'), + )..createSync(recursive: true); + final ok = await _buildModules( + preparedEmbedder.builder, + profile, + host, + target, + manifestDir, + preparedEmbedder.buildRoot, + libDir, + appDir: _appDir(appDirArg), + ); + if (!ok) return ExitCode.software.code; + } } // --deploy/--run without --build: ship what is already built rather than @@ -1510,11 +1549,24 @@ class CrossCommand extends Command { ); return ExitCode.usage.code; } - if (appPath == null && target.modules.isNotEmpty) { - _logger.warn( - ' modules : ${target.modules.map((m) => m.name).join(", ")} ' - 'declared but no --app bundle to stage them into (add --app).', + // Build modules standalone when no app bundle is being assembled. + // Skipped when preparedEmbedder is non-null: --prepare already built them + // into /modules/ in this invocation. + if (appPath == null && + target.modules.isNotEmpty && + preparedEmbedder == null) { + final libDir = Directory(p.join(buildRoot.path, 'modules')) + ..createSync(recursive: true); + final ok = await _buildModules( + builder, + profile, + host, + target, + manifestDir, + buildRoot, + libDir, ); + if (!ok) return ExitCode.software.code; } if (appPath != null) { final rc = await _runnable( @@ -3663,10 +3715,12 @@ class CrossCommand extends Command { return true; } - /// Build each app-owned [CrossTarget.modules] entry against the embedder's - /// cross toolchain ([builder]) and stage its declared `.so` artifacts into - /// [libDir] (the app bundle's `lib/`). Returns false — with an error logged — - /// on any build failure or missing declared artifact. + /// Build each [CrossTarget.modules] entry against the embedder's cross + /// toolchain ([builder]) and stage its declared `.so` artifacts into + /// [libDir]. Skips the compile step when the module source is unchanged + /// since the last successful build (stamp file matches source fingerprint). + /// Returns false — with an error logged — on any build failure or missing + /// declared artifact. Future _buildModules( CrossBuilder builder, CrossProfile profile, @@ -3682,9 +3736,15 @@ class CrossCommand extends Command { if (appDir != null) 'app_root': appDir, }; for (final m in target.modules) { - final src = Directory( - p.join(manifestDir.path, expandManifestVars(m.path, moduleVars)), - ); + final expanded = expandManifestVars(m.path, moduleVars); + if (appDir == null && hasUnresolvedVars(expanded)) { + _logger.info( + ' module ${m.name}: skipped' + r' (path has ${app_root} — pass --app to include)', + ); + continue; + } + final src = Directory(p.join(manifestDir.path, expanded)); if (!src.existsSync()) { _logger.err(' module ${m.name}: source dir not found: ${src.path}'); return false; @@ -3693,8 +3753,30 @@ class CrossCommand extends Command { final sw = Stopwatch()..start(); final gen = m.build.generator; + + // Artifact dir a prior build would have produced — computed without + // running the build so we can short-circuit when source is unchanged. + final cachedArtifactDir = gen != null + ? buildDir + : Directory( + p.join( + buildDir.path, + rustTriple(profile.targetTriple), + 'release', + ), + ); + + final stampFile = File(p.join(buildDir.path, '.module-stamp')); + final fingerprint = await _sourceFingerprint(src); + final upToDate = + stampFile.existsSync() && + stampFile.readAsStringSync().trim() == fingerprint && + cachedArtifactDir.existsSync(); + final Directory artifactDir; - if (gen != null) { + if (upToDate) { + artifactDir = cachedArtifactDir; + } else if (gen != null) { // cmake/meson: reuse the embedder's cross toolchain via CrossBuilder. final r = await builder.build( sourceDir: src, @@ -3707,11 +3789,13 @@ class CrossCommand extends Command { return false; } artifactDir = Directory(r.buildDir); + stampFile.writeAsStringSync(fingerprint); } else { // cargo: synthesize the cross env from the profile and run cargo. final dir = await _cargoModule(profile, host, m, src, buildDir); if (dir == null) return false; artifactDir = dir; + stampFile.writeAsStringSync(fingerprint); } for (final soname in m.artifacts) { @@ -3734,7 +3818,10 @@ class CrossCommand extends Command { } } _logger.info( - ' module ${m.name}: ${m.artifacts.join(", ")} → lib/ (${_secs(sw)})', + upToDate + ? ' module ${m.name}: up-to-date (cached)' + : ' module ${m.name}: ${m.artifacts.join(", ")} ' + '→ ${p.basename(libDir.path)}/ (${_secs(sw)})', ); } return true; diff --git a/lib/src/commands/matrix_command.dart b/lib/src/commands/matrix_command.dart index 2f99085a..4292d765 100644 --- a/lib/src/commands/matrix_command.dart +++ b/lib/src/commands/matrix_command.dart @@ -7,6 +7,7 @@ import 'package:emb_cli/src/cross/cross_project.dart'; import 'package:emb_cli/src/cross/cross_provider.dart'; import 'package:emb_cli/src/cross/cross_target.dart'; import 'package:emb_cli/src/host/host_info.dart'; +import 'package:emb_cli/src/manifest/build_config.dart'; import 'package:emb_cli/src/manifest/manifest_loader.dart'; import 'package:emb_cli/src/workspace/workspace.dart'; import 'package:mason_logger/mason_logger.dart'; @@ -107,6 +108,13 @@ class MatrixCommand extends Command { : hostFilter.where(supported.contains).toList(); if (hosts.isEmpty) continue; + if (declaresAppPath(manifest.raw['build'])) { + _logger.warn( + '${file.path}: build.app_path is deprecated and has no effect; ' + 'pass --app on the command line instead.', + ); + } + for (final entry in _targets(crossMap)) { if (targetFilter != null && entry.name != targetFilter) continue; final CrossTarget target; diff --git a/lib/src/manifest/build_config.dart b/lib/src/manifest/build_config.dart index 7f3a149a..2a1b75ee 100644 --- a/lib/src/manifest/build_config.dart +++ b/lib/src/manifest/build_config.dart @@ -51,3 +51,13 @@ class BuildConfig { String toString() => 'BuildConfig(app: $appPath, archs: $archs, modes: $modes)'; } + +/// True when a manifest's raw `build:` block declares `app_path`. +/// +/// Presence, not value: [BuildConfig.appPath] defaults to `.`, so once parsed a +/// manifest that wrote the default is indistinguishable from one that omitted +/// the key. The cross commands warn on the former, so they have to ask the raw +/// map. +bool declaresAppPath(Object? buildBlock) => + buildBlock is Map && + (buildBlock.containsKey('app_path') || buildBlock.containsKey('appPath')); diff --git a/test/src/commands/cross_command_test.dart b/test/src/commands/cross_command_test.dart index 260f1998..114991ad 100644 --- a/test/src/commands/cross_command_test.dart +++ b/test/src/commands/cross_command_test.dart @@ -732,6 +732,180 @@ void main() { ); }); + test('--dry-run plans a manifest with modules declared', () async { + final pkg = pkgWith( + 'withmod', + 'id: withmod\ntype: app\ncross:\n provider: arm-gnu\n' + ' toolchain_version: 12.3.rel1\n image_url: https://x/y.img.xz\n' + ' modules:\n - name: plugin_a\n path: native/plugin_a\n' + ' artifacts: [libplugin_a.so]\n', + ); + expect(await run(['cross', '--dry-run', pkg.path]), ExitCode.success.code); + }); + + test('--dry-run plans a manifest with app_root module paths', () async { + final pkg = pkgWith( + 'withappmod', + 'id: withappmod\ntype: app\ncross:\n provider: arm-gnu\n' + ' toolchain_version: 12.3.rel1\n image_url: https://x/y.img.xz\n' + ' modules:\n' + ' - name: app_plugin\n' + ' path: \${app_root}/native/app_plugin\n' + ' artifacts: [libapp_plugin.so]\n', + ); + expect(await run(['cross', '--dry-run', pkg.path]), ExitCode.success.code); + }); + + test( + r'--build --target local skips ${app_root} modules without --app', + () async { + final ws = Directory(p.join(tmp.path, 'ws'))..createSync(); + // Use arm-gnu manifest; --target local selects the native build path, + // which uses LocalCrossProvider (no toolchain download). + final pkg = pkgWith( + 'appmod', + 'id: appmod\ntype: app\ncross:\n provider: arm-gnu\n' + ' toolchain_version: 12.3.rel1\n image_url: https://x/y.img.xz\n' + ' modules:\n' + ' - name: app_plugin\n' + r' path: ${app_root}/native/app_plugin' + '\n' + ' artifacts: [libapp_plugin.so]\n', + ); + + // CrossCommand uses LocalCrossProvider for --target local, which sets + // targetTriple = machineArch-linux-gnu. The target itself comes from + // the manifest's cross: block (nativeCross), so provider stays arm-gnu + // for buildKey computation purposes. + final rawCross = { + 'provider': 'arm-gnu', + 'toolchain_version': '12.3.rel1', + 'image_url': 'https://x/y.img.xz', + 'modules': [ + { + 'name': 'app_plugin', + 'path': r'${app_root}/native/app_plugin', + 'artifacts': ['libapp_plugin.so'], + }, + ], + }; + final sourcePath = p.join(pkg.path, 'emb.yaml'); + final target = CrossTarget.fromMap(rawCross).withResolvedPaths( + sourcePath, + vars: {'embedder_root': p.normalize(pkg.path)}, + ); + final triple = '${_host.machineArch}-linux-gnu'; + final buildRootName = + 'cross-build-$triple-${buildKey(target)}-${projectKey(pkg.path)}'; + final buildRoot = Directory( + p.join(ws.path, '.config', 'flutter_workspace', buildRootName), + )..createSync(recursive: true); + + // Pre-seed stamp so _buildEmbedder's skipIfBuilt path exits early + // without running a real cmake/ninja build. + final fingerprint = await sourceFingerprint(pkg); + File( + p.join(buildRoot.path, '.emb-source-stamp'), + ).writeAsStringSync(fingerprint); + + // Fake ELF binary (with execute bit) so _artifactFor returns it. + final buildDir = Directory(p.join(buildRoot.path, 'build'))..createSync(); + final bin = File(p.join(buildDir.path, 'embedder')) + ..writeAsBytesSync([0x7f, 0x45, 0x4c, 0x46, 0, 0, 0, 0]); + await Process.run('chmod', ['+x', bin.path]); + + final logger = _CaptureLogger(); + final runner = CommandRunner('emb', 'test') + ..addCommand(CrossCommand(logger: logger, host: _host)); + final code = await runner.run([ + 'cross', + '--build', + '--target', + 'local', + '-w', + ws.path, + pkg.path, + ]); + + expect(code, ExitCode.success.code); + // The message, not just the token: `${app_root}` also appears in the + // "source dir not found" error this path produces if the skip regresses. + expect(logger.buffer.toString(), contains('module app_plugin: skipped')); + }, + ); + + test('--build warns that build.app_path has no effect', () async { + final ws = Directory(p.join(tmp.path, 'ws'))..createSync(); + final pkg = pkgWith( + 'apppathdep', + 'id: apppathdep\ntype: app\nbuild:\n app_path: .\ncross:\n' + ' provider: arm-gnu\n toolchain_version: 12.3.rel1\n' + ' image_url: https://x/y.img.xz\n', + ); + + // Same route into the build as the ${app_root} test above: --target local + // takes the native path, and a pre-seeded stamp plus an ELF-shaped binary + // satisfies skipIfBuilt so no real cmake/ninja runs. + final rawCross = { + 'provider': 'arm-gnu', + 'toolchain_version': '12.3.rel1', + 'image_url': 'https://x/y.img.xz', + }; + final target = CrossTarget.fromMap(rawCross).withResolvedPaths( + p.join(pkg.path, 'emb.yaml'), + vars: {'embedder_root': p.normalize(pkg.path)}, + ); + final triple = '${_host.machineArch}-linux-gnu'; + final buildRoot = Directory( + p.join( + ws.path, + '.config', + 'flutter_workspace', + 'cross-build-$triple-${buildKey(target)}-${projectKey(pkg.path)}', + ), + )..createSync(recursive: true); + File( + p.join(buildRoot.path, '.emb-source-stamp'), + ).writeAsStringSync(await sourceFingerprint(pkg)); + final buildDir = Directory(p.join(buildRoot.path, 'build'))..createSync(); + final bin = File(p.join(buildDir.path, 'embedder')) + ..writeAsBytesSync([0x7f, 0x45, 0x4c, 0x46, 0, 0, 0, 0]); + await Process.run('chmod', ['+x', bin.path]); + + final logger = _CaptureLogger(); + final runner = CommandRunner('emb', 'test') + ..addCommand(CrossCommand(logger: logger, host: _host)); + final code = await runner.run([ + 'cross', + '--build', + '--target', + 'local', + '-w', + ws.path, + pkg.path, + ]); + + expect(code, ExitCode.success.code); + expect(logger.errors.toString(), contains('build.app_path is deprecated')); + }); + + test('--dry-run does not warn about build.app_path', () async { + final pkg = pkgWith( + 'apppathdry', + 'id: apppathdry\ntype: app\nbuild:\n app_path: .\ncross:\n' + ' provider: arm-gnu\n toolchain_version: 12.3.rel1\n' + ' image_url: https://x/y.img.xz\n', + ); + final logger = _CaptureLogger(); + final runner = CommandRunner('emb', 'test') + ..addCommand(CrossCommand(logger: logger, host: _host)); + expect( + await runner.run(['cross', '--dry-run', pkg.path]), + ExitCode.success.code, + ); + expect(logger.errors.toString(), isNot(contains('build.app_path'))); + }); + test('--dry-run plans every target of the multi-platform example', () async { final example = p.join('examples', 'cross', 'raspberry-pi-family.emb.yaml'); for (final t in [ diff --git a/test/src/commands/matrix_command_test.dart b/test/src/commands/matrix_command_test.dart index 7a8bbe41..6447d67c 100644 --- a/test/src/commands/matrix_command_test.dart +++ b/test/src/commands/matrix_command_test.dart @@ -10,6 +10,13 @@ import 'package:mason_logger/mason_logger.dart'; import 'package:path/path.dart' as p; import 'package:test/test.dart'; +class _CapturingLogger extends Logger { + final StringBuffer warnings = StringBuffer(); + @override + void warn(String? message, {String tag = 'WARN', LogStyle? style}) => + warnings.writeln(message); +} + const _host = HostInfo( os: HostOs.linux, machineArch: 'x86_64', @@ -194,4 +201,106 @@ cross: expect(cell['args'], '${f.path} --target rpi5-bookworm'); }, ); + + test('host_dev_packages is not emitted in the matrix cell', () async { + final f = write('hostdev.emb.yaml', ''' +id: ivi-homescreen +type: app +supported_host_types: [ubuntu] +cross: + provider: arm-gnu + triple: aarch64-none-linux-gnu + toolchain_version: 12.3.rel1 + image_url: https://example.com/os.img.xz + host_dev_packages: [libpugixml-dev, libssl-dev] +'''); + final include = await render([f.path, '--host', 'ubuntu']); + + expect(include, hasLength(1)); + expect(include.single.containsKey('host_dev_packages'), isFalse); + }); + + test('non-default build.app_path emits a deprecation warning', () async { + final f = write('apppath.emb.yaml', ''' +id: ivi-homescreen +type: app +supported_host_types: [ubuntu] +build: + app_path: ../flutter_app +cross: + provider: arm-gnu + triple: aarch64-none-linux-gnu + toolchain_version: 12.3.rel1 + image_url: https://example.com/os.img.xz +'''); + final logger = _CapturingLogger(); + final outFile = p.join(tmp.path, 'matrix_apppath.json'); + final runner = CommandRunner('emb', 'test') + ..addCommand(MatrixCommand(logger: logger, host: _host)); + expect( + await runner.run(['matrix', f.path, '-o', outFile]), + ExitCode.success.code, + ); + expect( + logger.warnings.toString(), + contains('build.app_path is deprecated'), + ); + }); + + test( + 'build.app_path: . still emits a deprecation warning (key presence)', + () async { + final f = write('apppathdot.emb.yaml', ''' +id: ivi-homescreen +type: app +supported_host_types: [ubuntu] +build: + app_path: . +cross: + provider: arm-gnu + triple: aarch64-none-linux-gnu + toolchain_version: 12.3.rel1 + image_url: https://example.com/os.img.xz +'''); + final logger = _CapturingLogger(); + final outFile = p.join(tmp.path, 'matrix_apppathdot.json'); + final runner = CommandRunner('emb', 'test') + ..addCommand(MatrixCommand(logger: logger, host: _host)); + expect( + await runner.run(['matrix', f.path, '-o', outFile]), + ExitCode.success.code, + ); + expect( + logger.warnings.toString(), + contains('build.app_path is deprecated'), + ); + }, + ); + + test( + 'build block without app_path key does not emit a deprecation warning', + () async { + final f = write('buildnoapppath.emb.yaml', ''' +id: ivi-homescreen +type: app +supported_host_types: [ubuntu] +build: + archs: [arm64] +cross: + provider: arm-gnu + triple: aarch64-none-linux-gnu + toolchain_version: 12.3.rel1 + image_url: https://example.com/os.img.xz +'''); + final logger = _CapturingLogger(); + final outFile = p.join(tmp.path, 'matrix_buildnoapp.json'); + final runner = CommandRunner('emb', 'test') + ..addCommand(MatrixCommand(logger: logger, host: _host)); + expect( + await runner.run(['matrix', f.path, '-o', outFile]), + ExitCode.success.code, + ); + expect(logger.warnings.toString(), isEmpty); + }, + ); } diff --git a/test/src/manifest/build_config_test.dart b/test/src/manifest/build_config_test.dart index a53ed807..89efab22 100644 --- a/test/src/manifest/build_config_test.dart +++ b/test/src/manifest/build_config_test.dart @@ -1,5 +1,6 @@ import 'dart:io'; +import 'package:emb_cli/src/manifest/build_config.dart'; import 'package:emb_cli/src/manifest/manifest_loader.dart'; import 'package:path/path.dart' as p; import 'package:test/test.dart'; @@ -49,4 +50,26 @@ build: {} File(p.join(pkg.path, 'emb.yaml')).writeAsStringSync('id: x\ntype: app\n'); expect(loader.loadPackageDir(pkg)!.build, isNull); }); + + group('declaresAppPath', () { + test('true for either spelling, whatever the value', () { + expect(declaresAppPath({'app_path': '.'}), isTrue); + expect(declaresAppPath({'app_path': '../flutter_app'}), isTrue); + expect(declaresAppPath({'appPath': '.'}), isTrue); + // Declared but empty still counts: the key is what is deprecated. + expect(declaresAppPath({'app_path': null}), isTrue); + }); + + test('false without the key, and for a non-map build:', () { + expect( + declaresAppPath({ + 'archs': ['arm64'], + }), + isFalse, + ); + expect(declaresAppPath(const {}), isFalse); + expect(declaresAppPath(null), isFalse); + expect(declaresAppPath('nonsense'), isFalse); + }); + }); }