[build-tools] Allow running Maestro tests with maestro-runner

packages/build-tools/src/steps/functions/__tests__/maestroResultParser.test.ts

@@ -10,11 +10,92 @@ import {
   parseFailedFlowNamesFromJUnitFile,
   parseFailedFlowsFromFileAttrs,
   parseFailedFlowsFromJUnit,
+  parseFailedFlowsFromMaestroRunnerReport,
   parseJUnitTestCases,
   parseMaestroResults,
   parseMaestroResultsFromFileAttrs,
+  parseMaestroRunnerReport,
 } from '../maestroResultParser';
 
+describe(parseFailedFlowsFromMaestroRunnerReport, () => {
+  it('returns exact sourceFile paths for failed flows', async () => {
+    vol.fromJSON({
+      '/project/flows/pass.yml': '',
+      '/project/flows/nested/fail.yml': '',
+      '/reports/attempt-0/report.json': JSON.stringify({
+        flows: [
+          { name: 'Passing flow', sourceFile: 'flows/pass.yml', status: 'passed' },
+          {
+            name: 'Failing flow',
+            sourceFile: 'flows/nested/fail.yml',
+            status: 'failed',
+          },
+          { status: 'skipped' },
+        ],
+      }),
+    });
+
+    await expect(
+      parseFailedFlowsFromMaestroRunnerReport({
+        reportDirectory: '/reports/attempt-0',
+        workingDirectory: '/project',
+      })
+    ).resolves.toEqual(['flows/nested/fail.yml']);
+    await expect(parseMaestroRunnerReport('/reports/attempt-0')).resolves.toEqual({
+      flows: [
+        { name: 'Passing flow', sourceFile: 'flows/pass.yml', status: 'passed' },
+        {
+          name: 'Failing flow',
+          sourceFile: 'flows/nested/fail.yml',
+          status: 'failed',
+        },
+      ],
+    });
+  });
+
+  it.each([
+    {
+      name: 'incomplete flow',
+      report: {
+        version: '1.0.0',
+        status: 'failed',
+        summary: { total: 1 },
+        flows: [{ name: 'Failing flow', sourceFile: 'flows/fail.yml', status: 'running' }],
+      },
+    },
+  ])('returns null for an $name', async ({ report }) => {
+    vol.fromJSON({
+      '/project/flows/fail.yml': '',
+      '/reports/attempt-0/report.json': JSON.stringify(report),
+    });
+
+    await expect(
+      parseFailedFlowsFromMaestroRunnerReport({
+        reportDirectory: '/reports/attempt-0',
+        workingDirectory: '/project',
+      })
+    ).resolves.toBeNull();
+  });
+
+  it('returns null when a failed sourceFile does not exist', async () => {
+    vol.fromJSON({
+      '/reports/attempt-0/report.json': JSON.stringify({
+        version: '1.0.0',
+        status: 'failed',
+        summary: { total: 1 },
+        flows: [{ name: 'Failing flow', sourceFile: 'flows/missing.yml', status: 'failed' }],
+      }),
+    });
+
+    await expect(
+      parseFailedFlowsFromMaestroRunnerReport({
+        reportDirectory: '/reports/attempt-0',
+        workingDirectory: '/project',
+      })
+    ).resolves.toBeNull();
+  });
+});
+
 describe(parseFailedFlowNamesFromJUnitFile, () => {
   it('returns the names of failed testcases, keeping slashes', async () => {
     vol.fromJSON({
@@ -453,6 +534,20 @@ describe('junitFileHasFileAttrs', () => {
     expect(await junitFileHasFileAttrs('/junit/report.xml')).toBe(false);
   });
 
+  it('returns true for a maestro-runner report with a file property', async () => {
+    vol.fromJSON({
+      '/junit/report.xml': [
+        '<?xml version="1.0"?>',
+        '<testsuites><testsuite>',
+        '  <testcase name="a" time="1.0">',
+        '    <properties><property name="file" value="flows/a.yaml"/></properties>',
+        '  </testcase>',
+        '</testsuite></testsuites>',
+      ].join('\n'),
+    });
+    expect(await junitFileHasFileAttrs('/junit/report.xml')).toBe(true);
+  });
+
   it('returns false when the file cannot be read', async () => {
     expect(await junitFileHasFileAttrs('/missing.xml')).toBe(false);
   });
@@ -478,6 +573,41 @@ describe(parseJUnitTestCases, () => {
     expect(results[0].file).toBe('.maestro/login.yaml');
   });
 
+  it('parses maestro-runner file properties and standard JUnit pass status', async () => {
+    vol.fromJSON({
+      '/junit/report.xml': [
+        '<?xml version="1.0"?>',
+        '<testsuites><testsuite>',
+        '  <testcase name="login" time="1.0">',
+        '    <properties><property name="file" value="flows/login.yaml"/></properties>',
+        '  </testcase>',
+        '</testsuite></testsuites>',
+      ].join('\n'),
+    });
+
+    const results = await parseJUnitTestCases('/junit');
+
+    expect(results[0]).toEqual(
+      expect.objectContaining({ file: 'flows/login.yaml', status: 'passed' })
+    );
+  });
+
+  it('excludes skipped testcases instead of counting them as passed', async () => {
+    vol.fromJSON({
+      '/junit/report.xml': [
+        '<?xml version="1.0"?>',
+        '<testsuites><testsuite>',
+        '  <testcase name="passing" time="1.0"/>',
+        '  <testcase name="skipped-flow" time="0"><skipped/></testcase>',
+        '</testsuite></testsuites>',
+      ].join('\n'),
+    });
+
+    const results = await parseJUnitTestCases('/junit');
+
+    expect(results.map(r => r.name)).toEqual(['passing']);
+  });
+
   it('treats a missing or empty file= attribute as undefined', async () => {
     vol.fromJSON({
       '/junit/report.xml': [

packages/build-tools/src/steps/functions/__tests__/maestroScreenshots.test.ts

@@ -3,14 +3,160 @@ import os from 'os';
 import path from 'path';
 
 import { createMockLogger } from '../../../__tests__/utils/logger';
+import { Sentry } from '../../../sentry';
 import {
   type HarvestedScreenshot,
   computePureFailureFlowNames,
   harvestFailureScreenshotsAsync,
+  harvestMaestroRunnerFailureScreenshotsAsync,
   parseFailureScreenshotFilename,
   selectFailureScreenshots,
 } from '../maestroScreenshots';
 
+describe(harvestMaestroRunnerFailureScreenshotsAsync, () => {
+  const logger = createMockLogger();
+  let reportDirectory: string;
+  let captureSpy: jest.SpiedFunction<typeof Sentry.capture>;
+
+  beforeEach(async () => {
+    reportDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'maestro-runner-harvest-test-'));
+    await fs.mkdir(path.join(reportDirectory, 'flows'));
+    await fs.mkdir(path.join(reportDirectory, 'assets', 'flow-000'), { recursive: true });
+    captureSpy = jest.spyOn(Sentry, 'capture').mockImplementation(() => {});
+  });
+
+  afterEach(async () => {
+    await fs.rm(reportDirectory, { recursive: true, force: true });
+    captureSpy.mockRestore();
+  });
+
+  it('reads the failed command screenshot from a maestro-runner report bundle', async () => {
+    const capturedSinceMs = Date.now() - 1_000;
+    await fs.writeFile(
+      path.join(reportDirectory, 'report.json'),
+      JSON.stringify({
+        flows: [
+          {
+            name: 'authentication/Login',
+            status: 'failed',
+            dataFile: 'flows/flow-000.json',
+          },
+        ],
+      })
+    );
+    await fs.writeFile(
+      path.join(reportDirectory, 'flows', 'flow-000.json'),
+      JSON.stringify({
+        commands: [
+          {
+            status: 'failed',
+            artifacts: { screenshotAfter: 'assets/flow-000/cmd-001-after.png' },
+          },
+        ],
+      })
+    );
+    const screenshotPath = path.join(reportDirectory, 'assets', 'flow-000', 'cmd-001-after.png');
+    await fs.writeFile(screenshotPath, 'png');
+
+    const shots = await harvestMaestroRunnerFailureScreenshotsAsync({
+      reportDirectory,
+      capturedSinceMs,
+      attemptIndex: 1,
+      logger,
+    });
+
+    expect(shots).toEqual([
+      {
+        fileAbsPath: screenshotPath,
+        displayName: 'Failure Screenshot: authentication_Login (attempt 2)',
+        metadata: {
+          kind: 'maestro-test-screenshot',
+          flowName: 'authentication_Login',
+          attemptIndex: 1,
+          capturedAtMs: expect.any(Number),
+        },
+      },
+    ]);
+  });
+
+  it('rejects a screenshot path that resolves to the parent directory (exact "..")', async () => {
+    await fs.writeFile(
+      path.join(reportDirectory, 'report.json'),
+      JSON.stringify({
+        flows: [{ name: 'Login', status: 'failed', dataFile: 'flows/flow-000.json' }],
+      })
+    );
+    await fs.writeFile(
+      path.join(reportDirectory, 'flows', 'flow-000.json'),
+      JSON.stringify({
+        commands: [{ status: 'failed', artifacts: { screenshotAfter: '..' } }],
+      })
+    );
+
+    await expect(
+      harvestMaestroRunnerFailureScreenshotsAsync({
+        reportDirectory,
+        capturedSinceMs: 0,
+        attemptIndex: 0,
+        logger,
+      })
+    ).resolves.toEqual([]);
+  });
+
+  it('ignores passing flows and paths outside the report directory', async () => {
+    await fs.writeFile(
+      path.join(reportDirectory, 'report.json'),
+      JSON.stringify({
+        flows: [
+          { name: 'Passing', status: 'passed', dataFile: 'flows/flow-000.json' },
+          { name: 'Unsafe', status: 'failed', dataFile: '../outside.json' },
+        ],
+      })
+    );
+
+    await expect(
+      harvestMaestroRunnerFailureScreenshotsAsync({
+        reportDirectory,
+        capturedSinceMs: 0,
+        attemptIndex: 0,
+        logger,
+      })
+    ).resolves.toEqual([]);
+  });
+
+  it.each([
+    ['null', 'null'],
+    ['a non-object flows value', JSON.stringify({ flows: 5 })],
+    ['a null flow entry', JSON.stringify({ flows: [null] })],
+  ])('reports to Sentry and returns [] when report.json is %s', async (_label, contents) => {
+    await fs.writeFile(path.join(reportDirectory, 'report.json'), contents);
+
+    await expect(
+      harvestMaestroRunnerFailureScreenshotsAsync({
+        reportDirectory,
+        capturedSinceMs: 0,
+        attemptIndex: 0,
+        logger,
+      })
+    ).resolves.toEqual([]);
+    expect(captureSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it('returns [] without reporting to Sentry when report.json is not valid JSON', async () => {
+    await fs.writeFile(path.join(reportDirectory, 'report.json'), 'not-json');
+
+    await expect(
+      harvestMaestroRunnerFailureScreenshotsAsync({
+        reportDirectory,
+        capturedSinceMs: 0,
+        attemptIndex: 0,
+        logger,
+      })
+    ).resolves.toEqual([]);
+    expect(captureSpy).not.toHaveBeenCalled();
+  });
+});
+
 describe(parseFailureScreenshotFilename, () => {
   it('parses a plain failure screenshot', () => {
     expect(parseFailureScreenshotFilename('screenshot-❌-1781186692250-(Login Flow).png')).toEqual({