Skip to content

gh-239: Add metadata["phase"] #241

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maurycy wants to merge 7 commits into
base: main
Choose a base branch
from
Open

gh-239: Add metadata["phase"] #241

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
675e982
metadata['phase']
maurycy May 27, 2026
05696c0
test
maurycy May 27, 2026
e5b7e0a
teardown test
maurycy May 27, 2026
466a442
better teardown test
maurycy May 27, 2026
9ea6981
api.rst
maurycy May 27, 2026
4117e02
changelog
maurycy May 27, 2026
56c69cc
json test
maurycy May 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions doc/api.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,7 @@ HookBase class
Called when the hook is completed for a process.

May add any information collected to the passed-in ``metadata`` dictionary.
The ``phase`` metadata identifies the run phase and sample type.


Runner class
Expand Down Expand Up @@ -780,6 +781,9 @@ Other:

* ``perf_version``: Version of the ``pyperf`` module
* ``unit``: Unit of values: ``byte``, ``integer`` or ``second``
* ``phase`` (tuple or list of two non-empty strings): run phase and sample
type, set to one of ``("calibration", "loops")``, ``("calibration",
"warmup")``, ``("measurement", "warmup")`` or ``("measurement", "value")``.

@sergey-miryanov sergey-miryanov May 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IIUC, the user will never see ("measurement", "warmup") in the output JSON. It would be good to note this. Or we could keep only the four states in the HookBase.teardown docs and note the difference.

* ``calibrate_loops`` (``int >= 1``): number of loops computed in a loops
calibration run
* ``recalibrate_loops`` (``int >= 1``): number of loops computed in a loops
Expand Down
6 changes: 6 additions & 0 deletions doc/changelog.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
Changelog
=========

* Feature: pyperf now stores ``phase`` run metadata in JSON files to identify
calibration and measurement samples. Hooks can also inspect this metadata in
:meth:`HookBase.teardown` to distinguish calibration loops, calibration
warmups, measurement warmups, and measurement values.
Patch by Maurycy Pawłowski-Wieroński.

Version 2.10.0 (2026-02-07)
---------------------------

Expand Down
8 changes: 8 additions & 0 deletions pyperf/_metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ def is_tags(value):
return all(isinstance(x, str) and x not in ('all', '') for x in value)


def is_phase(value):
if not isinstance(value, (tuple, list)):
return False
return len(value) == 2 and all(isinstance(x, str) and x for x in value)


def parse_load_avg(value):
if isinstance(value, NUMBER_TYPES):
return value
Expand All @@ -69,6 +75,7 @@ def format_noop(value):
WARMUPS = _MetadataInfo(format_number, (int,), is_positive, 'integer')
SECONDS = _MetadataInfo(format_seconds, NUMBER_TYPES, is_positive, 'second')
TAGS = _MetadataInfo(format_generic, (list,), is_tags, 'tag')
PHASE = _MetadataInfo(format_generic, (tuple, list), is_phase, None)

# Registry of metadata keys
METADATA = {
Expand All @@ -92,6 +99,7 @@ def format_noop(value):
'calibrate_warmups': WARMUPS,
'recalibrate_warmups': WARMUPS,
'tags': TAGS,
'phase': PHASE,
}

DEFAULT_METADATA_INFO = _MetadataInfo(format_generic, METADATA_VALUE_TYPES, None, None)
Expand Down
9 changes: 9 additions & 0 deletions pyperf/_worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,15 @@ def _compute_values(self, values, nvalue,

index += 1

if calibrate_loops:
self.metadata["phase"] = ("calibration", "loops")
elif is_warmup and (args.calibrate_warmups or args.recalibrate_warmups):
self.metadata["phase"] = ("calibration", "warmup")
elif is_warmup:
self.metadata["phase"] = ("measurement", "warmup")
else:
self.metadata["phase"] = ("measurement", "value")

for hook in hook_managers.values():
hook.teardown(self.metadata)

Expand Down
54 changes: 54 additions & 0 deletions pyperf/tests/test_bench.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import datetime
import errno
import gzip
import json
import unittest

import pyperf
Expand Down Expand Up @@ -69,6 +70,25 @@ def test_constructor(self):
collect_metadata=False)
self.assertEqual(run.get_metadata()['load_avg_1min'], 0.0)

def test_phase_metadata(self):
for value in (('measurement', 'value'), ['calibration', 'loops']):
with self.subTest(value=value):
run = pyperf.Run([1.0], metadata={'phase': value},
collect_metadata=False)

@sergey-miryanov sergey-miryanov May 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we test for collect_metadata=True here?

self.assertEqual(run.get_metadata()['phase'], value)

invalid_values = (
'measurement:value',
('measurement',),
('measurement', ''),
('measurement', 1),
)
for value in invalid_values:
with self.subTest(value=value):
with self.assertRaises(ValueError):
pyperf.Run([1.0], metadata={'phase': value},
collect_metadata=False)

def test_name(self):
# name must be non-empty
with self.assertRaises(ValueError):
Expand Down Expand Up @@ -198,6 +218,40 @@ def test_dump_load(self):

self.check_benchmarks_equal(bench, bench2)

def test_dump_phase_metadata(self):
runs = [
pyperf.Run(
(),
warmups=[(1, 2.92e-7), (64, 2.65e-9)],
metadata={'name': 'bench',
'loops': 64,
'calibrate_loops': 64,
'phase': ('calibration', 'loops')},
collect_metadata=False),
pyperf.Run(
[2.5949900301497353e-9, 2.633662268304582e-9],
warmups=[(64, 2.6324198845045776e-9)],
metadata={'name': 'bench',
'loops': 64,
'phase': ('measurement', 'value')},
collect_metadata=False),
]
bench = pyperf.Benchmark(runs)

bench_json = tests.benchmark_as_json(bench)
data = json.loads(bench_json)
json_runs = data['benchmarks'][0]['runs']
self.assertEqual(json_runs[0]['metadata']['phase'],
['calibration', 'loops'])
self.assertEqual(json_runs[1]['metadata']['phase'],
['measurement', 'value'])

bench2 = pyperf.Benchmark.loads(bench_json)
phases = [run.get_metadata()['phase'] for run in bench2.get_runs()]
self.assertEqual(phases,
[['calibration', 'loops'],
['measurement', 'value']])

def test_dump_replace(self):
bench = self.create_dummy_benchmark()

Expand Down
42 changes: 42 additions & 0 deletions pyperf/tests/test_runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,48 @@ def test_hook_command(self):
self.assertEqual(bench.get_metadata()["hooks"],
"_test_hook")

def test_hook_teardown_phase_metadata(self):
def time_func(loops):
return 1.0

def hook_phases(args):
phases = []

class PhaseHook(HookBase):
name = "phase_hook"

@staticmethod
def load():
return PhaseHook

def teardown(self, metadata):
phases.append(metadata["phase"])

args = args + ["--hook", PhaseHook.name]
runner = self.create_runner(args, hooks=[PhaseHook])
with tests.capture_stdout():
runner.bench_time_func('bench', time_func)
return phases

cases = (
('measurement',
'-l1 -w1 -n1 --worker',
[('measurement', 'warmup'), ('measurement', 'value')]),
('calibrate loops',
'--calibrate-loops -w0 -n1 --min-time=1.0 --worker',
[('calibration', 'loops')]),
)
for name, args, expected in cases:
with self.subTest(name=name):
self.assertEqual(hook_phases(args.split()), expected)

with self.subTest(name='calibrate warmups'):
with mock.patch('pyperf._worker.WorkerTask.test_calibrate_warmups',
return_value=True):
self.assertEqual(
hook_phases('--calibrate-warmups -l1 -n1 --worker'.split()),
[('calibration', 'warmup')])

def test_custom_hook(self):
class State:
inited = 0
Expand Down
Loading