From c02a6fa2115a9072ddc34f292cec8ffd44cd9348 Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Sun, 10 Feb 2019 21:50:28 +0300 Subject: [PATCH 1/7] Fix deadlock in gc collector Signed-off-by: Krukov Dima --- prometheus_client/gc_collector.py | 103 ++++++++++++++++++++++-------- tests/test_gc_collector.py | 67 +++++++++++++------ 2 files changed, 122 insertions(+), 48 deletions(-) diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py index 8d7d7401..ee561b0c 100644 --- a/prometheus_client/gc_collector.py +++ b/prometheus_client/gc_collector.py @@ -5,15 +5,22 @@ import gc import os import time +from collections import defaultdict -from .metrics import Histogram +from .metrics_core import HistogramMetricFamily from .registry import REGISTRY +from .metrics import Histogram +from .utils import INF class GCCollector(object): """Collector for Garbage collection statistics.""" - def __init__(self, registry=REGISTRY, gc=gc): + _LATENCY = 'latency' + _COLLECTED = 'collected' + _UNCOLLECTABLE = 'uncollectable' + + def __init__(self, registry=REGISTRY): # To work around the deadlock issue described in # https://github.com/prometheus/client_python/issues/322, # the GC collector is always disabled in multiprocess mode. @@ -23,28 +30,17 @@ def __init__(self, registry=REGISTRY, gc=gc): if not hasattr(gc, 'callbacks'): return - collected = Histogram( - 'python_gc_collected_objects', - 'Objects collected during gc', - ['generation'], - buckets=[500, 1000, 5000, 10000, 50000], - registry=registry - ) - - uncollectable = Histogram( - 'python_gc_uncollectable_objects', - 'Uncollectable object found during GC', - ['generation'], - buckets=[500, 1000, 5000, 10000, 50000], - registry=registry - ) + self._buckets = [{ + self._LATENCY: defaultdict(lambda: 0.0), + self._COLLECTED: defaultdict(lambda: 0), + self._UNCOLLECTABLE: defaultdict(lambda: 0), + } for _ in [0, 1, 2]] - latency = Histogram( - 'python_gc_duration_seconds', - 'Time spent in garbage collection', - ['generation'], - registry=registry - ) + self._sums = [{ + self._LATENCY: 0.0, + self._COLLECTED: 0, + self._UNCOLLECTABLE: 0, + } for _ in [0, 1, 2]] times = {} @@ -67,15 +63,66 @@ def _cb(phase, info): if phase == 'stop': delta = time.time() - times[gen] - latency.labels(gen).observe(delta) - if 'collected' in info: - collected.labels(gen).observe(info['collected']) - if 'uncollectable' in info: - uncollectable.labels(gen).observe(info['uncollectable']) + self._add_to_bucket(self._LATENCY, gen, delta) + if 'collected' in info and info['collected'] != 0: + self._add_to_bucket(self._COLLECTED, gen, info['collected']) + if 'uncollectable' in info and info['uncollectable'] != 0: + self._add_to_bucket(self._UNCOLLECTABLE, gen, info['uncollectable']) finally: self.gc_cb_active = False gc.callbacks.append(_cb) + registry.register(self) + + def _add_to_bucket(self, bucket_name, gen, value): + bucket = self._buckets[gen][bucket_name] + self._sums[gen][bucket_name] += value + + for bound in self._get_bounds(gen, bucket_name): + if value <= bound: + bucket[INF] += 1 + bucket[bound] += 1 + break + + @staticmethod + def _get_bounds(gen, bucket_name): + if bucket_name == GC_COLLECTOR._LATENCY: + return Histogram.DEFAULT_BUCKETS + _max = gc.get_threshold()[gen] + return [int(_max/100), int(_max/50), int(_max/10), int(_max/5), int(_max/2), _max] + + def collect(self): + collected = HistogramMetricFamily( + 'python_gc_collected_objects', + 'Objects collected during gc', + labels=['generation'], + ) + uncollectable = HistogramMetricFamily( + 'python_gc_uncollectable_objects', + 'Uncollectable object found during GC', + labels=['generation'], + ) + + latency = HistogramMetricFamily( + 'python_gc_duration_seconds', + 'Time spent in garbage collection', + labels=['generation'], + ) + + for generation, buckets in enumerate(self._buckets): + _sums = self._sums[generation] + generation = str(generation) + + if _sums[self._LATENCY] != 0: + latency.add_metric([generation], buckets=list(buckets[self._LATENCY].items()), + sum_value=_sums[self._LATENCY]) + if _sums[self._COLLECTED] != 0: + collected.add_metric([generation], buckets=list(buckets[self._COLLECTED].items()), + sum_value=_sums[self._COLLECTED]) + if _sums[self._UNCOLLECTABLE] != 0: + uncollectable.add_metric([generation], buckets=list(buckets[self._UNCOLLECTABLE].items()), + sum_value=_sums[self._UNCOLLECTABLE]) + return [collected, uncollectable, latency] GC_COLLECTOR = GCCollector() diff --git a/tests/test_gc_collector.py b/tests/test_gc_collector.py index a317d605..ab9e01bf 100644 --- a/tests/test_gc_collector.py +++ b/tests/test_gc_collector.py @@ -1,5 +1,6 @@ from __future__ import unicode_literals +import gc import unittest from prometheus_client import CollectorRegistry, GCCollector @@ -7,48 +8,74 @@ class TestGCCollector(unittest.TestCase): def setUp(self): + gc.disable() + gc.collect() self.registry = CollectorRegistry() - self.gc = _MockGC() def test_working(self): - collector = GCCollector(registry=self.registry, gc=self.gc) - self.gc.start_gc({'generation': 0}) - self.gc.stop_gc({'generation': 0, 'collected': 10, 'uncollectable': 2}) + + GCCollector(registry=self.registry) + + # add targets for gc + a = [] + a.append(a) + del a + b = [] + b.append(b) + del b + + gc.collect(0) + self.registry.collect() self.assertEqual(1, self.registry.get_sample_value( 'python_gc_duration_seconds_count', labels={"generation": "0"})) - self.assertEqual(1, self.registry.get_sample_value( - 'python_gc_collected_objects_count', - labels={"generation": "0"})) + 'python_gc_duration_seconds_bucket', + labels={"generation": "0", "le": 0.005})) self.assertEqual(1, self.registry.get_sample_value( - 'python_gc_uncollectable_objects_count', + 'python_gc_collected_objects_count', labels={"generation": "0"})) - self.assertEqual(10, + self.assertEqual(2, self.registry.get_sample_value( 'python_gc_collected_objects_sum', labels={"generation": "0"})) + self.assertEqual(1, + self.registry.get_sample_value( + 'python_gc_collected_objects_bucket', + labels={"generation": "0", "le": 7})) - self.assertEqual(2, + def test_empty(self): + + GCCollector(registry=self.registry) + gc.collect(0) + self.registry.collect() + + self.assertEqual(1, self.registry.get_sample_value( - 'python_gc_uncollectable_objects_sum', + 'python_gc_duration_seconds_count', labels={"generation": "0"})) + self.assertEqual(1, + self.registry.get_sample_value( + 'python_gc_duration_seconds_bucket', + labels={"generation": "0", "le": 0.005})) + self.assertIsNone(self.registry.get_sample_value( + 'python_gc_collected_objects_count', + labels={"generation": "0"})) -class _MockGC(object): - def __init__(self): - self.callbacks = [] + self.assertIsNone(self.registry.get_sample_value( + 'python_gc_collected_objects_sum', + labels={"generation": "0"})) - def start_gc(self, info): - for cb in self.callbacks: - cb('start', info) + self.assertIsNone(self.registry.get_sample_value( + 'python_gc_collected_objects_bucket', + labels={"generation": "0", "le": 7})) - def stop_gc(self, info): - for cb in self.callbacks: - cb('stop', info) + def tearDown(self): + gc.enable() From 455b035882a46262acd28b312c76ffd5f82b7de4 Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Sun, 10 Feb 2019 22:00:10 +0300 Subject: [PATCH 2/7] Skip gccolect test for python < 3 Signed-off-by: Krukov Dima --- prometheus_client/gc_collector.py | 2 +- tests/test_gc_collector.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py index ee561b0c..d16024ea 100644 --- a/prometheus_client/gc_collector.py +++ b/prometheus_client/gc_collector.py @@ -7,9 +7,9 @@ import time from collections import defaultdict +from .metrics import Histogram from .metrics_core import HistogramMetricFamily from .registry import REGISTRY -from .metrics import Histogram from .utils import INF diff --git a/tests/test_gc_collector.py b/tests/test_gc_collector.py index ab9e01bf..507be9b1 100644 --- a/tests/test_gc_collector.py +++ b/tests/test_gc_collector.py @@ -1,11 +1,13 @@ from __future__ import unicode_literals import gc +import sys import unittest from prometheus_client import CollectorRegistry, GCCollector +@unittest.skipIf(sys.version_info < (3, ), "Test requires Python 3.+") class TestGCCollector(unittest.TestCase): def setUp(self): gc.disable() From b72da86387bcc71cb02dfc1d11480ccdfd5872e9 Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Sun, 10 Feb 2019 22:45:23 +0300 Subject: [PATCH 3/7] Fix python 2.6 skipIf Signed-off-by: Krukov Dima --- prometheus_client/gc_collector.py | 2 +- tests/test_gc_collector.py | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py index d16024ea..79f13864 100644 --- a/prometheus_client/gc_collector.py +++ b/prometheus_client/gc_collector.py @@ -89,7 +89,7 @@ def _get_bounds(gen, bucket_name): if bucket_name == GC_COLLECTOR._LATENCY: return Histogram.DEFAULT_BUCKETS _max = gc.get_threshold()[gen] - return [int(_max/100), int(_max/50), int(_max/10), int(_max/5), int(_max/2), _max] + return [int(_max / 100), int(_max / 50), int(_max / 10), int(_max / 5), int(_max / 2), _max] def collect(self): collected = HistogramMetricFamily( diff --git a/tests/test_gc_collector.py b/tests/test_gc_collector.py index 507be9b1..228df216 100644 --- a/tests/test_gc_collector.py +++ b/tests/test_gc_collector.py @@ -2,7 +2,12 @@ import gc import sys -import unittest + +if sys.version_info < (2, 7): + # We need the skip decorators from unittest2 on Python 2.6. + import unittest2 as unittest +else: + import unittest from prometheus_client import CollectorRegistry, GCCollector From 39a6362b518c1823b2a8669b80b6f6bb82d9062e Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Mon, 11 Feb 2019 01:08:20 +0300 Subject: [PATCH 4/7] Use bounds without INF and doubled threshold Signed-off-by: Krukov Dima --- prometheus_client/gc_collector.py | 7 +++---- tests/test_gc_collector.py | 5 ++++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py index 79f13864..2b16b247 100644 --- a/prometheus_client/gc_collector.py +++ b/prometheus_client/gc_collector.py @@ -7,7 +7,6 @@ import time from collections import defaultdict -from .metrics import Histogram from .metrics_core import HistogramMetricFamily from .registry import REGISTRY from .utils import INF @@ -87,9 +86,9 @@ def _add_to_bucket(self, bucket_name, gen, value): @staticmethod def _get_bounds(gen, bucket_name): if bucket_name == GC_COLLECTOR._LATENCY: - return Histogram.DEFAULT_BUCKETS - _max = gc.get_threshold()[gen] - return [int(_max / 100), int(_max / 50), int(_max / 10), int(_max / 5), int(_max / 2), _max] + return .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0 + _max = gc.get_threshold()[gen] * 2 + return int(_max / 100), int(_max / 50), int(_max / 10), int(_max / 5), int(_max / 2), _max def collect(self): collected = HistogramMetricFamily( diff --git a/tests/test_gc_collector.py b/tests/test_gc_collector.py index 228df216..ecf9a846 100644 --- a/tests/test_gc_collector.py +++ b/tests/test_gc_collector.py @@ -55,7 +55,10 @@ def test_working(self): self.assertEqual(1, self.registry.get_sample_value( 'python_gc_collected_objects_bucket', - labels={"generation": "0", "le": 7})) + labels={ + "generation": "0", + "le": gc.get_threshold()[0] * 2 / 100 + })) def test_empty(self): From 7589f93071dc861cd2c47644f799dc5596a6242c Mon Sep 17 00:00:00 2001 From: Krukov Dima Date: Tue, 12 Feb 2019 20:31:44 +0300 Subject: [PATCH 5/7] Use gc.stats for gcCollector metrics Signed-off-by: Krukov Dima --- prometheus_client/gc_collector.py | 100 ++++-------------------------- tests/test_gc_collector.py | 55 ++++------------ 2 files changed, 23 insertions(+), 132 deletions(-) diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py index 2b16b247..5ff6e4c4 100644 --- a/prometheus_client/gc_collector.py +++ b/prometheus_client/gc_collector.py @@ -4,124 +4,48 @@ import gc import os -import time -from collections import defaultdict -from .metrics_core import HistogramMetricFamily +from .metrics_core import GaugeMetricFamily from .registry import REGISTRY -from .utils import INF class GCCollector(object): """Collector for Garbage collection statistics.""" - _LATENCY = 'latency' - _COLLECTED = 'collected' - _UNCOLLECTABLE = 'uncollectable' - def __init__(self, registry=REGISTRY): - # To work around the deadlock issue described in - # https://github.com/prometheus/client_python/issues/322, # the GC collector is always disabled in multiprocess mode. if 'prometheus_multiproc_dir' in os.environ: return - if not hasattr(gc, 'callbacks'): + if not hasattr(gc, 'get_stats'): return - - self._buckets = [{ - self._LATENCY: defaultdict(lambda: 0.0), - self._COLLECTED: defaultdict(lambda: 0), - self._UNCOLLECTABLE: defaultdict(lambda: 0), - } for _ in [0, 1, 2]] - - self._sums = [{ - self._LATENCY: 0.0, - self._COLLECTED: 0, - self._UNCOLLECTABLE: 0, - } for _ in [0, 1, 2]] - - times = {} - - # Avoid _cb() being called re-entrantly - # by setting this flag and clearing it once - # the callback operation is complete. - # See https://github.com/prometheus/client_python/issues/322#issuecomment-438021132 - self.gc_cb_active = False - - def _cb(phase, info): - try: - if self.gc_cb_active: - return - self.gc_cb_active = True - - gen = info['generation'] - - if phase == 'start': - times[gen] = time.time() - - if phase == 'stop': - delta = time.time() - times[gen] - self._add_to_bucket(self._LATENCY, gen, delta) - if 'collected' in info and info['collected'] != 0: - self._add_to_bucket(self._COLLECTED, gen, info['collected']) - if 'uncollectable' in info and info['uncollectable'] != 0: - self._add_to_bucket(self._UNCOLLECTABLE, gen, info['uncollectable']) - finally: - self.gc_cb_active = False - - gc.callbacks.append(_cb) registry.register(self) - def _add_to_bucket(self, bucket_name, gen, value): - bucket = self._buckets[gen][bucket_name] - self._sums[gen][bucket_name] += value - - for bound in self._get_bounds(gen, bucket_name): - if value <= bound: - bucket[INF] += 1 - bucket[bound] += 1 - break - - @staticmethod - def _get_bounds(gen, bucket_name): - if bucket_name == GC_COLLECTOR._LATENCY: - return .005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0 - _max = gc.get_threshold()[gen] * 2 - return int(_max / 100), int(_max / 50), int(_max / 10), int(_max / 5), int(_max / 2), _max - def collect(self): - collected = HistogramMetricFamily( + collected = GaugeMetricFamily( 'python_gc_collected_objects', 'Objects collected during gc', labels=['generation'], ) - uncollectable = HistogramMetricFamily( + uncollectable = GaugeMetricFamily( 'python_gc_uncollectable_objects', 'Uncollectable object found during GC', labels=['generation'], ) - latency = HistogramMetricFamily( - 'python_gc_duration_seconds', - 'Time spent in garbage collection', + collections = GaugeMetricFamily( + 'python_gc_collections', + 'Number of times this generation was collected', labels=['generation'], ) - for generation, buckets in enumerate(self._buckets): - _sums = self._sums[generation] + for generation, stat in enumerate(gc.get_stats()): generation = str(generation) + collected.add_metric([generation], value=stat['collected']) + uncollectable.add_metric([generation], value=stat['uncollectable']) + collections.add_metric([generation], value=stat['collections']) - if _sums[self._LATENCY] != 0: - latency.add_metric([generation], buckets=list(buckets[self._LATENCY].items()), - sum_value=_sums[self._LATENCY]) - if _sums[self._COLLECTED] != 0: - collected.add_metric([generation], buckets=list(buckets[self._COLLECTED].items()), - sum_value=_sums[self._COLLECTED]) - if _sums[self._UNCOLLECTABLE] != 0: - uncollectable.add_metric([generation], buckets=list(buckets[self._UNCOLLECTABLE].items()), - sum_value=_sums[self._UNCOLLECTABLE]) - return [collected, uncollectable, latency] + return [collected, uncollectable, collections] GC_COLLECTOR = GCCollector() diff --git a/tests/test_gc_collector.py b/tests/test_gc_collector.py index ecf9a846..e08921a9 100644 --- a/tests/test_gc_collector.py +++ b/tests/test_gc_collector.py @@ -12,7 +12,7 @@ from prometheus_client import CollectorRegistry, GCCollector -@unittest.skipIf(sys.version_info < (3, ), "Test requires Python 3.+") +@unittest.skipIf(sys.version_info < (3, 4), "Test requires Python 3.4 +") class TestGCCollector(unittest.TestCase): def setUp(self): gc.disable() @@ -22,6 +22,8 @@ def setUp(self): def test_working(self): GCCollector(registry=self.registry) + self.registry.collect() + before = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) # add targets for gc a = [] @@ -34,58 +36,23 @@ def test_working(self): gc.collect(0) self.registry.collect() - self.assertEqual(1, - self.registry.get_sample_value( - 'python_gc_duration_seconds_count', - labels={"generation": "0"})) - self.assertEqual(1, - self.registry.get_sample_value( - 'python_gc_duration_seconds_bucket', - labels={"generation": "0", "le": 0.005})) - - self.assertEqual(1, + after = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) + self.assertEqual(2, after - before) + self.assertEqual(0, self.registry.get_sample_value( - 'python_gc_collected_objects_count', + 'python_gc_uncollectable_objects', labels={"generation": "0"})) - self.assertEqual(2, - self.registry.get_sample_value( - 'python_gc_collected_objects_sum', - labels={"generation": "0"})) - self.assertEqual(1, - self.registry.get_sample_value( - 'python_gc_collected_objects_bucket', - labels={ - "generation": "0", - "le": gc.get_threshold()[0] * 2 / 100 - })) - def test_empty(self): GCCollector(registry=self.registry) + self.registry.collect() + before = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) gc.collect(0) self.registry.collect() - self.assertEqual(1, - self.registry.get_sample_value( - 'python_gc_duration_seconds_count', - labels={"generation": "0"})) - self.assertEqual(1, - self.registry.get_sample_value( - 'python_gc_duration_seconds_bucket', - labels={"generation": "0", "le": 0.005})) - - self.assertIsNone(self.registry.get_sample_value( - 'python_gc_collected_objects_count', - labels={"generation": "0"})) - - self.assertIsNone(self.registry.get_sample_value( - 'python_gc_collected_objects_sum', - labels={"generation": "0"})) - - self.assertIsNone(self.registry.get_sample_value( - 'python_gc_collected_objects_bucket', - labels={"generation": "0", "le": 7})) + after = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) + self.assertEqual(0, after - before) def tearDown(self): gc.enable() From 9b12b4c800a7f74947ed5501d9eb7ca985a8a726 Mon Sep 17 00:00:00 2001 From: "dmitry.kryukov" Date: Wed, 13 Feb 2019 13:48:12 +0200 Subject: [PATCH 6/7] Allow multiprocess Signed-off-by: dmitry.kryukov --- prometheus_client/gc_collector.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py index 5ff6e4c4..6738321c 100644 --- a/prometheus_client/gc_collector.py +++ b/prometheus_client/gc_collector.py @@ -13,10 +13,6 @@ class GCCollector(object): """Collector for Garbage collection statistics.""" def __init__(self, registry=REGISTRY): - # the GC collector is always disabled in multiprocess mode. - if 'prometheus_multiproc_dir' in os.environ: - return - if not hasattr(gc, 'get_stats'): return registry.register(self) From 7bf2cdffd9041bed0258e547cd73dc4daf99c335 Mon Sep 17 00:00:00 2001 From: "dmitry.kryukov" Date: Thu, 14 Feb 2019 10:24:27 +0200 Subject: [PATCH 7/7] Use the counter as gc metrics and rename metrics names Signed-off-by: dmitry.kryukov --- prometheus_client/gc_collector.py | 13 ++++++------- tests/test_gc_collector.py | 14 +++++++++----- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/prometheus_client/gc_collector.py b/prometheus_client/gc_collector.py index 6738321c..de400d98 100644 --- a/prometheus_client/gc_collector.py +++ b/prometheus_client/gc_collector.py @@ -3,9 +3,8 @@ from __future__ import unicode_literals import gc -import os -from .metrics_core import GaugeMetricFamily +from .metrics_core import CounterMetricFamily from .registry import REGISTRY @@ -18,18 +17,18 @@ def __init__(self, registry=REGISTRY): registry.register(self) def collect(self): - collected = GaugeMetricFamily( - 'python_gc_collected_objects', + collected = CounterMetricFamily( + 'python_gc_objects_collected', 'Objects collected during gc', labels=['generation'], ) - uncollectable = GaugeMetricFamily( - 'python_gc_uncollectable_objects', + uncollectable = CounterMetricFamily( + 'python_gc_objects_uncollectable', 'Uncollectable object found during GC', labels=['generation'], ) - collections = GaugeMetricFamily( + collections = CounterMetricFamily( 'python_gc_collections', 'Number of times this generation was collected', labels=['generation'], diff --git a/tests/test_gc_collector.py b/tests/test_gc_collector.py index e08921a9..98f3ccde 100644 --- a/tests/test_gc_collector.py +++ b/tests/test_gc_collector.py @@ -23,7 +23,8 @@ def test_working(self): GCCollector(registry=self.registry) self.registry.collect() - before = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) + before = self.registry.get_sample_value('python_gc_objects_collected_total', + labels={"generation": "0"}) # add targets for gc a = [] @@ -36,22 +37,25 @@ def test_working(self): gc.collect(0) self.registry.collect() - after = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) + after = self.registry.get_sample_value('python_gc_objects_collected_total', + labels={"generation": "0"}) self.assertEqual(2, after - before) self.assertEqual(0, self.registry.get_sample_value( - 'python_gc_uncollectable_objects', + 'python_gc_objects_uncollectable_total', labels={"generation": "0"})) def test_empty(self): GCCollector(registry=self.registry) self.registry.collect() - before = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) + before = self.registry.get_sample_value('python_gc_objects_collected_total', + labels={"generation": "0"}) gc.collect(0) self.registry.collect() - after = self.registry.get_sample_value('python_gc_collected_objects', labels={"generation": "0"}) + after = self.registry.get_sample_value('python_gc_objects_collected_total', + labels={"generation": "0"}) self.assertEqual(0, after - before) def tearDown(self):