system, check: warn when a benchmark ran under a cgroup CPU quota

doc/system.rst

@@ -191,6 +191,12 @@ The :ref:`pyperf system command <system_cmd>` implements the following operation
 * "Power supply": check that the power cable is plugged. If the power cable is
   unplugged (a laptop running only on a battery), the CPU speed can change
   when the battery level becomes too low.
+* "CPU quota": report a cgroup CPU bandwidth limit (``docker run --cpus``, a
+  Kubernetes CPU limit, systemd ``CPUQuota=``) and whether the benchmark cgroup
+  has been throttled. A CPU quota throttles the benchmark, which moves the mean
+  without widening the relative spread, so a capped run can look statistically
+  stable while being wrong. ``pyperf check`` emits the same warning from the
+  ``cpu_quota`` metadata of a saved result.
 * "Turbo Boost (MSR)": use ``/dev/cpu/N/msr`` to read/write
   the Turbo Boost mode of Intel CPUs
 * "Turbo Boost (intel_pstate driver)": read from/write into

pyperf/_cli.py

@@ -493,6 +493,21 @@ def format_checks(bench, lines=None, check_too_many_processes=False):
             warn("See https://bugzilla.redhat.com/show_bug.cgi?id=1378529")
             break
 
+    # Warn if the benchmark ran under a cgroup CPU quota (cpu_quota metadata).
+    # A quota throttles the run, which moves the mean but leaves the relative
+    # spread alone, so the statistical checks above cannot see it: a capped run
+    # comes back looking stable.
+    for run in bench._runs:
+        cpu_quota = run._metadata.get('cpu_quota')
+        if cpu_quota:
+            empty_line(lines)
+            lines.append("WARNING: the benchmark ran under a CPU quota (%s)"
+                         % cpu_quota)
+            lines.append("A CPU quota throttles the benchmark: the result can "
+                         "look stable while being capped. Run without a CPU "
+                         "quota for reliable results.")
+            break
+
     return lines
 
 

pyperf/_collect_metadata.py

@@ -422,6 +422,79 @@ def collect_cpu_quota(metadata):
         metadata['cpu_quota'] = '%s CPUs' % format(quota, 'g')
 
 
+def _read_cgroup_cpu_stat(path):
+    # Parse a cgroup "cpu.stat" file ("key value" lines) into a dict of ints.
+    # Return None if the file cannot be read.
+    stat = {}
+    found = False
+    try:
+        with open_text(path) as fp:
+            for line in fp:
+                fields = line.split()
+                if len(fields) == 2:
+                    try:
+                        stat[fields[0]] = int(fields[1])
+                        found = True
+                    except ValueError:
+                        continue
+    except OSError:
+        return None
+    return stat if found else None
+
+
+def get_cgroup_cpu_throttling():
+    """Get the Linux cgroup CPU throttling counters of the current process.
+
+    Return a dict with 'nr_throttled' (number of enforcement periods in which
+    the cgroup was throttled) and 'throttled_usec' (total time spent throttled,
+    in microseconds), or None if the counters cannot be read (ex: not Linux, no
+    cgroup, or no cpu controller). The counters are cumulative since the cgroup
+    was created, so None means "could not be checked", which is distinct from
+    zero ("checked, never throttled").
+
+    With cgroup v2 a limit set on a parent cgroup also throttles, and the
+    throttling is recorded in the cgroup where the limit binds while the root
+    counters stay at zero, so the largest counters in the hierarchy are used.
+    """
+    nr_throttled = None
+    throttled_usec = None
+    for line in read_proc("self/cgroup"):
+        parts = line.split(":", 2)
+        if len(parts) != 3:
+            continue
+        hierarchy, controllers, path = parts
+        path = path.strip("/")
+        names = path.split("/") if path else []
+        if hierarchy == "0" and not controllers:
+            # cgroup v2: check the cgroup and each of its parents
+            bases = [os.path.join("/sys/fs/cgroup", *names[:index])
+                     for index in range(len(names), -1, -1)]
+        elif "cpu" in controllers.split(","):
+            # cgroup v1: the cgroup path is not visible inside a container
+            bases = [os.path.join("/sys/fs/cgroup/cpu", *names),
+                     "/sys/fs/cgroup/cpu"]
+        else:
+            continue
+        for base in bases:
+            stat = _read_cgroup_cpu_stat(os.path.join(base, "cpu.stat"))
+            if stat is None:
+                continue
+            if "nr_throttled" in stat:
+                nr_throttled = max(nr_throttled or 0, stat["nr_throttled"])
+            if "throttled_usec" in stat:
+                # cgroup v2
+                throttled_usec = max(throttled_usec or 0,
+                                     stat["throttled_usec"])
+            elif "throttled_time" in stat:
+                # cgroup v1 reports nanoseconds
+                throttled_usec = max(throttled_usec or 0,
+                                     stat["throttled_time"] // 1000)
+    if nr_throttled is None and throttled_usec is None:
+        return None
+    return {"nr_throttled": nr_throttled or 0,
+            "throttled_usec": throttled_usec or 0}
+
+
 def collect_cpu_model(metadata):
     for line in read_proc("cpuinfo"):
         if line.startswith('model name'):

pyperf/_system.py

@@ -983,6 +983,44 @@ def write(self, tune):
             self.log_action("Max sample rate set to %s per second" % new_rate)
 
 
+class CPUQuota(Operation):
+    """
+    Detect a cgroup CPU bandwidth limit: docker run --cpus, a Kubernetes CPU
+    limit, systemd CPUQuota=. A quota throttles the benchmark, which moves the
+    mean without widening the relative spread, so a capped run can look
+    statistically stable while being wrong. Nothing pyperf can tune fixes it,
+    so this only reports the state, like the power supply check.
+    """
+
+    @staticmethod
+    def available():
+        return OS_LINUX and os.path.exists('/sys/fs/cgroup')
+
+    def __init__(self, system):
+        super().__init__('CPU quota', system)
+
+    def show(self):
+        from pyperf._collect_metadata import (get_cgroup_cpu_quota,
+                                               get_cgroup_cpu_throttling)
+
+        quota = get_cgroup_cpu_quota()
+        if quota is None:
+            return
+
+        self.log_state('a CPU quota of %s CPUs is set' % format(quota, 'g'))
+
+        throttling = get_cgroup_cpu_throttling()
+        if throttling and throttling['nr_throttled']:
+            self.log_state('the benchmark cgroup was CPU-throttled %s times '
+                           '(%.1f ms total)'
+                           % (throttling['nr_throttled'],
+                              throttling['throttled_usec'] / 1000.0))
+
+        self.advice('A CPU quota throttles the benchmark and can make a '
+                    'capped run look stable; run without a CPU quota to get '
+                    'reliable results')
+
+
 OPERATIONS = [
     # Generic operations
     PerfEvent,
@@ -991,6 +1029,7 @@ def write(self, tune):
     CPUFrequency,
     IRQAffinity,
     PowerSupply,
+    CPUQuota,
 
     # Setting the CPU scaling governor resets no_turbo
     # and so must be set before Turbo Boost

pyperf/tests/test_cli.py

@@ -27,6 +27,26 @@ def test_format_result_calibration(self):
                          'Calibration: 100 loops')
         self.assertRaises(ValueError, bench.median)
 
+    def _quota_bench(self, metadata):
+        md = {'name': 'bench'}
+        md.update(metadata)
+        runs = [pyperf.Run([1.0, 1.0, 1.0], metadata=md,
+                           collect_metadata=False)
+                for _ in range(3)]
+        return pyperf.Benchmark(runs)
+
+    def test_checks_warns_on_cpu_quota(self):
+        # a run under a CPU quota must warn even though the values look stable
+        bench = self._quota_bench({'cpu_quota': '2 CPUs'})
+        text = '\n'.join(cli.format_checks(bench))
+        self.assertIn('CPU quota', text)
+        self.assertIn('2 CPUs', text)
+
+    def test_checks_no_cpu_quota(self):
+        bench = self._quota_bench({})
+        text = '\n'.join(cli.format_checks(bench))
+        self.assertNotIn('CPU quota', text)
+
 
 if __name__ == "__main__":
     unittest.main()

pyperf/tests/test_metadata.py

@@ -102,6 +102,59 @@ def test_cpu_quota_no_limit(self):
         # no cgroup information at all (ex: not Linux)
         self.assertIsNone(self.check_cpu_quota({}))
 
+    def check_cpu_throttling(self, files):
+        def mock_open(filename, *args, **kw):
+            filename = filename.replace('\\', '/')
+            if filename not in files:
+                raise FileNotFoundError(filename)
+            return io.StringIO(files[filename])
+
+        with mock.patch('pyperf._utils.open', create=True,
+                        side_effect=mock_open):
+            return perf_metadata.get_cgroup_cpu_throttling()
+
+    def test_cpu_throttling_cgroup_v2(self):
+        # the quota binds on the process cgroup and throttling is recorded
+        # there, while the root counters stay at zero
+        files = {
+            '/proc/self/cgroup': '0::/user.slice/run-1.scope\n',
+            '/sys/fs/cgroup/user.slice/run-1.scope/cpu.stat':
+                ('usage_usec 999\nnr_periods 2470\n'
+                 'nr_throttled 3306\nthrottled_usec 247631465\n'),
+            '/sys/fs/cgroup/user.slice/cpu.stat':
+                'usage_usec 0\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n',
+            '/sys/fs/cgroup/cpu.stat':
+                'usage_usec 0\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n',
+        }
+        self.assertEqual(self.check_cpu_throttling(files),
+                         {'nr_throttled': 3306, 'throttled_usec': 247631465})
+
+    def test_cpu_throttling_zero_is_not_none(self):
+        # checked and never throttled: zero, distinct from "could not check"
+        files = {
+            '/proc/self/cgroup': '0::/init.scope\n',
+            '/sys/fs/cgroup/init.scope/cpu.stat':
+                'usage_usec 5\nnr_periods 0\nnr_throttled 0\nthrottled_usec 0\n',
+        }
+        self.assertEqual(self.check_cpu_throttling(files),
+                         {'nr_throttled': 0, 'throttled_usec': 0})
+
+    def test_cpu_throttling_cgroup_v1(self):
+        # cgroup v1 reports throttled_time in nanoseconds
+        files = {
+            '/proc/self/cgroup': '5:cpu,cpuacct:/docker/abc\n',
+            '/sys/fs/cgroup/cpu/docker/abc/cpu.stat':
+                'nr_periods 100\nnr_throttled 7\nthrottled_time 456789000\n',
+        }
+        self.assertEqual(self.check_cpu_throttling(files),
+                         {'nr_throttled': 7, 'throttled_usec': 456789})
+
+    def test_cpu_throttling_no_cgroup(self):
+        # no cpu.stat to read (ex: not Linux): could not be checked
+        self.assertIsNone(self.check_cpu_throttling({}))
+        files = {'/proc/self/cgroup': '0::/init.scope\n'}
+        self.assertIsNone(self.check_cpu_throttling(files))
+
 
 class CpuFunctionsTests(unittest.TestCase):
     INTEL_CPU_INFO = textwrap.dedent("""

pyperf/tests/test_system.py

@@ -1,10 +1,49 @@
 import os.path
 import sys
 import unittest
+from unittest import mock
 
 from pyperf.tests import get_output
 
 
+class CPUQuotaTests(unittest.TestCase):
+    def _show(self, quota, throttling):
+        from pyperf._system import CPUQuota
+
+        system = mock.Mock()
+        op = CPUQuota(system)
+        with mock.patch('pyperf._collect_metadata.get_cgroup_cpu_quota',
+                        return_value=quota), \
+             mock.patch('pyperf._collect_metadata.get_cgroup_cpu_throttling',
+                        return_value=throttling):
+            op.show()
+        states = ' '.join(call.args[0]
+                          for call in system.log_state.call_args_list)
+        advices = ' '.join(call.args[0]
+                           for call in system.advice.call_args_list)
+        return states, advices
+
+    def test_no_quota_is_silent(self):
+        states, advices = self._show(None, None)
+        self.assertEqual(states, '')
+        self.assertEqual(advices, '')
+
+    def test_quota_set_but_not_throttled(self):
+        states, advices = self._show(2.0, {'nr_throttled': 0,
+                                           'throttled_usec': 0})
+        self.assertIn('2 CPUs', states)
+        self.assertNotIn('throttled', states)
+        self.assertIn('CPU quota', advices)
+
+    def test_quota_and_throttled(self):
+        states, advices = self._show(2.0, {'nr_throttled': 3306,
+                                           'throttled_usec': 247631465})
+        self.assertIn('2 CPUs', states)
+        self.assertIn('3306', states)
+        self.assertIn('throttled', states.lower())
+        self.assertIn('CPU quota', advices)
+
+
 class SystemTests(unittest.TestCase):
     def test_show(self):
         args = [sys.executable, '-m', 'pyperf', 'system', 'show']