Add force_mode_constant parameter to bypass psutil overhead

codecarbon/core/cpu.py

@@ -12,9 +12,16 @@
 from typing import Dict, Optional, Tuple
 
 import pandas as pd
-import psutil
 from rapidfuzz import fuzz, process, utils
 
+try:
+    import psutil
+
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+    psutil = None
+
 from codecarbon.core.rapl import RAPLFile
 from codecarbon.core.units import Time
 from codecarbon.core.util import detect_cpu_model
@@ -64,6 +71,9 @@ def is_rapl_available() -> bool:
 
 
 def is_psutil_available():
+    if not PSUTIL_AVAILABLE:
+        logger.debug("psutil module is not available.")
+        return False
     try:
         nice = psutil.cpu_times().nice
         if nice > 0.0001:

codecarbon/core/resource_tracker.py

@@ -45,6 +45,27 @@ def set_CPU_tracking(self):
             max_power = self.tracker._force_cpu_power
         else:
             max_power = tdp.tdp * cpu_number if tdp.tdp is not None else None
+
+        # Check for forced constant mode first
+        if self.tracker._conf.get("force_mode_constant", False):
+            logger.info(
+                "Force constant mode requested - bypassing psutil and using constant CPU power"
+            )
+            model = tdp.model
+            if max_power is None and self.tracker._force_cpu_power:
+                max_power = self.tracker._force_cpu_power
+                logger.debug(f"Using user input TDP for constant mode: {max_power} W")
+                self.cpu_tracker = "User Input TDP constant"
+            else:
+                self.cpu_tracker = "TDP constant"
+            logger.info(f"CPU Model on forced constant consumption mode: {model}")
+            self.tracker._conf["cpu_model"] = model
+            hardware_cpu = CPU.from_utils(
+                self.tracker._output_dir, "constant", model, max_power
+            )
+            self.tracker._hardware.append(hardware_cpu)
+            return
+
         if self.tracker._conf.get("force_mode_cpu_load", False) and (
             tdp.tdp is not None or self.tracker._force_cpu_power is not None
         ):

codecarbon/core/util.py

@@ -8,7 +8,14 @@
 from typing import Optional, Union
 
 import cpuinfo
-import psutil
+
+try:
+    import psutil
+
+    PSUTIL_AVAILABLE = True
+except ImportError:
+    PSUTIL_AVAILABLE = False
+    psutil = None
 
 from codecarbon.external.logger import logger
 
@@ -118,6 +125,12 @@ def count_physical_cpus():
 
 
 def count_cpus() -> int:
+    if not PSUTIL_AVAILABLE:
+        logger.warning("psutil not available, using fallback CPU count detection")
+        # Fallback to using os.cpu_count() or physical CPU count
+        cpu_count = os.cpu_count()
+        return cpu_count if cpu_count is not None else 1
+
     if SLURM_JOB_ID is None:
         return psutil.cpu_count()
 

codecarbon/emissions_tracker.py

@@ -180,6 +180,7 @@ def __init__(
         force_ram_power: Optional[int] = _sentinel,
         pue: Optional[int] = _sentinel,
         force_mode_cpu_load: Optional[bool] = _sentinel,
+        force_mode_constant: Optional[bool] = _sentinel,
         allow_multiple_runs: Optional[bool] = _sentinel,
     ):
         """
@@ -237,6 +238,7 @@ def __init__(
         :param force_ram_power: ram power to be used instead of automatic detection.
         :param pue: PUE (Power Usage Effectiveness) of the datacenter.
         :param force_mode_cpu_load: Force the addition of a CPU in MODE_CPU_LOAD
+        :param force_mode_constant: Force the addition of a CPU in constant mode, bypassing psutil
         :param allow_multiple_runs: Allow multiple instances of codecarbon running in parallel. Defaults to False.
         """
 
@@ -288,6 +290,7 @@ def __init__(
         self._set_from_conf(force_ram_power, "force_ram_power", None, float)
         self._set_from_conf(pue, "pue", 1.0, float)
         self._set_from_conf(force_mode_cpu_load, "force_mode_cpu_load", False, bool)
+        self._set_from_conf(force_mode_constant, "force_mode_constant", False, bool)
         self._set_from_conf(
             experiment_id, "experiment_id", "5b0fa12a-3dd7-45bb-9766-cc326314d9f1"
         )

docs/_sources/parameters.rst.txt

@@ -46,6 +46,15 @@ Input Parameters
        | Estimate it with ``sudo lshw -C memory -short | grep DIMM``
        | to get the number of RAM slots used, then do
        | *RAM power in W = Number of RAM Slots * 5 Watts*
+   * - force_mode_cpu_load
+     - | Force the use of CPU load mode for measuring CPU power consumption,
+       | defaults to ``False``. When enabled, uses psutil to monitor CPU load
+       | and estimates power consumption based on TDP and current CPU usage.
+   * - force_mode_constant
+     - | Force the use of constant mode for CPU power consumption measurement,
+       | defaults to ``False``. When enabled, bypasses psutil completely and
+       | uses a constant power consumption based on CPU TDP. Useful when
+       | psutil overhead is significant or psutil is unavailable.
    * - allow_multiple_runs
      - | Boolean variable indicating if multiple instance of CodeCarbon
        | on the same machine is allowed,

docs/edit/parameters.rst

@@ -46,6 +46,15 @@ Input Parameters
        | Estimate it with ``sudo lshw -C memory -short | grep DIMM``
        | to get the number of RAM slots used, then do
        | *RAM power in W = Number of RAM Slots * 5 Watts*
+   * - force_mode_cpu_load
+     - | Force the use of CPU load mode for measuring CPU power consumption,
+       | defaults to ``False``. When enabled, uses psutil to monitor CPU load
+       | and estimates power consumption based on TDP and current CPU usage.
+   * - force_mode_constant
+     - | Force the use of constant mode for CPU power consumption measurement,
+       | defaults to ``False``. When enabled, bypasses psutil completely and
+       | uses a constant power consumption based on CPU TDP. Useful when
+       | psutil overhead is significant or psutil is unavailable.
    * - allow_multiple_runs
      - | Boolean variable indicating if multiple instance of CodeCarbon
        | on the same machine is allowed,

tests/test_force_constant_mode.py

@@ -0,0 +1,146 @@
+import os
+import tempfile
+import time
+import unittest
+from unittest import mock
+
+import pandas as pd
+
+from codecarbon.emissions_tracker import (
+    EmissionsTracker,
+    OfflineEmissionsTracker,
+)
+
+
+def light_computation(run_time_secs: int = 1):
+    end_time: float = (
+        time.perf_counter() + run_time_secs
+    )  # Run for `run_time_secs` seconds
+    while time.perf_counter() < end_time:
+        pass
+
+
+class TestForceConstantMode(unittest.TestCase):
+    def setUp(self) -> None:
+        self.project_name = "project_TestForceConstantMode"
+        self.emissions_file = "emissions-test-TestForceConstantMode.csv"
+        self.emissions_path = tempfile.gettempdir()
+        self.emissions_file_path = os.path.join(
+            self.emissions_path, self.emissions_file
+        )
+        if os.path.isfile(self.emissions_file_path):
+            os.remove(self.emissions_file_path)
+
+    def tearDown(self) -> None:
+        if os.path.isfile(self.emissions_file_path):
+            os.remove(self.emissions_file_path)
+
+    def test_force_constant_mode_online(self):
+        """Test force_mode_constant parameter with online tracker"""
+        tracker = EmissionsTracker(
+            output_dir=self.emissions_path,
+            output_file=self.emissions_file,
+            force_mode_constant=True,
+        )
+        tracker.start()
+        light_computation(run_time_secs=1)
+        emissions = tracker.stop()
+
+        # Check that emissions were calculated
+        assert isinstance(emissions, float)
+        self.assertNotEqual(emissions, 0.0)
+
+        # Verify output file was created
+        self.verify_output_file(self.emissions_file_path)
+
+        # Check CSV content shows constant mode
+        df = pd.read_csv(self.emissions_file_path)
+        # The cpu_power should be a constant value (not varying like in load mode)
+        self.assertGreater(df["cpu_power"].iloc[0], 0)
+
+    def test_force_constant_mode_offline(self):
+        """Test force_mode_constant parameter with offline tracker"""
+        tracker = OfflineEmissionsTracker(
+            country_iso_code="USA",
+            output_dir=self.emissions_path,
+            output_file=self.emissions_file,
+            force_mode_constant=True,
+        )
+        tracker.start()
+        light_computation(run_time_secs=1)
+        emissions = tracker.stop()
+
+        assert isinstance(emissions, float)
+        self.assertNotEqual(emissions, 0.0)
+        self.verify_output_file(self.emissions_file_path)
+
+    def test_force_constant_mode_with_custom_cpu_power(self):
+        """Test force_mode_constant with custom CPU power"""
+        custom_cpu_power = 200  # 200W
+        tracker = EmissionsTracker(
+            output_dir=self.emissions_path,
+            output_file=self.emissions_file,
+            force_mode_constant=True,
+            force_cpu_power=custom_cpu_power,
+        )
+        tracker.start()
+        light_computation(run_time_secs=1)
+        emissions = tracker.stop()
+
+        assert isinstance(emissions, float)
+        self.assertNotEqual(emissions, 0.0)
+
+        # Check that the custom CPU power was used
+        df = pd.read_csv(self.emissions_file_path)
+        # CPU power should be 50% of the TDP (constant mode assumption)
+        expected_cpu_power = custom_cpu_power / 2
+        self.assertEqual(df["cpu_power"].iloc[0], expected_cpu_power)
+
+    @mock.patch("codecarbon.core.cpu.PSUTIL_AVAILABLE", False)
+    @mock.patch("codecarbon.core.util.PSUTIL_AVAILABLE", False)
+    def test_force_constant_mode_without_psutil(self):
+        """Test that force_mode_constant works when psutil is not available"""
+        tracker = EmissionsTracker(
+            output_dir=self.emissions_path,
+            output_file=self.emissions_file,
+            force_mode_constant=True,
+        )
+        tracker.start()
+        light_computation(run_time_secs=1)
+        emissions = tracker.stop()
+
+        assert isinstance(emissions, float)
+        self.assertNotEqual(emissions, 0.0)
+        self.verify_output_file(self.emissions_file_path)
+
+    def test_force_constant_mode_takes_precedence_over_cpu_load(self):
+        """Test that force_mode_constant takes precedence over force_mode_cpu_load"""
+        tracker = EmissionsTracker(
+            output_dir=self.emissions_path,
+            output_file=self.emissions_file,
+            force_mode_constant=True,
+            force_mode_cpu_load=True,  # This should be ignored
+        )
+        tracker.start()
+        light_computation(run_time_secs=1)
+        emissions = tracker.stop()
+
+        assert isinstance(emissions, float)
+        self.assertNotEqual(emissions, 0.0)
+        self.verify_output_file(self.emissions_file_path)
+
+    def verify_output_file(self, file_path: str) -> None:
+        """Verify that the output CSV file exists and has expected structure"""
+        with open(file_path, "r") as f:
+            lines = [line.rstrip() for line in f]
+        assert len(lines) == 2  # Header + 1 data row
+
+        # Check that it's a valid CSV with expected columns
+        df = pd.read_csv(file_path)
+        expected_columns = ["emissions", "cpu_power", "cpu_energy"]
+        for col in expected_columns:
+            self.assertIn(col, df.columns)
+
+
+if __name__ == "__main__":
+    unittest.main()