Allow Electricitymaps paid API key

src/CarbonIntensity.php

@@ -206,12 +206,12 @@ public function getFirstKnownDate(string $zone_name, string $source_name): ?Date
      * Download data for a single zone
      *
      * @param ClientInterface $data_source
-     * @param string $zone_name zone name
+     * @param array $source_zone source_zone row from database
      * @param int $limit maximum count of items to process
      * @param ProgressBar $progress_bar progress bar to update (CLI mode only)
      * @return int count of item downloaded
      */
-    public function downloadOneZone(ClientInterface $data_source, string $zone_name, int $limit = 0, ?ProgressBar $progress_bar = null): int
+    public function downloadOneZone(ClientInterface $data_source, array $source_zone, int $limit = 0, ?ProgressBar $progress_bar = null): int
     {
         $start_date = $this->getDownloadStartDate();
 
@@ -221,14 +221,14 @@ public function downloadOneZone(ClientInterface $data_source, string $zone_name,
         $source = new Source();
         $source->getFromDBByCrit(['name' => $data_source->getSourceName()]);
         $zone = new Zone();
-        $zone->getFromDBByCrit(['name' => $zone_name]);
+        $zone->getFromDBByCrit(['id' => $source_zone[Zone::getForeignKeyField()]]);
         $gaps = $this->findGaps($source->getID(), $zone->getID(), $start_date);
         if (count($gaps) === 0) {
             // Log a notice specifying the source and the zone
             trigger_error(sprintf(
                 "No gap to fill for source %s and zone %s between %s and %s",
                 $data_source->getSourceName(),
-                $zone_name,
+                $zone->fields['name'],
                 $start_date->format('Y-m-d'),
                 'now'
             ), E_USER_WARNING);
@@ -249,7 +249,7 @@ public function downloadOneZone(ClientInterface $data_source, string $zone_name,
         foreach ($gaps as $gap) {
             $gap_start = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $gap['start']);
             $gap_end = DateTimeImmutable::createFromFormat('Y-m-d H:i:s', $gap['end']);
-            $count = $data_source->fullDownload($zone_name, $gap_start, $gap_end, $this, $limit, $progress_bar);
+            $count = $data_source->fullDownload($source_zone, $gap_start, $gap_end, $this, $limit, $progress_bar);
             $total_count += $count;
             if ($total_count >= $limit) {
                 return $total_count;

src/CronTask.php

@@ -273,18 +273,17 @@ public static function downloadCarbonIntensityFromSource(GlpiCronTask $task, Cli
             $task->addVolume($done_count);
         }
 
-        $zones = $data_source->getZones(['is_download_enabled' => 1]);
-        if (count($zones) === 0) {
+        $sources_zones = $data_source->getSourceZones(['is_download_enabled' => 1]);
+        if (count($sources_zones) === 0) {
             trigger_error(__('No zone to download', 'carbon'), E_USER_WARNING);
             return 0;
         }
 
-        $limit_per_zone = max(1, floor(($remaining) / count($zones)));
+        $limit_per_zone = max(1, floor(($remaining) / count($sources_zones)));
         $count = 0;
-        foreach ($zones as $zone) {
-            $zone_name = $zone['name'];
+        foreach ($sources_zones as $source_zone) {
             try {
-                $added = $intensity->downloadOneZone($data_source, $zone_name, $limit_per_zone);
+                $added = $intensity->downloadOneZone($data_source, $source_zone, $limit_per_zone);
             } catch (RuntimeException $e) {
                 trigger_error($e->getMessage(), E_USER_WARNING);
                 continue;

src/DataSource/CarbonIntensity/AbstractClient.php

@@ -61,24 +61,24 @@
      * Download all data for a single day from the datasource
      *
      * @param DateTimeImmutable $day
-     * @param string $zone
+     * @param array $source_zone
      * @return array
      *
      * @throws AbortException if an error requires to stop all subsequent fetches
      */
-    abstract public function fetchDay(DateTimeImmutable $day, string $zone): array;
+    abstract public function fetchDay(DateTimeImmutable $day, array $source_zone): array;
 
     /**
      * Download a range if data from the data source
      *
      * @param DateTimeImmutable $start
      * @param DateTimeImmutable $stop
-     * @param string $zone
+     * @param array $source_zone
      * @return array
      *
      * @throws AbortException if an error requires to stop all subsequent fetches
      */
-    abstract public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array;
+    abstract public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, array $source_zone): array;
 
     public function disableCache()
     {
@@ -139,7 +139,46 @@
         $zone_fk = Zone::getForeignKeyField();
         $source_zone_table = Source_Zone::getTable();
         $iterator = $DB->request([
-            'SELECT' => Zone::getTableField('name'),
+            'SELECT' => [
+                Zone::getTableField('name')
+            ],
+            'FROM' => $zone_table,
+            'INNER JOIN' => [
+                $source_zone_table => [
+                    'ON' => [
+                        $zone_table => 'id',
+                        $source_zone_table => $zone_fk,
+                    ],
+                ],
+                $source_table => [
+                    'ON' => [
+                        $source_table => 'id',
+                        $source_zone_table => $source_fk,
+                    ],
+                ],
+            ],
+            'WHERE' => [
+                Source::getTableField('name') => $this->getSourceName(),
+            ] + $crit,
+        ]);
+
+        return iterator_to_array($iterator);
+    }
+
+    public function getSourceZones(array $crit = []): array
+    {
+        /** @var DBmysql $DB */
+        global $DB;
+
+        $source_table = Source::getTable();
+        $source_fk = Source::getForeignKeyField();
+        $zone_table = Zone::getTable();
+        $zone_fk = Zone::getForeignKeyField();
+        $source_zone_table = Source_Zone::getTable();
+        $iterator = $DB->request([
+            'SELECT' => [
+                getTableForItemType(Source_Zone::class) . '.*',
+            ],
             'FROM' => $zone_table,
             'INNER JOIN' => [
                 $source_zone_table => [
@@ -163,7 +202,7 @@
         return iterator_to_array($iterator);
     }
 
-    public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress_bar = null): int
+    public function fullDownload(array $source_zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress_bar = null): int
     {
         if ($start_date >= $stop_date) {
             return 0;
@@ -193,14 +232,14 @@
                 $data = $this->fetchRange(
                     DateTimeImmutable::createFromMutable($current_date),
                     DateTimeImmutable::createFromMutable($next_month),
-                    $zone
+                    $source_zone
                 );
             } catch (AbortException $e) {
                 break;
             }
             if (count($data) > 0) {
                 $data = $this->formatOutput($data, $this->step);
                 if (!isset($data[$zone])) {
                     break;
                 }
                 $saved = $intensity->save($zone, $this->getSourceName(), $data[$zone]);
@@ -222,20 +261,20 @@
         return $saved > 0 ? $count : -$count;
     }
 
-    public function incrementalDownload(string $zone, DateTimeImmutable $start_date, CarbonIntensity $intensity, int $limit = 0): int
+    public function incrementalDownload(array $source_zone, DateTimeImmutable $start_date, CarbonIntensity $intensity, int $limit = 0): int
     {
         $end_date = new DateTimeImmutable('now');
 
         $count = 0;
         $saved = 0;
         foreach ($this->sliceDateRangeByDay($start_date, $end_date) as $slice) {
             try {
-                $data = $this->fetchDay($slice, $zone);
+                $data = $this->fetchDay($slice, $source_zone);
             } catch (AbortException $e) {
                 throw $e;
             }
             $data = $this->formatOutput($data, $this->step);
             if (!isset($data[$zone])) {
                 continue;
             }
             $saved = $intensity->save($zone, $this->getSourceName(), $data[$zone]);

src/DataSource/CarbonIntensity/ClientInterface.php

@@ -161,6 +161,14 @@ public function getSupportedZones(): array;
      */
     public function getZones(array $crit = []): array;
 
+    /**
+     * get relations between the source and the zones
+     *
+     * @param array $crit
+     * @return array
+     */
+    public function getSourceZones(array $crit = []): array;
+
     public function getMaxIncrementalAge(): DateTimeImmutable;
 
     /**
@@ -175,24 +183,24 @@ public function getDataInterval(): string;
      * If the returned count is negative, then something went wrong
      * and the absolute value of the count tells how many items were saved
      *
-     * @param string $zone
+     * @param array $source_zone
      * @param DateTimeImmutable $start_date date where the download must start
      * @param DateTimeImmutable $stop_date date where the download must start
      * @param CarbonIntensity $intensity Instance used to update the database
      * @param int $limit
      * @param ProgressBar $progress progress bar to update during the download (CLI)
      * @return int count of successfully saved items
      */
-    public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress = null): int;
+    public function fullDownload(array $source_zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress = null): int;
 
     /**
      * Download recent carbon intensity data day by day
      *
-     * @param string $zone zone to process
+     * @param array $source_zone
      * @param DateTimeImmutable $start_date DAte where the downlos must begin
      * @param CarbonIntensity $intensity Instance of CarbonIntensity to use to save data
      * @param int $limit maximum count of items to process
      * @return int count of processed items. Negative count on failure
      */
-    public function incrementalDownload(string $zone, DateTimeImmutable $start_date, CarbonIntensity $intensity, int $limit = 0): int;
+    public function incrementalDownload(array $source_zone, DateTimeImmutable $start_date, CarbonIntensity $intensity, int $limit = 0): int;
 }

src/DataSource/CarbonIntensity/ElectricityMaps/Client.php

@@ -198,11 +198,9 @@
      *
      * The method fetches the intensities for the date range specified in argument.
      */
-    public function fetchDay(DateTimeImmutable $day, string $zone): array
+    public function fetchDay(DateTimeImmutable $day, array $source_zone): array
     {
-        $source_zone = new Source_Zone();
-        $source_zone->getFromDbBySourceAndZone($this->getSourceName(), $zone);
-        $zone_code = $source_zone->fields['code'];
+        $zone_code = $source_zone['code'];
 
         if ($zone_code === null) {
             throw new AbortException('Invalid zone');
@@ -252,7 +250,7 @@
 
         // Filter out already existing entries
         $carbon_intensity = new CarbonIntensity();
         $last_known_date = $carbon_intensity->getLastKnownDate($zone, $this->getSourceName());
         $intensities = array_filter($intensities, function ($intensity) use ($last_known_date) {
             $intensity_date = DateTime::createFromFormat('Y-m-d\TH:i:s', $intensity['datetime']);
             return $intensity_date > $last_known_date;
@@ -260,18 +258,15 @@
 
         return [
             'source' => $this->getSourceName(),
             $zone => $intensities,
         ];
     }
 
-    public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone): array
+    public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, array $source_zone): array
     {
-        // TODO: get zones from GLPI locations
-        $params = [
-            'zone' => $zone,
-        ];
-
-        $base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone;
+        $zone = new Zone();
+        $zone->getFromDBByCrit(['id' => $source_zone[Zone::getForeignKeyField()]]);
+        $base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone->fields['name'];
         $cache_file = $this->getCacheFilename(
             $base_path,
             $start,
@@ -293,24 +288,45 @@
         }
 
         // Set timezone to +00:00 and extend range by 12 hours on each side
-        $request_start = $start->setTimezone(new DateTimeZone('+0000'))->sub(new DateInterval('PT12H'));
-        $request_stop = $stop->setTimezone(new DateTimeZone('+0000'))->add(new DateInterval('PT12H'));
+        $timezone_z = new DateTimeZone('+0000');
+        $request_start = $start->setTimezone($timezone_z)->sub(new DateInterval('PT12H'));
+        $request_stop = $stop->setTimezone($timezone_z)->add(new DateInterval('PT14H'));
         $this->step = 60;
 
-        $step = new DateInterval('P10D');
+        $step = new DateInterval('PT240H');
         $full_response = [];
         $current_date = DateTime::createFromImmutable($request_start);
+        $glpikey = new GLPIKey();
+        $api_key = Config::getConfigurationValue('electricitymap_api_key');
+        $api_key = $glpikey->decrypt($api_key);
         while ($current_date < $request_stop) {
-            $response = $this->client->request('GET', $this->base_url . self::PAST_URL, ['query' => $params]);
+            $stop = clone $current_date;
+            $stop->add($step);
+            // For some reason, passing the parameters as a query stringthrough Guzzle
+            // Makes the request malformed from the point of view of Electricitymaps
+            // Workarounded by building here the query string
+            // $params = [
+            //     'zone' => $source_zone['code'],
+            //     'start' => $current_date->format('Y-m-d\+H:i'),
+            //     'end' => $stop->format('Y-m-d\+H:i'),
+            // ];
+            $url = $this->base_url . self::PAST_URL;
+            $url .= '?zone=' . $source_zone['code'];
+            $url .= '&start=' . $current_date->format('Y-m-d\+H:i');
+            $url .= '&end=' . $stop->format('Y-m-d\+H:i');
+            $response = $this->client->request('GET', $url, [/*'query' => $params,*/ 'headers' => ['auth-token' => $api_key]]);
+            if (isset($response['status']) && $response['status'] === 'error') {
+                trigger_error('Electricity maps API error: ' . $response['message'], E_USER_ERROR);
+            }
+            if (isset($response['error'])) {
+                trigger_error('Electricity maps API error: ' . $response['error'], E_USER_ERROR);
+            }
             if (!$full_response) {
                 $full_response = $response;
             } else {
                 $full_response['data'] = array_merge($full_response['data'], $response['data']);
             }
-            $current_date->add($step);
-            if ($current_date > $request_stop) {
-                $current_date = $request_stop;
-            }
+            $current_date = min($request_stop, $stop);
         }
         if (!$full_response) {
             return [];
@@ -355,7 +371,7 @@
     }
 
     /**
-     * Try ti determine the data quality of record
+     * Try to determine the data quality of record
      *
      * @param array $record
      * @return int
@@ -370,16 +386,16 @@
         return $data_quality;
     }
 
-    public function incrementalDownload(string $zone, DateTimeImmutable $start_date, CarbonIntensity $intensity, int $limit = 0): int
+    public function incrementalDownload(array $source_zone, DateTimeImmutable $start_date, CarbonIntensity $intensity, int $limit = 0): int
     {
         $count = 0;
         $saved = 0;
         try {
             $data = $this->fetchDay(new DateTimeImmutable(), $zone);
         } catch (AbortException $e) {
             throw $e;
         }
         $saved = $intensity->save($zone, $this->getSourceName(), $data[$zone]);
         $count += abs($saved);
         if ($limit > 0 && $count >= $limit) {
             return $saved > 0 ? $count : -$count;
@@ -388,14 +404,18 @@
         return $saved > 0 ? $count : -$count;
     }
 
-    public function fullDownload(string $zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress = null): int
+    public function fullDownload(array $source_zone, DateTimeImmutable $start_date, DateTimeImmutable $stop_date, CarbonIntensity $intensity, int $limit = 0, ?ProgressBar $progress = null): int
     {
-        // TODO : implement progress bar
+        $use_free_plan = (int) Config::getConfigurationValue('electricitymap_free_plan');
+        if ($use_free_plan === 0) {
+            return parent::fullDownload($source_zone, $start_date, $stop_date, $intensity, $limit);
+        }
+
         // Disable full download because we miss documentation for PAST_URL endpoint
         $start_date = new DateTime('24 hours ago');
         $start_date->setTime((int) $start_date->format('H'), 0, 0);
         $start_date = DateTimeImmutable::createFromMutable($start_date);
-        return $this->incrementalDownload($zone, $start_date, $intensity, $limit);
+        return $this->incrementalDownload($source_zone, $start_date, $intensity, $limit);
     }
 
     protected function sliceDateRangeByDay(DateTimeImmutable $start, DateTimeImmutable $stop)

src/DataSource/CarbonIntensity/ElectricityMaps/Config.php

@@ -62,6 +62,12 @@ public function getConfigTemplate(): string
             current_config['electricitymap_api_key'],
             __('Key for electricitymap.org API', 'carbon')
         ) }}
+
+        {{ fields.checkboxField(
+            'electricitymap_free_plan',
+            current_config['electricitymap_free_plan'] ?? 0,
+            __('Free plan', 'carbon')
+        ) }}
 TWIG;
 
         return $twig;

src/DataSource/CarbonIntensity/Rte/Client.php

@@ -164,10 +164,10 @@
      * Note that the HOUR-2 seems to be not fully guaranted.
      *
      * @param DateTimeImmutable $day date to download from [00::00:00 to 24:00:00[
-     * @param string $zone
+     * @param array $source_zone
      * @return array
      */
-    public function fetchDay(DateTimeImmutable $day, string $zone): array
+    public function fetchDay(DateTimeImmutable $day, array $source_zone): array
     {
         $stop = $day->add(new DateInterval('P1D'));
 
@@ -224,14 +224,16 @@
      *
      * @param DateTimeImmutable $start
      * @param DateTimeImmutable $stop
-     * @param string $zone
+     * @param array $source_zone
      * @param int $dataset
      * @return array
      */
-    public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, string $zone, int $dataset = self::DATASET_REALTIME): array
+    public function fetchRange(DateTimeImmutable $start, DateTimeImmutable $stop, array $source_zone, int $dataset = self::DATASET_REALTIME): array
     {
         // Build realtime and consolidated paths
-        $base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone;
+        $zone = new Zone();
+        $zone->getFromDBByCrit(['id' => $source_zone[Zone::getForeignKeyField()]]);
+        $base_path = GLPI_PLUGIN_DOC_DIR . '/carbon/carbon_intensity/' . $this->getSourceName() . '/' . $zone->fields['name'];
         $consolidated_dir = $base_path . '/consolidated';
         $realtime_dir = $base_path . '/realtime';
 
@@ -302,7 +304,7 @@
         $expected_samples_count = $expected_samples_hours * (60 / $this->step);
         $expected_samples_count--; // End boundary is excluded, decreasing the expeected count by 1
         if (($dataset === self::DATASET_REALTIME && abs(count($response) - $expected_samples_count) > (60 / $this->step))) {
             $alt_response = $this->fetchRange($start, $stop, $zone, self::DATASET_CONSOLIDATED);
             if (!isset($alt_response['error_code']) && count($alt_response) > count($response)) {
                 // Use the alternative response if more samples than the original response
                 $response = $alt_response;