Emissions

Gemfile

@@ -73,7 +73,7 @@ gem 'ruby-progressbar'
 
 # own gems
 gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit'
-gem 'atlas',         ref: 'f2e3f0d', github: 'quintel/atlas'
+gem 'atlas',         ref: 'c27c4f5', github: 'quintel/atlas'
 gem 'fever',         ref: '2afebd1', github: 'quintel/fever'
 gem 'refinery',      ref: '36b8e34', github: 'quintel/refinery'
 gem 'rubel',         ref: '9fe7010', github: 'quintel/rubel'

Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/quintel/atlas.git
-  revision: f2e3f0de7a7fc9cd864fcf47ed3624c812803e83
-  ref: f2e3f0d
+  revision: c27c4f58f302d68132e2d24159f05a60761a7a44
+  ref: c27c4f5
   specs:
     atlas (1.0.0)
       activemodel (>= 7)
@@ -281,7 +281,7 @@ GEM
     factory_bot_rails (6.5.1)
       factory_bot (~> 6.5)
       railties (>= 6.1.0)
-    faraday (2.14.2)
+    faraday (2.14.1)
       faraday-net_http (>= 2.0, < 3.5)
       json
       logger
@@ -340,7 +340,7 @@ GEM
       rails-dom-testing (>= 1, < 3)
       railties (>= 4.2.0)
       thor (>= 0.14, < 2.0)
-    json (2.19.5)
+    json (2.19.7)
     json-jwt (1.17.0)
       activesupport (>= 4.2)
       aes_key_wrap

app/controllers/api/v3/export_controller.rb

@@ -63,6 +63,26 @@ def storage_parameters
         )
       end
 
+      # GET /api/v3/scenarios/:id/direct_emissions_present
+      #
+      # Returns a CSV of emissions data for all nodes in each configured node group, present graph.
+      def direct_emissions_present
+        send_csv(
+          ConfiguredCSVSerializer.new(Etsource::Config.direct_emissions_csv, @scenario.gql, period: :present),
+          'direct_emissions_present.%d.csv'
+        )
+      end
+
+      # GET /api/v3/scenarios/:id/direct_emissions_future
+      #
+      # Returns a CSV of emissions data for all nodes in each configured node group, future graph.
+      def direct_emissions_future
+        send_csv(
+          ConfiguredCSVSerializer.new(Etsource::Config.direct_emissions_csv, @scenario.gql, period: :future),
+          'direct_emissions_future.%d.csv'
+        )
+      end
+
       # GET /api/v3/scenarios/:id/costs_parameters
       #
       # Returns a CSV file containing the cost paramaters of nodes belonging to costs groups.

app/lib/node_positions.rb

@@ -22,7 +22,9 @@ class NodePositions
     :buildings    => '#FF6666',
     :neighbor     => '#87CEEB',
     :bunkers      => '#5CD0A5',
-    :molecules    => '#4A69BD'
+    :molecules    => '#4A69BD',
+    :waste        => '#969A99',
+    :lulucf       => '#C377C3'
   }.with_indifferent_access
 
   def initialize(path, node_class)

app/models/etsource/config.rb

@@ -107,6 +107,15 @@ def storage_parameters_csv
       read('storage_parameters_csv')
     end
 
+    # Public: Contains a configuration for the emissions CSV export.
+    #
+    # See ConfiguredCSVSerializer.
+    #
+    # Returns a hash.
+    def direct_emissions_csv
+      read('direct_emissions_csv')
+    end
+
 
     # Public: Contains a configuration for the residual load CSV export.
     #

app/models/etsource/dataset.rb

@@ -48,5 +48,13 @@ def self.region_codes(refresh: false)
           .map    { |dataset| dataset.key.to_s }
       end
     end
+
+    def self.emissions_keys
+      NastyCache.instance.fetch('emission_keys') do
+        Atlas::Dataset.find(
+          Etsource::Config.default_dataset_key
+        ).emissions.to_hash.keys
+      end
+    end
   end
 end

app/models/etsource/dataset/import.rb

@@ -79,6 +79,7 @@ def load_dataset_hash
 
       {
         area: load_region_data,
+        emissions: load_emission_data,
         carriers: load_carrier_data(precalculated_objects),
         energy_graph: load_energy_graph_dataset(precalculated_objects),
         molecules_graph: load_molecules_graph_dataset(precalculated_objects)
@@ -114,6 +115,13 @@ def load_region_data
       { area_data: @atlas_ds.to_hash }
     end
 
+    # Internal: Loads the emission data.
+    #
+    # Returns a hash nested under emissions_data key for DatasetAttributes compatibility.
+    def load_emission_data
+      { emissions_data: @atlas_ds.emissions.to_hash }
+    end
+
     # Internal: Loads the carrier data.
     #
     # Returns a hash, each key-pair being a carrier.

app/models/etsource/loader.rb

@@ -45,6 +45,13 @@ def area_attributes(area_code)
       end
     end
 
+    def emissions(area_code)
+      cache("emissions/#{area_code}") do
+        area_emissions = Atlas::Dataset.find(area_code).emissions
+        IceNine.deep_freeze!(area_emissions.to_hash.with_indifferent_access)
+      end
+    end
+
     # @return [Qernel::Dataset] Dataset to be used for a country. Is in a uncalculated state.
     def dataset(country)
       instrument("etsource.loader: dataset(#{country.inspect})") do

app/models/gql/runtime/functions/lookup.rb

@@ -317,6 +317,56 @@ def DATASET_CURVE(key)
         curves = Qernel::Causality::Curves.new(scope.graph, rotate: 0)
         curves.curve(key, nil).to_a
       end
+
+      # Returns an attribute {Qernel::Emissions} or {Qernel::Emissions::ScopedSector} or a value.
+      #
+      # Emissions data is loaded from CSV files in ETSource with the following structure:
+      #   etm_sector, etm_subsector, use, ghg, unit, value
+      #
+      # Parameters:
+      #   - sector: ETM sector name (e.g., 'households', 'buildings_non_specified')
+      #             Dashes/dots in sector names are converted to underscores for key generation
+      #   - use: Emission use type (energetic, non_energetic) - REQUIRED when accessing values
+      #   - ghg: GHG type (co2, other_ghg) - optional
+      #   - year: Year of emission (e.g., 1990) - optional, reads from emissions_YEAR.csv files
+      #
+      # Key generation combines: sector_[subsector_]use_ghg[_year]
+      # Note: Unit column from CSV is not included in keys, blank values return nil
+      #
+      # EMISSIONS() without any keys returns {Qernel::Emissions}
+      #
+      #   EMISSIONS() # => <Qernel::Emissions>
+      #
+      # EMISSIONS(sector, use) returns {Qernel::Emissions::ScopedSector}
+      #
+      # Which can be used to update emission factors:
+      #   UPDATE(EMISSIONS(households, energetic), co2, VALUE )
+      #   UPDATE(EMISSIONS(buildings_non_specified, energetic), other_ghg, VALUE )
+      #
+      # Examples
+      #
+      #   EMISSIONS(buildings_non_specified, energetic)
+      #   # => <Qernel::Emissions::ScopedSector buildings_non_specified_energetic>
+      #
+      # EMISSIONS(sector, use, ghg) or EMISSIONS(sector, use, ghg, year) returns an emission value
+      #
+      # Examples
+      #   EMISSIONS(households, energetic, other_ghg) # => 12.0 (from emissions.csv)
+      #   EMISSIONS(households, energetic, co2, 1990) # => value (from emissions_1990.csv)
+      #   EMISSIONS(buildings_non_specified, energetic, other_ghg) # => 18.0
+      #
+      def EMISSIONS(*keys)
+        return scope.graph.emissions if keys.empty?
+
+        # Convert dashes/dots to underscores in the first key (sector)
+        keys[0] = keys.first.to_s.tr('-.', '_').to_sym
+
+        # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations
+        return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2
+
+        # EMISSIONS(sector, use, ghg [, year]) -> return value
+        scope.graph.emissions[keys.join('_').to_sym]
+      end
     end
   end
 end

app/models/gql/runtime/functions/update.rb

@@ -145,11 +145,7 @@ def update_something_by(*value_terms)
 
       # @private
       def update_element_with(object, attribute_name, value)
-        if object.is_a?(Hash) && object.key?(attribute_name.to_sym)
-          object[attribute_name.to_sym] = value
-        else
-          object.send("#{attribute_name}=", value)
-        end
+        object.send("#{attribute_name}=", value)
       end
 
       # Private: at the moment only takes care of percentages and absolute numbers.

app/models/qernel/dataset.rb

@@ -26,6 +26,7 @@ def initialize(id = nil)
 
       @data = {
         area: { area_data: {} },
+        emissions: {},
         energy_graph: { graph: {} },
         molecules_graph: { graph: {} }
       }

app/models/qernel/edge.rb

@@ -49,7 +49,7 @@ def initialize(id, lft, rgt, carrier, type, reversed = false, groups = [], circu
       @lft_node = lft
       @rgt_node = rgt
       @carrier       = carrier
-      @groups        = groups.freeze
+      @groups        = (groups || []).freeze
       @edge_type     = type.to_sym
 
       lft_node.add_input_edge(self)
@@ -134,6 +134,10 @@ def treat_as_loss?
       !!dataset_get(:treat_as_loss)
     end
 
+    def emissions_skip_crude_oil_mix?
+      @groups.include?(:emissions_skip_crude_oil_mix)
+    end
+
     # Calculation --------------------------------------------------------------
 
     def max_demand

app/models/qernel/edge_api/energy.rb

@@ -31,6 +31,10 @@ def potential_co2_per_mj
       def primary_co2_emission_of_bio_and_fossil_without_capture_factor
         rgt_node.query.primary_co2_emission_of_bio_and_fossil_without_capture_factor
       end
+
+      def direct_output_co2_composition
+        rgt_node.query.direct_output_co2_composition
+      end
     end
   end
 end

app/models/qernel/emissions.rb

@@ -0,0 +1,82 @@
+module Qernel
+  # Class for getting and setting emissions data
+  # Behaves much like Qernel::Area, can be seen as an extension of
+  # area attributes, scoped for emissions
+  #
+  # Emissions data is loaded from CSV files in ETSource with structure:
+  #   etm_sector, etm_subsector, use, ghg, unit, value
+  #
+  # Keys are generated as: sector_[subsector_]use_ghg[_year]
+  # Example: buildings_non_specified_energetic_co2
+  class Emissions
+    include DatasetAttributes
+
+    dataset_accessors ::Etsource::Dataset.emissions_keys
+    attr_accessor :graph
+
+    # Queryable object that provides scoped access to emissions data
+    #
+    # GQL uses this to scope the sector for easier queries and input/update
+    # statements in etsource
+    #
+    # Example:
+    #   EMISSIONS(households, energetic) returns a ScopedSector
+    #   Then UPDATE can call: scoped.co2 = 100.0
+    class ScopedSector
+      def initialize(emissions, scope)
+        @emissions = emissions
+        @scope = scope
+      end
+
+      def [](attr_name)
+        @emissions[scoped_method(attr_name).to_sym]
+      end
+
+      def []=(attr_name, value)
+        @emissions[scoped_method(attr_name).to_sym] = value
+      end
+
+      def inspect
+        "<Qernel::Emissions::ScopedSector #{@scope}>"
+      end
+
+      def scoped_method(method_name)
+        "#{@scope}_#{method_name}"
+      end
+
+      def respond_to_missing?(method_name, include_private = false)
+        data_key = scoped_method(method_name).split('=').first
+
+        @emissions.respond_to?(data_key) || super
+      end
+
+      def method_missing(method_name, *args)
+        data_key = scoped_method(method_name).split('=').first.to_sym
+
+        # Validate the key exists for both getters and setters
+        unless @emissions.respond_to?(data_key)
+          raise NoMethodError, "undefined method `#{method_name}' for #{inspect}"
+        end
+
+        if data_key.to_s == scoped_method(method_name)
+          @emissions[data_key]
+        else
+          @emissions[data_key] = args.first
+        end
+      end
+    end
+
+    def initialize(graph = nil)
+      self.graph = graph unless graph.nil?
+
+      @dataset_key = @key = :emissions_data
+    end
+
+    # Public: define the sector scope for access to the hashed emission keys
+    #
+    # Returns a scoped version of the emissions data
+    def scope(sector)
+      ScopedSector.new(self, sector)
+    end
+  end
+end

app/models/qernel/graph.rb

@@ -59,7 +59,8 @@ def graph
 
   attr_accessor :dataset,
                 :finished_nodes,
-                :area
+                :area,
+                :emissions
 
   delegate :weather_properties, to: :area
 
@@ -71,6 +72,7 @@ def self.dataset_group_with_name(name)
   def initialize(nodes = [], name = :anonymous)
     @logger = Qernel::Logger.new
     @area   = Qernel::Area.new(self)
+    @emissions = Qernel::Emissions.new(self)
     @name   = name.to_sym
 
     @nodes_by_group = {}
@@ -189,6 +191,8 @@ def remove_dataset_attributes
   def refresh_dataset_attributes
     # See Qernel::Dataset#assign_dataset_attributes to understand what's going on:
     call_on_each_qernel_object(:assign_dataset_attributes)
+    # Assign emissions data to the Emissions object via DatasetAttributes
+    @emissions.assign_dataset_attributes if @emissions
     reset_goals
   end
 

app/models/qernel/graph_parser.rb

@@ -10,7 +10,8 @@ class GraphParser
       'electricity'      => Carrier.new(id: 5, key: 'electricity', infinite: false),
       'cooling'          => Carrier.new(id: 6, key: 'cooling', infinite: false),
       'useable_heat'     => Carrier.new(id: 7, key: 'useable_heat', infinite: false),
-      'coupling_carrier' => Carrier.new(id: 8, key: 'coupling_carrier', infinite: false)
+      'coupling_carrier' => Carrier.new(id: 8, key: 'coupling_carrier', infinite: false),
+      'other_ghg'        => Carrier.new(id: 9, key: 'other_ghg',        infinite: false)
     }
 
     CARRIERS = {}.merge(CARRIERS_FOR_SPECS)