Fixed Code for 3 files

scripts/un/energy/download.py

@@ -83,20 +83,19 @@ def download_energy_dataset(
     Returns:
       A list of output files downloaded.
     """
+    output_files = []
     supported_datasets = get_all_energy_source_codes()
     if energy_dataset not in supported_datasets:
         logging.info(
             f'Dataset "{energy_dataset}" not in list of supported codes:' +
             str(supported_datasets))
         return output_files
     # Download data in batches of years as the download has a limit of 100k rows.
-    years_list = list(range(start_year, years_per_batch + 1))
     years_list = [str(y) for y in range(start_year, end_year + 1)]
     batch_years = [
         years_list[i:i + years_per_batch]
         for i in range(0, len(years_list), years_per_batch)
     ]
-    output_files = []
     for year_batch in batch_years:
         start_year = year_batch[0]
         end_year = year_batch[-1]
@@ -117,7 +116,8 @@ def download_energy_dataset(
         if download_successful:
             logging.info(f"Download of '{download_url}' completed.")
             for f in os.listdir(output):
-                output_files.append(os.path.join(output, f))
+                if f.endswith('.csv'):
+                    output_files.append(os.path.join(output, f))
         else:
             logging.fatal(f"Download or processing of '{download_url}' failed")
     return output_files
@@ -141,10 +141,11 @@ def download_un_energy_dataset() -> list:
     logging.info(f'Downloaded the following files: {output_files}')
     return output_files
 
-
 def main(_):
     download_un_energy_dataset()
 
-
 if __name__ == '__main__':
     app.run(main)
+
+
+

scripts/un/energy/process.py

@@ -545,6 +545,9 @@ def process(in_paths: list,
         with open(mcf_file_path, 'w+', newline='') as f_out_mcf:
             # Process each CSV input file, one row at a time.
             for in_file in in_paths:
+                if not in_file.endswith('.csv'):
+                    logging.info(f'Skipping non-CSV file: {in_file}')
+                    continue
                 logging.info(f'Processing data file: {in_file}')
                 with open(in_file) as csvfile:
                     counters['input_files'] += 1

scripts/un/energy/process_test.py

@@ -67,5 +67,4 @@ def test_un_energy_process(self):
 
 
 if __name__ == '__main__':
-    app.run()
     unittest.main()