Add support for OpenSearch API as ancillary data source

example-config.yaml

@@ -290,3 +290,21 @@ rules:
       command: 'swfo-convert mcd43a1 h5-md --fname {brdf_base}/{collection}/{ymd}/{basename}{hdf_ext} --outdir /tmp/data/conversion/BRDF/{collection}/{ymd}/ --filter-opts ''{{"aggression": 6}}'' --compression BLOSC_ZSTANDARD'
       input_files: ['^(?P<brdf_base>.*BRDF)/(?P<collection>.*)/(?P<ymd>[0-9]{4}\.[0-9]{2}\.[0-9]{2})/(?P<basename>.*)(?P<hdf_ext>.hdf)(?P<xml_ext>.xml)?', ['{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}', '{brdf_base}/{collection}/{ymd}/{basename}{hdf_ext}.xml']]
       expect_file: '/tmp/data/conversion/BRDF/{collection}/{ymd}/{basename}.h5'
+
+#
+#-------------------------------
+# Example using the OpenSearch API |
+#-------------------------------
+#
+  RESORB:
+    schedule: '0 */6 * * *'
+    source: !opensearch-api
+      username: <username>
+      password: <password>
+      api_url: https://scihub.copernicus.eu/gnss
+      query:
+        platformname: Sentinel-1
+        producttype: AUX_RESORB
+        generationdate: '[NOW-5DAYS TO NOW]'
+      filename_transform: !regexp-extract '(?P<platform>S1[A,B]{1})_[A-Z].*_[A-Z].*_(?P<product>[A-Z].*)_[A-Z].*_(?P<creation_date>[0-9,T]{15})_V(?P<valid_start_date>[0-9,T]{15})_(?P<valid_stop_date>[0-9,T]{15}).EOF'
+      target_dir: /tmp/data/sensor-specific/Sentinel-1/{product}/{platform}

fetch/load.py

@@ -13,7 +13,7 @@
 import yaml.resolver
 from croniter import croniter
 
-from . import ftp, http, ecmwf
+from . import ftp, http, ecmwf, opensearch
 from ._core import RegexpOutputPathTransform, DateRangeSource, DateFilenameTransform, \
     RsyncMirrorSource, SimpleObject, ShellFileProcessor
 from .util import remove_nones
@@ -367,6 +367,7 @@ def add_item_constructor(source, type_annotation, attribute):
     add_default_constructor(http.HttpPostAction, '!http-post')
     add_default_constructor(http.HttpAuthAction, '!http-auth')
     add_default_constructor(ecmwf.EcmwfApiSource, '!ecmwf-api')
+    add_default_constructor(opensearch.OpenSearchApiSource, '!opensearch-api')
     add_item_constructor(RegexpOutputPathTransform, '!regexp-extract', 'pattern')
     add_item_constructor(DateFilenameTransform, '!date-pattern', 'format_')
 

fetch/opensearch.py

@@ -0,0 +1,64 @@
+from __future__ import absolute_import
+
+import logging
+import os
+
+from sentinelsat import SentinelAPI
+
+from ._core import DataSource, fetch_file
+
+DEFAULT_CONNECT_TIMEOUT_SECS = 100
+
+_log = logging.getLogger(__name__)
+
+
+class OpenSearchApiSource(DataSource):
+    """
+    Class for data retrievals using the OpenSearch API.
+    """
+
+    def __init__(self, target_dir, api_url, username, password, query, show_progressbars=False,
+                 timeout=DEFAULT_CONNECT_TIMEOUT_SECS, filename_transform=None, override_existing=False):
+        self.target_dir = target_dir
+        self.filename_transform = filename_transform
+        self.override_existing = override_existing
+        self.api_url = api_url
+        self.username = username
+        self.password = password
+        self.query = query
+        self.show_progressbars = show_progressbars
+        self.timeout = timeout
+
+        self.api = SentinelAPI(self.username, self.password, self.api_url, self.show_progressbars, self.timeout)
+
+    def trigger(self, reporter):
+        """
+        :type reporter: ResultHandler
+        """
+
+        query_results = self.api.query(**self.query)
+
+        for (uuid, result) in query_results.items():
+            _log.info('Found %s with uuid %s', result['filename'], uuid)
+
+            def create_fetch_function(key):
+                def opensearch_fetch(target):
+                    download = self.api.download(key)
+
+                    # Workaround for fixed filename
+                    _log.debug('Renaming %s to %s', download['path'], target)
+                    os.rename(download['path'], target)
+
+                    return True
+
+                return opensearch_fetch
+
+            fetch_file(
+                result['link'].replace("'", '%27'),
+                create_fetch_function(uuid),
+                reporter,
+                os.path.basename(result['filename']),
+                self.target_dir,
+                filename_transform=self.filename_transform,
+                override_existing=self.override_existing
+            )

setup.py

@@ -23,6 +23,7 @@
           'croniter',
           'feedparser',
           'lxml',
+          'sentinelsat',
           'pathlib;python_version<"3.4"',
           'pyyaml<5.1',
           'requests>=2.21.0',