Add API pagination

README.rst

@@ -148,6 +148,21 @@ Delete a migration job:
 
 Note: only completed, failed or cancelled jobs can be deleted.
 
+The following Coriolis APIs support pagination:
+* transfers
+* transfer executions
+* deployments
+* endpoint instances (only marker and limit parameters)
+
+Pagination parameters:
+* ``sort_key`` - sort key, repeatable. `created_at` and `id` are used by default.
+* ``sort_dir`` - sort direction, repeatable. `asc` or `desc` (default).
+* ``marker`` - the last seen ID, omitted from the results.
+* ``limit`` - the maximum number of records to retrieve.
+
+Example:
+
+    GET http://server:7667/v1/transfers?marker=a7061715-e56c-470c-a6ac-80bb02f1f198&limit=2&sort_key=id&sort_dir=asc
 
 API Documentation
 -----------------

coriolis/api/common.py

@@ -10,3 +10,36 @@ def get_paging_params(req):
     if limit is not None:
         limit = utils.parse_int_value(limit)
     return marker, limit
+
+
+def get_sort_params(req,
+                    default_keys=('created_at', 'id'),
+                    default_dirs=('desc', 'desc')):
+    """Retrieves sort keys/directions parameters.
+
+    Processes the parameters to create a list of sort keys and sort directions
+    that correspond to the 'sort_key' and 'sort_dir' parameter values. These
+    sorting parameters can be specified multiple times in order to generate
+    the list of sort keys and directions.
+
+    The input parameters are not modified.
+
+    :param req: coriolis.api.wsgi.Request object
+    :param default_keys: default sort key values, added to the list if no
+                         'sort_key' parameters are supplied
+    :param default_dirs: default sort dir values, added to the list if no
+                         'sort_dir' parameters are supplied
+    :returns: list of sort keys, list of sort dirs
+    """
+    params = req.params.copy()
+    sort_keys = []
+    sort_dirs = []
+    while 'sort_key' in params:
+        sort_keys.append(params.pop('sort_key').strip())
+    while 'sort_dir' in params:
+        sort_dirs.append(params.pop('sort_dir').strip())
+    if len(sort_keys) == 0 and default_keys:
+        sort_keys.extend(default_keys)
+    if len(sort_dirs) == 0 and default_dirs:
+        sort_dirs.extend(default_dirs)
+    return sort_keys, sort_dirs

coriolis/api/v1/deployments.py

@@ -4,6 +4,7 @@
 from oslo_log import log as logging
 from webob import exc
 
+from coriolis.api import common
 from coriolis.api.v1 import utils as api_utils
 from coriolis.api.v1.views import deployment_view
 from coriolis.api import wsgi as api_wsgi
@@ -43,11 +44,17 @@ def _list(self, req):
         context.can(deployment_policies.get_deployments_policy_label("list"))
         include_task_info = api_utils.get_bool_url_arg(
             req, "include_task_info", default=False)
+
+        marker, limit = common.get_paging_params(req)
+        sort_keys, sort_dirs = common.get_sort_params(req)
+
         return deployment_view.collection(
             self._deployment_api.get_deployments(
                 context,
                 include_tasks=include_task_info,
-                include_task_info=include_task_info
+                include_task_info=include_task_info,
+                marker=marker, limit=limit,
+                sort_keys=sort_keys, sort_dirs=sort_dirs,
             ))
 
     def index(self, req):

coriolis/api/v1/transfer_tasks_executions.py

@@ -1,6 +1,7 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+from coriolis.api import common
 from coriolis.api.v1.views import transfer_tasks_execution_view
 from coriolis.api import wsgi as api_wsgi
 from coriolis import exception
@@ -31,9 +32,14 @@ def index(self, req, transfer_id):
         context.can(
             executions_policies.get_transfer_executions_policy_label("list"))
 
+        marker, limit = common.get_paging_params(req)
+        sort_keys, sort_dirs = common.get_sort_params(req)
+
         return transfer_tasks_execution_view.collection(
             self._transfer_tasks_execution_api.get_executions(
-                context, transfer_id, include_tasks=False))
+                context, transfer_id, include_tasks=False,
+                marker=marker, limit=limit,
+                sort_keys=sort_keys, sort_dirs=sort_dirs))
 
     def detail(self, req, transfer_id):
         context = req.environ["coriolis.context"]

coriolis/api/v1/transfers.py

@@ -1,6 +1,7 @@
 # Copyright 2016 Cloudbase Solutions Srl
 # All Rights Reserved.
 
+from coriolis.api import common
 from coriolis.api.v1 import utils as api_utils
 from coriolis.api.v1.views import transfer_tasks_execution_view
 from coriolis.api.v1.views import transfer_view
@@ -49,11 +50,16 @@ def _list(self, req):
         context.can(transfer_policies.get_transfers_policy_label("list"))
         include_task_info = api_utils.get_bool_url_arg(
             req, "include_task_info", default=False)
+        marker, limit = common.get_paging_params(req)
+        sort_keys, sort_dirs = common.get_sort_params(req)
         return transfer_view.collection(
             self._transfer_api.get_transfers(
                 context,
                 include_tasks_executions=include_task_info,
-                include_task_info=include_task_info))
+                include_task_info=include_task_info,
+                marker=marker, limit=limit,
+                sort_keys=sort_keys, sort_dirs=sort_dirs,
+            ))
 
     def index(self, req):
         return self._list(req)

coriolis/conductor/rpc/client.py

@@ -143,11 +143,20 @@ def execute_transfer_tasks(self, ctxt, transfer_id,
             shutdown_instances=shutdown_instances, auto_deploy=auto_deploy)
 
     def get_transfer_tasks_executions(self, ctxt, transfer_id,
-                                      include_tasks=False):
+                                      include_tasks=False,
+                                      marker=None,
+                                      limit=None,
+                                      sort_keys=None,
+                                      sort_dirs=None):
         return self._call(
             ctxt, 'get_transfer_tasks_executions',
             transfer_id=transfer_id,
-            include_tasks=include_tasks)
+            include_tasks=include_tasks,
+            marker=marker,
+            limit=limit,
+            sort_keys=sort_keys,
+            sort_dirs=sort_dirs,
+        )
 
     def get_transfer_tasks_execution(self, ctxt, transfer_id, execution_id,
                                      include_task_info=False):
@@ -197,11 +206,18 @@ def create_instances_transfer(self, ctxt,
             skip_os_morphing=skip_os_morphing)
 
     def get_transfers(self, ctxt, include_tasks_executions=False,
-                      include_task_info=False):
+                      include_task_info=False,
+                      marker=None, limit=None,
+                      sort_keys=None, sort_dirs=None):
         return self._call(
             ctxt, 'get_transfers',
             include_tasks_executions=include_tasks_executions,
-            include_task_info=include_task_info)
+            include_task_info=include_task_info,
+            marker=marker,
+            limit=limit,
+            sort_keys=sort_keys,
+            sort_dirs=sort_dirs,
+        )
 
     def get_transfer(self, ctxt, transfer_id, include_task_info=False):
         return self._call(
@@ -217,10 +233,17 @@ def delete_transfer_disks(self, ctxt, transfer_id):
             ctxt, 'delete_transfer_disks', transfer_id=transfer_id)
 
     def get_deployments(self, ctxt, include_tasks=False,
-                        include_task_info=False):
+                        include_task_info=False,
+                        marker=None, limit=None,
+                        sort_keys=None, sort_dirs=None):
         return self._call(
             ctxt, 'get_deployments', include_tasks=include_tasks,
-            include_task_info=include_task_info)
+            include_task_info=include_task_info,
+            marker=marker,
+            limit=limit,
+            sort_keys=sort_keys,
+            sort_dirs=sort_dirs,
+        )
 
     def get_deployment(self, ctxt, deployment_id, include_task_info=False):
         return self._call(

coriolis/conductor/rpc/server.py

@@ -1152,10 +1152,19 @@ def execute_transfer_tasks(self, ctxt, transfer_id, shutdown_instances,
     @transfer_synchronized
     def get_transfer_tasks_executions(self, ctxt, transfer_id,
                                       include_tasks=False,
-                                      include_task_info=False):
+                                      include_task_info=False,
+                                      marker=None,
+                                      limit=None,
+                                      sort_keys=None,
+                                      sort_dirs=None):
         return db_api.get_transfer_tasks_executions(
             ctxt, transfer_id, include_tasks,
-            include_task_info=include_task_info, to_dict=True)
+            include_task_info=include_task_info,
+            marker=marker,
+            limit=limit,
+            sort_keys=sort_keys,
+            sort_dirs=sort_dirs,
+            to_dict=True)
 
     @tasks_execution_synchronized
     def get_transfer_tasks_execution(self, ctxt, transfer_id, execution_id,
@@ -1206,10 +1215,17 @@ def _get_transfer_tasks_execution(ctxt, transfer_id, execution_id,
 
     @staticmethod
     def get_transfers(ctxt, include_tasks_executions=False,
-                      include_task_info=False):
+                      include_task_info=False,
+                      marker=None, limit=None,
+                      sort_keys=None, sort_dirs=None):
         return db_api.get_transfers(
             ctxt, include_tasks_executions=include_tasks_executions,
-            include_task_info=include_task_info, to_dict=True)
+            include_task_info=include_task_info,
+            marker=marker,
+            limit=limit,
+            sort_keys=sort_keys,
+            sort_dirs=sort_dirs,
+            to_dict=True)
 
     @transfer_synchronized
     def get_transfer(self, ctxt, transfer_id, include_task_info=False):
@@ -1365,10 +1381,16 @@ def _get_transfer(self, ctxt, transfer_id, include_task_info=False,
         return transfer
 
     @staticmethod
-    def get_deployments(ctxt, include_tasks, include_task_info=False):
+    def get_deployments(ctxt, include_tasks, include_task_info=False,
+                        marker=None, limit=None,
+                        sort_keys=None, sort_dirs=None):
         return db_api.get_deployments(
             ctxt, include_tasks,
             include_task_info=include_task_info,
+            marker=marker,
+            limit=limit,
+            sort_keys=sort_keys,
+            sort_dirs=sort_dirs,
             to_dict=True)
 
     @deployment_synchronized