SK-1889: Handled partial error cases when continue on error is true

Author: saileshwar-skyflow

skyflow/generated/rest/api/tokens_api.py

@@ -174,6 +174,7 @@ def record_service_detokenize_with_http_info(
 
         _response_types_map: Dict[str, Optional[str]] = {
             '200': "V1DetokenizeResponse",
+            '207': "V1DetokenizeResponse",
             '404': "object",
         }
         response_data = self.api_client.call_api(

skyflow/utils/_utils.py

@@ -12,7 +12,7 @@
 from urllib.parse import quote
 from skyflow.error import SkyflowError
 from skyflow.generated.rest import V1UpdateRecordResponse, V1BulkDeleteRecordResponse, \
-    V1DetokenizeResponse, V1TokenizeResponse, V1GetQueryResponse, V1BulkGetRecordResponse
+    V1DetokenizeResponse, V1TokenizeResponse, V1GetQueryResponse, V1BulkGetRecordResponse, ApiResponse
 from skyflow.utils.logger import log_error, log_error_log
 from . import SkyflowMessages, SDK_VERSION
 from .enums import Env, ContentType, EnvUrls
@@ -194,8 +194,9 @@ def parse_insert_response(api_response, continue_on_error):
     inserted_fields = []
     errors = []
     insert_response = InsertResponse()
+    response_data = json.loads(api_response.raw_data.decode('utf-8'))
     if continue_on_error:
-        for idx, response in enumerate(api_response.responses):
+        for idx, response in enumerate(response_data.get('responses', [])):
             if response['Status'] == 200:
                 body = response['Body']
                 if 'records' in body:
@@ -210,6 +211,7 @@ def parse_insert_response(api_response, continue_on_error):
                         inserted_fields.append(inserted_field)
             elif response['Status'] == 400:
                 error = {
+                    'request_id': api_response.headers.get('x-request-id'),
                     'request_index': idx,
                     'error': response['Body']['error']
                 }
@@ -219,7 +221,7 @@ def parse_insert_response(api_response, continue_on_error):
             insert_response.errors = errors
 
     else:
-        for record in api_response.records:
+        for record in response_data.get('records', []):
             field_data = {
                 'skyflow_id': record.skyflow_id
             }
@@ -264,13 +266,14 @@ def parse_get_response(api_response: V1BulkGetRecordResponse):
 
     return get_response
 
-def parse_detokenize_response(api_response: V1DetokenizeResponse):
+def parse_detokenize_response(api_response: ApiResponse[V1DetokenizeResponse]):
     detokenized_fields = []
     errors = []
 
-    for record in api_response.records:
+    for record in api_response.data.records:
         if record.error:
             errors.append({
+                "request_id": api_response.headers.get('x-request-id'),
                 "token": record.token,
                 "error": record.error
             })

skyflow/utils/enums/env.py

@@ -1,13 +1,13 @@
 from enum import Enum
 
 class Env(Enum):
-    DEV = 'DEV',
-    SANDBOX = 'SANDBOX',
+    DEV = 'DEV'
+    SANDBOX = 'SANDBOX'
     PROD = 'PROD'
     STAGE = 'STAGE'
 
 class EnvUrls(Enum):
-    PROD = "vault.skyflowapis.com",
-    SANDBOX = "vault.skyflowapis-preview.com",
+    PROD = "vault.skyflowapis.com"
+    SANDBOX = "vault.skyflowapis-preview.com"
     DEV = "vault.skyflowapis.dev"
     STAGE = "vault.skyflowapis.tech"

skyflow/utils/validations/_validations.py

@@ -514,15 +514,15 @@ def validate_detokenize_request(logger, request):
         raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENS_LIST_VALUE.value, invalid_input_error_code)
 
     for item in request.data:
-        if 'token' not in item or 'redaction' not in item:
+        if 'token' not in item:
             raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_LIST_VALUE.value(type(request.data)), invalid_input_error_code)
         token = item.get('token')
-        redaction = item.get('redaction')
+        redaction = item.get('redaction', RedactionType.PLAIN_TEXT)
 
         if not isinstance(token, str) or not token:
             raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_TYPE.value.format("DETOKENIZE"), invalid_input_error_code)
 
-        if not isinstance(redaction, RedactionType) or not redaction:
+        if redaction is not None and not isinstance(redaction, RedactionType):
             raise SkyflowError(SkyflowMessages.Error.INVALID_REDACTION_TYPE.value.format(type(redaction)), invalid_input_error_code)
 
 def validate_tokenize_request(logger, request):

skyflow/vault/controller/_vault.py

@@ -6,6 +6,7 @@
 from skyflow.utils import SkyflowMessages, parse_insert_response, \
     handle_exception, parse_update_record_response, parse_delete_response, parse_detokenize_response, \
     parse_tokenize_response, parse_query_response, parse_get_response, encode_column_values
+from skyflow.utils.enums import RedactionType
 from skyflow.utils.logger import log_info, log_error_log
 from skyflow.utils.validations import validate_insert_request, validate_delete_request, validate_query_request, \
     validate_get_request, validate_update_request, validate_detokenize_request, validate_tokenize_request
@@ -89,7 +90,7 @@ def insert(self, request: InsertRequest):
             log_info(SkyflowMessages.Info.INSERT_TRIGGERED.value, self.__vault_client.get_logger())
 
             if request.continue_on_error:
-                api_response = records_api.record_service_batch_operation(self.__vault_client.get_vault_id(),
+                api_response = records_api.record_service_batch_operation_with_http_info(self.__vault_client.get_vault_id(),
                                                                           insert_body)
 
             else:
@@ -230,14 +231,17 @@ def detokenize(self, request: DetokenizeRequest):
         log_info(SkyflowMessages.Info.DETOKENIZE_REQUEST_RESOLVED.value, self.__vault_client.get_logger())
         self.__initialize()
         tokens_list = [
-            V1DetokenizeRecordRequest(token=item.get('token'), redaction=item.get('redaction').value)
+            V1DetokenizeRecordRequest(
+                token=item.get('token'),
+                redaction=item.get('redaction').value if item.get('redaction') else RedactionType.PLAIN_TEXT.value
+            )
             for item in request.data
         ]
         payload = V1DetokenizePayload(detokenization_parameters=tokens_list, continue_on_error=request.continue_on_error)
         tokens_api = self.__vault_client.get_tokens_api()
         try:
             log_info(SkyflowMessages.Info.DETOKENIZE_TRIGGERED.value, self.__vault_client.get_logger())
-            api_response = tokens_api.record_service_detokenize(
+            api_response = tokens_api.record_service_detokenize_with_http_info(
                 self.__vault_client.get_vault_id(),
                 detokenize_payload=payload
             )