Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

* Fixed binary file downloads.
* Fixed credentials cache handling.
* Added missing `env` init parameter to the `Upload` SDK class.

## 3.3.0 - 2025-06-20

Expand Down
50 changes: 23 additions & 27 deletions okdata/sdk/data/upload.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,47 +4,43 @@
from okdata.sdk import SDK
from okdata.sdk.exceptions import ApiAuthenticateError


log = logging.getLogger()


class Upload(SDK):
def __init__(self, config=None, auth=None):
def __init__(self, config=None, auth=None, env=None):
self.__name__ = "upload"
super().__init__(config, auth)
super().__init__(config, auth, env)

def upload(self, fileName, datasetid, versionid, editionid, retries=0):
def upload(self, filename, dataset_id, version, edition, retries=0):
url = self.config.get("s3BucketUrl")
log.info(f"Uploading {fileName} to {datasetid} on: {url}")
log.info(f"Uploading {filename} to {dataset_id} on: {url}")
if url is None:
raise KeyError("No s3 Bucket URL set")
raise KeyError("No S3 bucket URL set")

s3SignedData = self.create_s3_signed_data(
fileName, datasetid, versionid, editionid, retries=retries
s3_signed_data = self.create_s3_signed_data(
filename, dataset_id, version, edition, retries=retries
)
s3Data = {}
if "message" in s3SignedData:
# TODO: very specific error raised by the Lambda function, remove later
raise ApiAuthenticateError(s3SignedData["message"])
s3_data = {}
if "message" in s3_signed_data:
# TODO: Very specific error raised by the Lambda function, remove later
raise ApiAuthenticateError(s3_signed_data["message"])

for var in s3SignedData["fields"]:
s3Data[var] = s3SignedData["fields"][var]
for var in s3_signed_data["fields"]:
s3_data[var] = s3_signed_data["fields"][var]

with open(fileName, "rb") as file:
with open(filename, "rb") as file:
files = {"file": file}
upload_session = self.prepared_request_with_retries(retries=retries)
result = upload_session.post(url, data=s3Data, files=files)
trace_id = s3SignedData.get("trace_id")
data = {"result": result.status_code == 204, "trace_id": trace_id}
return data

def create_s3_signed_data(
self, fileName, datasetid, versionid, editionid, retries=0
):
edition = f"{datasetid}/{versionid}/{editionid}"
data = {"filename": os.path.basename(fileName), "editionId": edition}
result = upload_session.post(url, data=s3_data, files=files)
trace_id = s3_signed_data.get("trace_id")
return {"result": result.status_code == 204, "trace_id": trace_id}

def create_s3_signed_data(self, filename, dataset_id, version, edition, retries=0):
edition_id = f"{dataset_id}/{version}/{edition}"
data = {"filename": os.path.basename(filename), "editionId": edition_id}
url = self.config.get("uploadUrl")
log.info(f"Creating s3 signed data with payload: {data} on: {url}")
log.info(f"Creating S3 signed data with payload: {data} on: {url}")
if url is None:
raise KeyError("No Signed S3 URL set")
raise KeyError("No signed S3 URL set")
return self.post(url, data, retries=retries).json()