Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions arkouda/numpy/pdarrayclass.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from functools import reduce
from math import ceil
from sys import modules
from typing import TYPE_CHECKING, List, Optional, Tuple, Union, cast, overload
from typing import TYPE_CHECKING, List, Literal, Optional, Tuple, Union, cast, overload

import numpy as np

Expand Down Expand Up @@ -2831,7 +2831,7 @@ def to_parquet(
self,
prefix_path: str,
dataset: str = "array",
mode: str = "truncate",
mode: Literal["truncate", "append"] = "truncate",
compression: Optional[str] = None,
) -> str:
"""
Expand All @@ -2846,7 +2846,7 @@ def to_parquet(
Directory and filename prefix that all output files share
dataset : str
Name of the dataset to create in files (must not already exist)
mode : str {'truncate' | 'append'}
mode : {'truncate', 'append'}
By default, truncate (overwrite) output files, if they exist.
If 'append', attempt to create new dataset in existing files.
compression : str (Optional)
Expand Down Expand Up @@ -2914,8 +2914,8 @@ def to_hdf(
self,
prefix_path: str,
dataset: str = "array",
mode: str = "truncate",
file_type: str = "distribute",
mode: Literal["truncate", "append"] = "truncate",
file_type: Literal["single", "distribute"] = "distribute",
) -> str:
"""
Save the pdarray to HDF5.
Expand All @@ -2927,10 +2927,10 @@ def to_hdf(
Directory and filename prefix that all output files share
dataset : str
Name of the dataset to create in files (must not already exist)
mode : str {'truncate' | 'append'}
mode : {'truncate', 'append'}
By default, truncate (overwrite) output files, if they exist.
If 'append', attempt to create new dataset in existing files.
file_type: str ("single" | "distribute")
file_type: {"single", "distribute"}
Default: "distribute"
When set to single, dataset is written to a single file.
When distribute, dataset is written on a file per locale.
Expand Down
4 changes: 2 additions & 2 deletions arkouda/numpy/segarray.py
Original file line number Diff line number Diff line change
Expand Up @@ -838,8 +838,8 @@ def to_hdf(
self,
prefix_path,
dataset="segarray",
mode="truncate",
file_type="distribute",
mode: Literal["truncate", "append"] = "truncate",
file_type: Literal["single", "distribute"] = "distribute",
):
"""
Save the SegArray to HDF5. The result is a collection of HDF5 files, one file
Expand Down
6 changes: 3 additions & 3 deletions arkouda/numpy/timeclass.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import datetime
import json

from typing import Optional, Union
from typing import Literal, Optional, Union

import numpy as np

Expand Down Expand Up @@ -226,8 +226,8 @@ def to_hdf(
self,
prefix_path: str,
dataset: str = "array",
mode: str = "truncate",
file_type: str = "distribute",
mode: Literal["truncate", "append"] = "truncate",
file_type: Literal["single", "distribute"] = "distribute",
):
"""Override of the pdarray to_hdf to store the special dtype."""
from arkouda.client import generic_msg
Expand Down
5 changes: 3 additions & 2 deletions arkouda/pandas/categorical.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
Any,
Dict,
List,
Literal,
Optional,
Sequence,
Tuple,
Expand Down Expand Up @@ -1233,8 +1234,8 @@ def to_hdf(
self,
prefix_path,
dataset="categorical_array",
mode="truncate",
file_type="distribute",
mode: Literal["truncate", "append"] = "truncate",
file_type: Literal["single", "distribute"] = "distribute",
):
"""
Save the Categorical to HDF5.
Expand Down
5 changes: 4 additions & 1 deletion arkouda/pandas/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
Callable,
Dict,
List,
Literal,
Optional,
Tuple,
TypeVar,
Expand Down Expand Up @@ -2782,7 +2783,9 @@ def _prep_data(self, index=False, columns=None):
data["Index"] = self.index.values
return data

def to_hdf(self, path, index=False, columns=None, file_type="distribute"):
def to_hdf(
self, path, index=False, columns=None, file_type: Literal["single", "distribute"] = "distribute"
):
"""
Save DataFrame to disk as hdf5, preserving column names.

Expand Down
5 changes: 3 additions & 2 deletions arkouda/pandas/groupbyclass.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
TYPE_CHECKING,
Dict,
List,
Literal,
Optional,
Sequence,
Tuple,
Expand Down Expand Up @@ -470,8 +471,8 @@ def to_hdf(
self,
prefix_path,
dataset="groupby",
mode="truncate",
file_type="distribute",
mode: Literal["truncate", "append"] = "truncate",
file_type: Literal["single", "distribute"] = "distribute",
):
"""
Save the GroupBy to HDF5.
Expand Down
8 changes: 6 additions & 2 deletions arkouda/pandas/io.py
Original file line number Diff line number Diff line change
Expand Up @@ -1561,20 +1561,24 @@ def to_hdf(
mode = "append"


def _get_hdf_filetype(filename: str) -> str:
def _get_hdf_filetype(filename: str) -> Literal["single", "distribute"]:
from arkouda.client import generic_msg

if not (filename and filename.strip()):
raise ValueError("filename cannot be an empty string")

cmd = "hdffileformat"
return cast(
result = cast(
str,
generic_msg(
cmd=cmd,
args={"filename": filename},
),
)
if result not in ("single", "distribute"):
raise ValueError(f"Unexpected file type: {result!r}")

return cast(Literal["single", "distribute"], result)


def _repack_hdf(prefix_path: str):
Expand Down