Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 44 additions & 27 deletions packages/data/.pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,32 +1,49 @@
---
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v3.2.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: check-yaml
- id: check-added-large-files
args: ['--maxkb=10485760']
- id: check-byte-order-marker
- id: check-executables-have-shebangs
- repo: https://github.com/psf/black
rev: 22.3.0
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 22.10.0
hooks:
- id: black
args: ['--line-length', '120']
# - repo: https://github.com/PyCQA/flake8
# rev: 3.9.2
# hooks:
# - id: flake8
# args: ['--max-line-length', '120', "--extend-ignore", "E203"]
- repo: https://github.com/Quantco/pre-commit-mirrors-shellcheck
rev: 0.7.1
- id: black
- repo: https://github.com/pycqa/isort
rev: 6.0.1
hooks:
- id: shellcheck-conda
- repo: https://github.com/antonbabenko/pre-commit-terraform
rev: v1.64.0
hooks:
- id: terraform_validate
- id: isort


# ---
# # See https://pre-commit.com for more information
# # See https://pre-commit.com/hooks.html for more hooks
# repos:
# - repo: https://github.com/pre-commit/pre-commit-hooks
# rev: v3.2.0
# hooks:
# - id: trailing-whitespace
# - id: end-of-file-fixer
# - id: check-yaml
# - id: check-added-large-files
# args: ['--maxkb=10485760']
# - id: check-byte-order-marker
# - id: check-executables-have-shebangs
# - repo: https://github.com/psf/black
# rev: 22.3.0
# hooks:
# - id: black
# # - repo: https://github.com/PyCQA/flake8
# # rev: 3.9.2
# # hooks:
# # - id: flake8
# # args: ['--max-line-length', '120', "--extend-ignore", "E203"]
# - repo: https://github.com/Quantco/pre-commit-mirrors-shellcheck
# rev: 0.7.1
# hooks:
# - id: shellcheck-conda
# - repo: https://github.com/antonbabenko/pre-commit-terraform
# rev: v1.64.0
# hooks:
# - id: terraform_validate

88 changes: 47 additions & 41 deletions packages/data/src/pyearthtools/data/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,65 +78,71 @@

__version__ = "0.1.0"

from pyearthtools.data import logger
from pyearthtools.data import config

from pyearthtools.data.time import Petdt, TimeResolution, TimeDelta, TimeRange
from pyearthtools.data.time import Petdt as datetime
import warnings as __python_warnings

from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError
from pyearthtools.data.warnings import (
IndexWarning,
pyearthtoolsDataWarning,
AccessorRegistrationWarning,
import pyearthtools.utils
from pyearthtools.data import (
archive,
config,
derived,
download,
indexes,
logger,
modifications,
)


from pyearthtools.data import operations
from pyearthtools.data import operations as op
from pyearthtools.data import (
patterns,
save,
static,
)
from pyearthtools.data import transforms
from pyearthtools.data import transforms as transform
from pyearthtools.data import (
utils,
)
from pyearthtools.data.archive.utils import auto_import
from pyearthtools.data.collection import Collection, LabelledCollection

# from pyearthtools.data.catalog import Catalog, CatalogEntry

from pyearthtools.data.exceptions import DataNotFoundError, InvalidIndexError
from pyearthtools.data.indexes import (
Index,
DataIndex,
FileSystemIndex,
TimeIndex,
AdvancedTimeIndex,
AdvancedTimeDataIndex,
AdvancedTimeIndex,
ArchiveIndex,
BaseTimeIndex,
CachingForecastIndex,
CachingIndex,
DataFileSystemIndex,
ArchiveIndex,
DataIndex,
FileSystemIndex,
ForecastIndex,
StaticDataIndex,
CachingIndex,
CachingForecastIndex,
Index,
IntakeIndex,
IntakeIndexCache,
StaticDataIndex,
TimeIndex,
register_accessor,
)
from pyearthtools.data import indexes
from pyearthtools.data.indexes import register_accessor

from pyearthtools.data import operations as op

from pyearthtools.data import archive, operations, static, transforms, patterns, download, modifications, derived, utils
from pyearthtools.data import transforms as transform
from pyearthtools.data.load import load
from pyearthtools.data.patterns import PatternIndex

from pyearthtools.data.save import ManageFiles, ManageTemp
from pyearthtools.data.time import Petdt
from pyearthtools.data.time import Petdt as datetime
from pyearthtools.data.time import TimeDelta, TimeRange, TimeResolution
from pyearthtools.data.transforms.derive import evaluate
from pyearthtools.data.transforms.transform import (
FunctionTransform,
Transform,
TransformCollection,
FunctionTransform,
)
from pyearthtools.data.transforms.derive import evaluate
from pyearthtools.data import save
from pyearthtools.data.save import ManageFiles, ManageTemp

from pyearthtools.data.load import load
from pyearthtools.data.warnings import (
AccessorRegistrationWarning,
IndexWarning,
pyearthtoolsDataWarning,
)

import pyearthtools.utils
import warnings as __python_warnings
# from pyearthtools.data.catalog import Catalog, CatalogEntry

from pyearthtools.data.archive.utils import auto_import

"""Auto import archives if available"""

Expand Down
4 changes: 1 addition & 3 deletions packages/data/src/pyearthtools/data/archive/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,7 @@
"""

from pyearthtools.data.archive.extensions import register_archive

from pyearthtools.data.archive.root import set_root, reset_root, config_root

from pyearthtools.data.archive.root import config_root, reset_root, set_root

ZARR_IMPORTED = True
try:
Expand Down
5 changes: 2 additions & 3 deletions packages/data/src/pyearthtools/data/archive/extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,9 @@

from __future__ import annotations

from types import ModuleType
from typing import Callable, Any

import warnings
from types import ModuleType
from typing import Any, Callable

import pyearthtools.data
from pyearthtools.data import archive
Expand Down
2 changes: 1 addition & 1 deletion packages/data/src/pyearthtools/data/archive/root.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@

from __future__ import annotations

import warnings
import logging
import warnings

import pyearthtools.data
from pyearthtools.data import archive
Expand Down
9 changes: 4 additions & 5 deletions packages/data/src/pyearthtools/data/archive/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,13 @@

from __future__ import annotations

from pathlib import Path
import os
import re

import importlib
import logging
from typing import Any
import os
import re
import warnings
from pathlib import Path
from typing import Any

from pyearthtools.data import archive
from pyearthtools.data.indexes.utilities.fileload import open_static
Expand Down
15 changes: 6 additions & 9 deletions packages/data/src/pyearthtools/data/archive/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,23 +21,20 @@

from __future__ import annotations

import logging
from os import PathLike
from pathlib import Path
from typing import Any, Literal
from os import PathLike

import xarray as xr
import dask
import logging

import pyearthtools.data
from pyearthtools.data.time import Petdt
from pyearthtools.data.transforms import Transform, TransformCollection
import xarray as xr
from pyearthtools.data.indexes.indexes import DataFileSystemIndex, TimeIndex

from pyearthtools.data.indexes.utilities.dimensions import identify_time_dimension
from pyearthtools.data.utils import parse_path

from pyearthtools.data.save import save
from pyearthtools.data.time import Petdt
from pyearthtools.data.transforms import Transform, TransformCollection
from pyearthtools.data.utils import parse_path

LOG = logging.getLogger("pyearthtools.data")

Expand Down
20 changes: 8 additions & 12 deletions packages/data/src/pyearthtools/data/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,26 +40,22 @@
"""

from __future__ import annotations
from collections import OrderedDict

from typing import Optional

import yaml
import io
import json
from pathlib import Path
from typing import Any, Callable, Optional
import types
import warnings
from collections import OrderedDict
from functools import lru_cache
import io

from pyearthtools.utils.parsing import function_name
from pyearthtools.utils.initialisation.imports import dynamic_import
from pathlib import Path
from typing import Any, Callable, Optional

import pyearthtools.data
import yaml
from pyearthtools.data.collection import Collection, LabelledCollection

from pyearthtools.utils.decorators import alias_arguments
from pyearthtools.utils.initialisation.imports import dynamic_import
from pyearthtools.utils.parsing import function_name

UTILS_REPR = False
try:
Expand Down Expand Up @@ -195,7 +191,7 @@ def convert(**kwargs):
Converts non serialisable to serialisable
"""
for key, value in kwargs.items():
if isinstance(value, (pyearthtools.data.TimeDelta, pyearthtools.data.TimeResolution, Path)):
if isinstance(value, (pyearthtools.data.time.TimeDelta, pyearthtools.data.TimeResolution, Path)):
kwargs[key] = str(value)
return kwargs

Expand Down
5 changes: 3 additions & 2 deletions packages/data/src/pyearthtools/data/collection.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,11 @@
"""

from __future__ import annotations
from _collections_abc import dict_keys

from typing import Any
from collections import OrderedDict
from typing import Any

from _collections_abc import dict_keys


class Collection(tuple):
Expand Down
3 changes: 2 additions & 1 deletion packages/data/src/pyearthtools/data/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
"""

from __future__ import annotations

from pathlib import Path

import click
Expand Down Expand Up @@ -98,8 +99,8 @@ def create_structure(top, blacklisted, save, verbose):
top: Path
Location to generate structure for
"""
from pyearthtools.data.indexes.utilities.structure import structure
import yaml
from pyearthtools.data.indexes.utilities.structure import structure

structure_dict: dict[str, dict | list] = {}
structure_d: dict[str, dict[str, Any] | list[str]] = structure(top, blacklisted=blacklisted, verbose=verbose) # type: ignore
Expand Down
7 changes: 5 additions & 2 deletions packages/data/src/pyearthtools/data/derived/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@
"""


from pyearthtools.data.derived.derived import DerivedValue, TimeDerivedValue, AdvancedTimeDerivedValue
from pyearthtools.data.derived.derived import (
AdvancedTimeDerivedValue,
DerivedValue,
TimeDerivedValue,
)
from pyearthtools.data.derived.insolation import Insolation


__all__ = ["DerivedValue", "TimeDerivedValue", "AdvancedTimeDerivedValue", "Insolation"]
8 changes: 3 additions & 5 deletions packages/data/src/pyearthtools/data/derived/derived.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,13 @@

from __future__ import annotations

from typing import Union
import inspect
from abc import abstractmethod, ABCMeta
from abc import ABCMeta, abstractmethod
from typing import Union

import xarray as xr


from pyearthtools.data.indexes import AdvancedTimeDataIndex, DataIndex, TimeDataIndex
from pyearthtools.data.time import Petdt, TimeDelta, TimeRange
from pyearthtools.data.indexes import DataIndex, TimeDataIndex, AdvancedTimeDataIndex


class DerivedValue(DataIndex, metaclass=ABCMeta):
Expand Down
4 changes: 1 addition & 3 deletions packages/data/src/pyearthtools/data/derived/insolation.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,9 @@

from __future__ import annotations


import numpy as np
import xarray as xr
import pandas as pd

import xarray as xr
from pyearthtools.data.derived.derived import AdvancedTimeDerivedValue

DASK_IMPORTED = True
Expand Down
Loading