Merge pull request #13 from SamhammerAG/KIT-4026

KIT-4026 update

Author: oschulzSam

.flake8

@@ -7,4 +7,5 @@ exclude =
     build,
     dist
 per-file-ignores =
-    __init__.py: F401
+    __init__.py: D100, D101, D102, D103, D104, D106, D107, F401
+    /*: D100, D101, D102, D103, D104, D106, D107

.gitignore

@@ -13,6 +13,7 @@
 .idea/sqlDataSources.xml
 .idea/dynamic.xml
 .idea/dictionaries
+.idea/ai_data_preprocessing_queue.iml
 
 ## File-based project format
 *.ipr

.idea/ai_data_preprocessing_queue.iml

@@ -14,7 +14,8 @@ def __init__(self, name: str, step_data: str | None) -> None:
         module_name = f".{self.name}"
         self.module = importlib.import_module(module_name, package_name)
 
-        assert self.module.step is not None
+        if self.module.step is None:  # pragma: no cover
+            raise AssertionError("self.module.step must not be None")
 
     def run(self, item: Any, item_state: dict[str, Any], global_state: dict[str, Any] | None = None) -> Any:
         return self.module.step(item, item_state, global_state, self.step_data or "")

ai_data_preprocessing_queue/StepProcessor.py

@@ -1,5 +1,6 @@
 """
-Detects one of the following languages and writes the language to local state
+Detects one of the following languages and writes the language to local state.
+
 af, ar, bg, bn, ca, cs, cy, da, de, el, en, es, et, fa, fi, fr, gu, he,
 hi, hr, hu, id, it, ja, kn, ko, lt, lv, mk, ml, mr, ne, nl, no, pa, pl,
 pt, ro, ru, sk, sl, so, sq, sv, sw, ta, te, th, tl, tr, uk, ur, vi,

ai_data_preprocessing_queue/Steps/language_detect.py

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "ai-data-preprocessing-queue"
-version = "0.2.0"
+version = "1.6.0"
 description = "A collection of different text processing steps that can be enabled or disabled dynamically."
 authors = ["KI-Team"]
 license = "MIT"
@@ -14,15 +14,13 @@ pandas = "*"
 numpy = "*"
 
 [tool.poetry.group.dev.dependencies]
+coverage-lcov = "*"
 flake8-bandit = "*"
 flake8-pydocstyle = "*"
 mypy = "*"
 build = "*"
 pytest = "*"
-pytest-asyncio = "*"
 pytest-cov = "*"
-coverage-lcov = "*"
-mock = "*"
 types-mock = "*"
 
 [build-system]

pyproject.toml

@@ -1,12 +1,9 @@
 -r requirements.txt
+coverage-lcov
+build
 flake8-bandit
 flake8-pydocstyle
 mypy
-build
-
 pytest
-pytest-asyncio
 pytest-cov
-coverage-lcov
-mock
-types-mock
+

requirements-dev.txt

@@ -1,4 +1,4 @@
 langdetect~=1.0.9
-nltk>=3.7
-pandas>=1.5.1
-numpy>=1.23.4
+nltk>=3.9.0, <4.0
+pandas>=2.0.0, <3.0
+numpy>=2.0.0, <3.0

requirements.txt

@@ -11,7 +11,7 @@
 
 setuptools.setup(
     name="ai-data-preprocessing-queue",
-    version="1.5.0",
+    version="1.6.0",
     description="Can be used to pre process data before ai processing",
     long_description=LONG_DESCRIPTION,
     long_description_content_type="text/markdown",