Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions .github/workflows/QA.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -62,16 +62,16 @@ jobs:
- name: Install dependencies (and project)
run: |
pip install -U pip
pip install -e .[lint,scripts,test,check]
pip install -e .[qa,scripts,test]

- name: Check black formatting
run: inv lint-black
- name: Check Linting
run: inv check-lint

- name: Check ruff
run: inv lint-ruff
- name: Check Formatting
run: inv check-format

- name: Check pyright
run: inv check-pyright
- name: Check Type
run: inv check-type

check-javascript-qa:
runs-on: ubuntu-24.04
Expand Down
3 changes: 1 addition & 2 deletions .github/workflows/Tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -133,10 +133,9 @@ jobs:
python-version-file: pyproject.toml
architecture: x64

- name: Install dependencies (and project)
- name: Install dependencies
run: |
pip install -U pip hatch
pip install -e .[docs]

- name: Build mkdoc documentation
run: hatch run docs:build
Expand Down
9 changes: 3 additions & 6 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,11 @@ repos:
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- repo: https://github.com/psf/black
rev: '25.11.0'
hooks:
- id: black
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.14.5
rev: v0.15.12
hooks:
- id: ruff
- id: ruff-check
- id: ruff-format
- repo: https://github.com/RobertCraigie/pyright-python
rev: v1.1.407
hooks:
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Changed

- Upgrade to wombat 3.10.3 (#286)
- Adopt openzim/\_python-bootstrap 2.0.0 (#303)

## [5.3.0] - 2025-11-14

Expand Down
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# zimscraperlib

[![QA Status](https://github.com/openzim/python-scraperlib/workflows/QA/badge.svg?query=branch%3Amain)](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
[![Tests Status](https://github.com/openzim/python-scraperlib/workflows/Tests/badge.svg?query=branch%3Amain)](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
[![QA Status](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml/badge.svg?branch=main)](https://github.com/openzim/python-scraperlib/actions/workflows/QA.yaml)
[![Tests Status](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml/badge.svg?branch=main)](https://github.com/openzim/python-scraperlib/actions/workflows/Tests.yaml)
[![CodeFactor](https://www.codefactor.io/repository/github/openzim/python-scraperlib/badge)](https://www.codefactor.io/repository/github/openzim/python-scraperlib)
[![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
[![PyPI version shields.io](https://img.shields.io/pypi/v/zimscraperlib.svg)](https://pypi.org/project/zimscraperlib/)
Expand Down Expand Up @@ -67,7 +67,7 @@ apk add ffmpeg gifsicle libmagic wget cairo

This project adheres to openZIM's [Contribution Guidelines](https://github.com/openzim/overview/wiki/Contributing).

This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v1.0.2**.
This project has implemented openZIM's [Python bootstrap, conventions and policies](https://github.com/openzim/_python-bootstrap/docs/Policy.md) **v2.0.0**.

All instructions below must be run from the root of your local clone of this repository.

Expand Down
39 changes: 15 additions & 24 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,11 +57,8 @@ scripts = [
"PyYAML==6.0.3",

]
lint = [
"black==25.11.0",
"ruff==0.14.5",
]
check = [
qa = [
"ruff==0.15.12",
"pyright==1.1.407",
"pytest==9.0.1",
]
Expand All @@ -71,6 +68,7 @@ test = [
"coverage==7.11.3",
]
docs = [
"ruff==0.15.12",
"mkdocs==1.6.1",
"mkdocs-include-markdown-plugin==7.2.0",
"mkdocs-material==9.7.0",
Expand All @@ -84,9 +82,8 @@ dev = [
"ipython==9.7.0",
"pre-commit==4.4.0",
"zimscraperlib[scripts]",
"zimscraperlib[lint]",
"zimscraperlib[qa]",
"zimscraperlib[test]",
"zimscraperlib[check]",
"zimscraperlib[docs]",
]

Expand Down Expand Up @@ -129,25 +126,19 @@ report-cov = "inv report-cov"
coverage = "inv coverage --args '{args}'"
html = "inv coverage --html --args '{args}'"

[tool.hatch.envs.lint]
template = "lint"
[tool.hatch.envs.qa]
template = "qa"
skip-install = false
features = ["scripts", "lint"]

[tool.hatch.envs.lint.scripts]
black = "inv lint-black --args '{args}'"
ruff = "inv lint-ruff --args '{args}'"
all = "inv lintall --args '{args}'"
fix-black = "inv fix-black --args '{args}'"
fix-ruff = "inv fix-ruff --args '{args}'"
fixall = "inv fixall --args '{args}'"

[tool.hatch.envs.check]
features = ["scripts", "check"]
features = ["scripts", "qa", "test"]

[tool.hatch.envs.check.scripts]
pyright = "inv check-pyright --args '{args}'"
all = "inv checkall --args '{args}'"
[tool.hatch.envs.qa.scripts]
check-lint = "inv check-lint --args '{args}'"
check-format = "inv check-format --args '{args}'"
check-type = "inv check-type --args '{args}'"
check-all = "inv check-all --args '{args}'"
fix-format = "inv fix-format --args '{args}'"
fix-lint = "inv fix-lint --args '{args}'"
fix-all = "inv fix-all --args '{args}'"

[tool.hatch.envs.docs]
features = ["scripts", "docs"]
Expand Down
7 changes: 3 additions & 4 deletions src/zimscraperlib/html.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,14 +40,13 @@ def find_language_in(content: str | BinaryIO | TextIO, mime_type: str) -> str:
node = soup.find(nodename)
if node:
if not isinstance(
node, element.Tag # pyright:ignore[reportUnnecessaryIsInstance]
node,
element.Tag, # pyright:ignore[reportUnnecessaryIsInstance]
) or not node.has_attr(key):
continue
if (
nodename == "meta"
and not node.attrs.get(
"http-equiv", ""
).lower() # pyright:ignore[reportUnknownMemberType, reportAttributeAccessIssue]
and not node.attrs.get("http-equiv", "").lower() # pyright:ignore[reportUnknownMemberType, reportAttributeAccessIssue]
== "content-language"
):
continue
Expand Down
14 changes: 7 additions & 7 deletions src/zimscraperlib/i18n.py
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,13 @@ def __eq__(self, value: object) -> bool:

def __hash__(self):
return hash(
f"{getattr(self, "iso_639_1", None)}$"
f"{getattr(self, "iso_639_2b", None)}$"
f"{getattr(self, "iso_639_2t", None)}$"
f"{getattr(self, "iso_639_3", None)}$"
f"{getattr(self, "iso_639_5", None)}$"
f"{getattr(self, "english", None)}$"
f"{getattr(self, "native", None)}"
f"{getattr(self, 'iso_639_1', None)}$"
f"{getattr(self, 'iso_639_2b', None)}$"
f"{getattr(self, 'iso_639_2t', None)}$"
f"{getattr(self, 'iso_639_3', None)}$"
f"{getattr(self, 'iso_639_5', None)}$"
f"{getattr(self, 'english', None)}$"
f"{getattr(self, 'native', None)}"
)


Expand Down
4 changes: 3 additions & 1 deletion src/zimscraperlib/image/probing.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ def solarize(r: int, g: int, b: int) -> tuple[int, int, int]:
sr, sg, sb = solarize(mr, mg, mb) # pyright: ignore[reportUnknownArgumentType]

return rgb_to_hex(
mr, mg, mb # pyright: ignore[reportUnknownArgumentType]
mr, # pyright: ignore[reportUnknownArgumentType]
mg, # pyright: ignore[reportUnknownArgumentType]
mb, # pyright: ignore[reportUnknownArgumentType]
), rgb_to_hex(sr, sg, sb)


Expand Down
6 changes: 1 addition & 5 deletions src/zimscraperlib/misc.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
"""Miscelaneous utils"""

from typing import TypeVar

T = TypeVar("T")


def first(*args: T | None, default: T = "") -> T:
def first[T](*args: T | None, default: T = "") -> T:
"""Return the first non-None value from *args; fallback to an empty string."""
return next((item for item in args if item is not None), default)
2 changes: 1 addition & 1 deletion src/zimscraperlib/rewriting/js.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ def rewrite(self, text: str | bytes, opts: dict[str, Any] | None = None) -> str:
# Modules are always strict mode
if is_module:
opts["isStrict"] = True
elif "isStrict" not in opts:
elif "isStrict" not in opts: # pragma: no branch
# Detect strict mode from the code itself
opts["isStrict"] = self._detect_strict_mode(text)

Expand Down
1 change: 0 additions & 1 deletion src/zimscraperlib/video/encoding.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ def reencode(
"""

with path_from(existing_tmp_path or tempfile.TemporaryDirectory()) as tmp_dir:

tmp_path = pathlib.Path(tmp_dir).joinpath(f"video.tmp{dst_path.suffix}")
args = _build_ffmpeg_args(
src_path=src_path,
Expand Down
11 changes: 7 additions & 4 deletions src/zimscraperlib/zim/creator.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,9 @@ def mimetype_for(
mimetype = (
get_file_mimetype(fpath)
if fpath
else get_content_mimetype(content[:2048]) if content else None
else get_content_mimetype(content[:2048])
if content
else None
)
# try to guess more-defined mime if it's text
if (
Expand Down Expand Up @@ -125,7 +127,9 @@ def __init__(
self.ignore_duplicates = ignore_duplicates

def config_indexing(
self, indexing: bool, language: str | None = None # noqa: FBT001
self,
indexing: bool, # noqa: FBT001
language: str | None = None,
):
"""Toggle full-text indexing of entries

Expand All @@ -143,7 +147,6 @@ def config_indexing(
def _log_metadata(self):
"""Log in DEBUG level all metadata key and value"""
for name, metadata in sorted(self._metadata.items()):

if not hasattr(metadata, "value"):
logger.debug(
f"Metadata: {name} is improper metadata type: "
Expand Down Expand Up @@ -211,7 +214,7 @@ def start(self):
]
raise ValueError(
"Mandatory metadata are not all set. Missing metadata: "
f'{",".join(missing_keys)}. You should prefer to use '
f"{','.join(missing_keys)}. You should prefer to use "
"StandardMetadataList if possible."
)

Expand Down
2 changes: 0 additions & 2 deletions src/zimscraperlib/zim/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,6 @@ def x_prefixed[U: AnyMetadata](cls: type[U]):


class Metadata(MetadataBase[bytes]):

def get_binary_from(
self,
value: bytes | SupportsRead[bytes] | SupportsSeekableRead[bytes] | io.BytesIO,
Expand Down Expand Up @@ -463,7 +462,6 @@ class RelationMetadata(TextBasedMetadata):

@dataclass
class StandardMetadataList:

Name: NameMetadata
Language: LanguageMetadata
Title: TitleMetadata
Expand Down
74 changes: 35 additions & 39 deletions tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,72 +42,68 @@ def coverage(ctx: Context, args: str = "", *, html: bool = False):
report_cov(ctx, html=html)


@task(optional=["args"], help={"args": "black additional arguments"})
def lint_black(ctx: Context, args: str = "."):
def _lint(ctx: Context, args: str = "."):
args = args or "." # needed for hatch script
ctx.run("black --version", pty=use_pty)
ctx.run(f"black --check --diff {args}", pty=use_pty)
ctx.run("ruff --version", pty=use_pty)
ctx.run(f"ruff check {args}", pty=use_pty)


@task(optional=["args"], help={"args": "ruff additional arguments"})
def lint_ruff(ctx: Context, args: str = "."):
def check_lint(ctx: Context, args: str = "."):
"""check linting with ruff"""
args = args or "." # needed for hatch script
ctx.run("ruff --version", pty=use_pty)
ctx.run(f"ruff check {args}", pty=use_pty)
_lint(ctx, args)


@task(
optional=["args"],
help={
"args": "linting tools (black, ruff) additional arguments, typically a path",
},
)
def lintall(ctx: Context, args: str = "."):
"""Check linting"""
@task(optional=["args"], help={"args": "ruff additional arguments"})
def fix_lint(ctx: Context, args: str = "."):
"""fix linting issues with ruff"""
args = args or "." # needed for hatch script
lint_black(ctx, args)
lint_ruff(ctx, args)
_lint(ctx, f"--fix {args}")


@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"})
def check_pyright(ctx: Context, args: str = ""):
def check_type(ctx: Context, args: str = ""):
"""check static types with pyright"""
ctx.run("pyright --version")
ctx.run(f"pyright {args}", pty=use_pty)


@task(optional=["args"], help={"args": "check tools (pyright) additional arguments"})
def checkall(ctx: Context, args: str = ""):
"""check static types"""
check_pyright(ctx, args)
def _format(ctx: Context, args: str = "."):
args = args or "." # needed for hatch script
ctx.run("ruff --version", pty=use_pty)
ctx.run(f"ruff format {args}", pty=use_pty)


@task(optional=["args"], help={"args": "black additional arguments"})
def fix_black(ctx: Context, args: str = "."):
"""fix black formatting"""
@task(optional=["args"], help={"args": "ruff additional arguments"})
def check_format(ctx: Context, args: str = "."):
"""check formatting with ruff"""
args = args or "." # needed for hatch script
ctx.run(f"black {args}", pty=use_pty)
_format(ctx, f"--check {args}")


@task(optional=["args"], help={"args": "ruff additional arguments"})
def fix_ruff(ctx: Context, args: str = "."):
"""fix all ruff rules"""
def fix_format(ctx: Context, args: str = "."):
"""fix formatting with ruff"""
args = args or "." # needed for hatch script
ctx.run(f"ruff check --fix {args}", pty=use_pty)
_format(ctx, args)


@task(
optional=["args"],
help={
"args": "linting tools (black, ruff) additional arguments, typically a path",
},
)
def fixall(ctx: Context, args: str = "."):
@task(optional=["args"], help={"args": "additional arguments"})
def check_all(ctx: Context, args: str = ""):
"""check linting, formatting and static types"""
args = args or "." # needed for hatch script
check_lint(ctx, args)
check_format(ctx, args)
check_type(ctx, args)


@task(optional=["args"], help={"args": "additional arguments"})
def fix_all(ctx: Context, args: str = ""):
"""Fix everything automatically"""
args = args or "." # needed for hatch script
fix_black(ctx, args)
fix_ruff(ctx, args)
lintall(ctx, args)
fix_lint(ctx, args)
fix_format(ctx, args)


@task(optional=["args"], help={"args": "mkdocs build additional arguments"})
Expand Down
Loading
Loading