WIP: Upgrade python tooling and deps

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version-file: .python-version
+
+      - name: Configure git for private deps
+        run: |
+          git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install -U pip
+          python -m pip install -r requirements-dev.txt
+
+      - name: Lint
+        run: |
+          flake8 validator tests
+
+      - name: Test
+        run: |
+          pytest -q

.python-version

@@ -1 +1 @@
-3.6.8
+3.12.12

Makefile

@@ -2,7 +2,9 @@
 
 PYTHON=venv/bin/python3
 PIP=venv/bin/pip
-NOSE=venv/bin/nosetests
+COVERAGE=venv/bin/coverage
+TEST_RUNNER=venv/bin/pytest
+TEST_RUNNER_FLAGS=-s --durations=3 --durations-min=0.005
 FLAKE=venv/bin/flake8
 PYPICLOUD_HOST=pypicloud.getkeepsafe.local
 PIP_ARGS=--extra-index=http://$(PYPICLOUD_HOST)/simple/ --trusted-host $(PYPICLOUD_HOST)
@@ -30,14 +32,16 @@ flake:
 	$(FLAKE) validator tests
 
 test: flake
-	$(NOSE) -s $(FLAGS)
+	$(COVERAGE) run -m pytest $(TEST_RUNNER_FLAGS)
 
 vtest:
-	$(NOSE) -s -v $(FLAGS)
+	$(COVERAGE) run -m pytest -v $(TEST_RUNNER_FLAGS)
+
+testloop:
+	while sleep 1; do $(TEST_RUNNER) -s --lf $(TEST_RUNNER_FLAGS); done
 
 cov cover coverage:
-	$(NOSE) -s --with-cover --cover-html --cover-html-dir ./coverage $(FLAGS)
-	echo "open file://`pwd`/coverage/index.html"
+	$(COVERAGE) report -m
 
 clean:
 	rm -rf `find . -name __pycache__`

requirements-dev.txt

@@ -1,4 +1,4 @@
 -r requirements.txt
-flake8==3.6.0
-nose
-coverage
+flake8==7.1.1
+pytest>=8
+coverage==7.6.1

requirements.txt

@@ -1,6 +1,7 @@
-beautifulsoup4==4.4.1
+beautifulsoup4==4.14.3
+sdiff @ git+https://github.com/KeepSafe/html-structure-diff.git@1.0.0#egg=sdiff
 Markdown
 html2text==2014.12.29
-lxml==3.5
+lxml==6.0.2
 parse==1.8.2
-aiohttp==3.1.3
+aiohttp==3.13.2

setup.cfg

@@ -7,3 +7,9 @@ ignore = F403
 
 [pep8]
 max-line-length = 120
+
+[coverage:run]
+branch = True
+
+[coverage:report]
+fail_under = 96

setup.py

@@ -1,27 +1,26 @@
 import os
 from setuptools import setup, find_packages
 
-
-version = '0.7.2'
+version = '1.0.0'
 
 
 def read(f):
     return open(os.path.join(os.path.dirname(__file__), f)).read().strip()
 
 
 install_requires = [
-    'sdiff @ git+https://github.com/KeepSafe/html-structure-diff.git@0.4.1#egg=sdiff',
-    'aiohttp >=3, <3.4',
+    'sdiff @ git+https://github.com/KeepSafe/html-structure-diff.git@1.0.0#egg=sdiff',
+    'aiohttp==3.13.2',
     'Markdown',
     'parse <= 1.8.2',
     'beautifulsoup4 >=4, <5',
-    'lxml >=3',
+    'lxml==6.0.2',
 ]
 
 tests_require = [
-    'nose',
-    'flake8==3.6.0',
-    'coverage',
+    'pytest >= 8',
+    'coverage==7.6.1',
+    'flake8==7.1.1',
 ]
 
 devtools_require = [
@@ -32,6 +31,7 @@ def read(f):
 setup(
     name='content-validator',
     version=version,
+    python_requires='>=3.11',
     description=('Content validator looks at text content and preforms different validation tasks'),
     classifiers=[
         'License :: OSI Approved :: BSD License', 'Intended Audience :: Developers', 'Programming Language :: Python'

tests/test_parser.py

@@ -1,5 +1,7 @@
 from unittest import TestCase
 
+import pytest
+
 from validator import parsers
 
 from tests.utils import read
@@ -10,3 +12,18 @@ def test_xml_parsing(self):
         content = read('tests/fixtures/bugs/parser_bug.xml')
         parser = parsers.XmlParser()
         parser.parse(content)
+
+
+def test_xml_parser_empty_returns_empty():
+    parser = parsers.XmlParser()
+
+    assert parser.parse('   ') == ''
+
+
+def test_chain_parser_wraps_errors():
+    parser = parsers.ChainParser([parsers.XmlParser()])
+
+    with pytest.raises(parsers.ParserError) as exc:
+        parser.parse('<root><broken></root>')
+
+    assert 'error in content' in str(exc.value)

tests/test_reports.py

@@ -0,0 +1,31 @@
+from validator.errors import ContentData, MdDiff, UrlDiff
+from validator.reports import HtmlReporter
+
+
+def test_html_reporter_handles_url_diff(tmp_path):
+    reporter = HtmlReporter(output_directory=str(tmp_path))
+    error = UrlDiff('http://example.com', files=['errors/source.md'], status_code=404)
+
+    reporter.report([error])
+
+    report_path = tmp_path / 'errors' / 'source.html'
+    assert report_path.exists()
+    assert 'http://example.com returned with code 404' in report_path.read_text()
+
+
+def test_html_reporter_writes_markdown_diff(tmp_path):
+    reporter = HtmlReporter(output_directory=str(tmp_path))
+    base = ContentData('base.md', 'Base', '<del>Base</del>', '<p>Base</p>')
+    other = ContentData('other.md', 'Other', '<ins>Other</ins>', '<p>Other</p>')
+    error = MdDiff(base, other, ['Missing content'])
+
+    reporter.report([error])
+
+    report_path = tmp_path / 'other.html'
+    assert report_path.exists()
+    content = report_path.read_text()
+    assert 'Missing content' in content
+    assert '<del>' in content
+    assert '<ins>' in content
+    assert 'Base' in content
+    assert 'Other' in content

tests/utils.py

@@ -1,4 +1,3 @@
-
 def read(path):
     with open(path) as fp:
         return fp.read()

validator/__init__.py

@@ -3,7 +3,7 @@
 from . import parsers, checks, reports, fs
 
 
-class Validator(object):
+class Validator:
     def __init__(self, contents, parser, reader, check, reporter=None):
         self.contents = contents
         self.parser = parser
@@ -24,7 +24,7 @@ async def async_validate(self):
         return errors
 
 
-class ReportBuilder(object):
+class ReportBuilder:
     def __init__(self, contents, parser, reader, check):
         self.contents = contents
         self.parser = parser
@@ -49,7 +49,7 @@ def validate(self):
         return Validator(self.contents, self.parser, self.reader, self.check, reporter).validate()
 
 
-class CheckBuilder(object):
+class CheckBuilder:
     def __init__(self, contents, content_type, parser, reader):
         self.contents = contents
         self.content_type = content_type
@@ -89,7 +89,7 @@ async def async_validate(self):
         return res
 
 
-class ParserBuilder(object):
+class ParserBuilder:
     def __init__(self, contents, reader=None):
         self.contents = contents
         self.content_type = 'txt'
@@ -120,7 +120,7 @@ def check(self):
         return CheckBuilder(self.contents, self.content_type, parser, self.reader)
 
 
-class ContentBuilder(object):
+class ContentBuilder:
     def files(self, pattern, **kwargs):
         contents = fs.files(pattern, **kwargs)
         return ParserBuilder(contents, parsers.FileReader())

validator/checks/__init__.py

@@ -1,5 +1,3 @@
-from typing import Type
-
 from sdiff import MdParser
 
 from .md import MarkdownComparator
@@ -21,7 +19,7 @@ def url_occurences(filetype):
     return UrlOccurenciesValidator()
 
 
-def markdown(filetype, md_parser_cls: Type[MdParser] = MdParser):
+def markdown(filetype, md_parser_cls: type[MdParser] = MdParser):
     if filetype not in ['txt', 'html']:
         raise UndefinedCheckTypeError('got filetype %s' % filetype)
     return MarkdownComparator(md_parser_cls)
@@ -33,7 +31,7 @@ def java_args(filetype):
     return JavaComparator()
 
 
-class ChainCheck(object):
+class ChainCheck:
     def __init__(self, checks):
         self.checks = checks
 

validator/checks/java.py

@@ -6,7 +6,7 @@
 REF_PATTERN = r'@string/\w+'
 
 
-class JavaComparator(object):
+class JavaComparator:
     def _get_args(self, content):
         return re.findall(ARG_PATTERN, content)
 

validator/checks/md.py

@@ -1,5 +1,4 @@
 import re
-from typing import Type
 
 from sdiff import diff, renderer, MdParser
 from markdown import markdown
@@ -14,8 +13,8 @@ def save_file(content, filename):
         fp.write(content)
 
 
-class MarkdownComparator(object):
-    def __init__(self, md_parser_cls: Type[MdParser] = MdParser):
+class MarkdownComparator:
+    def __init__(self, md_parser_cls: type[MdParser] = MdParser):
         self._md_parser_cls = md_parser_cls
 
     def check(self, data, parser, reader):