Fix integration tests to be compatible with poetry and change the linter to ruff

Author: AntoniaSzecsi

.pre-commit-config.yaml

@@ -1,7 +1,13 @@
 repos:
-  - repo: https://github.com/python/black
-    rev: 19.3b0
+  - repo: local
     hooks:
-      - id: black
-        language_version: python3.9
-        exclude_types: ['markdown', 'ini', 'toml', 'rst']
+      - id: ruff-check
+        name: ruff
+        entry: poetry run ruff check --fix
+        language: system
+        types: [python]
+      - id: ruff-format
+        name: ruff format
+        entry: poetry run ruff format
+        language: system
+        types: [python]

Makefile

@@ -5,32 +5,28 @@ target:
 
 .PHONY: init
 init:
-	python scripts/dev.py init
+	python3 scripts/dev.py init
 
 .PHONY: test
 test:
-	python scripts/dev.py test
+	python3 scripts/dev.py test
 
 .PHONY: lint
 lint:
-	python scripts/dev.py lint
+	python3 scripts/dev.py lint
 
 .PHONY: clean
 clean:
-	python scripts/dev.py clean
+	python3 scripts/dev.py clean
 
 .PHONY: build
 build: clean
 ifeq ($(shell uname),Linux)
-	BUILD=true python scripts/dev.py build
+	BUILD=true python3 scripts/dev.py build
 else
-	python scripts/dev.py build
+	python3 scripts/dev.py build
 endif
 
-.PHONY: local-test
-local-test:
-	python scripts/dev.py local-test
-
 .PHONY: setup-codebuild-agent
 setup-codebuild-agent:
 	docker build -t codebuild-agent - < tests/integration/codebuild-local/Dockerfile.agent
@@ -81,6 +77,4 @@ TARGETS
 	lint        	Run all linters via scripts/dev.py.
 	test-smoke  	Run smoke tests inside Docker.
 	test-integ  	Run all integration tests.
-	local-test		Run the Lambda handler locally using AWS RIE.
-
 endef

poetry.lock

@@ -17,7 +17,48 @@ coverage = ">=4.4.0"
 pytest = ">=3.0.7"
 mock = ">=2.0.0"
 parameterized = ">=0.9.0"
+ruff = "^0.1.0"
+bandit = "^1.7.5"
+pre-commit = "^3.0.0"
+
+# Development scripts
+[tool.poetry.scripts]
+init = "scripts.dev:init"
+test = "scripts.dev:test"
+lint = "scripts.dev:lint"
+format = "scripts.dev:format_code"
+clean = "scripts.dev:clean"
+build = "scripts.dev:build"
+local-test = "scripts.dev:local_test"
 
 [build-system]
 requires = ["poetry-core>=2.0.0,<3.0.0", "setuptools>=68", "wheel"]
 build-backend = "poetry.core.masonry.api"
+
+# Ruff configuration
+[tool.ruff]
+target-version = "py39"
+line-length = 88
+
+[tool.ruff.lint]
+select = [
+    "E",   # pycodestyle errors
+    "W",   # pycodestyle warnings  
+    "F",   # pyflakes
+    "I",   # isort
+    "UP",  # pyupgrade
+    "C90", # mccabe complexity
+]
+
+# Ignore rules that are too strict for existing codebases
+ignore = [
+    "E501",    # Line too long (handled by formatter)
+    "PLR0913", # Too many arguments
+    "E722",    # Bare except 
+    "PLW0603", # Global statement 
+    "UP031",   # % formatting vs f-strings
+    "E402",    # Module import not at top
+]
+
+[tool.ruff.format]
+quote-style = "double"

pyproject.toml

@@ -9,7 +9,7 @@
 ROOT = Path(__file__).resolve().parent.parent
 
 def run(cmd, check=True, env=None):
-    print(f"\n$ {' '.join(cmd) if isinstance(cmd, list) else cmd}")
+    print("\n$ {}".format(' '.join(cmd) if isinstance(cmd, list) else cmd))
     result = subprocess.run(cmd, shell=isinstance(cmd, str), check=check, env=env)
     if result.returncode != 0 and check:
         sys.exit(result.returncode)
@@ -26,51 +26,55 @@ def test():
 
 
 def lint():
-    print("Running lint checks")
-    run(["poetry", "run", "flake8", "awslambdaric"])
-    run(["poetry", "run", "pylint", "awslambdaric"])
-    run(["poetry", "run", "black", "--check", "awslambdaric"])
-    run(["poetry", "run", "bandit", "-r", "awslambdaric"])
+    print("Running linters")
+    run(["poetry", "run", "ruff", "check", "awslambdaric/", "tests/"])
+
+
+def format_code():
+    print("Formatting code")
+    run(["poetry", "run", "black", "awslambdaric/", "tests/"])
 
 
 def clean():
-    print("Cleaning build and cache files")
-    patterns = ["__pycache__", "*.pyc", "*.pyo", ".pytest_cache", ".mypy_cache"]
-    for pattern in patterns:
-        for path in ROOT.rglob(pattern):
+    print("Cleaning build artifacts")
+    dirs_to_remove = ["build", "dist", "*.egg-info"]
+    for pattern in dirs_to_remove:
+        for path in ROOT.glob(pattern):
             if path.is_dir():
                 shutil.rmtree(path)
+                print("Removed directory: {}".format(path))
             elif path.is_file():
                 path.unlink()
+                print("Removed file: {}".format(path))
 
-    for folder in ["dist", "build", "awslambdaric.egg-info"]:
-        dir_path = ROOT / folder
-        if dir_path.exists():
-            shutil.rmtree(dir_path)
 
 def build():
     print("Building package")
     env = os.environ.copy()
-    if sys.platform.startswith("linux"):
-        env["BUILD"] = "true"
+    env["BUILD"] = "true"
     run([sys.executable, "setup.py", "sdist", "bdist_wheel"], env=env)
 
-def local_test():
-    print("Running local tests")
-    # will be implemented later using RIE
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Development command-line tool")
-    parser.add_argument("command", choices=["init", "test", "lint", "clean", "build", "local-test"])
+def main():
+    parser = argparse.ArgumentParser(description="Development scripts")
+    parser.add_argument("command", choices=[
+        "init", "test", "lint", "format", "clean", "build"
+    ])
+    
     args = parser.parse_args()
-    commands = {
-    "init": init,
-    "test": test,
-    "lint": lint,
-    "clean": clean,
-    "build": build,
-    "local-test": local_test,
-}
-
-commands[args.command]()
+    
+    command_map = {
+        "init": init,
+        "test": test,
+        "lint": lint,
+        "format": format_code,
+        "clean": clean,
+        "build": build,
+    }
+    
+    command_map[args.command]()
+
+
+if __name__ == "__main__":
+    main()
 

scripts/dev.py

@@ -1,7 +1,37 @@
 import os
 import platform
+import sys
 from subprocess import check_call, check_output
-from setuptools import Extension, setup
+from setuptools import Extension, setup, find_packages
+
+if sys.version_info >= (3, 11):
+    import tomllib
+else:
+    try:
+        import tomli as tomllib
+    except ImportError:
+        import subprocess
+        subprocess.check_call([sys.executable, "-m", "pip", "install", "tomli"])
+        import tomli as tomllib
+
+def get_metadata():
+    with open("pyproject.toml", "rb") as f:
+        pyproject = tomllib.load(f)
+    
+    poetry_config = pyproject["tool"]["poetry"]
+    return {
+        "name": poetry_config["name"],
+        "version": poetry_config["version"],
+        "description": poetry_config["description"],
+        "author": poetry_config["authors"][0] if poetry_config["authors"] else "",
+        "license": poetry_config["license"],
+        "python_requires": poetry_config["dependencies"]["python"],
+        "install_requires": [
+            f"{pkg}{version}" if not version.startswith("^") and not version.startswith("~") else f"{pkg}>={version[1:]}"
+            for pkg, version in poetry_config["dependencies"].items()
+            if pkg != "python"
+        ]
+    }
 
 def get_curl_extra_linker_flags():
     if platform.system() != "Linux" or os.getenv("BUILD") != "true":
@@ -25,6 +55,16 @@ def get_runtime_client_extension():
         include_dirs=["deps/artifacts/include"],
     )]
 
+metadata = get_metadata()
+
 setup(
+    name=metadata["name"],
+    version=metadata["version"],
+    description=metadata["description"],
+    author=metadata["author"],
+    license=metadata["license"],
+    packages=find_packages(),
+    python_requires=metadata["python_requires"],
+    install_requires=metadata["install_requires"],
     ext_modules=get_runtime_client_extension(),
 )

setup.py

@@ -22,7 +22,8 @@ RUN apk add --no-cache \
     elfutils-dev \
     make \
     cmake \
-    libcurl
+    libcurl \
+    curl
 
 # Include global args in this stage of the build
 ARG RIC_BUILD_DIR="/home/build/"
@@ -32,8 +33,13 @@ RUN mkdir -p ${RIC_BUILD_DIR}
 WORKDIR ${RIC_BUILD_DIR}
 COPY . .
 RUN pip3 install setuptools
-RUN make init build test && \
-    mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
+# Install Poetry
+RUN curl -sSL https://install.python-poetry.org | python3 - && \
+    ln -s /root/.local/bin/poetry /usr/local/bin/poetry
+
+RUN make init build
+
+RUN ls -la ./dist/
 
 # Include global args in this stage of the build
 ARG FUNCTION_DIR="/home/app/"
@@ -45,7 +51,7 @@ COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR}
 # Install the function's dependencies
 WORKDIR ${FUNCTION_DIR}
 RUN python${RUNTIME_VERSION} -m pip install \
-        ${RIC_BUILD_DIR}/dist/awslambdaric-test.tar.gz \
+        ${RIC_BUILD_DIR}/dist/*.whl \
         --target ${FUNCTION_DIR}
 
 

tests/.DS_Store

@@ -22,7 +22,8 @@ RUN yum install -y \
   openssl11-devel \
   bzip2-devel \
   libffi-devel \
-  sqlite-devel
+  sqlite-devel \
+  curl
 
 RUN RUNTIME_LATEST_VERSION=${RUNTIME_VERSION}.$(curl -s https://www.python.org/ftp/python/ | \
                                                 grep -oE "href=\"$(echo ${RUNTIME_VERSION} | sed "s/\\./\\\./g")\.[0-9]+" | \
@@ -72,6 +73,9 @@ RUN wget --quiet -O cmake-install https://github.com/Kitware/CMake/releases/down
 ENV PATH=/usr/local/bin:$PATH
 ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 
+# Create symlinks for consistent python usage
+RUN ln -sf /usr/local/bin/python${RUNTIME_VERSION} /usr/local/bin/python3 && \
+    ln -sf /usr/local/bin/python${RUNTIME_VERSION} /usr/local/bin/python
 
 # Include global args in this stage of the build
 ARG RIC_BUILD_DIR="/home/build/"
@@ -81,24 +85,29 @@ RUN mkdir -p ${RIC_BUILD_DIR}
 WORKDIR ${RIC_BUILD_DIR}
 COPY . .
 
-RUN make init build test && \
-    mv ./dist/awslambdaric-*.tar.gz ./dist/awslambdaric-test.tar.gz
+# Install wheel and poetry using the correct python version
+RUN /usr/local/bin/python${RUNTIME_VERSION} -m pip install wheel poetry
+
+# Configure poetry to use the correct python version
+RUN poetry config virtualenvs.create false
+RUN poetry env use /usr/local/bin/python${RUNTIME_VERSION}
+
+RUN make init build
+
+RUN ls -la ./dist/
 
 # Include global args in this stage of the build
 ARG FUNCTION_DIR="/home/app/"
 # Create function directory
 RUN mkdir -p ${FUNCTION_DIR}
 # Copy function code
 COPY tests/integration/test-handlers/echo/* ${FUNCTION_DIR}
-# Copy Runtime Interface Client .tgz
-RUN cp ./dist/awslambdaric-test.tar.gz ${FUNCTION_DIR}/awslambdaric-test.tar.gz
 
 # Install the function's dependencies
 WORKDIR ${FUNCTION_DIR}
 RUN python${RUNTIME_VERSION} -m pip install \
-        awslambdaric-test.tar.gz \
-        --target ${FUNCTION_DIR} && \
-    rm awslambdaric-test.tar.gz
+        ${RIC_BUILD_DIR}/dist/*.whl \
+        --target ${FUNCTION_DIR}
 
 
 # Stage 4 - final runtime interface client image