Automate migrating notebooks to the landing page

.github/workflows/pre-commit.yaml

@@ -0,0 +1,31 @@
+name: Pre-commit checks
+
+on: [ pull_request ]
+
+jobs:
+  pre-commit:
+    name: pre-commit
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'poetry'
+
+      - name: Cache pre-commit
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pre-commit
+          key: pre-commit-3|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
+
+      - name: Install pre-commit
+        run: pip3 install pre-commit
+
+      - name: Run pre-commit
+        run: pre-commit run --all-files --show-diff-on-failure --color always

.github/workflows/pull-request-notebooks.yaml

@@ -0,0 +1,107 @@
+name: Pull Request notebooks to landing page
+
+on:
+  pull_request:
+#    types: [ closed ]
+#    branches:
+#      - main
+#      - master
+    paths:
+      - '**.ipynb'
+
+env:
+  TARGET_REPO: 'kacperlukawski/landing_page'
+  TARGET_BRANCH: 'examples/${{ github.event.pull_request.head.ref }}-pr-${{ github.event.pull_request.number }}'
+
+jobs:
+  convert-notebooks:
+    name: Push notebooks to landing page
+    runs-on: ubuntu-latest
+
+    # TODO: uncomment once it's stable
+    # This job will only run if the PR has been merged
+    # if: github.event.pull_request.merged == true
+
+    steps:
+      - name: Checkout current repository
+        uses: actions/checkout@v4
+        with:
+          path: 'examples'
+          ref: ${{ github.head_ref }}
+          fetch-depth: 0
+
+      - name: Get all changed markdown files
+        id: changed-jupyter-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            **.ipynb
+            */**.ipynb
+
+      - name: List all changed Jupyter files
+        if: steps.changed-jupyter-files.outputs.any_changed == 'true'
+        env:
+          ALL_CHANGED_FILES: ${{ steps.changed-jupyter-files.outputs.all_changed_files }}
+        run: |
+          for file in ${ALL_CHANGED_FILES}; do
+            echo "$file was changed"
+          done
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'poetry'
+      - run: |
+          cd examples
+          poetry install --with dev
+
+      # TODO: convert just the notebooks modified in this PR, not all of them
+      - name: Convert notebooks
+        run: |
+          cd examples
+          poetry run python .scripts/convert-all-notebooks-to-hugo-markdown.py
+
+      - name: Checkout landing page repository
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ env.TARGET_REPO }}
+          path: 'landing_page'
+          token: ${{ secrets.API_TOKEN_GITHUB }}
+
+      - name: Checkout the target branch of the landing page repository
+        run: |
+          cd landing_page
+          git fetch origin $TARGET_BRANCH:$TARGET_BRANCH || git checkout -b $TARGET_BRANCH
+
+      - name: Copy converted notebooks to landing page
+        run: |
+          # cp --update=none -r examples/.dist/* landing_page/
+          cp -r examples/.dist/* landing_page/
+
+      - name: Commit changes to a new branch and start a PR
+        env:
+          GH_TOKEN: ${{ secrets.API_TOKEN_GITHUB }}
+          USER_NAME: '${{ github.actor }}'
+          USER_EMAIL: '${{ github.actor }}@users.noreply.github.com'
+        run: |
+          cd landing_page
+          git config user.name $USER_NAME
+          git config user.email $USER_EMAIL
+          git add .
+          git commit -m "${{ github.event.pull_request.title }}"
+          git push --set-upstream origin $TARGET_BRANCH
+
+          # Check if there is an existing PR for this branch
+          PR_ID=$(gh pr list --repo "${{ env.TARGET_REPO }}" --state open --base master --head "${{ env.TARGET_BRANCH }}" --json number | jq -r '.[0].number')
+          if [ -n "$PR_ID" ]; then
+            gh pr create \
+              --title "${{ github.event.pull_request.title }}" \
+              --body "Add notebooks from PR ${{ github.event.repository.name }}#${{ github.event.pull_request.number }}" \
+              --base master \
+              --head $TARGET_BRANCH
+          else
+            echo "PR already exists: $PR_ID"
+          fi

.github/workflows/run-notebooks.yaml

@@ -0,0 +1,22 @@
+name: Run notebooks
+
+on: [ workflow_dispatch ]
+
+jobs:
+  run-notebooks:
+    name: Run notebooks
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'poetry'
+
+      - run: poetry install --with dev

.gitignore

@@ -1,12 +1,183 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# PyPI configuration file
+.pypirc
+
 data/sample/*
 qdrant_storage/
 qdrant_101/notes
-.ipynb_checkpoints
-qdrant_101_image_data/vecs.npy
+101-foundations/qdrant_101_image_data/vecs.npy
 data
-qdrant_101_image_data/vectors.npy
-qdrant_101_image_data/image_search_app.py
+101-foundations/qdrant_101_image_data/vectors.npy
+101-foundations/qdrant_101_image_data/image_search_app.py
 *.json
 */*.log
 */.env
-.venv/
+.venv/

.pre-commit-config.yaml

@@ -0,0 +1,36 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+default_language_version:
+  python: python3.10
+
+ci:
+  autofix_prs: true
+  autoupdate_commit_msg: "[pre-commit.ci] pre-commit suggestions"
+  autoupdate_schedule: quarterly
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+        exclude_types:
+          - csv
+      - id: trailing-whitespace
+        exclude_types:
+          - csv
+      - id: check-added-large-files
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.8.1
+    hooks:
+      - id: ruff
+        args: [ --fix ]
+      - id: ruff-format
+
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: "Sort Imports"
+        args: [ "--profile", "black" ]