feat: Add e2e test for server-to-server migration

.github/workflows/e2e.yml

@@ -1,3 +1,5 @@
+permissions:
+  contents: read
 name: End-to-End Tests
 
 on:
@@ -27,3 +29,37 @@ jobs:
         run: |
           chmod +x ./run_e2e_tests.sh
           ./run_e2e_tests.sh ${{ github.event.inputs.odoo_version }}
+
+  e2e-migration:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          python -m pip install .
+
+      - name: Run migration e2e tests for Odoo ${{ github.event.inputs.odoo_version }}
+        run: |
+          chmod +x ./run_migration_e2e_tests.sh
+          ./run_migration_e2e_tests.sh ${{ github.event.inputs.odoo_version }}
+
+  e2e-advanced:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools
+          python -m pip install .
+
+      - name: Run advanced e2e tests for Odoo ${{ github.event.inputs.odoo_version }}
+        run: |
+          chmod +x ./run_advanced_e2e_tests.sh
+          ./run_advanced_e2e_tests.sh ${{ github.event.inputs.odoo_version }}

docker-compose.advanced.yml

@@ -0,0 +1,46 @@
+services:
+  db-source:
+    image: postgres:15
+    container_name: odf_adv_db_source
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_PASSWORD=odoo
+      - POSTGRES_USER=odoo
+
+  odoo-source:
+    image: odoo:16.0
+    container_name: odf_adv_odoo_source
+    depends_on:
+      - db-source
+    ports:
+      - "8069:8069"
+    environment:
+      - HOST=db-source
+      - PORT=5432
+      - USER=odoo
+      - PASSWORD=odoo
+    volumes:
+      - .:/odoo-data-flow:z
+
+  db-target:
+    image: postgres:15
+    container_name: odf_adv_db_target
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_PASSWORD=odoo
+      - POSTGRES_USER=odoo
+
+  odoo-target:
+    image: odoo:16.0
+    container_name: odf_adv_odoo_target
+    depends_on:
+      - db-target
+    ports:
+      - "8070:8069"
+    environment:
+      - HOST=db-target
+      - PORT=5432
+      - USER=odoo
+      - PASSWORD=odoo
+    volumes:
+      - .:/odoo-data-flow:z

docker-compose.yml

@@ -0,0 +1,25 @@
+
+services:
+  db:
+    image: postgres:15
+    environment:
+      - POSTGRES_DB=postgres
+      - POSTGRES_PASSWORD=odoo
+      - POSTGRES_USER=odoo
+    ports:
+      - "5432:5432"
+
+  odoo:
+    image: odoo:16.0
+    depends_on:
+      - db
+    ports:
+      - "8069:8069"
+    environment:
+      - PGHOST=db
+      - PGPORT=5432
+      - PGUSER=odoo
+      - PGPASSWORD=odoo
+
+    volumes:
+      - .:/odoo-data-flow:z

docs/guides/server_to_server_migration.md

@@ -1,5 +1,11 @@
 # Guide: Direct Server-to-Server Migration
 
+```{warning}
+This is an experimental feature.
+While it has been unit-tested, it has not been extensively tested in a variety of real-world scenarios.
+Please use with caution and always back up your data before proceeding.
+```
+
 The `odoo-data-flow` library includes a powerful `migrate` command designed to perform a **direct, in-memory** data migration from one Odoo database to another. This is an advanced feature that chains together the export, transform, and import processes into a single step, without creating intermediate CSV files.
 
 > **When to use this?** This method is fast and convenient for simple, one-shot migrations where you don't need to inspect or modify the data mid-process.

noxfile.py

@@ -150,7 +150,7 @@ def mypy(session: nox.Session) -> None:
 
     session.install("mypy")
     session.install("pytest")
-    session.install("httpx")
+    session.install("requests", "types-requests")
     session.install("-e", ".")
     session.run("mypy", *args)
     if not session.posargs:
@@ -171,7 +171,7 @@ def tests(session: nox.Session) -> None:
         external=True,
     )
 
-    session.install("pytest", "coverage", "pytest-mock")
+    session.install("pytest", "coverage", "pytest-mock", "polars")
     session.install("-e", ".")
     session.run("pytest", *session.posargs)
 
@@ -197,7 +197,7 @@ def coverage(session: nox.Session) -> None:
         "coverage[toml]",
         "pytest-cov",
         "pytest-mock",
-        "httpx",
+        "requests",
         "rich",
         "polars",
         "click",

pyproject.toml

@@ -4,7 +4,7 @@ version = "0.0.1"
 description = "Odoo Data Flow"
 readme = "README.md"
 requires-python = ">=3.9"
-license = { file = "LICENSE" }
+license = "LGPL-3.0-only"
 authors = [
     { name = "bosd", email = "c5e2fd43-d292-4c90-9d1f-74ff3436329a@anonaddy.me" },
 ]

run_advanced_e2e_tests.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+set -e
+
+ODOO_VERSION=$1
+if [ -z "$ODOO_VERSION" ]; then
+    echo "Usage: $0 <odoo_version>"
+    exit 1
+fi
+
+COMPOSE_FILE="docker-compose.advanced.yml"
+
+# Function to handle failures
+handle_failure() {
+    echo "--- An error occurred. Dumping container logs. ---"
+    docker-compose -f $COMPOSE_FILE logs
+    echo "--- Tearing down containers ---"
+    docker-compose -f $COMPOSE_FILE down --volumes
+    exit 1
+}
+
+# Trap errors
+trap 'handle_failure' ERR
+
+
+# Function to replace Odoo version in compose file
+replace_odoo_version() {
+    sed "s/image: odoo:.*/image: odoo:$ODOO_VERSION/g" $COMPOSE_FILE > docker-compose.advanced.yml.tmp && mv docker-compose.advanced.yml.tmp $COMPOSE_FILE
+}
+
+# Clean up previous run
+echo "--- Cleaning up previous run ---"
+docker-compose -f $COMPOSE_FILE down --volumes || true
+rm -f testdata/res_partner_advanced.csv
+rm -f testdata/res_partner_category_advanced.csv
+rm -rf .odf_cache
+mkdir -p conf conf_target testdata
+
+# Replace Odoo version in compose file
+replace_odoo_version
+
+# Start Odoo containers
+echo "--- Starting containers for Odoo $ODOO_VERSION ---"
+docker-compose -f $COMPOSE_FILE up -d --build
+
+# Wait for databases to be ready
+echo "--- Waiting for databases to be ready ---"
+sleep 15 # Initial wait
+
+echo "Waiting for Odoo Source to be ready..."
+timeout 600 bash -c 'until curl -s http://localhost:8069/web/login > /dev/null; do echo -n "."; sleep 5; done'
+echo "Odoo Source is ready!"
+
+echo "Waiting for Odoo Target to be ready..."
+timeout 600 bash -c 'until curl -s http://localhost:8070/web/login > /dev/null; do echo -n "."; sleep 5; done'
+echo "Odoo Target is ready!"
+
+# Create and initialize databases
+echo "--- Creating and initializing databases ---"
+docker-compose -f $COMPOSE_FILE exec -T odoo-source odoo -d odoo -i base --stop-after-init --db_host=db-source --db_user=odoo --db_password=odoo
+docker-compose -f $COMPOSE_FILE exec -T odoo-target odoo -d odoo -i base --stop-after-init --db_host=db-target --db_user=odoo --db_password=odoo
+
+
+# Install dependencies in containers
+echo "--- Installing dependencies ---"
+docker-compose -f $COMPOSE_FILE exec -T --user root odoo-source bash -c "apt-get update && apt-get install -y git && python3 -m pip install --upgrade pip setuptools && pip install /odoo-data-flow"
+docker-compose -f $COMPOSE_FILE exec -T --user root odoo-target bash -c "apt-get update && apt-get install -y git && python3 -m pip install --upgrade pip setuptools && pip install /odoo-data-flow"
+
+# Seed the source database
+echo "--- Seeding source database ---"
+docker-compose -f $COMPOSE_FILE exec -T odoo-source python3 /odoo-data-flow/tests/e2e/seed_advanced_database.py odoo
+
+# Create connection configs
+cat << EOF > conf/connection.conf
+[Connection]
+hostname = localhost
+port = 8069
+login = admin
+password = admin
+database = odoo
+protocol = jsonrpc
+EOF
+
+cat << EOF > conf_target/connection.conf
+[Connection]
+hostname = localhost
+port = 8070
+login = admin
+password = admin
+database = odoo
+protocol = jsonrpc
+EOF
+
+# Run the export for categories
+echo "--- Exporting categories ---"
+docker-compose -f $COMPOSE_FILE exec -T --user root odoo-source bash -c "chown -R odoo:odoo /odoo-data-flow"
+docker-compose -f $COMPOSE_FILE exec -T --user odoo odoo-source bash -c "cd /odoo-data-flow && odoo-data-flow export --config conf/connection.conf --model res.partner.category --domain \"[('name', 'like', 'Test Category%')]\" --fields \"id,name\" --output testdata/res_partner_category_advanced.csv"
+
+# Run the export for partners
+echo "--- Exporting partners ---"
+docker-compose -f $COMPOSE_FILE exec -T --user root odoo-source bash -c "chown -R odoo:odoo /odoo-data-flow"
+docker-compose -f $COMPOSE_FILE exec -T --user odoo odoo-source bash -c "cd /odoo-data-flow && odoo-data-flow export --config conf/connection.conf --model res.partner --domain \"[('name', 'like', 'Advanced Test Partner%')]\" --fields \"id,name,category_id/.id\" --output testdata/res_partner_advanced.csv"
+
+# Modify the partner export header for import
+docker-compose -f $COMPOSE_FILE exec -T --user odoo odoo-source bash -c "cd /odoo-data-flow && sed -i 's/category_id\/.id/category_id/g' testdata/res_partner_advanced.csv"
+
+# Run the import for categories into the target
+echo "--- Importing categories into target ---"
+docker-compose -f $COMPOSE_FILE exec -T --user root odoo-target bash -c "chown -R odoo:odoo /odoo-data-flow"
+docker-compose -f $COMPOSE_FILE exec -T --user odoo odoo-target bash -c "cd /odoo-data-flow && odoo-data-flow import --config conf_target/connection.conf --file testdata/res_partner_category_advanced.csv"
+
+# Run the import for partners into the target
+echo "--- Importing partners into target ---"
+docker-compose -f $COMPOSE_FILE exec -T --user root odoo-target bash -c "chown -R odoo:odoo /odoo-data-flow"
+docker-compose -f $COMPOSE_FILE exec -T --user odoo odoo-target bash -c "cd /odoo-data-flow && odoo-data-flow import --config conf_target/connection.conf --file testdata/res_partner_advanced.csv --strategy relational"
+
+# Verify the data in the target database
+echo "--- Verifying data in target database ---"
+docker-compose -f $COMPOSE_FILE exec -T odoo-target python3 /odoo-data-flow/tests/e2e/verify_advanced_data.py odoo
+
+# Tear down the containers
+echo "--- Tearing down containers ---"
+docker-compose -f $COMPOSE_FILE down --volumes
+
+echo "--- Advanced e2e tests completed successfully! ---"