Refactor Web Scraping for Printers on Campus

.gitignore

@@ -2,6 +2,7 @@
 build/
 logs/
 node_modules/
+__pycache__/
 
 # Specific Files
 config.json

package.json

@@ -6,12 +6,15 @@
   "type": "module",
   "scripts": {
     "start:dev": "nodemon --ignore src/data/notifRequests.json src/index.js",
-    "start": "node src/index.js"
+    "start": "node src/index.js",
+    "migrate": "node src/data/scripts/run-migrations.js",
+    "populate:db": "npm run migrate && python3 src/data/scripts/populate_db.py"
   },
   "keywords": [],
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "better-sqlite3": "^12.4.1",
     "dotenv": "^16.4.7",
     "express": "^4.21.2",
     "firebase-admin": "^13.1.0",

src/data/db/database.py

@@ -32,18 +32,54 @@ def insert_library(location, address, latitude, longitude):
     conn.close()
 
 
-def insert_printer(location, description, latitude, longitude):
+def insert_printer(location, description, labels, latitude, longitude):
     """Insert a printer into the database."""
     conn = get_db_connection()
     cursor = conn.cursor()
 
+    # We remove the "OR IGNORE" because we acknoledge that several printers may have the same location and description (i.e., same building and room), so we rely on the unique printer_id to identify the printer
     cursor.execute(
         """
-        INSERT OR IGNORE INTO printers (location, description, latitude, longitude)
+        INSERT INTO printers (location, description, latitude, longitude)
         VALUES (?, ?, ?, ?)
     """,
         (location, description, latitude, longitude),
     )
 
+    # To get the printer_id, we do NOT rely on the location/description/coordinates, but rather on the printer_id that was just inserted (lastrowid), as several printers may have the same location and description (i.e., same building and room)
+    printer_id = cursor.lastrowid
+
+    # Insert labels into the labels table and get their IDs
+    label_ids = []
+    for label in labels:
+        cursor.execute(
+            """
+            INSERT OR IGNORE INTO labels (label)
+            VALUES (?)
+        """,
+            (label,),
+        )
+        cursor.execute(
+            """
+            SELECT id FROM labels WHERE label = ?
+        """,
+            (label,),
+        )
+        result = cursor.fetchone()
+        if result is None:
+            raise ValueError(f"Failed to find label: {label}")
+        label_id = result[0]
+        label_ids.append(label_id)
+
+    # Insert into junction table
+    for label_id in label_ids:
+        cursor.execute(
+            """
+            INSERT OR IGNORE INTO printer_labels (printer_id, label_id)
+            VALUES (?, ?)
+        """,
+            (printer_id, label_id),
+        )
+
     conn.commit()
     conn.close()

src/data/db/models.py

@@ -31,7 +31,7 @@ def create_tables():
         """
         CREATE TABLE IF NOT EXISTS printers (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
-            location TEXT UNIQUE,
+            location TEXT,
             description TEXT,
             latitude REAL,
             longitude REAL

src/data/migrations/2025117_1854_create_labels.sql

@@ -0,0 +1,6 @@
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE IF NOT EXISTS labels (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    label TEXT UNIQUE NOT NULL
+);

src/data/migrations/2025117_1859_create_printer_labels.sql

@@ -0,0 +1,9 @@
+PRAGMA foreign_keys = ON;
+
+CREATE TABLE IF NOT EXISTS printer_labels (
+    printer_id INTEGER NOT NULL,
+    label_id   INTEGER NOT NULL,
+    PRIMARY KEY (printer_id, label_id),
+    FOREIGN KEY (printer_id) REFERENCES printers(id) ON DELETE CASCADE,
+    FOREIGN KEY (label_id) REFERENCES labels(id) ON DELETE CASCADE
+);