Include ability to connect either sqllite or postgresql to store data in a database

.gitignore

@@ -0,0 +1,5 @@
+venv/
+.venv/
+.idea/
+.env
+**.db

USE_DB.md

@@ -0,0 +1,49 @@
+# How to Use the PySpectrometer DB Module
+
+## Requirements:
+1. Update, install, and upgrade pip:
+   ```shell
+   sudo apt update
+   sudo apt install python3-pip
+   pip3 --version
+   sudo pip3 install --upgrade pip
+   ```
+2. Set up the .env file:
+   ```shell
+   sudo cp src/.env.example src/.env
+   ```
+
+## Installation:
+1. Create a new virtual environment:
+   ```shell
+   python3 -m venv .venv
+   ```
+2. Activate the virtual environment:
+   ```shell
+   source .venv/bin/activate
+   ```
+3. Install requirements in the virtual environment:
+   ```shell
+   pip3 install -r requirements.txt
+   ```
+
+## Initialize Database:
+### Local / Development Environment:
+1. Run the `db.py` module with the initialize argument:
+   ```shell
+   python3 src/db.py --initialize
+   ```
+   > This creates a new local SQLite3 database in the root folder of this project.  
+   ![db_file.png](media/db_file.png)
+2. Run the `db.py` module again with the test argument:
+   ```shell
+   python3 src/db.py --test
+   ```
+   > This will create two test entries in the newly created database in the `measurements` table.  
+   You can delete them after verifying that it works.  
+   ![db_test_entries.png](media/db_test_entries.png)
+
+### Production Environment:
+> Coming soon...
+
+Now, if you run any of the PySpectrometer2 scripts, it should, based on the DB_TYPE (`sqlite3`, `postgresql`) from the .env file, select the correct database and write data to the database whenever a new .csv file is generated.

requirements.in

@@ -0,0 +1,3 @@
+psycopg2-binary
+python-dotenv
+opencv-python

requirements.txt

@@ -0,0 +1,14 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile
+#
+numpy==2.0.1
+    # via opencv-python
+opencv-python==4.10.0.84
+    # via -r requirements.in
+psycopg2-binary==2.9.9
+    # via -r requirements.in
+python-dotenv==1.0.1
+    # via -r requirements.in

src/.env.example

@@ -0,0 +1,6 @@
+DB_TYPE=sqlite3
+DB_HOST=
+DB_PORT=5432
+DB_NAME=
+DB_USERNAME=
+DB_PASSWORD=

src/PySpectrometer2-Picam2-v1.0.py

@@ -32,6 +32,8 @@
 import base64
 import argparse
 from picamera2 import Picamera2
+from src.db import snapshot_database
+
 
 parser = argparse.ArgumentParser()
 group = parser.add_mutually_exclusive_group()
@@ -149,7 +151,8 @@ def snapshot(savedata):
 	now = time.strftime("%Y%m%d--%H%M%S")
 	timenow = time.strftime("%H:%M:%S")
 	imdata1 = savedata[0]
-	graphdata = savedata[1]
+	wavelengths = savedata[1][0]
+	intensities = savedata[1][1]
 	if dispWaterfall == True:
 		imdata2 = savedata[2]
 		cv2.imwrite("waterfall-" + now + ".png",imdata2)
@@ -158,11 +161,14 @@ def snapshot(savedata):
 	#print(graphdata[1]) #intensities
 	f = open("Spectrum-"+now+'.csv','w')
 	f.write('Wavelength,Intensity\r\n')
-	for x in zip(graphdata[0],graphdata[1]):
+	for x in zip(wavelengths, intensities):
 		f.write(str(x[0])+','+str(x[1])+'\r\n')
 	f.close()
+
+	snapshot_database(now, wavelengths, intensities)
+
 	message = "Last Save: "+timenow
-	return(message)
+	return message
 
 
 while True:

src/PySpectrometer2-USB-v1.0.py

@@ -33,6 +33,8 @@
 import base64
 import argparse
 
+from src.db import snapshot_database
+
 parser = argparse.ArgumentParser()
 parser.add_argument("--device", type=int, default=0, help="Video Device number e.g. 0, use v4l2-ctl --list-devices")
 parser.add_argument("--fps", type=int, default=30, help="Frame Rate e.g. 30")
@@ -153,7 +155,8 @@ def snapshot(savedata):
 	now = time.strftime("%Y%m%d--%H%M%S")
 	timenow = time.strftime("%H:%M:%S")
 	imdata1 = savedata[0]
-	graphdata = savedata[1]
+	wavelengths = savedata[1][0]
+	intensities = savedata[1][1]
 	if dispWaterfall == True:
 		imdata2 = savedata[2]
 		cv2.imwrite("waterfall-" + now + ".png",imdata2)
@@ -162,11 +165,14 @@ def snapshot(savedata):
 	#print(graphdata[1]) #intensities
 	f = open("Spectrum-"+now+'.csv','w')
 	f.write('Wavelength,Intensity\r\n')
-	for x in zip(graphdata[0],graphdata[1]):
+	for x in zip(wavelengths, intensities):
 		f.write(str(x[0])+','+str(x[1])+'\r\n')
 	f.close()
+
+	snapshot_database(now, wavelengths, intensities)
+
 	message = "Last Save: "+timenow
-	return(message)
+	return message
 
 while(cap.isOpened()):
 	# Capture frame-by-frame

src/db.py

@@ -0,0 +1,120 @@
+import os
+import sys
+import time
+import sqlite3
+import argparse
+import psycopg2
+from dotenv import load_dotenv
+from psycopg2.extras import execute_values
+
+load_dotenv()
+
+SQLITE3_DB = 'spectrometer_data.db'
+DB_TYPE = os.getenv('DB_TYPE', 'sqlite3')
+
+if DB_TYPE != 'sqlite3':
+    DB_HOST = os.environ.get('DB_HOST', 'localhost')
+    DB_PORT = os.environ.get('DB_PORT', '5432')
+    DB_NAME = os.environ.get('DB_NAME', 'postgres')
+    DB_USERNAME = os.environ.get('DB_USERNAME', 'postgres')
+    DB_PASSWORD = os.environ.get('DB_PASSWORD', 'postgres')
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        '-i', '--initialize', action='store_true', help='Initialize SQLite3 database',
+    )
+    parser.add_argument(
+        '-t', '--test', action='store_true', help='Test SQLite3 database measurements insertion.',
+    )
+    return parser.parse_args()
+
+
+def snapshot_database(timestamp, wavelengths, intensities):
+    data_to_insert = prepare_snapshot_db_data(timestamp, wavelengths, intensities)
+
+    if DB_TYPE == "postgresql":
+        save_measurements_to_postgresql(
+            '''
+                    INSERT INTO measurements (timestamp, wavelength, intensity)
+                    VALUES %s
+                ''' % data_to_insert,
+            data_to_insert,
+        )
+    else:
+        save_measurements_to_sqlite3(
+            '''INSERT INTO measurements (timestamp, wavelength, intensity) VALUES (?, ?, ?)''',
+            data_to_insert,
+        )
+
+
+def prepare_snapshot_db_data(timestamp, wavelengths, intensities):
+    return [(timestamp, wl, inten) for wl, inten in zip(wavelengths, intensities)]
+
+
+def connect_to_sqlite3() -> sqlite3.Connection:
+    return sqlite3.connect(SQLITE3_DB)
+
+
+def connect_to_postgresql() -> psycopg2.connect:
+    return psycopg2.connect(
+        host=DB_HOST,
+        database=DB_NAME,
+        user=DB_USERNAME,
+        password=DB_PASSWORD,
+        port=DB_PORT,
+    )
+
+
+def save_measurements_to_sqlite3(query, data_to_insert):
+    conn = connect_to_sqlite3()
+    cursor = conn.cursor()
+    cursor.executemany(query, data_to_insert)
+    conn.commit()
+    conn.close()
+
+
+def save_measurements_to_postgresql(query, data_to_insert):
+    conn = connect_to_postgresql()
+    cursor = conn.cursor()
+    execute_values(cursor, query, data_to_insert)
+    conn.commit()
+    cursor.close()
+    conn.close()
+
+
+def create_sqlite3_db():
+    conn = connect_to_sqlite3()
+    cursor = conn.cursor()
+
+    cursor.execute('''
+        CREATE TABLE IF NOT EXISTS measurements (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            timestamp TEXT,
+            wavelength REAL,
+            intensity REAL
+        )
+    ''')
+
+    conn.commit()
+    conn.close()
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+    if args.initialize:
+        create_sqlite3_db()
+    elif args.test:
+        test_data = prepare_snapshot_db_data(
+            time.strftime("%Y%m%d--%H%M%S"),  # <- timestamp
+            [395.6, 396.1],  # <- wavelengths
+            [2, 1]  # <- intensities
+        )
+
+        save_measurements_to_sqlite3(
+            '''INSERT INTO measurements (timestamp, wavelength, intensity) VALUES (?, ?, ?)''',
+            test_data,
+        )
+
+    sys.exit(0)