diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fab3812 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +venv/ +.venv/ +.idea/ +.env +**.db \ No newline at end of file diff --git a/USE_DB.md b/USE_DB.md new file mode 100644 index 0000000..51c6c64 --- /dev/null +++ b/USE_DB.md @@ -0,0 +1,49 @@ +# How to Use the PySpectrometer DB Module + +## Requirements: +1. Update, install, and upgrade pip: + ```shell + sudo apt update + sudo apt install python3-pip + pip3 --version + sudo pip3 install --upgrade pip + ``` +2. Set up the .env file: + ```shell + sudo cp src/.env.example src/.env + ``` + +## Installation: +1. Create a new virtual environment: + ```shell + python3 -m venv .venv + ``` +2. Activate the virtual environment: + ```shell + source .venv/bin/activate + ``` +3. Install requirements in the virtual environment: + ```shell + pip3 install -r requirements.txt + ``` + +## Initialize Database: +### Local / Development Environment: +1. Run the `db.py` module with the initialize argument: + ```shell + python3 src/db.py --initialize + ``` + > This creates a new local SQLite3 database in the root folder of this project. + ![db_file.png](media/db_file.png) +2. Run the `db.py` module again with the test argument: + ```shell + python3 src/db.py --test + ``` + > This will create two test entries in the newly created database in the `measurements` table. + You can delete them after verifying that it works. + ![db_test_entries.png](media/db_test_entries.png) + +### Production Environment: +> Coming soon... + +Now, if you run any of the PySpectrometer2 scripts, it should, based on the DB_TYPE (`sqlite3`, `postgresql`) from the .env file, select the correct database and write data to the database whenever a new .csv file is generated. diff --git a/media/db_file.png b/media/db_file.png new file mode 100644 index 0000000..0fdcbb2 Binary files /dev/null and b/media/db_file.png differ diff --git a/media/db_test_entries.png b/media/db_test_entries.png new file mode 100644 index 0000000..fbbaa35 Binary files /dev/null and b/media/db_test_entries.png differ diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..af010ed --- /dev/null +++ b/requirements.in @@ -0,0 +1,3 @@ +psycopg2-binary +python-dotenv +opencv-python diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..687f469 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,14 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile +# +numpy==2.0.1 + # via opencv-python +opencv-python==4.10.0.84 + # via -r requirements.in +psycopg2-binary==2.9.9 + # via -r requirements.in +python-dotenv==1.0.1 + # via -r requirements.in diff --git a/src/.env.example b/src/.env.example new file mode 100644 index 0000000..e1824db --- /dev/null +++ b/src/.env.example @@ -0,0 +1,6 @@ +DB_TYPE=sqlite3 +DB_HOST= +DB_PORT=5432 +DB_NAME= +DB_USERNAME= +DB_PASSWORD= diff --git a/src/PySpectrometer2-Picam2-v1.0.py b/src/PySpectrometer2-Picam2-v1.0.py index c0965ef..7b31f64 100644 --- a/src/PySpectrometer2-Picam2-v1.0.py +++ b/src/PySpectrometer2-Picam2-v1.0.py @@ -32,6 +32,8 @@ import base64 import argparse from picamera2 import Picamera2 +from src.db import snapshot_database + parser = argparse.ArgumentParser() group = parser.add_mutually_exclusive_group() @@ -149,7 +151,8 @@ def snapshot(savedata): now = time.strftime("%Y%m%d--%H%M%S") timenow = time.strftime("%H:%M:%S") imdata1 = savedata[0] - graphdata = savedata[1] + wavelengths = savedata[1][0] + intensities = savedata[1][1] if dispWaterfall == True: imdata2 = savedata[2] cv2.imwrite("waterfall-" + now + ".png",imdata2) @@ -158,11 +161,14 @@ def snapshot(savedata): #print(graphdata[1]) #intensities f = open("Spectrum-"+now+'.csv','w') f.write('Wavelength,Intensity\r\n') - for x in zip(graphdata[0],graphdata[1]): + for x in zip(wavelengths, intensities): f.write(str(x[0])+','+str(x[1])+'\r\n') f.close() + + snapshot_database(now, wavelengths, intensities) + message = "Last Save: "+timenow - return(message) + return message while True: diff --git a/src/PySpectrometer2-USB-v1.0.py b/src/PySpectrometer2-USB-v1.0.py index 8bb6adc..1b4fa2f 100644 --- a/src/PySpectrometer2-USB-v1.0.py +++ b/src/PySpectrometer2-USB-v1.0.py @@ -33,6 +33,8 @@ import base64 import argparse +from src.db import snapshot_database + parser = argparse.ArgumentParser() parser.add_argument("--device", type=int, default=0, help="Video Device number e.g. 0, use v4l2-ctl --list-devices") parser.add_argument("--fps", type=int, default=30, help="Frame Rate e.g. 30") @@ -153,7 +155,8 @@ def snapshot(savedata): now = time.strftime("%Y%m%d--%H%M%S") timenow = time.strftime("%H:%M:%S") imdata1 = savedata[0] - graphdata = savedata[1] + wavelengths = savedata[1][0] + intensities = savedata[1][1] if dispWaterfall == True: imdata2 = savedata[2] cv2.imwrite("waterfall-" + now + ".png",imdata2) @@ -162,11 +165,14 @@ def snapshot(savedata): #print(graphdata[1]) #intensities f = open("Spectrum-"+now+'.csv','w') f.write('Wavelength,Intensity\r\n') - for x in zip(graphdata[0],graphdata[1]): + for x in zip(wavelengths, intensities): f.write(str(x[0])+','+str(x[1])+'\r\n') f.close() + + snapshot_database(now, wavelengths, intensities) + message = "Last Save: "+timenow - return(message) + return message while(cap.isOpened()): # Capture frame-by-frame diff --git a/src/db.py b/src/db.py new file mode 100644 index 0000000..82749e7 --- /dev/null +++ b/src/db.py @@ -0,0 +1,120 @@ +import os +import sys +import time +import sqlite3 +import argparse +import psycopg2 +from dotenv import load_dotenv +from psycopg2.extras import execute_values + +load_dotenv() + +SQLITE3_DB = 'spectrometer_data.db' +DB_TYPE = os.getenv('DB_TYPE', 'sqlite3') + +if DB_TYPE != 'sqlite3': + DB_HOST = os.environ.get('DB_HOST', 'localhost') + DB_PORT = os.environ.get('DB_PORT', '5432') + DB_NAME = os.environ.get('DB_NAME', 'postgres') + DB_USERNAME = os.environ.get('DB_USERNAME', 'postgres') + DB_PASSWORD = os.environ.get('DB_PASSWORD', 'postgres') + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument( + '-i', '--initialize', action='store_true', help='Initialize SQLite3 database', + ) + parser.add_argument( + '-t', '--test', action='store_true', help='Test SQLite3 database measurements insertion.', + ) + return parser.parse_args() + + +def snapshot_database(timestamp, wavelengths, intensities): + data_to_insert = prepare_snapshot_db_data(timestamp, wavelengths, intensities) + + if DB_TYPE == "postgresql": + save_measurements_to_postgresql( + ''' + INSERT INTO measurements (timestamp, wavelength, intensity) + VALUES %s + ''' % data_to_insert, + data_to_insert, + ) + else: + save_measurements_to_sqlite3( + '''INSERT INTO measurements (timestamp, wavelength, intensity) VALUES (?, ?, ?)''', + data_to_insert, + ) + + +def prepare_snapshot_db_data(timestamp, wavelengths, intensities): + return [(timestamp, wl, inten) for wl, inten in zip(wavelengths, intensities)] + + +def connect_to_sqlite3() -> sqlite3.Connection: + return sqlite3.connect(SQLITE3_DB) + + +def connect_to_postgresql() -> psycopg2.connect: + return psycopg2.connect( + host=DB_HOST, + database=DB_NAME, + user=DB_USERNAME, + password=DB_PASSWORD, + port=DB_PORT, + ) + + +def save_measurements_to_sqlite3(query, data_to_insert): + conn = connect_to_sqlite3() + cursor = conn.cursor() + cursor.executemany(query, data_to_insert) + conn.commit() + conn.close() + + +def save_measurements_to_postgresql(query, data_to_insert): + conn = connect_to_postgresql() + cursor = conn.cursor() + execute_values(cursor, query, data_to_insert) + conn.commit() + cursor.close() + conn.close() + + +def create_sqlite3_db(): + conn = connect_to_sqlite3() + cursor = conn.cursor() + + cursor.execute(''' + CREATE TABLE IF NOT EXISTS measurements ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT, + wavelength REAL, + intensity REAL + ) + ''') + + conn.commit() + conn.close() + + +if __name__ == '__main__': + args = parse_arguments() + if args.initialize: + create_sqlite3_db() + elif args.test: + test_data = prepare_snapshot_db_data( + time.strftime("%Y%m%d--%H%M%S"), # <- timestamp + [395.6, 396.1], # <- wavelengths + [2, 1] # <- intensities + ) + + save_measurements_to_sqlite3( + '''INSERT INTO measurements (timestamp, wavelength, intensity) VALUES (?, ?, ?)''', + test_data, + ) + + sys.exit(0)