Smdb auto upload

.gitignore

@@ -209,3 +209,5 @@ __marimo__/
 # pixi environments
 .pixi
 *.egg-info
+.vscode/launch.json
+.vscode/settings.json

demux/pixi.toml

@@ -14,3 +14,5 @@ matplotlib = ">=3.10.0,<4"
 samshee = ">=0.2.0,<0.3"
 pandas = ">=2.3.0,<3"
 numpy = ">=2.3.0,<3"
+sqlalchemy = ">=2.0.45,<3"
+psycopg2 = ">=2.9.11,<3"

demux/smdb-upload/environment.yaml

@@ -0,0 +1,8 @@
+name: smdb_upload_env
+channels:
+  - defaults
+dependencies:
+  - python=3.11 
+  - pandas 
+  - sqlalchemy  
+  - psycopg2

demux/smdb-upload/requirements.txt

@@ -0,0 +1,6 @@
+# Requirements for smdb_upload.py
+
+# List your package dependencies here, for example:
+pandas
+sqlalchemy
+psycopg2

demux/smdb-upload/smdb_upload.py

@@ -0,0 +1,133 @@
+import subprocess
+import argparse
+from sqlalchemy import create_engine
+import pandas as pd
+from datetime import datetime
+from uuid import uuid4
+from zoneinfo import ZoneInfo
+
+parser = argparse.ArgumentParser(
+        description="Upload demultiplex stats, run info, and sample sheet metadata to SMDB."
+    )
+parser.add_argument("-c", "--path_to_demultiplex_stats", required=True, help="Path to demultiplex stats CSV")
+parser.add_argument("-r", "--path_to_run_info", required=True, help="Path to run info XML")
+parser.add_argument("-x", "--path_to_sample_sheet", required=True, help="Path to sample sheet CSV")
+parser.add_argument("-n", "--db_name", required=True, help="Database name")
+parser.add_argument("-s", "--schema_name", required=True, help="Target schema name")
+parser.add_argument("-u", "--db_user", required=True, help="Database user")
+parser.add_argument("-p", "--db_password", required=True, help="Database password")
+parser.add_argument("-d", "--db_host", required=True, help="Database host")
+parser.add_argument("-o", "--db_port", required=True, type=int, help="Database port")
+parser.add_argument("-t", "--table_name", required=True, help="Target table name")
+parser.add_argument("-e", "--send_upload_receipts_to", required=True, help="Space-separated emails for upload receipts")
+
+args = parser.parse_args()
+
+def upload_demultiplex_stats(path_to_demultiplex_stats, path_to_run_info, path_to_sample_sheet, db_name, schema_name, db_user, db_password, db_host, db_port, table_name, send_upload_receipts_to):
+    """
+    Connects to a PostgreSQL database, queries a view, and exports the results to a TSV file.
+
+    Args:
+        path_to_demultiplex_stats (str): Path to the demultiplex stats CSV file.
+        path_to_run_info (str): Path to the run info CSV file.
+        path_to_sample_sheet (str): Path to the sample sheet CSV file.
+        db_name (str): Database name.
+        db_user (str): Database user.
+        db_password (str): Database password.
+        db_port (int): Database port (default is 5432).
+        table_name (str): Name of the PostgreSQL table to export.
+        output_file (str): Path to the output TSV file.
+    """
+
+    dmux_stats = pd.read_csv(path_to_demultiplex_stats)
+    run_info = pd.read_xml(path_to_run_info, parser='etree')
+    sample_sheet = pd.read_csv(path_to_sample_sheet)
+
+    sample_sheet = (
+        sample_sheet.iloc[0:17]
+        .dropna(how="all", axis=1)
+        .dropna(how="any", axis=0)
+        .rename(columns={"[Header]": "Attribute", "Unnamed: 1": "Value"})
+        .set_index("Attribute")
+    )
+
+    send_upload_receipts_to =  send_upload_receipts_to.replace(" ", "; ")
+    upload_sheets = str(path_to_demultiplex_stats) + "; " + str(path_to_run_info) + "; " + str(path_to_sample_sheet)
+    timestamp = datetime.now(ZoneInfo("Europe/Copenhagen")) 
+    seq_run_id = run_info.at[0, 'Id']
+    seq_run_number = run_info.at[0, 'Number']
+    seq_machine_id = run_info.at[0, 'Instrument']
+    flowcell_position = run_info.at[0, 'Id'].split('_')[3][0]
+    flowcell_id = run_info.at[0, 'Flowcell']
+
+    unformatted_seq_date = run_info.at[0, 'Date']
+    dt = datetime.strptime(unformatted_seq_date, "%m/%d/%Y %I:%M:%S %p")
+    formatted_date = dt.strftime("%Y-%m-%d")
+
+    dmux_stats['database_insert_by'] = send_upload_receipts_to
+    dmux_stats['upload_sheet'] = upload_sheets
+    dmux_stats['database_insert_datetime_utc'] = timestamp
+    dmux_stats['upload_uuid'] = uuid4()
+    dmux_stats['sequencing_run_id'] = seq_run_id
+    dmux_stats['sequencing_run_number'] = seq_run_number
+    dmux_stats['sequencing_machine_id'] = seq_machine_id
+    dmux_stats['sequencing_run_date'] = str(formatted_date)
+    dmux_stats['sequencing_tube_tag'] = None
+    dmux_stats['flowcell_position'] = flowcell_position    
+    dmux_stats['flowcell'] = flowcell_id
+
+    # Add sequencing pool based on lane
+    pool_lanes = [sample_sheet.loc['PoolLane1', 'Value'],
+                  sample_sheet.loc['PoolLane2', 'Value'],
+                  sample_sheet.loc['PoolLane3', 'Value'],
+                  sample_sheet.loc['PoolLane4', 'Value']]
+    num_lanes = len(dmux_stats['Lane'].unique())
+    assert num_lanes == 4, 'Expected 4 lanes in the demultiplex stats file. got ' + str(num_lanes)
+    dmux_stats['sequencing_tube_tag'] = dmux_stats['Lane'].apply(lambda x: pool_lanes[x - 1])
+
+    ENGINE = create_engine(f"postgresql://{db_user}:{db_password}@{db_host}:{db_port}/{db_name}")    
+
+    q = 'select column_name_db, column_name_sheet from name_maps.column_names where table_id = 47;'
+
+    renamer = pd.read_sql(q, ENGINE)
+    rename_dict = dict(zip(renamer['column_name_sheet'], renamer['column_name_db']))
+    dmux_stats = dmux_stats.rename(columns=rename_dict)
+
+    dmux_stats.to_sql(table_name, ENGINE, schema=schema_name, if_exists="append", index=False)
+
+    email_cmd = [
+        "mail",
+        "-s",
+        "Sequencing stats successfully uploaded to SMDB",
+        "-a",
+        str(path_to_demultiplex_stats),
+        "-a",
+        str(path_to_run_info),
+        "-a",
+        str(path_to_sample_sheet)
+    ]
+    email_cmd.extend(send_upload_receipts_to.split("; "))
+    subprocess.run(
+        email_cmd,
+        input="The appended sequencing stats has been successfully uploaded to SMDB.",
+        text=True,
+        check=False,
+    )
+
+
+upload_demultiplex_stats(
+    path_to_demultiplex_stats=args.path_to_demultiplex_stats,
+    path_to_run_info=args.path_to_run_info,
+    path_to_sample_sheet=args.path_to_sample_sheet,
+    db_name=args.db_name,
+    schema_name=args.schema_name,
+    db_user=args.db_user,
+    db_password=args.db_password,
+    db_port=args.db_port,
+    table_name=args.table_name,
+    db_host=args.db_host,
+    send_upload_receipts_to=args.send_upload_receipts_to
+    )
+
+
+

demux/wrapper_bclconvert.sh

@@ -24,6 +24,8 @@ python3 -c 'import pathlib'
 python3 -c 'import collections'
 python3 -c 'import plotly'
 python3 -c 'import matplotlib'
+python3 -c 'import psycopg2'
+python3 -c 'import sqlalchemy'
 
 
 ## Functions
@@ -64,8 +66,10 @@ EXTRA=$@
 RUN=20`basename $IN_FOLDER`
 if [[ `realpath $OUT_FOLDER` == "/maps/datasets/caeg_fastq" ]]; then
     OUT_FOLDER=$OUT_FOLDER/${RUN:0:4}/$RUN
+    CAEG_DATA=true
 else
     OUT_FOLDER=$OUT_FOLDER/$RUN
+    CAEG_DATA=false
 fi
 
 # Check if output folder exists
@@ -143,9 +147,42 @@ mkdir -p $OUT_FOLDER
 	cd ../
     done
 
+	if [ $CAEG_DATA = true ]; then
+		echo `date`" [$RUN] uploading metadata to SMDB"
+		SMDB_UPLOAD_SCRIPT="$BASEDIR/smdb-upload/smdb_upload.py"
+
+		DEMUX_STATS_CSV="$OUT_FOLDER/Reports/Demultiplex_Stats.csv"
+		RUNINFO_XML="$OUT_FOLDER/Reports/RunInfo.xml"
+		UPLOAD_RECEIPTS_TO="julie.bitz-thorsen@sund.ku.dk"
+
+		# Required DB env vars (do not hardcode secrets in the script)
+		: "${DB_NAME:?Set DB_NAME in environment}"
+		: "${DB_SCHEMA:?Set DB_SCHEMA in environment}"
+		: "${DB_USER:?Set DB_USER in environment}"
+		: "${DB_PASSWORD:?Set DB_PASSWORD in environment}"
+		: "${DB_HOST:?Set DB_HOST in environment}"
+		: "${DB_PORT:?Set DB_PORT in environment (integer)}"
+		: "${DB_TABLE:?Set DB_TABLE in environment}"
+
+		python3 "$SMDB_UPLOAD_SCRIPT" \
+				--path_to_demultiplex_stats "$DEMUX_STATS_CSV" \
+				--path_to_run_info "$RUNINFO_XML" \
+				--path_to_sample_sheet "$SS" \
+				--db_name "$DB_NAME" \
+				--schema_name "$DB_SCHEMA" \
+				--db_user "$DB_USER" \
+				--db_password "$DB_PASSWORD" \
+				--db_host "$DB_HOST" \
+				--db_port "$DB_PORT" \
+				--table_name "$DB_TABLE" \
+				--send_upload_receipts_to "$UPLOAD_RECEIPTS_TO"
+	fi
+
     TIMESTAMP=`date "+%Y%m%d_%H%M%S"`
     touch seqcenter.$TIMESTAMP.done
     cd ../
 } 2>&1 | tee $OUT_FOLDER/$RUN.demux.log
 
+
+
 exit 0