Updates to website issue

requirements.txt

@@ -1,6 +1,6 @@
 astrodbkit==2.5
 astropy==7.1.0
-bokeh==3.7.3
+bokeh==3.8.2
 Flask==3.1.1
 Flask-Cors==6.0.0
 Flask-WTF==1.2.2
@@ -9,9 +9,10 @@ multiprocess==0.70.17
 numpy==1.26.4
 pandas==2.0.3
 pytest==8.3.4
-requests==2.32.4
+requests==2.33.0
 specutils==2.2.0
 SQLAlchemy==2.0.38
 tqdm==4.67.1
-Werkzeug==3.1.4
+Werkzeug==3.1.5
 WTForms==3.2.1
+simple @ git+https://github.com/SIMPLE-AstroDB/SIMPLE-db

simple_app/app_simple.py

@@ -170,12 +170,21 @@ def raw_query():
         except (ResourceClosedError, OperationalError, IndexError, SqliteWarning, BadSQLError, ProgrammingError):
             results = pd.DataFrame()
 
-        results = reference_handle(results, db_file, True)
+        if 'access_url' in results.columns:
+            url_links = [f'<a href="{url}" target="_blank">Link</a>' for url in results.access_url.values]
+            results.drop(columns=['access_url'], inplace=True)
+            download_col = '<a href="/write_sql_spectra" target="_blank">download</a>'
+            results.insert(1, download_col, url_links)
+
+        results = reference_handle(results, db_file)
+        res_len = len(results)
         stringed_results = one_df_query(results)
-        return render_template('raw_query.html', form=form, results=stringed_results, version_str=version_str)
+        return render_template('raw_query.html', form=form, results=stringed_results, query=query,
+                               res_len=res_len, version_str=version_str)
 
     else:
-        return render_template('raw_query.html', form=form, results=None, query='', version_str=version_str)
+        return render_template('raw_query.html', form=form, results=None, res_len=0, query='',
+                               version_str=version_str)
 
 
 @app_simple.route('/solo_result/<query>')
@@ -338,13 +347,8 @@ def create_spectra_files_for_download():
     results: pd.DataFrame = getattr(everything, 'spectra')
 
     # write all spectra for object to zipped file
-    zipped = write_spec_files(results.access_url.values)
-    if zipped is not None:
-        response = Response(zipped, mimetype='application/zip')
-        response = control_response(response, app_type='zip')
-        return response
-
-    abort(400, 'Could not download fits')
+    response = zip_spectra(results.access_url)
+    return response if response is not None else abort(400, 'Could not download fits')
 
 
 @app_simple.route('/write_multi_spectra', methods=['GET'])
@@ -360,13 +364,27 @@ def create_multi_spectra_files_for_download():
     spectra_df: pd.DataFrame = resultdict['Spectra']
 
     # write all spectra for object to zipped file
-    zipped = write_spec_files(spectra_df.access_url.values)
-    if zipped is not None:
-        response = Response(zipped, mimetype='application/zip')
-        response = control_response(response, app_type='zip')
-        return response
+    response = zip_spectra(spectra_df.access_url)
+    return response if response is not None else abort(400, 'Could not download fits')
+
+
+@app_simple.route('/write_sql_spectra', methods=['GET', 'POST'])
+def create_sql_spectra_files_for_download():
+    """
+    Downloads all spectra from a raw SQL query if an access_url column is present.
+    """
+    query = session.get('query')
+    db = SimpleDB(db_file)
+
+    # query database via sql
+    results: Optional[pd.DataFrame] = db.sql_query(query, fmt='pandas')
 
-    abort(400, 'Could not download fits')
+    if 'access_url' not in results.columns:
+        abort(400, 'No url column in SQL results')
+
+    # write all spectra in SQL result to zipped file
+    response = zip_spectra(results.access_url)
+    return response if response is not None else abort(400, 'Could not download fits')
 
 
 @app_simple.route('/write_filt', methods=['GET'])
@@ -474,6 +492,17 @@ def create_file_for_sql_download():
     return response
 
 
+@app_simple.route('/download_sqlite', methods=['GET'])
+def download_sqlite():
+    """
+    Downloads the SIMPLE.db file.
+    """
+    local_db_file = db_file.replace('sqlite:///', '')
+    directory = os.path.dirname(local_db_file) or '..'  # code runs in simple_app dir, we keep the binary one dir up
+    filename = os.path.basename(local_db_file)
+    return send_from_directory(directory, filename, as_attachment=True)
+
+
 args, db_file, photometric_filters, all_results, all_results_full, version_str, \
     all_photometry, all_bands, all_parallaxes, all_spectral_types = main_utils()
 night_sky_theme, js_callbacks = main_plots()

simple_app/simports.py

@@ -12,7 +12,7 @@
 from sqlite3 import Warning as SqliteWarning  # errors from sqlite
 from time import localtime, strftime  # time stuff for naming files
 from typing import Dict, Generator, List, Optional, Tuple, Union  # type hinting (good in IDEs)
-from urllib.parse import quote  # handling strings into url friendly form
+from urllib.parse import quote, unquote, urlparse  # handling strings into url friendly form
 from zipfile import ZipFile  # zipping files together
 
 import astropy.units as u  # units
@@ -67,3 +67,4 @@
 from tqdm import tqdm  # progress bars
 from werkzeug.exceptions import HTTPException  # underlying http
 from wtforms import StringField, SubmitField, TextAreaField, ValidationError  # web forms
+from simple import REFERENCE_TABLES

simple_app/templates/about.html

@@ -55,7 +55,9 @@ <h2 class="display-3">Holdings</h2>
         This database uses the <a href="https://docs.sqlalchemy.org/en/14/orm/index.html" target="_blank">
         SQLAlchemy ORM</a> and is designed to be interacted with via the
         <a href="https://pypi.org/project/astrodbkit2/" target="_blank"> astrodbkit2</a> package.
-        The full database can be downloaded from the
+        The full database can be downloaded from
+        <a href="{{ url_for('download_sqlite') }}">here</a>,
+        or the
         <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-binary" target="_blank">binary repo</a>
         and manipulated via local SQLite software.
         The change log following the different versions of the database can be read

simple_app/templates/base.html

@@ -23,8 +23,8 @@
     <script type="text/javascript" src="{{ url_for('static', filename='js/bootstrap.js') }}"></script>
 
     <!--Bokeh -->
-    <script type="text/javascript" src="{{ url_for('static', filename='js/bokeh-3.7.3.min.js') }}"></script>
-    <script type="text/javascript" src="{{ url_for('static', filename='js/bokeh-widgets-3.7.3.min.js') }}"></script>
+    <script type="text/javascript" src="{{ url_for('static', filename='js/bokeh-3.8.2.min.js') }}"></script>
+    <script type="text/javascript" src="{{ url_for('static', filename='js/bokeh-widgets-3.8.2.min.js') }}"></script>
     <script type="text/javascript">Bokeh.set_log_level("info");</script>
 
 

simple_app/templates/raw_query.html

@@ -35,6 +35,8 @@ <h3 class="display-4">Examples:</h3>
                     or read the
                     <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/tree/main/documentation" target="_blank">
                         documentation</a>.
+                    Download the live SQLite file
+                    <a href="{{ url_for('download_sqlite') }}">here</a>.
                     <ul style="list-style-type: none" class="display-6 sql-list">
                         <li>
                             select * from Sources
@@ -49,6 +51,24 @@ <h3 class="display-4">Examples:</h3>
                             using (source)<br>
                             where source like "%twa%"
                         </li>
+                        <li>
+                            select * from SpectralTypes<br>
+                            join Spectra using (source)<br>
+                            where spectral_type_string like "%sd%"
+                        </li>
+                        <li>
+                            select * from SpectralTypes<br>
+                            join Spectra using (source)<br>
+                            where (spectral_type_string like "%L%" or<br>
+                            spectral_type_string like "%T%" or<br>
+                            spectral_type_string like "%Y%") and<br>
+                            instrument like "SpeX"
+                        </li>
+                        <li>
+                            select * from Spectra<br>
+                            where telescope like "JWST"<br>
+                            and instrument like "NIRspec"
+                        </li>
                     </ul>
                     Note that the query backend is SQLite, be aware of any
                     <a href="https://www.sqlite.org/lang_keywords.html" target="_blank"> syntax differences</a>.
@@ -83,10 +103,18 @@ <h3 class="display-4">Examples:</h3>
     <div class="container table-responsive">
         {% if query != '' and results is not none %}
             <h2 class="display-3"><a href="{{ url_for('create_file_for_sql_download') }}">Results</a>:</h2>
+            {% if res_len > 10_000 %}
+                <p class="display-6">There are a large number of rows returned ({{ res_len }}),
+                please be patient whilst the table constructs itself.
+                You may wish to consider downloading the database and querying locally. </p>
+            {% endif %}
+            <p class="display-6">{{ results|safe }}</p>
         {% else %}
             <h2 class="display-3">Results:</h2>
+            {% if query != '' and results is none %}
+                <p class="display-5">No results found.</p>
+            {% endif %}
         {% endif %}
-        <p class="display-6">{{ results|safe if results is not none }}</p>
         {% if query != '' %}
         <p class="display-6">Are we missing any expected data? Please open an
             <a href="https://github.com/SIMPLE-AstroDB/SIMPLE-db/issues/new?assignees=&labels=data+ingestion&projects=&template=missing-source.md&title=Ingest+request%3A+%3Csource+name%3E" target="_blank">
@@ -97,8 +125,9 @@ <h2 class="display-3">Results:</h2>
         $(document).ready(function() {
             $(".table").each(function() {
                 $(this).DataTable( {
-                    responsive: true,
-                    autoWidth: false
+                    responsive: false,
+                    autoWidth: false,
+                    scrollX: true
                 });
             });
             }

simple_app/templates/solo_result.html

@@ -3,20 +3,26 @@
 {% block content %}
     <div class="container">
         <div class="row table-responsive">
-            <div class="col-10">
+            <div class="col-9">
                 <h1 class="display-2">
                     <a href="{{ url_for('create_files_for_solo_download') }}">
                         {{ query|safe }}
                     </a>
                 </h1>
             </div>
-            <div class="col-1">
+            <div class="col-1" style="justify-content: center; display: inline-flex">
+                <button onclick="window.open('https://github.com/SIMPLE-AstroDB/SIMPLE-db/search?q={{ query|urlencode }}.json&type=code', '_blank')"
+                        class="btn btn-success" style="align-self: flex-start" title="Find source JSON in SIMPLE-db">
+                    JSON
+                </button>
+            </div>
+            <div class="col-1" style="justify-content: center; display: inline-flex">
                 <button onclick="window.open('https://simbad.u-strasbg.fr/simbad/sim-basic?Ident={{ query|urlencode }}&submit=SIMBAD+search', '_blank')"
-                        class="btn btn-success" style="float: right" title="Link to SIMBAD">SIMBAD</button>
+                        class="btn btn-success" style="align-self: flex-start" title="Link to SIMBAD">SIMBAD</button>
             </div>
-            <div class="col-1">
-                <button onclick="share()" type="button" class="btn btn-success"
-                        style="float: right" title="Copy link to object">Share</button>
+            <div class="col-1" style="justify-content: center; display: inline-flex">
+                <button onclick="share()" type="button" class="btn btn-success" style="align-self: flex-start"
+                        title="Copy link to object">Share</button>
             </div>
         </div>
         <p class="display-6">Are we missing any expected data? Please open an