Stream JSON results to client

Author: Sebastian Benjamin

jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx

@@ -68,7 +68,6 @@ const VariantTableWidget = observer(props => {
 
             return obj
         }))
-        setTotalHits(data.totalHits)
         setDataLoaded(true)
     }
 
@@ -96,12 +95,35 @@ const VariantTableWidget = observer(props => {
         currentUrl.searchParams.set("sortDirection", sort.toString());
 
         if (pushToHistory) {
-          window.history.pushState(null, "", currentUrl.toString());
+            window.history.pushState(null, "", currentUrl.toString());
         }
 
         setFilters(passedFilters);
-        setDataLoaded(false)
-        fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
+        setDataLoaded(false);
+        setFeatures([]);
+
+        fetchLuceneQuery(
+            passedFilters,
+            sessionId,
+            trackGUID,
+            page,
+            pageSize,
+            field,
+            sort,
+            (row) => {
+                setFeatures(prev => {
+                    row.id = prev.length;
+                    row.trackId = trackId;
+                    return [...prev, row];
+                });
+            },
+            () => setDataLoaded(true),
+            (error) => {
+                console.error("Stream error:", error);
+                setError(error);
+                setDataLoaded(true);
+            }
+        );
     }
 
     const handleExport = () => {
@@ -274,7 +296,6 @@ const VariantTableWidget = observer(props => {
 
     const [filterModalOpen, setFilterModalOpen] = useState(false);
     const [filters, setFilters] = useState([]);
-    const [totalHits, setTotalHits] = useState(0);
     const [fieldTypeInfo, setFieldTypeInfo] = useState<FieldModel[]>([]);
     const [allowedGroupNames, setAllowedGroupNames] = useState<string[]>([]);
     const [promotedFilters, setPromotedFilters] = useState<Map<string, Filter[]>>(null);

jbrowse/src/client/JBrowse/utils.ts

@@ -328,23 +328,24 @@ export function serializeLocationToEncodedSearchString(contig, start, end) {
     return createEncodedFilterString(filters)
 }
 
-export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) {
+export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString,
+    handleRow, handleComplete, handleError) {
     if (!offset) {
         offset = 0
     }
 
     if (!sessionId) {
-        failureCallback("There was an error: " + "Lucene query: no session ID")
+        handleError("There was an error: " + "Lucene query: no session ID")
         return
     }
 
     if (!trackGUID) {
-        failureCallback("There was an error: " + "Lucene query: no track ID")
+        handleError("There was an error: " + "Lucene query: no track ID")
         return
     }
 
     if (!filters) {
-        failureCallback("There was an error: " + "Lucene query: no filters")
+        handleError("There was an error: " + "Lucene query: no filters")
         return
     }
 
@@ -358,27 +359,59 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
         sortReverse = false
     }
 
-    return Ajax.request({
-        url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'),
-        method: 'GET',
-        success: async function(res){
-            let jsonRes = JSON.parse(res.response);
-            successCallback(jsonRes)
-        },
-        failure: function(res) {
-            console.error("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId)
-            failureCallback("There was an error: status " + res.status)
-        },
-        params: {
-            "searchString": encoded,
-            "sessionId": sessionId,
-            "trackId": trackGUID,
-            "offset": offset,
-            "pageSize": pageSize,
-            "sortField": sortField ?? "genomicPosition",
-            "sortReverse": sortReverse 
-        },
+    const params = new URLSearchParams({
+        searchString: encoded,
+        sessionId,
+        trackId: trackGUID,
+        offset: offset,
+        pageSize: pageSize,
+        sortField: sortField ?? "genomicPosition",
+        sortReverse: sortReverse,
     });
+
+    try {
+        const response = await fetch(`/jbrowse/luceneQuery.api?${params.toString()}`);
+        if (!response.ok || !response.body) {
+            throw new Error(`HTTP error ${response.status}`);
+        }
+
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder("utf-8");
+        let buffer = '';
+
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) break;
+
+            buffer += decoder.decode(value, { stream: true });
+
+            let boundary;
+            while ((boundary = buffer.indexOf('\n')) >= 0) {
+                const line = buffer.slice(0, boundary).trim();
+                buffer = buffer.slice(boundary + 1);
+                if (line) {
+                    try {
+                        const parsed = JSON.parse(line);
+                        handleRow(parsed);
+                    } catch (err) {
+                        console.error('Failed to parse line:', line, err);
+                    }
+                }
+            }
+        }
+
+        if (buffer.trim()) {
+            try {
+                handleRow(JSON.parse(buffer));
+            } catch (err) {
+                console.error('Final line parse error:', buffer, err);
+            }
+        }
+
+        handleComplete();
+    } catch (error) {
+        handleError(error.toString());
+    }
 }
 
 export class FieldModel {

jbrowse/src/org/labkey/jbrowse/JBrowseController.java

@@ -962,14 +962,18 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors)
 
             try
             {
-                return new ApiSimpleResponse(searcher.doSearchJSON(
+                HttpServletResponse response = getViewContext().getResponse();
+                response.setContentType("application/x-ndjson");
+                searcher.doSearchJSON(
                         getUser(),
                         PageFlowUtil.decode(form.getSearchString()),
                         form.getPageSize(),
                         form.getOffset(),
                         form.getSortField(),
-                        form.getSortReverse()
-                ));
+                        form.getSortReverse(),
+                        response
+                );
+                return null;
             }
             catch (Exception e)
             {

jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java

@@ -1,6 +1,7 @@
 package org.labkey.jbrowse;
 
 import jakarta.servlet.http.HttpServletResponse;
+import org.apache.catalina.connector.Response;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Logger;
 import org.apache.lucene.analysis.Analyzer;
@@ -202,9 +203,9 @@ public String extractFieldName(String queryString)
         return parts.length > 0 ? parts[0].trim() : null;
     }
 
-    public JSONObject doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException {
+    public void doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException {
         SearchConfig searchConfig = createSearchConfig(u, searchString, pageSize, offset, sortField, sortReverse);
-        return paginateJSON(searchConfig);
+        paginateJSON(searchConfig, response);
     }
 
     public void doSearchCSV(User u, String searchString, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException {
@@ -330,32 +331,26 @@ else if (numericQueryParserFields.containsKey(fieldName))
         return new SearchConfig(cacheEntry, query, pageSize, offset, sort, fieldsList);
     }
 
-    private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseException {
+    private void paginateJSON(SearchConfig c, HttpServletResponse response) throws IOException, ParseException {
         IndexSearcher searcher = c.cacheEntry.indexSearcher;
         TopDocs topDocs;
+        PrintWriter writer = response.getWriter();
 
         if (c.offset == 0) {
             topDocs = searcher.search(c.query, c.pageSize, c.sort);
         } else {
             TopFieldDocs prev = searcher.search(c.query, c.pageSize * c.offset, c.sort);
-            long totalHits = prev.totalHits.value;
             ScoreDoc[] prevHits = prev.scoreDocs;
 
             if (prevHits.length < c.pageSize * c.offset)
             {
-                JSONObject results = new JSONObject();
-                results.put("data", Collections.emptyList());
-                results.put("totalHits", totalHits);
-                return results;
+                return;
             }
 
             ScoreDoc lastDoc = prevHits[c.pageSize * c.offset - 1];
             topDocs = searcher.searchAfter(lastDoc, c.query, c.pageSize, c.sort);
         }
 
-        JSONObject results = new JSONObject();
-        List<JSONObject> data = new ArrayList<>(topDocs.scoreDocs.length);
-
         for (ScoreDoc sd : topDocs.scoreDocs)
         {
             Document doc = searcher.storedFields().document(sd.doc);
@@ -366,12 +361,10 @@ private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseExcepti
                 String[] vals = doc.getValues(name);
                 elem.put(name, vals.length > 1 ? Arrays.asList(vals) : vals[0]);
             }
-            data.add(elem);
-        }
 
-        results.put("data", data);
-        results.put("totalHits", topDocs.totalHits.value);
-        return results;
+            writer.println(elem);
+            writer.flush();
+        }
     }
 
     private void exportCSV(SearchConfig c, HttpServletResponse response) throws IOException
@@ -648,8 +641,9 @@ public void cacheDefaultQuery()
     {
         try
         {
+            HttpServletResponse response = new Response();
             JBrowseLuceneSearch.clearCache(_jsonFile.getObjectId());
-            doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false);
+            doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false, response);
         }
         catch (ParseException | IOException e)
         {