Skip to content

Commit 9c6d114

Browse files
hextrazaSebastian Benjaminbbimber
authored
Stream JSON results to client (#330)
* Stream JSON results to client * Use ActionURL to build lucene queries * Switch to ExportAction --------- Co-authored-by: Sebastian Benjamin <sebastiancbenjamin@gmail.com> Co-authored-by: bbimber <bbimber@gmail.com>
1 parent 1ac2cb4 commit 9c6d114

File tree

4 files changed

+106
-54
lines changed

4 files changed

+106
-54
lines changed

‎jbrowse/src/client/JBrowse/VariantSearch/components/VariantTableWidget.tsx‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,6 @@ const VariantTableWidget = observer(props => {
6868

6969
return obj
7070
}))
71-
setTotalHits(data.totalHits)
7271
setDataLoaded(true)
7372
}
7473

@@ -96,12 +95,35 @@ const VariantTableWidget = observer(props => {
9695
currentUrl.searchParams.set("sortDirection", sort.toString());
9796

9897
if (pushToHistory) {
99-
window.history.pushState(null, "", currentUrl.toString());
98+
window.history.pushState(null, "", currentUrl.toString());
10099
}
101100

102101
setFilters(passedFilters);
103-
setDataLoaded(false)
104-
fetchLuceneQuery(passedFilters, sessionId, trackGUID, page, pageSize, field, sort, (json)=>{handleSearch(json)}, (error) => {setDataLoaded(true); setError(error)});
102+
setDataLoaded(false);
103+
setFeatures([]);
104+
105+
fetchLuceneQuery(
106+
passedFilters,
107+
sessionId,
108+
trackGUID,
109+
page,
110+
pageSize,
111+
field,
112+
sort,
113+
(row) => {
114+
setFeatures(prev => {
115+
row.id = prev.length;
116+
row.trackId = trackId;
117+
return [...prev, row];
118+
});
119+
},
120+
() => setDataLoaded(true),
121+
(error) => {
122+
console.error("Stream error:", error);
123+
setError(error);
124+
setDataLoaded(true);
125+
}
126+
);
105127
}
106128

107129
const handleExport = () => {
@@ -274,7 +296,6 @@ const VariantTableWidget = observer(props => {
274296

275297
const [filterModalOpen, setFilterModalOpen] = useState(false);
276298
const [filters, setFilters] = useState([]);
277-
const [totalHits, setTotalHits] = useState(0);
278299
const [fieldTypeInfo, setFieldTypeInfo] = useState<FieldModel[]>([]);
279300
const [allowedGroupNames, setAllowedGroupNames] = useState<string[]>([]);
280301
const [promotedFilters, setPromotedFilters] = useState<Map<string, Filter[]>>(null);

‎jbrowse/src/client/JBrowse/utils.ts‎

Lines changed: 59 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -328,23 +328,24 @@ export function serializeLocationToEncodedSearchString(contig, start, end) {
328328
return createEncodedFilterString(filters)
329329
}
330330

331-
export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString, successCallback, failureCallback) {
331+
export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pageSize, sortField, sortReverseString,
332+
handleRow, handleComplete, handleError) {
332333
if (!offset) {
333334
offset = 0
334335
}
335336

336337
if (!sessionId) {
337-
failureCallback("There was an error: " + "Lucene query: no session ID")
338+
handleError("There was an error: " + "Lucene query: no session ID")
338339
return
339340
}
340341

341342
if (!trackGUID) {
342-
failureCallback("There was an error: " + "Lucene query: no track ID")
343+
handleError("There was an error: " + "Lucene query: no track ID")
343344
return
344345
}
345346

346347
if (!filters) {
347-
failureCallback("There was an error: " + "Lucene query: no filters")
348+
handleError("There was an error: " + "Lucene query: no filters")
348349
return
349350
}
350351

@@ -358,27 +359,60 @@ export async function fetchLuceneQuery(filters, sessionId, trackGUID, offset, pa
358359
sortReverse = false
359360
}
360361

361-
return Ajax.request({
362-
url: ActionURL.buildURL('jbrowse', 'luceneQuery.api'),
363-
method: 'GET',
364-
success: async function(res){
365-
let jsonRes = JSON.parse(res.response);
366-
successCallback(jsonRes)
367-
},
368-
failure: function(res) {
369-
console.error("There was an error: " + res.status + "\n Status Body: " + res.responseText + "\n Session ID:" + sessionId)
370-
failureCallback("There was an error: status " + res.status)
371-
},
372-
params: {
373-
"searchString": encoded,
374-
"sessionId": sessionId,
375-
"trackId": trackGUID,
376-
"offset": offset,
377-
"pageSize": pageSize,
378-
"sortField": sortField ?? "genomicPosition",
379-
"sortReverse": sortReverse
380-
},
381-
});
362+
const params = {
363+
searchString: encoded,
364+
sessionId,
365+
trackId: trackGUID,
366+
offset: offset,
367+
pageSize: pageSize,
368+
sortField: sortField ?? "genomicPosition",
369+
sortReverse: sortReverse,
370+
};
371+
372+
try {
373+
const url = ActionURL.buildURL('jbrowse', 'luceneQuery.api', null, params);
374+
const response = await fetch(url);
375+
if (!response.ok || !response.body) {
376+
throw new Error(`HTTP error ${response.status}`);
377+
}
378+
379+
const reader = response.body.getReader();
380+
const decoder = new TextDecoder("utf-8");
381+
let buffer = '';
382+
383+
while (true) {
384+
const { done, value } = await reader.read();
385+
if (done) break;
386+
387+
buffer += decoder.decode(value, { stream: true });
388+
389+
let boundary;
390+
while ((boundary = buffer.indexOf('\n')) >= 0) {
391+
const line = buffer.slice(0, boundary).trim();
392+
buffer = buffer.slice(boundary + 1);
393+
if (line) {
394+
try {
395+
const parsed = JSON.parse(line);
396+
handleRow(parsed);
397+
} catch (err) {
398+
console.error('Failed to parse line:', line, err);
399+
}
400+
}
401+
}
402+
}
403+
404+
if (buffer.trim()) {
405+
try {
406+
handleRow(JSON.parse(buffer));
407+
} catch (err) {
408+
console.error('Final line parse error:', buffer, err);
409+
}
410+
}
411+
412+
handleComplete();
413+
} catch (error) {
414+
handleError(error.toString());
415+
}
382416
}
383417

384418
export class FieldModel {

‎jbrowse/src/org/labkey/jbrowse/JBrowseController.java‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.json.JSONObject;
2929
import org.labkey.api.action.ApiResponse;
3030
import org.labkey.api.action.ApiSimpleResponse;
31+
import org.labkey.api.action.ExportAction;
3132
import org.labkey.api.action.MutatingApiAction;
3233
import org.labkey.api.action.ReadOnlyApiAction;
3334
import org.labkey.api.action.SimpleApiJsonForm;
@@ -944,10 +945,10 @@ else if (!isValidUUID(form.getTrackId()))
944945
}
945946

946947
@RequiresPermission(ReadPermission.class)
947-
public static class LuceneQueryAction extends ReadOnlyApiAction<LuceneQueryForm>
948+
public static class LuceneQueryAction extends ExportAction<LuceneQueryForm>
948949
{
949950
@Override
950-
public ApiResponse execute(LuceneQueryForm form, BindException errors)
951+
public void export(LuceneQueryForm form, HttpServletResponse response, BindException errors) throws Exception
951952
{
952953
JBrowseLuceneSearch searcher;
953954
try
@@ -957,30 +958,31 @@ public ApiResponse execute(LuceneQueryForm form, BindException errors)
957958
catch (IllegalArgumentException e)
958959
{
959960
errors.reject(ERROR_MSG, e.getMessage());
960-
return null;
961+
return;
961962
}
962963

963964
try
964965
{
965-
return new ApiSimpleResponse(searcher.doSearchJSON(
966+
response.setContentType("application/x-ndjson");
967+
searcher.doSearchJSON(
966968
getUser(),
967969
PageFlowUtil.decode(form.getSearchString()),
968970
form.getPageSize(),
969971
form.getOffset(),
970972
form.getSortField(),
971-
form.getSortReverse()
972-
));
973+
form.getSortReverse(),
974+
response
975+
);
973976
}
974977
catch (Exception e)
975978
{
976979
_log.error("Error in JBrowse lucene query", e);
977980
errors.reject(ERROR_MSG, e.getMessage());
978-
return null;
979981
}
980982
}
981983

982984
@Override
983-
public void validateForm(LuceneQueryForm form, Errors errors)
985+
public void validate(LuceneQueryForm form, BindException errors)
984986
{
985987
if ((form.getSearchString() == null || form.getSessionId() == null || form.getTrackId() == null))
986988
{

‎jbrowse/src/org/labkey/jbrowse/JBrowseLuceneSearch.java‎

Lines changed: 11 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.labkey.jbrowse;
22

33
import jakarta.servlet.http.HttpServletResponse;
4+
import org.apache.catalina.connector.Response;
45
import org.apache.commons.lang3.StringUtils;
56
import org.apache.logging.log4j.Logger;
67
import org.apache.lucene.analysis.Analyzer;
@@ -202,9 +203,9 @@ public String extractFieldName(String queryString)
202203
return parts.length > 0 ? parts[0].trim() : null;
203204
}
204205

205-
public JSONObject doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse) throws IOException, ParseException {
206+
public void doSearchJSON(User u, String searchString, final int pageSize, final int offset, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException {
206207
SearchConfig searchConfig = createSearchConfig(u, searchString, pageSize, offset, sortField, sortReverse);
207-
return paginateJSON(searchConfig);
208+
paginateJSON(searchConfig, response);
208209
}
209210

210211
public void doSearchCSV(User u, String searchString, String sortField, boolean sortReverse, HttpServletResponse response) throws IOException, ParseException {
@@ -330,32 +331,26 @@ else if (numericQueryParserFields.containsKey(fieldName))
330331
return new SearchConfig(cacheEntry, query, pageSize, offset, sort, fieldsList);
331332
}
332333

333-
private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseException {
334+
private void paginateJSON(SearchConfig c, HttpServletResponse response) throws IOException, ParseException {
334335
IndexSearcher searcher = c.cacheEntry.indexSearcher;
335336
TopDocs topDocs;
337+
PrintWriter writer = response.getWriter();
336338

337339
if (c.offset == 0) {
338340
topDocs = searcher.search(c.query, c.pageSize, c.sort);
339341
} else {
340342
TopFieldDocs prev = searcher.search(c.query, c.pageSize * c.offset, c.sort);
341-
long totalHits = prev.totalHits.value;
342343
ScoreDoc[] prevHits = prev.scoreDocs;
343344

344345
if (prevHits.length < c.pageSize * c.offset)
345346
{
346-
JSONObject results = new JSONObject();
347-
results.put("data", Collections.emptyList());
348-
results.put("totalHits", totalHits);
349-
return results;
347+
return;
350348
}
351349

352350
ScoreDoc lastDoc = prevHits[c.pageSize * c.offset - 1];
353351
topDocs = searcher.searchAfter(lastDoc, c.query, c.pageSize, c.sort);
354352
}
355353

356-
JSONObject results = new JSONObject();
357-
List<JSONObject> data = new ArrayList<>(topDocs.scoreDocs.length);
358-
359354
for (ScoreDoc sd : topDocs.scoreDocs)
360355
{
361356
Document doc = searcher.storedFields().document(sd.doc);
@@ -366,12 +361,11 @@ private JSONObject paginateJSON(SearchConfig c) throws IOException, ParseExcepti
366361
String[] vals = doc.getValues(name);
367362
elem.put(name, vals.length > 1 ? Arrays.asList(vals) : vals[0]);
368363
}
369-
data.add(elem);
364+
365+
writer.println(elem);
370366
}
371367

372-
results.put("data", data);
373-
results.put("totalHits", topDocs.totalHits.value);
374-
return results;
368+
writer.flush();
375369
}
376370

377371
private void exportCSV(SearchConfig c, HttpServletResponse response) throws IOException
@@ -648,8 +642,9 @@ public void cacheDefaultQuery()
648642
{
649643
try
650644
{
645+
HttpServletResponse response = new Response();
651646
JBrowseLuceneSearch.clearCache(_jsonFile.getObjectId());
652-
doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false);
647+
doSearchJSON(_user, ALL_DOCS, 100, 0, GENOMIC_POSITION, false, response);
653648
}
654649
catch (ParseException | IOException e)
655650
{

0 commit comments

Comments
 (0)