Skip to content

Commit 07f5c64

Browse files
ivictborivictbor
committed
fix: dont mutate filters (fixes bug with get_resource_data on sqlite with listed field involved in filters), and normalize them handling across data connectors for consistent query execution
1 parent 865d64f commit 07f5c64

6 files changed

Lines changed: 123 additions & 59 deletions

File tree

‎adminforth/dataConnectors/baseConnector.ts‎

Lines changed: 102 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,14 @@ import { randomUUID } from "crypto";
1313
import dayjs from "dayjs";
1414
import { afLogger } from '../modules/logger.js';
1515

16+
type AdminForthFilterNode = IAdminForthSingleFilter | IAdminForthAndOrFilter;
17+
type AdminForthFilterInput = AdminForthFilterNode | AdminForthFilterNode[];
18+
type AdminForthFilterNormalizationResult = {
19+
ok: boolean;
20+
error: string;
21+
normalizedFilters?: AdminForthFilterInput;
22+
};
23+
1624

1725
export default class AdminForthBaseConnector implements IAdminForthDataSourceConnectorBase {
1826

@@ -49,6 +57,22 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
4957
return data.length > 0 ? data[0] : null;
5058
}
5159

60+
cloneFilterNode(filter: AdminForthFilterNode): AdminForthFilterNode {
61+
if ((filter as IAdminForthAndOrFilter).subFilters) {
62+
const complexFilter = filter as IAdminForthAndOrFilter;
63+
return {
64+
...complexFilter,
65+
subFilters: complexFilter.subFilters.map((subFilter) => this.cloneFilterNode(subFilter)),
66+
};
67+
}
68+
69+
const singleFilter = filter as IAdminForthSingleFilter;
70+
return {
71+
...singleFilter,
72+
...(Array.isArray(singleFilter.value) ? { value: [...singleFilter.value] } : {}),
73+
};
74+
}
75+
5276
validateAndNormalizeInputFilters(filter: IAdminForthSingleFilter | IAdminForthAndOrFilter | Array<IAdminForthSingleFilter | IAdminForthAndOrFilter> | undefined): IAdminForthAndOrFilter {
5377
if (!filter) {
5478
// if no filter, return empty "and" filter
@@ -59,77 +83,82 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
5983
}
6084
if (Array.isArray(filter)) {
6185
// if filter is an array, combine them using "and" operator
62-
return { operator: AdminForthFilterOperators.AND, subFilters: filter };
86+
return { operator: AdminForthFilterOperators.AND, subFilters: filter.map((subFilter) => this.cloneFilterNode(subFilter)) };
6387
}
6488
if ((filter as IAdminForthAndOrFilter).subFilters) {
6589
// if filter is already AndOr filter - return as is
66-
return filter as IAdminForthAndOrFilter;
90+
return this.cloneFilterNode(filter as IAdminForthAndOrFilter) as IAdminForthAndOrFilter;
6791
}
6892

6993
// by default, assume filter is Single filter, turn it into AndOr filter
70-
return { operator: AdminForthFilterOperators.AND, subFilters: [filter] };
94+
return { operator: AdminForthFilterOperators.AND, subFilters: [this.cloneFilterNode(filter as AdminForthFilterNode)] };
7195
}
7296

73-
validateAndNormalizeFilters(filters: IAdminForthSingleFilter | IAdminForthAndOrFilter | Array<IAdminForthSingleFilter | IAdminForthAndOrFilter>, resource: AdminForthResource): { ok: boolean, error: string } {
97+
validateAndNormalizeFilters(filters: AdminForthFilterInput, resource: AdminForthResource): AdminForthFilterNormalizationResult {
7498
if (Array.isArray(filters)) {
75-
// go through all filters in array and call validation+normalization for each
76-
// as soon as error is encountered, there is no point in calling validation for other filters
77-
// if error is not encountered all filters will be validated and normalized
78-
return filters.reduce((result, f, fIndex) => {
79-
if (!result.ok) {
80-
return result;
99+
const normalizedFilters: AdminForthFilterNode[] = [];
100+
101+
for (const filter of filters) {
102+
const filterValidation = this.validateAndNormalizeFilters(filter, resource);
103+
if (!filterValidation.ok) {
104+
return filterValidation;
81105
}
82106

83-
const filterValidation = this.validateAndNormalizeFilters(f, resource);
107+
let normalizedFilter = filterValidation.normalizedFilters as AdminForthFilterNode;
108+
const normalizedSingleFilter = normalizedFilter as IAdminForthSingleFilter;
84109

85110
// in case column isArray and enumerator/foreign resource - IN filter must be transformed into OR filter
86-
if (filterValidation.ok && f.operator == AdminForthFilterOperators.IN) {
87-
const column = resource.dataSourceColumns.find((col) => col.name == (f as IAdminForthSingleFilter).field);
111+
if (normalizedSingleFilter.field && normalizedSingleFilter.operator == AdminForthFilterOperators.IN) {
112+
const column = resource.dataSourceColumns.find((col) => col.name == normalizedSingleFilter.field);
88113
if (column.isArray?.enabled && (column.enum || column.foreignResource)) {
89-
filters[fIndex] = {
114+
normalizedFilter = {
90115
operator: AdminForthFilterOperators.OR,
91-
subFilters: f.value.map((v: any) => {
116+
subFilters: normalizedSingleFilter.value.map((v: any) => {
92117
return { field: column.name, operator: AdminForthFilterOperators.LIKE, value: v };
93118
}),
94119
};
95120
}
96121
}
97122

98-
return filterValidation;
99-
}, { ok: true, error: '' });
123+
normalizedFilters.push(normalizedFilter);
124+
}
125+
126+
return { ok: true, error: '', normalizedFilters };
100127
}
101128

102129
const filtersAsSingle = filters as IAdminForthSingleFilter;
103130
if (filtersAsSingle.field) {
131+
const normalizedFilter = this.cloneFilterNode(filters) as IAdminForthSingleFilter;
132+
104133
// if "field" is present, filter must be Single
105-
if (!filters.operator) {
134+
if (!normalizedFilter.operator) {
106135
return { ok: false, error: `Field "operator" not specified in filter object: ${JSON.stringify(filters)}` };
107136
}
108137
// Either compare with value or with rightField (field-to-field). If rightField is set, value must be undefined.
109-
const comparingWithRightField = filtersAsSingle.rightField !== undefined && filtersAsSingle.rightField !== null;
110-
const isEmptyOperator = filters.operator === AdminForthFilterOperators.IS_EMPTY || filters.operator === AdminForthFilterOperators.IS_NOT_EMPTY;
138+
const comparingWithRightField = normalizedFilter.rightField !== undefined && normalizedFilter.rightField !== null;
139+
const isEmptyOperator = normalizedFilter.operator === AdminForthFilterOperators.IS_EMPTY || normalizedFilter.operator === AdminForthFilterOperators.IS_NOT_EMPTY;
111140

112-
if (!comparingWithRightField && !isEmptyOperator && filtersAsSingle.value === undefined) {
141+
if (!comparingWithRightField && !isEmptyOperator && normalizedFilter.value === undefined) {
113142
return { ok: false, error: `Field "value" not specified in filter object: ${JSON.stringify(filters)}` };
114143
}
115-
if (comparingWithRightField && filtersAsSingle.value !== undefined) {
144+
if (comparingWithRightField && normalizedFilter.value !== undefined) {
116145
return { ok: false, error: `Specify either "value" or "rightField", not both: ${JSON.stringify(filters)}` };
117146
}
118-
if (filtersAsSingle.insecureRawSQL) {
147+
if (normalizedFilter.insecureRawSQL) {
119148
return { ok: false, error: `Field "insecureRawSQL" should not be specified in filter object alongside "field": ${JSON.stringify(filters)}` };
120149
}
121-
if (filtersAsSingle.insecureRawNoSQL) {
150+
if (normalizedFilter.insecureRawNoSQL) {
122151
return { ok: false, error: `Field "insecureRawNoSQL" should not be specified in filter object alongside "field": ${JSON.stringify(filters)}` };
123152
}
124153
if (![AdminForthFilterOperators.EQ, AdminForthFilterOperators.NE, AdminForthFilterOperators.GT,
125154
AdminForthFilterOperators.LT, AdminForthFilterOperators.GTE, AdminForthFilterOperators.LTE,
126155
AdminForthFilterOperators.LIKE, AdminForthFilterOperators.ILIKE, AdminForthFilterOperators.IN,
127-
AdminForthFilterOperators.NIN, AdminForthFilterOperators.IS_EMPTY, AdminForthFilterOperators.IS_NOT_EMPTY].includes(filters.operator)) {
156+
AdminForthFilterOperators.NIN, AdminForthFilterOperators.IS_EMPTY, AdminForthFilterOperators.IS_NOT_EMPTY].includes(normalizedFilter.operator)) {
128157
return { ok: false, error: `Field "operator" has wrong value in filter object: ${JSON.stringify(filters)}` };
129158
}
130-
const fieldObj = resource.dataSourceColumns.find((col) => col.name == filtersAsSingle.field);
159+
const fieldObj = resource.dataSourceColumns.find((col) => col.name == normalizedFilter.field);
131160
if (!fieldObj) {
132-
const similar = suggestIfTypo(resource.dataSourceColumns.map((col) => col.name), filtersAsSingle.field);
161+
const similar = suggestIfTypo(resource.dataSourceColumns.map((col) => col.name), normalizedFilter.field);
133162

134163
let isPolymorphicTarget = false;
135164
if (global.adminforth?.config?.resources) {
@@ -142,68 +171,85 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
142171
);
143172
}
144173
if (isPolymorphicTarget) {
145-
afLogger.trace(`⚠️ Field '${filtersAsSingle.field}' not found in polymorphic target resource '${resource.resourceId}', allowing query to proceed.`);
146-
return { ok: true, error: '' };
174+
afLogger.trace(`⚠️ Field '${normalizedFilter.field}' not found in polymorphic target resource '${resource.resourceId}', allowing query to proceed.`);
175+
return { ok: true, error: '', normalizedFilters: normalizedFilter };
147176
} else {
148-
throw new Error(`Field '${filtersAsSingle.field}' not found in resource '${resource.resourceId}'. ${similar ? `Did you mean '${similar}'?` : ''}`);
177+
throw new Error(`Field '${normalizedFilter.field}' not found in resource '${resource.resourceId}'. ${similar ? `Did you mean '${similar}'?` : ''}`);
149178
}
150179
}
151180
// value normalization
152181
if (comparingWithRightField) {
153182
// ensure rightField exists in resource
154-
const rightFieldObj = resource.dataSourceColumns.find((col) => col.name == filtersAsSingle.rightField);
183+
const rightFieldObj = resource.dataSourceColumns.find((col) => col.name == normalizedFilter.rightField);
155184
if (!rightFieldObj) {
156-
const similar = suggestIfTypo(resource.dataSourceColumns.map((col) => col.name), filtersAsSingle.rightField as string);
157-
throw new Error(`Field '${filtersAsSingle.rightField}' not found in resource '${resource.resourceId}'. ${similar ? `Did you mean '${similar}'?` : ''}`);
185+
const similar = suggestIfTypo(resource.dataSourceColumns.map((col) => col.name), normalizedFilter.rightField as string);
186+
throw new Error(`Field '${normalizedFilter.rightField}' not found in resource '${resource.resourceId}'. ${similar ? `Did you mean '${similar}'?` : ''}`);
158187
}
159188
// No value conversion needed for field-to-field comparison here
160189
} else if (isEmptyOperator) {
161190
// IS_EMPTY and IS_NOT_EMPTY don't need value normalization
162191
// Set value to null if not already set
163-
if (filtersAsSingle.value === undefined) {
164-
filtersAsSingle.value = null;
192+
if (normalizedFilter.value === undefined) {
193+
normalizedFilter.value = null;
165194
}
166-
} else if (filters.operator == AdminForthFilterOperators.IN || filters.operator == AdminForthFilterOperators.NIN) {
167-
if (!Array.isArray(filters.value)) {
168-
return { ok: false, error: `Value for operator '${filters.operator}' should be an array, in filter object: ${JSON.stringify(filters) }` };
195+
} else if (normalizedFilter.operator == AdminForthFilterOperators.IN || normalizedFilter.operator == AdminForthFilterOperators.NIN) {
196+
if (!Array.isArray(normalizedFilter.value)) {
197+
return { ok: false, error: `Value for operator '${normalizedFilter.operator}' should be an array, in filter object: ${JSON.stringify(filters) }` };
169198
}
170-
if (filters.value.length === 0) {
199+
if (normalizedFilter.value.length === 0) {
171200
// nonsense, and some databases might not accept IN []
172-
const colType = resource.dataSourceColumns.find((col) => col.name == filtersAsSingle.field)?.type;
201+
const colType = resource.dataSourceColumns.find((col) => col.name == normalizedFilter.field)?.type;
173202
if (colType === AdminForthDataTypes.STRING || colType === AdminForthDataTypes.TEXT) {
174-
filters.value = [randomUUID()];
175-
return { ok: true, error: `` };
203+
normalizedFilter.value = [randomUUID()];
204+
return { ok: true, error: '', normalizedFilters: normalizedFilter };
176205
} else {
177-
return { ok: false, error: `Value for operator '${filters.operator}' should not be empty array, in filter object: ${JSON.stringify(filters) }` };
206+
return { ok: false, error: `Value for operator '${normalizedFilter.operator}' should not be empty array, in filter object: ${JSON.stringify(filters) }` };
178207
}
179208
}
180-
filters.value = filters.value.map((val: any) => this.validateAndSetFieldValue(fieldObj, val));
209+
normalizedFilter.value = normalizedFilter.value.map((val: any) => this.validateAndSetFieldValue(fieldObj, val));
181210
} else {
182-
filtersAsSingle.value = this.validateAndSetFieldValue(fieldObj, filtersAsSingle.value);
211+
normalizedFilter.value = this.validateAndSetFieldValue(fieldObj, normalizedFilter.value);
183212
}
213+
214+
return { ok: true, error: '', normalizedFilters: normalizedFilter };
184215
} else if (filtersAsSingle.insecureRawSQL || filtersAsSingle.insecureRawNoSQL) {
216+
const normalizedFilter = this.cloneFilterNode(filters) as IAdminForthSingleFilter;
217+
185218
// if "insecureRawSQL" filter is insecure sql string
186-
if (filtersAsSingle.operator) {
219+
if (normalizedFilter.operator) {
187220
return { ok: false, error: `Field "operator" should not be specified in filter object alongside "insecureRawSQL" or "insecureRawNoSQL": ${JSON.stringify(filters)}` };
188221
}
189-
if (filtersAsSingle.value !== undefined) {
222+
if (normalizedFilter.value !== undefined) {
190223
return { ok: false, error: `Field "value" should not be specified in filter object alongside "insecureRawSQL" or "insecureRawNoSQL": ${JSON.stringify(filters)}` };
191224
}
225+
return { ok: true, error: '', normalizedFilters: normalizedFilter };
192226
} else if ((filters as IAdminForthAndOrFilter).subFilters) {
227+
const complexFilter = filters as IAdminForthAndOrFilter;
228+
193229
// if "subFilters" is present, filter must be AndOr
194-
if (!filters.operator) {
230+
if (!complexFilter.operator) {
195231
return { ok: false, error: `Field "operator" not specified in filter object: ${JSON.stringify(filters)}` };
196232
}
197-
if (![AdminForthFilterOperators.AND, AdminForthFilterOperators.OR].includes(filters.operator)) {
233+
if (![AdminForthFilterOperators.AND, AdminForthFilterOperators.OR].includes(complexFilter.operator)) {
198234
return { ok: false, error: `Field "operator" has wrong value in filter object: ${JSON.stringify(filters)}` };
199235
}
200236

201-
return this.validateAndNormalizeFilters((filters as IAdminForthAndOrFilter).subFilters, resource);
237+
const subFiltersValidation = this.validateAndNormalizeFilters(complexFilter.subFilters, resource);
238+
if (!subFiltersValidation.ok) {
239+
return subFiltersValidation;
240+
}
241+
242+
return {
243+
ok: true,
244+
error: '',
245+
normalizedFilters: {
246+
...complexFilter,
247+
subFilters: subFiltersValidation.normalizedFilters as AdminForthFilterNode[],
248+
},
249+
};
202250
} else {
203251
return { ok: false, error: `Fields "field" or "subFilters" are not specified in filter object: ${JSON.stringify(filters)}` };
204252
}
205-
206-
return { ok: true, error: '' };
207253
}
208254

209255
getDataWithOriginalTypes({ resource, limit, offset, sort, filters }: {
@@ -469,14 +515,17 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
469515
filters: IAdminForthAndOrFilter,
470516
getTotals: boolean,
471517
}): Promise<{ data: any[], total: number }> {
518+
let normalizedFilters = filters;
519+
472520
if (filters) {
473521
const filterValidation = this.validateAndNormalizeFilters(filters, resource);
474522
if (!filterValidation.ok) {
475523
throw new Error(filterValidation.error);
476524
}
525+
normalizedFilters = filterValidation.normalizedFilters as IAdminForthAndOrFilter;
477526
}
478527

479-
const promises: Promise<any>[] = [this.getDataWithOriginalTypes({ resource, limit, offset, sort, filters })];
528+
const promises: Promise<any>[] = [this.getDataWithOriginalTypes({ resource, limit, offset, sort, filters: normalizedFilters })];
480529
if (getTotals) {
481530
promises.push(this.getCount({ resource, filters }));
482531
} else {

‎adminforth/dataConnectors/clickhouse.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -498,14 +498,17 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
498498
filters: IAdminForthAndOrFilter;
499499
}): Promise<number> {
500500
const tableName = resource.table;
501+
let normalizedFilters = filters;
502+
501503
// validate and normalize in case this method is called from dataAPI
502504
if (filters) {
503505
const filterValidation = this.validateAndNormalizeFilters(filters, resource);
504506
if (!filterValidation.ok) {
505507
throw new Error(filterValidation.error);
506508
}
509+
normalizedFilters = filterValidation.normalizedFilters as IAdminForthAndOrFilter;
507510
}
508-
const { where, params } = this.whereClause(resource, filters);
511+
const { where, params } = this.whereClause(resource, normalizedFilters);
509512

510513
const countQ = await this.client.query({
511514
query: `SELECT COUNT(*) as count FROM ${tableName} ${where}`,

‎adminforth/dataConnectors/mongo.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -340,15 +340,18 @@ class MongoConnector extends AdminForthBaseConnector implements IAdminForthDataS
340340
resource: AdminForthResource,
341341
filters: IAdminForthAndOrFilter,
342342
}): Promise<number> {
343+
let normalizedFilters = filters;
344+
343345
if (filters) {
344346
// validate and normalize in case this method is called from dataAPI
345347
const filterValidation = this.validateAndNormalizeFilters(filters, resource);
346348
if (!filterValidation.ok) {
347349
throw new Error(filterValidation.error);
348350
}
351+
normalizedFilters = filterValidation.normalizedFilters as IAdminForthAndOrFilter;
349352
}
350353
const collection = this.client.db().collection(resource.table);
351-
const query = filters.subFilters.length ? this.getFilterQuery(resource, filters) : {};
354+
const query = normalizedFilters.subFilters.length ? this.getFilterQuery(resource, normalizedFilters) : {};
352355
return await collection.countDocuments(query);
353356
}
354357

‎adminforth/dataConnectors/mysql.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,14 +365,17 @@ class MysqlConnector extends AdminForthBaseConnector implements IAdminForthDataS
365365

366366
async getCount({ resource, filters }: { resource: AdminForthResource; filters: IAdminForthAndOrFilter; }): Promise<number> {
367367
const tableName = resource.table;
368+
let normalizedFilters = filters;
369+
368370
// validate and normalize in case this method is called from dataAPI
369371
if (filters) {
370372
const filterValidation = this.validateAndNormalizeFilters(filters, resource);
371373
if (!filterValidation.ok) {
372374
throw new Error(filterValidation.error);
373375
}
376+
normalizedFilters = filterValidation.normalizedFilters as IAdminForthAndOrFilter;
374377
}
375-
const { sql: where, values: filterValues } = this.whereClauseAndValues(filters);
378+
const { sql: where, values: filterValues } = this.whereClauseAndValues(normalizedFilters);
376379
const q = `SELECT COUNT(*) FROM ${tableName} ${where}`;
377380
dbLogger.trace(`🪲📜 MySQL Q: ${q} values: ${JSON.stringify(filterValues)}`);
378381
const [results] = await this.client.execute(q, filterValues);

0 commit comments

Comments
 (0)