Skip to content

Commit c7c0400

Browse files
ivictborivictbor
authored
Merge pull request #163 from devforth/and-or-filters
feat: implement and/or operators for filters
2 parents a9a9be5 + b1071b5 commit c7c0400

File tree

12 files changed

+530
-300
lines changed

12 files changed

+530
-300
lines changed

‎adminforth/dataConnectors/baseConnector.ts‎

Lines changed: 77 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import {
22
AdminForthResource, IAdminForthDataSourceConnectorBase,
33
AdminForthResourceColumn,
4-
IAdminForthSort, IAdminForthFilter
4+
IAdminForthSort, IAdminForthSingleFilter, IAdminForthAndOrFilter
55
} from "../types/Back.js";
66

77

@@ -32,27 +32,84 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
3232
}
3333

3434
async getRecordByPrimaryKeyWithOriginalTypes(resource: AdminForthResource, id: string): Promise<any> {
35-
const data = await this.getDataWithOriginalTypes({
36-
resource,
37-
limit: 1,
35+
const data = await this.getDataWithOriginalTypes({
36+
resource,
37+
limit: 1,
3838
offset: 0,
39-
sort: [],
40-
filters: [{ field: this.getPrimaryKey(resource), operator: AdminForthFilterOperators.EQ, value: id }],
39+
sort: [],
40+
filters: { operator: AdminForthFilterOperators.AND, subFilters: [{ field: this.getPrimaryKey(resource), operator: AdminForthFilterOperators.EQ, value: id }]},
4141
});
4242
return data.length > 0 ? data[0] : null;
4343
}
4444

45+
validateAndNormalizeFilters(filters: IAdminForthSingleFilter | IAdminForthAndOrFilter | Array<IAdminForthSingleFilter | IAdminForthAndOrFilter>, resource: AdminForthResource): { ok: boolean, error: string } {
46+
if (Array.isArray(filters)) {
47+
// go through all filters in array and call validation+normalization for each
48+
// as soon as error is encountered, there is no point in calling validation for other filters
49+
// if error is not encountered all filters will be validated and normalized
50+
return filters.reduce((result, f) => {
51+
if (!result.ok) {
52+
return result;
53+
}
54+
55+
return this.validateAndNormalizeFilters(f, resource);
56+
}, { ok: true, error: '' });
57+
}
58+
59+
if (!filters.operator) {
60+
return { ok: false, error: `Field "operator" not specified in filter object: ${JSON.stringify(filters)}` };
61+
}
62+
63+
if ((filters as IAdminForthSingleFilter).field) {
64+
// if "field" is present, filter must be Single
65+
if (![AdminForthFilterOperators.EQ, AdminForthFilterOperators.NE, AdminForthFilterOperators.GT,
66+
AdminForthFilterOperators.LT, AdminForthFilterOperators.GTE, AdminForthFilterOperators.LTE,
67+
AdminForthFilterOperators.LIKE, AdminForthFilterOperators.ILIKE, AdminForthFilterOperators.IN,
68+
AdminForthFilterOperators.NIN].includes(filters.operator)) {
69+
return { ok: false, error: `Field "operator" has wrong value in filter object: ${JSON.stringify(filters)}` };
70+
}
71+
const fieldObj = resource.dataSourceColumns.find((col) => col.name == (filters as IAdminForthSingleFilter).field);
72+
if (!fieldObj) {
73+
const similar = suggestIfTypo(resource.dataSourceColumns.map((col) => col.name), (filters as IAdminForthSingleFilter).field);
74+
throw new Error(`Field '${(filters as IAdminForthSingleFilter).field}' not found in resource '${resource.resourceId}'. ${similar ? `Did you mean '${similar}'?` : ''}`);
75+
}
76+
if (filters.operator == AdminForthFilterOperators.IN || filters.operator == AdminForthFilterOperators.NIN) {
77+
if (!Array.isArray(filters.value)) {
78+
return { ok: false, error: `Value for operator '${filters.operator}' should be an array, in filter object: ${JSON.stringify(filters) }` };
79+
}
80+
if (filters.value.length === 0) {
81+
// nonsense
82+
return { ok: false, error: `Filter has IN operator but empty value: ${JSON.stringify(filters)}` };
83+
}
84+
filters.value = filters.value.map((val: any) => this.setFieldValue(fieldObj, val));
85+
} else {
86+
(filters as IAdminForthSingleFilter).value = this.setFieldValue(fieldObj, (filters as IAdminForthSingleFilter).value);
87+
}
88+
} else if ((filters as IAdminForthAndOrFilter).subFilters) {
89+
// if "subFilters" is present, filter must be AndOr
90+
if (![AdminForthFilterOperators.AND, AdminForthFilterOperators.OR].includes(filters.operator)) {
91+
return { ok: false, error: `Field "operator" has wrong value in filter object: ${JSON.stringify(filters)}` };
92+
}
93+
94+
return this.validateAndNormalizeFilters((filters as IAdminForthAndOrFilter).subFilters, resource);
95+
} else {
96+
return { ok: false, error: `Fields "field" or "subFilters" are not specified in filter object: ${JSON.stringify(filters)}` };
97+
}
98+
99+
return { ok: true, error: '' };
100+
}
101+
45102
getDataWithOriginalTypes({ resource, limit, offset, sort, filters }: {
46103
resource: AdminForthResource,
47104
limit: number,
48105
offset: number,
49106
sort: IAdminForthSort[],
50-
filters: IAdminForthFilter[],
107+
filters: IAdminForthAndOrFilter,
51108
}): Promise<any[]> {
52109
throw new Error('Method not implemented.');
53110
}
54111

55-
getCount({ resource, filters }: { resource: AdminForthResource; filters: { field: string; operator: AdminForthFilterOperators; value: any; }[]; }): Promise<number> {
112+
getCount({ resource, filters }: { resource: AdminForthResource; filters: IAdminForthAndOrFilter; }): Promise<number> {
56113
throw new Error('Method not implemented.');
57114
}
58115

@@ -80,7 +137,7 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
80137
process.env.HEAVY_DEBUG && console.log('☝️🪲🪲🪲🪲 checkUnique|||', column, value);
81138
const existingRecord = await this.getData({
82139
resource,
83-
filters: [{ field: column.name, operator: AdminForthFilterOperators.EQ, value }],
140+
filters: { operator: AdminForthFilterOperators.AND, subFilters: [{ field: column.name, operator: AdminForthFilterOperators.EQ, value }]},
84141
limit: 1,
85142
sort: [],
86143
offset: 0,
@@ -130,7 +187,12 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
130187
}
131188

132189
process.env.HEAVY_DEBUG && console.log('🪲🆕 creating record',JSON.stringify(recordWithOriginalValues));
133-
const pkValue = await this.createRecordOriginalValues({ resource, record: recordWithOriginalValues });
190+
let pkValue = await this.createRecordOriginalValues({ resource, record: recordWithOriginalValues });
191+
if (recordWithOriginalValues[this.getPrimaryKey(resource)] !== undefined) {
192+
// some data sources always return some value for pk, even if it is was not auto generated
193+
// this check prevents wrong value from being used later in get request
194+
pkValue = recordWithOriginalValues[this.getPrimaryKey(resource)];
195+
}
134196

135197
let createdRecord = recordWithOriginalValues;
136198
if (pkValue) {
@@ -175,38 +237,19 @@ export default class AdminForthBaseConnector implements IAdminForthDataSourceCon
175237
throw new Error('Method not implemented.');
176238
}
177239

178-
179240
async getData({ resource, limit, offset, sort, filters, getTotals }: {
180241
resource: AdminForthResource,
181242
limit: number,
182243
offset: number,
183244
sort: { field: string, direction: AdminForthSortDirections }[],
184-
filters: { field: string, operator: AdminForthFilterOperators, value: any }[],
245+
filters: IAdminForthAndOrFilter,
185246
getTotals: boolean,
186247
}): Promise<{ data: any[], total: number }> {
187248
if (filters) {
188-
for (const f of filters) {
189-
if (!f.field) {
190-
throw new Error(`Field "field" not specified in filter object: ${JSON.stringify(f)}`);
191-
}
192-
if (!f.operator) {
193-
throw new Error(`Field "operator" not specified in filter object: ${JSON.stringify(f)}`);
194-
}
195-
const fieldObj = resource.dataSourceColumns.find((col) => col.name == f.field);
196-
if (!fieldObj) {
197-
const similar = suggestIfTypo(resource.dataSourceColumns.map((col) => col.name), f.field);
198-
throw new Error(`Field '${f.field}' not found in resource '${resource.resourceId}'. ${similar ? `Did you mean '${similar}'?` : ''}`);
199-
}
200-
if (f.operator == AdminForthFilterOperators.IN || f.operator == AdminForthFilterOperators.NIN) {
201-
f.value = f.value.map((val) => this.setFieldValue(fieldObj, val));
202-
} else {
203-
f.value = this.setFieldValue(fieldObj, f.value);
204-
}
205-
if (f.operator === AdminForthFilterOperators.IN && f.value.length === 0) {
206-
// nonsense
207-
return { data: [], total: 0 };
208-
}
209-
};
249+
const filterValidation = this.validateAndNormalizeFilters(filters, resource);
250+
if (!filterValidation.ok) {
251+
throw new Error(filterValidation.error);
252+
}
210253
}
211254

212255
const promises: Promise<any>[] = [this.getDataWithOriginalTypes({ resource, limit, offset, sort, filters })];

‎adminforth/dataConnectors/clickhouse.ts‎

Lines changed: 86 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { IAdminForthDataSourceConnector, AdminForthResource, AdminForthResourceColumn } from '../types/Back.js';
1+
import { IAdminForthDataSourceConnector, IAdminForthSingleFilter, IAdminForthAndOrFilter, AdminForthResource, AdminForthResourceColumn } from '../types/Back.js';
22
import AdminForthBaseConnector from './baseConnector.js';
33
import dayjs from 'dayjs';
44
import { createClient } from '@clickhouse/client'
@@ -162,60 +162,107 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
162162
[AdminForthFilterOperators.ILIKE]: 'ILIKE',
163163
[AdminForthFilterOperators.IN]: 'IN',
164164
[AdminForthFilterOperators.NIN]: 'NOT IN',
165+
[AdminForthFilterOperators.AND]: 'AND',
166+
[AdminForthFilterOperators.OR]: 'OR',
165167
};
166168

167169
SortDirectionsMap = {
168170
[AdminForthSortDirections.asc]: 'ASC',
169171
[AdminForthSortDirections.desc]: 'DESC',
170172
};
173+
174+
getFilterString(resource: AdminForthResource, filter: IAdminForthSingleFilter | IAdminForthAndOrFilter): string {
175+
if ((filter as IAdminForthSingleFilter).field) {
176+
// filter is a Single filter
177+
let field = (filter as IAdminForthSingleFilter).field;
178+
const column = resource.dataSourceColumns.find((col) => col.name == field);
179+
let placeholder = `{f$?:${column._underlineType}}`;
180+
let operator = this.OperatorsMap[filter.operator];
181+
if (filter.operator == AdminForthFilterOperators.IN || filter.operator == AdminForthFilterOperators.NIN) {
182+
placeholder = `(${filter.value.map((_, j) => `{p$?:${column._underlineType}}`).join(', ')})`;
183+
}
184+
185+
return `${field} ${operator} ${placeholder}`;
186+
}
187+
188+
// filter is a AndOr filter
189+
return (filter as IAdminForthAndOrFilter).subFilters.map((f) => {
190+
if ((f as IAdminForthSingleFilter).field) {
191+
// subFilter is a Single filter
192+
return this.getFilterString(resource, f);
193+
}
194+
195+
// subFilter is a AndOr filter - add parentheses
196+
return `(${this.getFilterString(resource, f)})`;
197+
}).join(` ${this.OperatorsMap[filter.operator]} `);
198+
}
171199

172-
whereClause(
173-
resource: AdminForthResource,
174-
filters: { field: string, operator: AdminForthFilterOperators, value: any }[]
175-
): string {
176-
return filters.length ? `WHERE ${filters.map((f, i) => {
177-
const column = resource.dataSourceColumns.find((col) => col.name == f.field);
178-
let placeholder = `{f${i}:${column._underlineType}}`;
179-
let field = f.field;
180-
let operator = this.OperatorsMap[f.operator];
181-
if (f.operator == AdminForthFilterOperators.IN || f.operator == AdminForthFilterOperators.NIN) {
182-
placeholder = `(${f.value.map((_, j) => `{p${i}_${j}:${
183-
column._underlineType
184-
}}`).join(', ')})`;
200+
getFilterParams(filter: IAdminForthSingleFilter | IAdminForthAndOrFilter): any[] {
201+
if ((filter as IAdminForthSingleFilter).field) {
202+
// filter is a Single filter
203+
if (filter.operator == AdminForthFilterOperators.LIKE || filter.operator == AdminForthFilterOperators.ILIKE) {
204+
return [{ 'f': `%${filter.value}%` }];
205+
} else if (filter.operator == AdminForthFilterOperators.IN || filter.operator == AdminForthFilterOperators.NIN) {
206+
return [{ 'p': filter.value }];
207+
} else {
208+
return [{ 'f': (filter as IAdminForthSingleFilter).value }];
185209
}
210+
}
186211

187-
return `${field} ${operator} ${placeholder}`
188-
}).join(' AND ')}` : '';
212+
// filter is a AndOrFilter
213+
return (filter as IAdminForthAndOrFilter).subFilters.reduce((params: any[], f: IAdminForthSingleFilter | IAdminForthAndOrFilter) => {
214+
return params.concat(this.getFilterParams(f));
215+
}, []);
189216
}
190217

191-
whereParams(
192-
filters: { field: string, operator: AdminForthFilterOperators, value: any }[]
193-
): any {
194-
const params = {};
195-
filters.length ? filters.forEach((f, i) => {
196-
// for arrays do set in map
197-
const v = f.value;
198-
199-
if (f.operator == AdminForthFilterOperators.LIKE || f.operator == AdminForthFilterOperators.ILIKE) {
200-
params[`f${i}`] = `%${v}%`;
201-
} else if (f.operator == AdminForthFilterOperators.IN || f.operator == AdminForthFilterOperators.NIN) {
202-
v.forEach((_, j) => {
203-
params[`p${i}_${j}`] = v[j];
204-
});
205-
} else {
206-
params[`f${i}`] = v;
218+
whereParams(filters: IAdminForthAndOrFilter): any {
219+
if (filters.subFilters.length === 0) {
220+
return {};
221+
}
222+
const paramsArray = this.getFilterParams(filters);
223+
const params = paramsArray.reduce((acc, param, paramIndex) => {
224+
if (param.f !== undefined) {
225+
acc[`f${paramIndex}`] = param.f;
226+
}
227+
else if (param.p !== undefined) {
228+
param.p.forEach((paramValue: any, paramValueIndex: number) => acc[`p${paramIndex}_${paramValueIndex}`] = paramValue);
207229
}
208-
}) : [];
230+
231+
return acc;
232+
}, {});
209233

210234
return params;
235+
}
236+
237+
whereClause(
238+
resource: AdminForthResource,
239+
filters: IAdminForthAndOrFilter
240+
): {
241+
where: string,
242+
params: any,
243+
} {
244+
if (filters.subFilters.length === 0) {
245+
return {
246+
where: '',
247+
params: {},
248+
}
249+
}
250+
const params = this.whereParams(filters);
251+
const where = Object.keys(params).reduce((w, paramKey) => {
252+
// remove first char of string (will be "f" or "p") to leave only index
253+
const keyIndex = paramKey.substring(1);
254+
return w.replace('$?', keyIndex);
255+
}, `WHERE ${this.getFilterString(resource, filters)}`);
256+
257+
return { where, params };
211258
}
212259

213260
async getDataWithOriginalTypes({ resource, limit, offset, sort, filters }: {
214261
resource: AdminForthResource,
215262
limit: number,
216263
offset: number,
217264
sort: { field: string, direction: AdminForthSortDirections }[],
218-
filters: { field: string, operator: AdminForthFilterOperators, value: any }[],
265+
filters: IAdminForthAndOrFilter,
219266
}): Promise<any[]> {
220267
const columns = resource.dataSourceColumns.map((col) => {
221268
// for decimal cast to string
@@ -226,9 +273,7 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
226273
}).join(', ');
227274
const tableName = resource.table;
228275

229-
const where = this.whereClause(resource, filters);
230-
231-
const params = this.whereParams(filters);
276+
const { where, params } = this.whereClause(resource, filters);
232277

233278
const orderBy = sort.length ? `ORDER BY ${sort.map((s) => `${s.field} ${this.SortDirectionsMap[s.direction]}`).join(', ')}` : '';
234279

@@ -262,16 +307,15 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
262307
filters,
263308
}: {
264309
resource: AdminForthResource;
265-
filters: { field: string, operator: AdminForthFilterOperators, value: any }[];
310+
filters: IAdminForthAndOrFilter;
266311
}): Promise<number> {
267312
const tableName = resource.table;
268-
const where = this.whereClause(resource, filters);
269-
const d = this.whereParams(filters);
313+
const { where, params } = this.whereClause(resource, filters);
270314

271315
const countQ = await this.client.query({
272316
query: `SELECT COUNT(*) as count FROM ${tableName} ${where}`,
273317
format: 'JSONEachRow',
274-
query_params: d,
318+
query_params: params,
275319
});
276320
const countResp = await countQ.json()
277321
return +countResp[0]['count'];
@@ -303,7 +347,7 @@ class ClickhouseConnector extends AdminForthBaseConnector implements IAdminForth
303347
columns: columns,
304348
values: [Object.values(record)],
305349
});
306-
return ''; // todo
350+
return '';
307351
}
308352

309353
async updateRecordOriginalValues({ resource, recordId, newValues }: { resource: AdminForthResource, recordId: any, newValues: any }) {

0 commit comments

Comments
 (0)