Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions apps/nestjs-backend/scripts/validate-dual-db-cutover.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -162,16 +162,24 @@ const main = async () => {
]);

const schemaDiff = diffSets(sourceSchemas, targetSchemas);
const schemaTableCountDiffs = compareCountMaps(sourceSchemaTableCounts, targetSchemaTableCounts);
const schemaTableCountDiffs = compareCountMaps(
sourceSchemaTableCounts,
targetSchemaTableCounts
);

const [sourceDataCounts, targetDataCounts, sourceMetaCounts, targetMetaCounts, undoFunctionExists] =
await Promise.all([
getTableCounts(sourceClient, DATA_PLANE_TABLES),
getTableCounts(dataClient, DATA_PLANE_TABLES),
getTableCounts(sourceClient, META_PLANE_TABLES),
getTableCounts(metaClient, META_PLANE_TABLES),
getFunctionExists(dataClient, '__teable_capture_undo_row'),
]);
const [
sourceDataCounts,
targetDataCounts,
sourceMetaCounts,
targetMetaCounts,
undoFunctionExists,
] = await Promise.all([
getTableCounts(sourceClient, DATA_PLANE_TABLES),
getTableCounts(dataClient, DATA_PLANE_TABLES),
getTableCounts(sourceClient, META_PLANE_TABLES),
getTableCounts(metaClient, META_PLANE_TABLES),
getFunctionExists(dataClient, '__teable_capture_undo_row'),
]);

const dataCountDiffs = compareCountMaps(sourceDataCounts, targetDataCounts);
const metaCountDiffs = compareCountMaps(sourceMetaCounts, targetMetaCounts);
Expand Down
1 change: 1 addition & 0 deletions apps/nestjs-backend/src/configs/threshold.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ export const thresholdConfig = registerAs('threshold', () => ({
jitter: Number(process.env.BACKEND_DB_DEADLOCK_JITTER ?? 1.0),
},
baseNodeMaxFolderDepth: Number(process.env.BASE_NODE_MAX_FOLDER_DEPTH ?? 2),
maxOwnedSpaceCount: Number(process.env.MAX_SPACE_OWNER_COUNT ?? 10),
changeEmailSendCodeMailRate: Number(process.env.BACKEND_CHANGE_EMAIL_SEND_CODE_MAIL_RATE ?? 30),
resetPasswordSendMailRate: Number(process.env.BACKEND_RESET_PASSWORD_SEND_MAIL_RATE ?? 30),
signupVerificationSendCodeMailRate: Number(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import type { IFilter, IGroup, StatisticsFunc } from '@teable/core';
import type { IFilter, IGroup, ISortItem, StatisticsFunc } from '@teable/core';
import type {
IAggregationField,
IQueryBaseRo,
Expand Down Expand Up @@ -32,6 +32,9 @@ export interface IAggregationService {
withView?: IWithView;
search?: [string, string?, boolean?];
useQueryModel?: boolean;
skip?: number;
take?: number;
orderBy?: ISortItem[];
}): Promise<IRawAggregationValue>;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,14 @@ import {
identify,
IdPrefix,
mergeWithDefaultFilter,
mergeWithDefaultSort,
nullsToUndefined,
ViewType,
} from '@teable/core';
import type { IGridColumnMeta, IFilter, IGroup } from '@teable/core';
import type { IGridColumnMeta, IFilter, IGroup, ISortItem } from '@teable/core';
import { DataPrismaService } from '@teable/db-data-prisma';
import type { Prisma } from '@teable/db-main-prisma';
import { PrismaService } from '@teable/db-main-prisma';
import { DataPrismaService } from '@teable/db-data-prisma';
import { StatisticsFunc } from '@teable/openapi';
import type {
IAggregationField,
Expand Down Expand Up @@ -59,6 +60,10 @@ type IStatisticsData = {
viewId?: string;
filter?: IFilter;
statisticFields?: IAggregationField[];
// Resolved view.sort merged with caller-supplied orderBy. Used for the BASE
// CTE order when skip/take is applied; aggregation itself is order-invariant
// so this is undefined unless the caller is paginating.
sort?: ISortItem[];
};
/**
* Version 2 implementation of the aggregation service
Expand Down Expand Up @@ -92,21 +97,39 @@ export class AggregationService implements IAggregationService {
withView?: IWithView;
search?: [string, string?, boolean?];
useQueryModel?: boolean;
// Optional row-range slice: when provided, the aggregation is computed
// over rows [skip, skip+take) of the view's filtered + sorted output. Used
// by the grid selection statistic endpoint; existing callers (footer
// aggregation, row count) leave them undefined for unchanged behavior.
skip?: number;
take?: number;
orderBy?: ISortItem[];
}): Promise<IRawAggregationValue> {
const { tableId, withFieldIds, withView, search, useQueryModel } = params;
const { tableId, withFieldIds, withView, search, useQueryModel, skip, take, orderBy } = params;
// Retrieve the current user's ID to build user-related query conditions
const currentUserId = this.cls.get('user.id');

const { statisticsData, fieldInstanceMap } = await this.fetchStatisticsParams({
tableId,
withView,
withFieldIds,
extraOrderBy: orderBy,
});

const dbTableName = await this.getDbTableName(this.prisma, tableId);

const { filter, statisticFields } = statisticsData;
const { filter, statisticFields, sort: resolvedSort } = statisticsData;
const groupBy = withView?.groupBy;

// When paginating, BASE CTE needs a deterministic ORDER BY so [skip, take)
// matches what the grid renders. Sort = group + view-sort, falling back to
// the view's row-order column (or __auto_number) for a stable tiebreaker.
const isPaginated = take !== undefined;
const baseSort = isPaginated ? [...(groupBy ?? []), ...(resolvedSort ?? [])] : undefined;
const defaultOrderField = isPaginated
? await this.recordService.getBasicOrderIndexField(dbTableName, withView?.viewId)
: undefined;

const rawAggregationData = await this.handleAggregation({
dbTableName,
fieldInstanceMap,
Expand All @@ -117,6 +140,10 @@ export class AggregationService implements IAggregationService {
withUserId: currentUserId,
withView,
useQueryModel,
skip,
take,
sort: baseSort && baseSort.length ? baseSort : undefined,
defaultOrderField,
});

const aggregationResult = rawAggregationData && rawAggregationData[0];
Expand Down Expand Up @@ -210,6 +237,13 @@ export class AggregationService implements IAggregationService {
withUserId?: string;
withView?: IWithView;
useQueryModel?: boolean;
// Optional row-range slice + ordering. Only set when the caller is
// paginating (selection aggregation); footer/row-count callers leave them
// undefined and the BASE CTE sees the full filtered set.
skip?: number;
take?: number;
sort?: ISortItem[];
defaultOrderField?: string;
}) {
const {
dbTableName,
Expand All @@ -222,6 +256,10 @@ export class AggregationService implements IAggregationService {
withView,
tableId,
useQueryModel,
skip,
take,
sort,
defaultOrderField,
} = params;

if (!statisticFields?.length) {
Expand Down Expand Up @@ -267,6 +305,10 @@ export class AggregationService implements IAggregationService {
projection,
useQueryModel,
builder: permissionProbe.builder,
sort,
defaultOrderField,
limit: take,
offset: skip,
}
);

Expand Down Expand Up @@ -490,6 +532,7 @@ export class AggregationService implements IAggregationService {
});
return tableMeta.dbTableName;
}

private async handleRowCount(params: {
tableId: string;
dbTableName: string;
Expand Down Expand Up @@ -585,11 +628,12 @@ export class AggregationService implements IAggregationService {
tableId: string;
withView?: IWithView;
withFieldIds?: string[];
extraOrderBy?: ISortItem[];
}): Promise<{
statisticsData: IStatisticsData;
fieldInstanceMap: Record<string, IFieldInstance>;
}> {
const { tableId, withView, withFieldIds } = params;
const { tableId, withView, withFieldIds, extraOrderBy } = params;

const viewRaw = await this.findView(tableId, withView);

Expand All @@ -600,7 +644,12 @@ export class AggregationService implements IAggregationService {
withFieldIds
);

const statisticsData = this.buildStatisticsData(filteredFieldInstances, viewRaw, withView);
const statisticsData = this.buildStatisticsData(
filteredFieldInstances,
viewRaw,
withView,
extraOrderBy
);

return { statisticsData, fieldInstanceMap };
}
Expand All @@ -616,6 +665,7 @@ export class AggregationService implements IAggregationService {
id: true,
type: true,
filter: true,
sort: true,
group: true,
options: true,
columnMeta: true,
Expand Down Expand Up @@ -652,10 +702,12 @@ export class AggregationService implements IAggregationService {
id: string | undefined;
columnMeta: string | undefined;
filter: string | undefined;
sort: string | undefined;
group: string | undefined;
}
| undefined,
withView?: IWithView
withView?: IWithView,
extraOrderBy?: ISortItem[]
) {
let statisticsData: IStatisticsData = {
viewId: viewRaw?.id,
Expand All @@ -666,6 +718,12 @@ export class AggregationService implements IAggregationService {
statisticsData = { ...statisticsData, filter };
}

if (viewRaw?.sort || extraOrderBy) {
// Same recipe as record list: caller's orderBy overrides view.sort.
const sort = mergeWithDefaultSort(viewRaw?.sort, extraOrderBy);
statisticsData = { ...statisticsData, sort };
}

if (viewRaw?.id || withView?.customFieldStats) {
const statisticFields = this.getStatisticFields(
filteredFieldInstances,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import {
searchIndexByQueryRoSchema,
IRecordIndexRo,
recordIndexRoSchema,
ISelectionAggregationRo,
selectionAggregationRoSchema,
} from '@teable/openapi';
import { ClsService } from 'nestjs-cls';
import { PerformanceCacheService } from '../../../performance-cache';
Expand Down Expand Up @@ -176,6 +178,16 @@ export class AggregationOpenApiController {
);
}

@Get('/selection')
@Permissions('table|read')
async getSelectionAggregation(
@Param('tableId') tableId: string,
@Query(new ZodValidationPipe(selectionAggregationRoSchema), TqlPipe)
query: ISelectionAggregationRo
): Promise<IAggregationVo> {
return await this.aggregationOpenApiService.getSelectionAggregation(tableId, query);
}

@Get('/task-status-collection')
@Permissions('table|read')
async getTaskStatusCollection(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import { Module } from '@nestjs/common';
import { RecordModule } from '../../record/record.module';
import { AggregationModule } from '../aggregation.module';
import { AggregationOpenApiController } from './aggregation-open-api.controller';
import { AggregationOpenApiService } from './aggregation-open-api.service';

@Module({
controllers: [AggregationOpenApiController],
imports: [AggregationModule],
imports: [AggregationModule, RecordModule],
providers: [AggregationOpenApiService],
exports: [AggregationOpenApiService],
})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,19 @@ import type {
ISearchCountRo,
IRecordIndexRo,
IRecordIndexVo,
ISelectionAggregationRo,
} from '@teable/openapi';
import { forIn, isEmpty, map } from 'lodash';
import { RecordService } from '../../record/record.service';
import { IAggregationService } from '../aggregation.service.interface';
import type { IWithView } from '../aggregation.service.interface';
import { InjectAggregationService } from '../aggregation.service.provider';

@Injectable()
export class AggregationOpenApiService {
constructor(
@InjectAggregationService() private readonly aggregationService: IAggregationService
@InjectAggregationService() private readonly aggregationService: IAggregationService,
private readonly recordService: RecordService
) {}

async getAggregation(tableId: string, query?: IAggregationRo): Promise<IAggregationVo> {
Expand Down Expand Up @@ -133,4 +136,88 @@ export class AggregationOpenApiService {
) {
return await this.aggregationService.getRecordIndexBySearchOrder(tableId, queryRo, projection);
}

// Selection aggregation = the existing aggregation flow + a row-range slice.
// Same recipe as getAggregation: build customFieldStats, validate them, then
// delegate to performAggregation. Two deltas:
// 1. skip/take/orderBy thread through to scope the BASE CTE to the slice.
// 2. groupBy (if any) is folded INTO orderBy as a sort prefix and NOT
// passed via withView. Two reasons:
// a. `performGroupedAggregation` keys aggregations by fieldId, so a
// request asking multiple funcs for the same field (chip asks
// Sum + Filled) loses all but the last entry. Bypassing it keeps
// every (fieldId, aggFunc) result intact.
// b. The same routine re-runs handleAggregation without skip/take,
// which would compute group totals over the whole view instead of
// the slice β€” pointless work for the chip, which only reads
// `total`.
// The group prefix in orderBy preserves grid row order (records list
// uses [...groupBy, ...orderBy] for its sort, mirrored here).
async getSelectionAggregation(
tableId: string,
query: ISelectionAggregationRo
): Promise<IAggregationVo> {
const {
viewId,
filter: customFilter,
field: aggregationFields,
groupBy,
collapsedGroupIds,
ignoreViewQuery,
skip,
take,
orderBy,
} = query;

const sortWithGroup = [...(groupBy ?? []), ...(orderBy ?? [])];

// Translate collapsedGroupIds into a SQL filter (records in collapsed
// groups are excluded from the BASE CTE) so skip/take indexes the same
// visible-record sequence the grid renders. Same recipe records list uses.
let filterWithCollapsed = customFilter;
if (groupBy?.length && collapsedGroupIds?.length) {
const { filter } = await this.recordService.getGroupRelatedData(tableId, {
viewId,
ignoreViewQuery,
filter: customFilter,
groupBy,
collapsedGroupIds,
search: query.search,
});
filterWithCollapsed = filter;
}

let withView: IWithView = {
viewId: ignoreViewQuery ? undefined : viewId,
customFilter: filterWithCollapsed,
// Intentionally NOT passing groupBy (folded into orderBy above).
};

const fieldStatistics: Array<{ fieldId: string; statisticFunc: StatisticsFunc }> = [];
forIn(aggregationFields, (value: string[], key) => {
const fieldStats = map(value, (item) => ({
fieldId: item,
statisticFunc: key as StatisticsFunc,
}));
fieldStatistics.push(...fieldStats);
});

const validFieldStats = await this.validFieldStats(tableId, fieldStatistics);
if (validFieldStats) {
withView = { ...withView, customFieldStats: validFieldStats };
}

const result = await this.aggregationService.performAggregation({
tableId,
withView,
search: query.search,
// useQueryModel must stay false here: the tableCache path skips BASE CTE
// pagination, which would silently aggregate the entire view.
useQueryModel: false,
skip,
take,
orderBy: sortWithGroup.length ? sortWithGroup : undefined,
});
return { aggregations: result?.aggregations };
}
}
Loading
Loading