Skip to content
15 changes: 7 additions & 8 deletions conf/openmetadata.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -225,12 +225,12 @@ database:
# These work across both PostgreSQL and MySQL drivers
properties:
# HikariCP connection pool settings
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-300000} # 300 seconds
idleTimeout: ${DB_IDLE_TIMEOUT:-600000} # 10 minutes
maxLifetime: ${DB_MAX_LIFETIME:-1800000} # 30 minutes
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-600000} # 10 minute
connectionTimeout: ${DB_CONNECTION_TIMEOUT:-30000} # 30 seconds - fail fast if pool exhausted
idleTimeout: ${DB_IDLE_TIMEOUT:-120000} # 2 minutes - reclaim idle connections faster
maxLifetime: ${DB_MAX_LIFETIME:-600000} # 10 minutes - rotate connections (cloud DB friendly)
leakDetectionThreshold: ${DB_LEAK_DETECTION_THRESHOLD:-60000} # 1 minute - detect leaks early
keepaliveTime: ${DB_KEEPALIVE_TIME:-0} # 0 = disabled (set to 30000 for Aurora)
validationTimeout: ${DB_VALIDATION_TIMEOUT:-300000} # 300 seconds
validationTimeout: ${DB_VALIDATION_TIMEOUT:-5000} # 5 seconds - HikariCP default
# PostgreSQL specific - these are ignored by MySQL driver
prepareThreshold: ${DB_PG_PREPARE_THRESHOLD:-1} # Use prepared statements immediately
preparedStatementCacheQueries: ${DB_PG_PREP_STMT_CACHE_QUERIES:-500} # Cache more statements
Expand Down Expand Up @@ -580,10 +580,9 @@ limits:
limitsConfigFile: ${LIMITS_CONFIG_FILE:-""}

# Bulk Operation Configuration
# Controls parallelism and resource usage for bulk API operations (e.g., bulk import/export)
# Uses a bounded thread pool to prevent connection pool exhaustion
# Controls parallelism and admission control for bulk API operations.
bulkOperation:
# Max threads for bulk operations (recommendations: 2 vCore=5-8, 4 vCore=8-15, 8 vCore=15-25)
# Max threads for processing
maxThreads: ${BULK_OPERATION_MAX_THREADS:-10}
# Max queued operations before rejection (returns 503)
queueSize: ${BULK_OPERATION_QUEUE_SIZE:-1000}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import org.openmetadata.schema.type.Field;
import org.openmetadata.schema.type.FieldDataType;
import org.openmetadata.schema.type.TagLabel;
import org.openmetadata.schema.type.api.BulkOperationResult;
import org.openmetadata.sdk.client.OpenMetadataClient;
import org.openmetadata.sdk.exceptions.InvalidRequestException;
import org.openmetadata.sdk.models.ListParams;
Expand All @@ -50,6 +51,7 @@ public class APIEndpointResourceIT extends BaseEntityIT<APIEndpoint, CreateAPIEn
{
supportsLifeCycle = true;
supportsListHistoryByTimestamp = true;
supportsBulkAPI = true;
}

// ===================================================================
Expand Down Expand Up @@ -705,4 +707,25 @@ private static void assertField(Field expectedField, Field actualField) {
assertFields(expectedField.getChildren(), actualField.getChildren());
}
}

// ===================================================================
// BULK API SUPPORT
// ===================================================================

@Override
protected BulkOperationResult executeBulkCreate(List<CreateAPIEndpoint> createRequests) {
return SdkClients.adminClient().apiEndpoints().bulkCreateOrUpdate(createRequests);
}

@Override
protected BulkOperationResult executeBulkCreateAsync(List<CreateAPIEndpoint> createRequests) {
return SdkClients.adminClient().apiEndpoints().bulkCreateOrUpdateAsync(createRequests);
}

@Override
protected CreateAPIEndpoint createInvalidRequestForBulk(TestNamespace ns) {
CreateAPIEndpoint request = new CreateAPIEndpoint();
request.setName(ns.prefix("invalid_api_endpoint"));
return request;
}
}
Loading
Loading