Skip to content

Update workflow settings for integration tests #25713

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
edg956 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Update workflow settings for integration tests #25713

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 79 additions & 15 deletions openmetadata-integration-tests/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,9 @@
<artifactId>maven-failsafe-plugin</artifactId>
<version>${maven.failsafe.version}</version>
<executions>
<!-- Phase 1: Run TagRecognizerFeedbackIT sequentially in integration-test phase -->
<!-- Run sequential tests -->
<execution>
<id>sequential-test</id>
<id>sequential-tests</id>
<goals>
<goal>integration-test</goal>
</goals>
Expand All @@ -176,6 +176,12 @@
<include>**/TagRecognizerFeedbackIT.java</include>
</includes>
<systemPropertyVariables>
<databaseType>mysql</databaseType>
<databaseImage>mysql:8.3.0</databaseImage>
<searchType>elasticsearch</searchType>
<searchImage>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>false</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand All @@ -184,7 +190,7 @@
<useFile>true</useFile>
</configuration>
</execution>
<!-- Phase 2: Run all other tests in parallel in integration-test phase -->
<!-- Run parallel tests -->
<execution>
<id>parallel-tests</id>
<goals>
Expand All @@ -202,6 +208,12 @@
<exclude>**/TagRecognizerFeedbackIT.java</exclude>
</excludes>
<systemPropertyVariables>
<databaseType>mysql</databaseType>
<databaseImage>mysql:8.3.0</databaseImage>
<searchType>elasticsearch</searchType>
<searchImage>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>true</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand Down Expand Up @@ -232,9 +244,9 @@
<artifactId>maven-failsafe-plugin</artifactId>
<version>${maven.failsafe.version}</version>
<executions>
<!-- Phase 1: Run TagRecognizerFeedbackIT sequentially in integration-test phase -->
<!-- Run sequential tests -->
<execution>
<id>sequential-test</id>
<id>sequential-tests</id>
<goals>
<goal>integration-test</goal>
</goals>
Expand All @@ -246,6 +258,12 @@
<include>**/TagRecognizerFeedbackIT.java</include>
</includes>
<systemPropertyVariables>
<databaseType>postgres</databaseType>
<databaseImage>postgres:15</databaseImage>
<searchType>opensearch</searchType>
<searchImage>opensearchproject/opensearch:2.19.0</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>false</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand All @@ -254,7 +272,7 @@
<useFile>true</useFile>
</configuration>
</execution>
<!-- Phase 2: Run all other tests in parallel in integration-test phase -->
<!-- Run parallel tests -->
<execution>
<id>parallel-tests</id>
<goals>
Expand All @@ -272,6 +290,12 @@
<exclude>**/TagRecognizerFeedbackIT.java</exclude>
</excludes>
<systemPropertyVariables>
<databaseType>postgres</databaseType>
<databaseImage>postgres:15</databaseImage>
<searchType>opensearch</searchType>
<searchImage>opensearchproject/opensearch:2.19.0</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>true</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand Down Expand Up @@ -302,9 +326,9 @@
<artifactId>maven-failsafe-plugin</artifactId>
<version>${maven.failsafe.version}</version>
<executions>
<!-- Phase 1: Run TagRecognizerFeedbackIT sequentially in integration-test phase -->
<!-- Run sequential tests -->
<execution>
<id>sequential-test</id>
<id>sequential-tests</id>
<goals>
<goal>integration-test</goal>
</goals>
Expand All @@ -316,6 +340,12 @@
<include>**/TagRecognizerFeedbackIT.java</include>
</includes>
<systemPropertyVariables>
<databaseType>postgres</databaseType>
<databaseImage>postgres:15</databaseImage>
<searchType>elasticsearch</searchType>
<searchImage>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>false</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand All @@ -324,7 +354,7 @@
<useFile>true</useFile>
</configuration>
</execution>
<!-- Phase 2: Run all other tests in parallel in integration-test phase -->
<!-- Run parallel tests -->
<execution>
<id>parallel-tests</id>
<goals>
Expand All @@ -342,6 +372,12 @@
<exclude>**/TagRecognizerFeedbackIT.java</exclude>
</excludes>
<systemPropertyVariables>
<databaseType>postgres</databaseType>
<databaseImage>postgres:15</databaseImage>
<searchType>elasticsearch</searchType>
<searchImage>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>true</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand Down Expand Up @@ -372,9 +408,9 @@
<artifactId>maven-failsafe-plugin</artifactId>
<version>${maven.failsafe.version}</version>
<executions>
<!-- Phase 1: Run TagRecognizerFeedbackIT sequentially in integration-test phase -->
<!-- Run sequential tests -->
<execution>
<id>sequential-test</id>
<id>sequential-tests</id>
<goals>
<goal>integration-test</goal>
</goals>
Expand All @@ -386,6 +422,12 @@
<include>**/TagRecognizerFeedbackIT.java</include>
</includes>
<systemPropertyVariables>
<databaseType>mysql</databaseType>
<databaseImage>mysql:8.3.0</databaseImage>
<searchType>opensearch</searchType>
<searchImage>opensearchproject/opensearch:2.19.0</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>false</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand All @@ -394,7 +436,7 @@
<useFile>true</useFile>
</configuration>
</execution>
<!-- Phase 2: Run all other tests in parallel in integration-test phase -->
<!-- Run parallel tests -->
<execution>
<id>parallel-tests</id>
<goals>
Expand All @@ -412,6 +454,12 @@
<exclude>**/TagRecognizerFeedbackIT.java</exclude>
</excludes>
<systemPropertyVariables>
<databaseType>mysql</databaseType>
<databaseImage>mysql:8.3.0</databaseImage>
<searchType>opensearch</searchType>
<searchImage>opensearchproject/opensearch:2.19.0</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>true</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand Down Expand Up @@ -442,9 +490,9 @@
<artifactId>maven-failsafe-plugin</artifactId>
<version>${maven.failsafe.version}</version>
<executions>
<!-- Phase 1: Run TagRecognizerFeedbackIT sequentially in integration-test phase -->
<!-- Run sequential tests -->
<execution>
<id>sequential-test</id>
<id>sequential-tests</id>
<goals>
<goal>integration-test</goal>
</goals>
Expand All @@ -456,6 +504,14 @@
<include>**/TagRecognizerFeedbackIT.java</include>
</includes>
<systemPropertyVariables>
<enableRdf>true</enableRdf>
<rdfContainerImage>stain/jena-fuseki:latest</rdfContainerImage>
<databaseType>postgres</databaseType>
<databaseImage>postgres:15</databaseImage>
<searchType>elasticsearch</searchType>
<searchImage>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>false</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand All @@ -464,7 +520,7 @@
<useFile>true</useFile>
</configuration>
</execution>
<!-- Phase 2: Run all other tests in parallel in integration-test phase -->
<!-- Run parallel tests -->
<execution>
<id>parallel-tests</id>
<goals>
Expand All @@ -482,6 +538,14 @@
<exclude>**/TagRecognizerFeedbackIT.java</exclude>
</excludes>
<systemPropertyVariables>
<enableRdf>true</enableRdf>
<rdfContainerImage>stain/jena-fuseki:latest</rdfContainerImage>
<databaseType>postgres</databaseType>
<databaseImage>postgres:15</databaseImage>
<searchType>elasticsearch</searchType>
<searchImage>docker.elastic.co/elasticsearch/elasticsearch:8.11.4</searchImage>
<openmetadata.flowable.historyMode>sync</openmetadata.flowable.historyMode>

<junit.jupiter.extensions.autodetection.enabled>true</junit.jupiter.extensions.autodetection.enabled>
<junit.jupiter.execution.parallel.enabled>true</junit.jupiter.execution.parallel.enabled>
</systemPropertyVariables>
Expand Down
59 changes: 57 additions & 2 deletions ...ata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import io.dropwizard.testing.ResourceHelpers;
import io.dropwizard.testing.junit5.DropwizardAppExtension;
import jakarta.validation.Validator;
import jakarta.ws.rs.core.Response;
import java.io.IOException;
import java.time.Duration;
import java.util.concurrent.atomic.AtomicBoolean;
Expand All @@ -43,7 +44,10 @@
import org.openmetadata.schema.api.configuration.pipelineServiceClient.Parameters;
import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration;
import org.openmetadata.schema.api.configuration.rdf.RdfConfiguration;
import org.openmetadata.schema.configuration.WorkflowSettings;
import org.openmetadata.schema.service.configuration.elasticsearch.ElasticSearchConfiguration;
import org.openmetadata.schema.settings.Settings;
import org.openmetadata.schema.settings.SettingsType;
import org.openmetadata.schema.type.IndexMappingLanguage;
import org.openmetadata.search.IndexMappingLoader;
import org.openmetadata.service.Entity;
Expand All @@ -56,6 +60,7 @@
import org.openmetadata.service.governance.workflows.WorkflowHandler;
import org.openmetadata.service.jdbi3.CollectionDAO;
import org.openmetadata.service.jdbi3.HikariCPDataSourceFactory;
import org.openmetadata.service.jdbi3.SystemRepository;
import org.openmetadata.service.jdbi3.locator.ConnectionAwareAnnotationSqlLocator;
import org.openmetadata.service.jdbi3.locator.ConnectionType;
import org.openmetadata.service.jobs.JobDAO;
Expand Down Expand Up @@ -464,7 +469,8 @@ private void validateAndRunSystemDataMigrations(
String nativeMigrationSQLPath,
String extensionSQLScriptRootPath,
String flywayPath,
boolean forceMigrations) {
boolean forceMigrations)
throws ConfigurationException, IOException {
DatasourceConfig.initialize(connType.label);
MigrationWorkflow workflow =
new MigrationWorkflow(
Expand All @@ -482,7 +488,9 @@ private void validateAndRunSystemDataMigrations(
Entity.initializeRepositories(config, jdbi);
workflow.loadMigrations();
workflow.runMigrationWorkflows(false);
WorkflowHandler.initialize(config);

initializeTestWorkflowSettings();
WorkflowHandler.initialize(config, true);
SettingsCache.initialize(config);
ApplicationHandler.initialize(config);
ApplicationContext.initialize();
Expand All @@ -497,6 +505,43 @@ private void createIndices() {
searchRepository.createIndexes();
}

private WorkflowSettings readTestWorkflowSettings(String path)
throws IOException, ConfigurationException {
ObjectMapper objectMapper = Jackson.newObjectMapper();
Validator validator = Validators.newValidator();

YamlConfigurationFactory<WorkflowSettings> factory =
new YamlConfigurationFactory<>(WorkflowSettings.class, validator, objectMapper, "dw");

return factory.build(
new SubstitutingSourceProvider(
new FileConfigurationSourceProvider(), new EnvironmentVariableSubstitutor(false)),
path);
}

private void initializeTestWorkflowSettings() throws IOException, ConfigurationException {
String workflowSettingsPath = ResourceHelpers.resourceFilePath("test-workflow-settings.yaml");

WorkflowSettings testSettings = readTestWorkflowSettings(workflowSettingsPath);

SystemRepository repo = new SystemRepository();
WorkflowSettings existingSettings = repo.getWorkflowSettingsOrDefault();

Settings settings =
new Settings()
.withConfigType(SettingsType.WORKFLOW_SETTINGS)
.withConfigValue(
existingSettings.withExecutorConfiguration(
testSettings.getExecutorConfiguration()));

try (Response res = repo.createNewSetting(settings)) {
if (!res.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) {
throw new RuntimeException(
String.format("Failed to create test workflow settings: %s", res.getEntity()));
}
}
}

private ElasticSearchConfiguration getSearchConfig() {
ElasticSearchConfiguration config = new ElasticSearchConfiguration();
ElasticSearchConfiguration.SearchType type =
Expand Down Expand Up @@ -576,6 +621,16 @@ private void cleanup() {
LOG.warn("Error cleaning up shared entities", e);
}

try {
if (WorkflowHandler.isInitialized()) {
LOG.info("Shutting down Flowable ProcessEngine...");
org.flowable.engine.ProcessEngines.destroy();
LOG.info("Flowable ProcessEngine shut down successfully");
}
} catch (Exception e) {
LOG.warn("Error shutting down Flowable ProcessEngine", e);
}

try {
if (APP != null) {
APP.after();
Expand Down
2 changes: 1 addition & 1 deletion ...ta-integration-tests/src/test/java/org/openmetadata/it/tests/TagRecognizerFeedbackIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
@ExtendWith(TestNamespaceExtension.class)
@Execution(ExecutionMode.SAME_THREAD)
public class TagRecognizerFeedbackIT {
private static final long TIMEOUT_MINUTES = 3;
private static final long TIMEOUT_MINUTES = 5;
private static final long POLL_INTERVAL_SECONDS = 3;

@BeforeAll
Expand Down
8 changes: 8 additions & 0 deletions openmetadata-integration-tests/src/test/resources/test-workflow-settings.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
executorConfiguration:
corePoolSize: 32
maxPoolSize: 32
queueSize: 10000
tasksDuePerAcquisition: 500
jobLockTimeInMillis: 1296000000
asyncJobAcquisitionInterval: 500
timerJobAcquisitionInterval: 500
Loading
Loading