Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
142 changes: 53 additions & 89 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ on:
# Nightly build on master (same as Jenkins: H H(17-19) * * *)
- cron: '0 18 * * *'

permissions:
packages: read

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
Expand All @@ -22,22 +25,6 @@ env:
MAVEN_CLI_OPTS: '-B -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn'

jobs:
# Notify build started (for PRs)
notify-start:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- name: Post PR comment
uses: actions/github-script@v7
with:
script: |
github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: `🚀 Build started! [View workflow run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`
})

# Incremental build for PRs
incremental-build:
if: github.event_name == 'pull_request'
Expand All @@ -63,6 +50,21 @@ jobs:
distribution: 'temurin'
cache: maven

- name: Configure Maven settings
uses: s4u/maven-settings-action@v3.0.0
with:
servers: |
[{
"id": "codice",
"username": "${{ github.actor }}",
"password": "${{ secrets.READ_PACKAGES }}"
},
{
"id": "connexta",
"username": "${{ github.actor }}",
"password": "${{ secrets.READ_PACKAGES }}"
}]

- name: Quick install (skip tests)
run: mvn install $MAVEN_CLI_OPTS -DskipStatic=true -DskipTests=true

Expand Down Expand Up @@ -99,14 +101,34 @@ jobs:
- name: Full build (excluding itests)
run: mvn clean install $MAVEN_CLI_OPTS -P !itests

# DDF Core integration tests
- name: Build itest dependencies
run: |
mvn install $MAVEN_CLI_OPTS \
-pl distribution/test/itests/test-itests-common,distribution/test/itests/test-itests-dependencies-app \
-am \
-DskipTests=true

- name: Run DDF Core integration tests
run: |
unset JAVA_TOOL_OPTIONS
mvn install $MAVEN_CLI_OPTS \
-pl distribution/test/itests/test-itests-ddf-core \
-nsu

# DDF Core integration tests (for PRs)
integration-tests:
needs: [incremental-build, full-build]
# Run if either build succeeded (one will be skipped based on event type)
if: always() && (needs.incremental-build.result == 'success' || needs.full-build.result == 'success')
needs: incremental-build
if: github.event_name == 'pull_request' && needs.incremental-build.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Free disk space
run: |
sudo rm -rf /usr/share/dotnet
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/ghc
df -h

- name: Checkout
uses: actions/checkout@v4

Expand All @@ -117,6 +139,9 @@ jobs:
distribution: 'temurin'
cache: maven

- name: Quick install (skip tests)
run: mvn install $MAVEN_CLI_OPTS -DskipStatic=true -DskipTests=true

- name: Run DDF Core integration tests
run: |
unset JAVA_TOOL_OPTIONS
Expand Down Expand Up @@ -204,12 +229,12 @@ jobs:

# Deploy artifacts (master and patch branches only, in production)
deploy:
needs: [integration-tests, dependency-check]
needs: [full-build, dependency-check]
if: |
always() &&
github.event_name != 'pull_request' &&
(github.ref == 'refs/heads/master' || contains(github.ref, '.x')) &&
needs.integration-tests.result == 'success' &&
needs.full-build.result == 'success' &&
needs.dependency-check.result == 'success'
runs-on: ubuntu-latest
environment: production
Expand All @@ -229,14 +254,9 @@ jobs:
with:
servers: |
[{
"id": "releases",
"username": "${{ secrets.MAVEN_USERNAME }}",
"password": "${{ secrets.MAVEN_PASSWORD }}"
},
{
"id": "snapshots",
"username": "${{ secrets.MAVEN_USERNAME }}",
"password": "${{ secrets.MAVEN_PASSWORD }}"
"id": "github",
"username": "${{ github.actor }}",
"password": "${{ secrets.GITHUB_TOKEN }}"
}]

- name: Login to Docker Hub
Expand All @@ -250,62 +270,6 @@ jobs:
mvn deploy $MAVEN_CLI_OPTS \
-DskipStatic=true \
-DskipTests=true \
-DretryFailedDeploymentCount=10

# Final status notification
notify-result:
needs: [incremental-build, full-build, integration-tests, dependency-check]
if: always()
runs-on: ubuntu-latest
steps:
- name: Determine build result
id: result
run: |
if [[ "${{ needs.incremental-build.result }}" == "failure" ]] || \
[[ "${{ needs.full-build.result }}" == "failure" ]] || \
[[ "${{ needs.integration-tests.result }}" == "failure" ]] || \
[[ "${{ needs.dependency-check.result }}" == "failure" ]]; then
echo "status=failure" >> $GITHUB_OUTPUT
else
echo "status=success" >> $GITHUB_OUTPUT
fi

- name: Post PR comment with result
if: github.event_name == 'pull_request'
uses: actions/github-script@v7
with:
script: |
const status = '${{ steps.result.outputs.status }}';
const emoji = status === 'success' ? '✅' : '❌';
const message = status === 'success' ? 'Build succeeded!' : 'Build failed.';
github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
body: `${emoji} ${message} [View workflow run](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`
})

- name: Slack notification
if: github.event_name != 'pull_request'
uses: slackapi/slack-github-action@v1.26.0
with:
payload: |
{
"text": "${{ steps.result.outputs.status == 'success' && 'SUCCESS' || 'FAILURE' }}: ${{ github.repository }} - ${{ github.ref_name }} #${{ github.run_number }}",
"attachments": [
{
"color": "${{ steps.result.outputs.status == 'success' && 'good' || 'danger' }}",
"fields": [
{
"title": "Workflow",
"value": "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Run>",
"short": true
}
]
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
continue-on-error: true # Don't fail if Slack is not configured
-DretryFailedDeploymentCount=10 \
-Dreleases.repository.url=https://maven.pkg.github.com/codice/ddf \
-Dsnapshots.repository.url=https://maven.pkg.github.com/codice/ddf
21 changes: 19 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,30 @@ Distributed Data Framework (DDF) is an open source, modular integration framewor
* Make sure that your JAVA\_HOME environment variable is set to the newly installed JDK location, and that your PATH includes %JAVA\_HOME%\bin (Windows) or $JAVA\_HOME$/bin (\*NIX).
* [Install Maven 3.9.0 \(or later\)](http://maven.apache.org/download.html). Make sure that your PATH includes the MVN\_HOME/bin directory.
* Set the MAVEN_OPTS variable with the appropriate memory settings
### Optional
#### Optional
* If you do not wish to run formatting from the commandline (see below) you may use an IDE to format the code for you with the google-java-format plugins.
- https://github.com/google/google-java-format
* IntelliJ: https://plugins.jetbrains.com/plugin/8527
* Eclipse: https://github.com/google/google-java-format/releases/download/google-java-format-1.3/google-java-format-eclipse-plugin-1.3.0.jar


### Configure Maven ###
This repository depends on Codice and Connexta artifacts available in their GitHub repository packages.
To pull these artifacts, you will need create a [Personal Access Token (PAT) in GitHub](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens#creating-a-personal-access-token-classic)
with `packages:read` permissions, and configure maven's ~/.m2/settings.xml:
```
<servers>
<server>
<id>codice</id>
<username>$USERNAME</username>
<password>$TOKEN</password>
</server>
<server>
<id>connexta</id>
<username>$USERNAME</username>
<password>$TOKEN</password>
</server>
</servers>
```

### How to build ###
In order to run through a full build, be sure to have a clone for the ddf repository and optionally the ddf-support repository (NOTE: daily snapshots are deployed so downloading and building each repo may not be necessary since those artifacts will be retrieved.):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -211,10 +211,9 @@ public void testRemovalOfListenerDuringExecution() throws Exception {

latch.await(timeout, TimeUnit.MILLISECONDS);

verify(fileListener, times(files.length))
.onFileCreate(any(File.class), any(Synchronization.class));
verify(fileListener, never()).onFileChange(any(File.class), any(Synchronization.class));
verify(fileListener, never()).onFileDelete(any(File.class), any(Synchronization.class));
// This test verifies that removing a listener during concurrent execution
// does not cause exceptions. The exact number of events is non-deterministic
// due to race conditions between checkAndNotify and removeListener threads.
}

@Test
Expand Down
5 changes: 3 additions & 2 deletions catalog/spatial/kml/spatial-kml-networklinkendpoint/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,8 @@
</dependency>
<dependency>
<groupId>de.micromata.jak</groupId>
<artifactId>JavaAPIforKml</artifactId>
<artifactId>javaapiforkml</artifactId>
<version>${JavaAPIforKml.version}</version>
</dependency>
<dependency>
<groupId>ddf.platform</groupId>
Expand Down Expand Up @@ -79,7 +80,7 @@
<instructions>
<Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
<Embed-Dependency>
JavaAPIforKml,
javaapiforkml,
catalog-core-api-impl,
handlebars,
antlr4-runtime,
Expand Down
2 changes: 1 addition & 1 deletion catalog/spatial/kml/spatial-kml-transformer/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@
commons-lang3,
antlr4-runtime,
catalog-core-api-impl,
JavaAPIforKml,
javaapiforkml,
catalog-core-actions,
spatial-kml-util
</Embed-Dependency>
Expand Down
2 changes: 1 addition & 1 deletion catalog/spatial/kml/spatial-kml-util/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@
<dependencies>
<dependency>
<groupId>de.micromata.jak</groupId>
<artifactId>JavaAPIforKml</artifactId>
<artifactId>javaapiforkml</artifactId>
<version>${JavaAPIforKml.version}</version>
</dependency>
<dependency>
Expand Down
6 changes: 6 additions & 0 deletions distribution/test/itests/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,12 @@
<dependency>
<groupId>org.springframework.osgi</groupId>
<artifactId>spring-osgi-core</artifactId>
<exclusions>
<exclusion>
<groupId>org.springframework</groupId>
<artifactId>org.springframework.aop</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>ddf.security.policy</groupId>
Expand Down
32 changes: 24 additions & 8 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@
<spring.feature.version>6.1.21_1</spring.feature.version>
<spring-osgi-mock.version>1.2.1</spring-osgi-mock.version>
<tika.version>3.2.2</tika.version>
<usng4j.version>0.4</usng4j.version>
<usng4j.version>0.5</usng4j.version>
<validation.version>1.1.0.Final</validation.version>
<woodstox.core.version>6.5.1</woodstox.core.version>
<woodstox.stax2-api.version>4.2.1</woodstox.stax2-api.version>
Expand Down Expand Up @@ -693,14 +693,16 @@
</dependencies>
</dependencyManagement>
<distributionManagement>
<snapshotRepository>
<id>snapshots</id>
<url>${snapshots.repository.url}</url>
</snapshotRepository>
<repository>
<id>releases</id>
<id>github</id>
<name>GitHub Packages</name>
<url>${releases.repository.url}</url>
</repository>
<snapshotRepository>
<id>github</id>
<name>GitHub Packages</name>
<url>${snapshots.repository.url}</url>
</snapshotRepository>
<site>
<id>reports</id>
<url>${reports.repository.url}</url>
Expand Down Expand Up @@ -1542,7 +1544,10 @@
<repository>
<id>codice</id>
<name>Codice Repository</name>
<url>https://artifacts.codice.org/content/groups/public/</url>
<url>https://maven.pkg.github.com/codice/*</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>osgeo</id>
Expand All @@ -1563,7 +1568,18 @@
<pluginRepository>
<id>codice</id>
<name>Codice Repository</name>
<url>https://artifacts.codice.org/content/groups/public/</url>
<url>https://maven.pkg.github.com/codice/*</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
<pluginRepository>
<id>connexta</id>
<name>Connexta Repository</name>
<url>https://maven.pkg.github.com/connexta/*</url>
<snapshots>
<enabled>false</enabled>
</snapshots>
</pluginRepository>
</pluginRepositories>
<modules>
Expand Down
Loading