Add Airbyte stream configuration job for .NET apps

aks/airbyte/resources.tf

@@ -90,7 +90,9 @@ module "streams_init_job" {
   cpu            = var.cpu
 }
 
-module "streams_update_job" {
+module "rails_streams_update_job" {
+  count = var.is_rails_application ? 1 : 0
+
   source = "../job_configuration"
 
   depends_on = [module.streams_init_job]
@@ -109,6 +111,47 @@ module "streams_update_job" {
   cpu            = var.cpu
 }
 
+module "dotnet_streams_update_job" {
+  count = var.is_dotnet_application ? 1 : 0
+
+  source = "../job_configuration"
+
+  namespace    = var.namespace
+  environment  = var.environment
+  service_name = var.service_name
+  docker_image = var.docker_image
+  commands     = ["/bin/sh"]
+  arguments = [
+    "-f",
+    "${coalesce(trimsuffix(var.dotnet_application_directory, "/"), ".")}/dfe-analytics/apply-config.sh",
+    "--connection-string",
+    "Server=${var.host_name};Database=${var.database_name};User Id=airbyte_replication;Password='${local.replication_password}';Ssl Mode=Require;Trust Server Certificate=true",
+    "--google-credentials",
+    local.gcp_credentials,
+    "--project-id",
+    data.google_project.main.project_id,
+    "--dataset-id",
+    local.gcp_dataset_name,
+    "--hidden-policy-tag-name",
+    local.gcp_policy_tag_name,
+    "--airbyte-api-base-address",
+    var.server_url,
+    "--airbyte-client-id",
+    var.client_id,
+    "--airbyte-client-secret",
+    var.client_secret,
+    "--airbyte-connection-id",
+    airbyte_connection.connection.connection_id
+  ]
+  job_name       = "airbyte-stream-update"
+  enable_logit   = true
+  enable_gcp_wif = true
+
+  config_map_ref = var.config_map_ref
+  secret_ref     = var.secret_ref
+  cpu            = var.cpu
+}
+
 resource "kubernetes_secret" "airbyte-sql" {
   metadata {
     name      = "${var.service_short}${var.environment}absql${local.sql_secret_hash}"

aks/airbyte/tfdocs.md

@@ -20,8 +20,9 @@
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_cluster_data"></a> [cluster\_data](#module\_cluster\_data) | ../cluster_data | n/a |
+| <a name="module_dotnet_streams_update_job"></a> [dotnet\_streams\_update\_job](#module\_dotnet\_streams\_update\_job) | ../job_configuration | n/a |
+| <a name="module_rails_streams_update_job"></a> [rails\_streams\_update\_job](#module\_rails\_streams\_update\_job) | ../job_configuration | n/a |
 | <a name="module_streams_init_job"></a> [streams\_init\_job](#module\_streams\_init\_job) | ../job_configuration | n/a |
-| <a name="module_streams_update_job"></a> [streams\_update\_job](#module\_streams\_update\_job) | ../job_configuration | n/a |
 
 ## Resources
 
@@ -65,6 +66,7 @@
 | <a name="input_cpu"></a> [cpu](#input\_cpu) | formerly: module.cluster\_data.configuration\_map.cpu\_min | `string` | n/a | yes |
 | <a name="input_database_name"></a> [database\_name](#input\_database\_name) | database name | `string` | n/a | yes |
 | <a name="input_docker_image"></a> [docker\_image](#input\_docker\_image) | Current application environment | `string` | n/a | yes |
+| <a name="input_dotnet_application_directory"></a> [dotnet\_application\_directory](#input\_dotnet\_application\_directory) | The path to application containing the dfe-analytics directory | `string` | `""` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | Current application environment | `string` | n/a | yes |
 | <a name="input_gcp_bq_sa"></a> [gcp\_bq\_sa](#input\_gcp\_bq\_sa) | Name of BQ service account | `string` | `null` | no |
 | <a name="input_gcp_dataset"></a> [gcp\_dataset](#input\_gcp\_dataset) | Name of an existing dataset. Optional: if not provided, create a new dataset | `string` | `null` | no |
@@ -75,6 +77,8 @@
 | <a name="input_gcp_table_deletion_protection"></a> [gcp\_table\_deletion\_protection](#input\_gcp\_table\_deletion\_protection) | Prevents deletion of the event table. Default: true | `bool` | `true` | no |
 | <a name="input_gcp_taxonomy_id"></a> [gcp\_taxonomy\_id](#input\_gcp\_taxonomy\_id) | Policy tags taxonomy ID. Required when creating the dataset | `number` | `null` | no |
 | <a name="input_host_name"></a> [host\_name](#input\_host\_name) | Host name | `string` | n/a | yes |
+| <a name="input_is_dotnet_application"></a> [is\_dotnet\_application](#input\_is\_dotnet\_application) | Whether to run the Airbyte deployment script in the application container | `bool` | `false` | no |
+| <a name="input_is_rails_application"></a> [is\_rails\_application](#input\_is\_rails\_application) | Whether to run the Airbyte deployment rake task in the application container | `bool` | `true` | no |
 | <a name="input_namespace"></a> [namespace](#input\_namespace) | AKS Namespace where the service is deployed to. Required | `string` | n/a | yes |
 | <a name="input_postgres_url"></a> [postgres\_url](#input\_postgres\_url) | Postgres connection url | `string` | n/a | yes |
 | <a name="input_postgres_version"></a> [postgres\_version](#input\_postgres\_version) | postgres version | `string` | n/a | yes |

aks/airbyte/variables.tf

@@ -121,6 +121,24 @@ variable "gcp_bq_sa" {
   default     = null
 }
 
+variable "is_rails_application" {
+  type        = bool
+  default     = true
+  description = "Whether to run the Airbyte deployment rake task in the application container"
+}
+
+variable "is_dotnet_application" {
+  type        = bool
+  default     = false
+  description = "Whether to run the Airbyte deployment script in the application container"
+}
+
+variable "dotnet_application_directory" {
+  type        = string
+  default     = ""
+  description = "The path to application containing the dfe-analytics directory"
+}
+
 locals {
   source_name      = "${var.azure_resource_prefix}-${var.service_short}-${var.environment}-pg-source"
   destination_name = "${var.azure_resource_prefix}-${var.service_short}-${var.environment}-bq-destination"