Skip to content

Commit 1ff494d

Browse files
alhendricksonalhendrickson
authored
feat(iac): openstack kubernetes module improvements (#37)
* feat(k8s): Make name prefix optional * feat(iac): Output the security group * feat(iac): In k8s module, added support for network configuration through a new variable 'network' * feat(iac): Support network id input * feat(iac): Extract provisioner to null_resource in order to make optional * feat(iac): fix nullable network name * feat(iac): floating_ip for k8s server. Set SAN * feat(iac): format * feat(iac): add readme for floating ip setup * feat(iac): ignore user_data changes to not recreate after initialised * feat(iac): cleanup
1 parent 3a564e1 commit 1ff494d

File tree

12 files changed

+241
-48
lines changed

12 files changed

+241
-48
lines changed

‎deployment/terraform/examples/openstack-kubernetes/k3s-cluster/main.tf‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
module "openstack_cogstack_infra" {
22
source = "../../../modules/openstack-kubernetes-infra"
33
host_instances = [
4-
{ name = "cogstack-k3s", is_controller = true },
4+
{
5+
name = "cogstack-k3s",
6+
is_controller = true,
7+
# floating_ip = {
8+
# use_floating_ip = true,
9+
# address = "10.10.10.10"
10+
# }
11+
},
512
{
613
name = "cogstack-k3s-node-2"
714
flavour = "2cpu4ram"
@@ -11,4 +18,9 @@ module "openstack_cogstack_infra" {
1118
]
1219
allowed_ingress_ips_cidr = var.allowed_ingress_ips_cidr
1320
ubuntu_immage_name = var.ubuntu_immage_name
21+
# generate_random_name_prefix = false
22+
# prefix = "dev"
23+
# network = {
24+
# network_id = "some-id"
25+
# }
1426
}

‎deployment/terraform/modules/openstack-cogstack-infra/compute.tf‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,3 @@ data "openstack_images_image_v2" "ubuntu" {
125125
most_recent = true
126126
}
127127

128-
data "openstack_networking_secgroup_v2" "er_https_from_lbs" {
129-
name = "er_https_from_lbs"
130-
}

‎deployment/terraform/modules/openstack-kubernetes-infra/README.md‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,54 @@ module "openstack_kubernetes_cluster" {
3535
}
3636
```
3737

38+
## Existing Network and Floating IPs
39+
You can use this module with a custom OpenStack network by specifying the `network` variable. By default, it will use the network named `"external_4003"`, but you can override this with your own network name or network ID.
40+
41+
Using a custom OpenStack network with your own subnet allows you to improve security by keeping most nodes private. This ensures worker nodes and other internal resources are not directly exposed, reducing security risks. Only the controller node needs a floating IP to be accessible in order to use the k8s api server (eg for kubectl to work from outside the network). Using floating ips will also have the advantage of being stable, so you can destroy/create VMs without having to update anywhere the IP address is referenced by reassigning the floating ip.
42+
43+
To use this configuration, you will probably need to assign a floating IP to the controller node so that it is accessible. You can configure floating IP assignment per node using the `floating_ip` block within each entry in the `host_instances` variable
44+
45+
Please see this example for using thism module with a custom network and floating IPs.
46+
47+
```hcl
48+
host_instances = [
49+
{
50+
name = "controller"
51+
is_controller = true
52+
floating_ip = {
53+
use_floating_ip = true
54+
address = "203.0.113.10" # Address of an existing floating_ip in openstack
55+
}
56+
},
57+
{
58+
name = "worker"
59+
# Optionally also assign a floating IP here, or leave blank to keep it internal to the network
60+
}
61+
network = {
62+
network_id = openstack_networking_network_v2.example_network.id
63+
}
64+
]
65+
66+
67+
resource "openstack_networking_network_v2" "example_network" {
68+
name = "dev-example-network"
69+
admin_state_up = "true"
70+
}
71+
72+
resource "openstack_networking_subnet_v2" "example_subnet"{
73+
name = "dev-example-subnet"
74+
network_id = openstack_networking_network_v2.example_network.id
75+
cidr = "192.168.0.0/24"
76+
ip_version = 4
77+
}
78+
79+
resource "openstack_networking_router_v2" "example_router" {
80+
name = "test-router"
81+
admin_state_up = true
82+
external_network_id = data.openstack_networking_network_v2.external_4003.id
83+
}
84+
```
85+
86+
When using a non-default network, ensure the controller host has a floating IP so Terraform can access it for provisioning.
87+
3888

‎deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-agent.yaml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,13 @@ runcmd:
3737

3838
# Run K3s
3939
- echo "Installing K3S"
40-
- curl -sfL https://get.k3s.io | K3S_URL=https://${TF_K3S_SERVER_IP_ADDRESS}:6443 K3S_TOKEN="${TF_K3S_TOKEN}" sh -
40+
- |
41+
INSTALL_K3S_EXEC=""
42+
TF_K3S_NODE_EXTERNAL_IP=${TF_K3S_NODE_EXTERNAL_IP}
43+
if [ -n "$TF_K3S_NODE_EXTERNAL_IP" ]; then
44+
INSTALL_K3S_EXEC="--node-external-ip $TF_K3S_NODE_EXTERNAL_IP"
45+
fi
46+
curl -sfL https://get.k3s.io | K3S_URL=https://${TF_K3S_SERVER_IP_ADDRESS}:6443 K3S_TOKEN="${TF_K3S_TOKEN}" INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC" sh -
4147
- echo "Completed Installing K3S"
4248
# - sudo chmod 644 /etc/rancher/k3s/k3s.yaml
4349

‎deployment/terraform/modules/openstack-kubernetes-infra/cloud-init-k3s-server.yaml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,17 @@ runcmd:
3737

3838
# Run K3s
3939
- echo "Installing K3S"
40-
- sudo curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="server" K3S_TOKEN="${TF_K3S_TOKEN}" sh -
40+
- |
41+
TF_K3S_TLS_SAN=${TF_K3S_TLS_SAN}
42+
TF_K3S_NODE_EXTERNAL_IP=${TF_K3S_NODE_EXTERNAL_IP}
43+
INSTALL_K3S_EXEC="server"
44+
if [ -n "$TF_K3S_TLS_SAN" ]; then
45+
INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --tls-san $TF_K3S_TLS_SAN"
46+
fi
47+
if [ -n "$TF_K3S_NODE_EXTERNAL_IP" ]; then
48+
INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC --node-external-ip $TF_K3S_NODE_EXTERNAL_IP"
49+
fi
50+
sudo curl -sfL https://get.k3s.io | K3S_TOKEN="${TF_K3S_TOKEN}" INSTALL_K3S_EXEC="$INSTALL_K3S_EXEC" sh -
4151
- echo "Completed Installing K3S"
4252
- sudo chmod 644 /etc/rancher/k3s/k3s.yaml
4353

‎deployment/terraform/modules/openstack-kubernetes-infra/compute-keypair.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ locals {
1414
}
1515

1616
resource "openstack_compute_keypair_v2" "compute_keypair" {
17-
name = "${local.random_prefix}-cogstack_keypair"
17+
name = local.prefix != "" ? "${local.prefix}-cogstack_keypair" : "cogstack_keypair"
1818
public_key = local.is_using_existing_ssh_keypair ? file(var.ssh_key_pair.public_key_file) : null
1919
}
2020

@@ -30,4 +30,4 @@ resource "local_file" "public_key" {
3030
content = openstack_compute_keypair_v2.compute_keypair.public_key
3131
filename = "${local.output_file_directory}/${openstack_compute_keypair_v2.compute_keypair.name}-rsa.pub"
3232
file_permission = "0600"
33-
}
33+
}

‎deployment/terraform/modules/openstack-kubernetes-infra/compute.tf‎

Lines changed: 51 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
resource "openstack_compute_instance_v2" "kubernetes_server" {
22

3-
name = "${local.random_prefix}-${local.controller_host.name}"
3+
name = local.prefix != "" ? "${local.prefix}-${local.controller_host.name}" : local.controller_host.name
44
flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[local.controller_host.flavour].id
55
key_pair = openstack_compute_keypair_v2.compute_keypair.name
66
region = "RegionOne"
77

88
user_data = data.cloudinit_config.init_docker_controller.rendered
9+
910
security_groups = ["default",
1011
openstack_networking_secgroup_v2.cogstack_apps_security_group.name
1112
]
1213

1314
network {
14-
uuid = data.openstack_networking_network_v2.external_4003.id
15+
uuid = local.network_id
1516
}
1617

1718
block_device {
@@ -23,9 +24,17 @@ resource "openstack_compute_instance_v2" "kubernetes_server" {
2324
delete_on_termination = true
2425
}
2526

27+
lifecycle {
28+
ignore_changes = [user_data]
29+
}
30+
}
31+
32+
resource "null_resource" "kubernetes_server_provisioner" {
33+
depends_on = [openstack_compute_instance_v2.kubernetes_server, openstack_networking_floatingip_associate_v2.kubernetes_server_fip]
34+
2635
connection {
2736
user = "ubuntu"
28-
host = self.access_ip_v4
37+
host = local.controller_host_instance.ip_address
2938
private_key = file(local.ssh_keys.private_key_file)
3039
}
3140

@@ -37,34 +46,55 @@ resource "openstack_compute_instance_v2" "kubernetes_server" {
3746
}
3847

3948
resource "openstack_compute_instance_v2" "kubernetes_nodes" {
40-
depends_on = [openstack_compute_instance_v2.kubernetes_server]
41-
for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller }
42-
name = "${local.random_prefix}-${each.value.name}"
43-
flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id
44-
key_pair = openstack_compute_keypair_v2.compute_keypair.name
45-
region = "RegionOne"
49+
depends_on = [
50+
openstack_compute_instance_v2.kubernetes_server
51+
]
52+
53+
for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller }
54+
55+
name = local.prefix != "" ? "${local.prefix}-${each.value.name}" : each.value.name
56+
flavor_id = data.openstack_compute_flavor_v2.available_compute_flavors[each.value.flavour].id
57+
key_pair = openstack_compute_keypair_v2.compute_keypair.name
58+
region = "RegionOne"
4659

47-
user_data = data.cloudinit_config.init_docker.rendered
60+
user_data = data.cloudinit_config.init_docker[each.key].rendered
4861
security_groups = ["default",
4962
openstack_networking_secgroup_v2.cogstack_apps_security_group.name
5063
]
5164

5265
network {
53-
uuid = data.openstack_networking_network_v2.external_4003.id
66+
uuid = local.network_id
5467
}
5568

5669
block_device {
57-
uuid = each.value.image_uuid == null ? data.openstack_images_image_v2.ubuntu.id : each.value.image_uuid
70+
uuid = each.value.image_uuid == null ? data.openstack_images_image_v2.ubuntu.id : each.value.image_uuid
5871
source_type = "image"
5972
volume_size = each.value.volume_size
6073
boot_index = 0
6174
destination_type = "volume"
6275
delete_on_termination = true
6376
}
6477

78+
lifecycle {
79+
ignore_changes = [user_data]
80+
}
81+
}
82+
83+
locals {
84+
is_default_network = var.network != null && var.network == { name = "external_4003" }
85+
nodes_with_exernal_ip = { for node in local.created_nodes : node.name => node if local.is_default_network || node.use_floating_ip }
86+
87+
}
88+
resource "null_resource" "kubernetes_nodes_provisioner" {
89+
# Provisioner is only used to check for node readiness. Skip for any nodes that are not in the external network,
90+
# TODO: Filter this provisioner to only run on nodes that have a floating IP if the network is not default
91+
for_each = local.nodes_with_exernal_ip
92+
93+
depends_on = [openstack_compute_instance_v2.kubernetes_nodes, openstack_networking_floatingip_associate_v2.kubernetes_nodes_fip]
94+
6595
connection {
6696
user = "ubuntu"
67-
host = self.access_ip_v4
97+
host = each.value.ip_address
6898
private_key = file(local.ssh_keys.private_key_file)
6999
}
70100

@@ -75,8 +105,6 @@ resource "openstack_compute_instance_v2" "kubernetes_nodes" {
75105
}
76106
}
77107

78-
79-
80108
# TODO: Read content from files and put into cloud-init config
81109
# data "local_file" "install_docker_sh" {
82110
# filename = "${path.module}/resources/install-docker.sh"
@@ -88,6 +116,8 @@ resource "openstack_compute_instance_v2" "kubernetes_nodes" {
88116

89117

90118
data "cloudinit_config" "init_docker" {
119+
for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller }
120+
91121
depends_on = [openstack_compute_instance_v2.kubernetes_server]
92122
part {
93123
filename = "cloud-init-k3s-agent.yaml"
@@ -96,6 +126,7 @@ data "cloudinit_config" "init_docker" {
96126
{
97127
TF_K3S_TOKEN = random_password.k3s_token.result
98128
TF_K3S_SERVER_IP_ADDRESS = openstack_compute_instance_v2.kubernetes_server.access_ip_v4
129+
TF_K3S_NODE_EXTERNAL_IP = each.value.floating_ip != null && each.value.floating_ip.use_floating_ip ? each.value.floating_ip.address : ""
99130
}
100131
)
101132
}
@@ -111,7 +142,9 @@ data "cloudinit_config" "init_docker_controller" {
111142
content_type = "text/cloud-config"
112143
content = templatefile("${path.module}/cloud-init-k3s-server.yaml",
113144
{
114-
TF_K3S_TOKEN = random_password.k3s_token.result
145+
TF_K3S_TOKEN = random_password.k3s_token.result
146+
TF_K3S_TLS_SAN = local.controller_host_has_floating_ip ? local.controller_host.floating_ip.address : ""
147+
TF_K3S_NODE_EXTERNAL_IP = local.controller_host_has_floating_ip ? local.controller_host.floating_ip.address : ""
115148
}
116149
)
117150
}
@@ -123,16 +156,12 @@ data "openstack_compute_flavor_v2" "available_compute_flavors" {
123156
}
124157

125158

126-
data "openstack_networking_network_v2" "external_4003" {
127-
name = "external_4003"
159+
data "openstack_networking_network_v2" "network" {
160+
name = var.network != null && var.network.name != null ? var.network.name : "external_4003"
128161
}
129162

130163
data "openstack_images_image_v2" "ubuntu" {
131164
name = var.ubuntu_immage_name
132165
most_recent = true
133166
}
134167

135-
data "openstack_networking_secgroup_v2" "er_https_from_lbs" {
136-
name = "er_https_from_lbs"
137-
}
138-

‎deployment/terraform/modules/openstack-kubernetes-infra/kubeconfig-extraction.tf‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
resource "null_resource" "copy_kubeconfig" {
2-
depends_on = [openstack_compute_instance_v2.kubernetes_server]
2+
depends_on = [openstack_compute_instance_v2.kubernetes_server, null_resource.kubernetes_server_provisioner]
33

44
provisioner "local-exec" {
55
# Copy the kubeconfig file from the host to a local file using SCP.
66
# Use ssh-keyscan to prevent interactive prompt on unknown host
77
# Use sed to replace the localhost address in the KUBECONFIG file with the actual IP adddress of the created VM.
88
command = <<EOT
99
mkdir -p ${path.root}/.build/ && \
10-
ssh-keyscan -H ${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} >> ${path.root}/.build/.known_hosts_cogstack && \
10+
ssh-keyscan -H ${local.controller_host_instance.ip_address} >> ${path.root}/.build/.known_hosts_cogstack && \
1111
ssh -o UserKnownHostsFile=${path.root}/.build/.known_hosts_cogstack -o StrictHostKeyChecking=yes \
1212
-i ${local.ssh_keys.private_key_file} \
13-
ubuntu@${openstack_compute_instance_v2.kubernetes_server.access_ip_v4} \
13+
ubuntu@${local.controller_host_instance.ip_address} \
1414
"sudo cat /etc/rancher/k3s/k3s.yaml" > ${local.kubeconfig_file} && \
15-
sed -i "s/127.0.0.1/${openstack_compute_instance_v2.kubernetes_server.access_ip_v4}/" ${local.kubeconfig_file}
15+
sed -i "s/127.0.0.1/${local.controller_host_instance.ip_address}/" ${local.kubeconfig_file}
1616
EOT
1717
}
1818
}

‎deployment/terraform/modules/openstack-kubernetes-infra/networking.tf‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ locals {
1313
}
1414

1515
resource "openstack_networking_secgroup_v2" "cogstack_apps_security_group" {
16-
name = "${local.random_prefix}-cogstack-services"
16+
name = local.prefix != "" ? "${local.prefix}-cogstack-services" : "cogstack-services"
1717
description = "Cogstack Apps and Services Group"
1818
}
1919

@@ -29,3 +29,31 @@ resource "openstack_networking_secgroup_rule_v2" "cogstack_apps_port_rules" {
2929
security_group_id = openstack_networking_secgroup_v2.cogstack_apps_security_group.id
3030
}
3131

32+
33+
34+
# Look up ports by device_id and network_id
35+
data "openstack_networking_port_v2" "server_port" {
36+
count = local.controller_host_has_floating_ip ? 1 : 0
37+
device_id = openstack_compute_instance_v2.kubernetes_server.id
38+
network_id = openstack_compute_instance_v2.kubernetes_server.network[0].uuid
39+
}
40+
41+
data "openstack_networking_port_v2" "nodes_port" {
42+
for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller && vm.floating_ip != null && vm.floating_ip.use_floating_ip }
43+
device_id = openstack_compute_instance_v2.kubernetes_nodes[each.key].id
44+
network_id = openstack_compute_instance_v2.kubernetes_nodes[each.key].network[0].uuid
45+
}
46+
47+
# Associate floating IP with kubernetes server
48+
resource "openstack_networking_floatingip_associate_v2" "kubernetes_server_fip" {
49+
count = local.controller_host_has_floating_ip ? 1 : 0
50+
floating_ip = local.controller_host.floating_ip.address
51+
port_id = data.openstack_networking_port_v2.server_port[0].id
52+
}
53+
54+
# Associate floating IPs with kubernetes nodes
55+
resource "openstack_networking_floatingip_associate_v2" "kubernetes_nodes_fip" {
56+
for_each = { for vm in var.host_instances : vm.name => vm if !vm.is_controller && vm.floating_ip != null && vm.floating_ip.use_floating_ip }
57+
floating_ip = each.value.floating_ip.address
58+
port_id = data.openstack_networking_port_v2.nodes_port[each.key].id
59+
}

‎deployment/terraform/modules/openstack-kubernetes-infra/outputs.tf‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,6 @@
11

22
output "created_hosts" {
3-
value = merge({ for k, value in openstack_compute_instance_v2.kubernetes_nodes : k => {
4-
ip_address = value.access_ip_v4
5-
unique_name = value.name
6-
name = k
7-
} },
3+
value = merge(local.created_nodes,
84
{
95
(local.controller_host.name) : local.controller_host_instance
106
})
@@ -29,3 +25,8 @@ output "kubeconfig_file" {
2925
value = abspath(local.kubeconfig_file)
3026
description = "Path to the generated KUBECONFIG file used to connect to kubernetes"
3127
}
28+
29+
output "created_security_group" {
30+
value = openstack_networking_secgroup_v2.cogstack_apps_security_group
31+
description = "Security group associated to the created hosts"
32+
}

0 commit comments

Comments
 (0)