Skip to content
Draft
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
97c5c36
cluster_test
okankoAMZ Feb 17, 2025
4def8bf
added container insights test
okankoAMZ Feb 18, 2025
6ae15ad
fixed fillEKS for rosa
okankoAMZ Feb 18, 2025
976127e
fixed helm args
okankoAMZ Feb 18, 2025
741d2e9
added k8smode
okankoAMZ Feb 19, 2025
65f8fec
made sample app optional
okankoAMZ Feb 19, 2025
7bb61cd
added security test
okankoAMZ Mar 4, 2025
ba23de2
updated rosa validation test
okankoAMZ Mar 4, 2025
0ae2d3a
update gomod
okankoAMZ Mar 4, 2025
12d6c58
update generator
okankoAMZ Mar 4, 2025
263725e
added test name
okankoAMZ Mar 4, 2025
946a0fb
fixed rosa cluster validation
okankoAMZ Mar 4, 2025
a6eb637
updated the generator
okankoAMZ Mar 4, 2025
571765d
added diskio exception
okankoAMZ Mar 4, 2025
f59a3bc
change test-shell requirement
okankoAMZ Mar 4, 2025
0f582ff
added terraform generation
okankoAMZ Mar 5, 2025
add1529
change vpc naming
okankoAMZ Mar 5, 2025
4d2c5a5
changed iam to be per cluster
okankoAMZ Mar 5, 2025
520e311
removed random string gen
okankoAMZ Mar 5, 2025
3cf7d6d
appsig test
okankoAMZ Mar 10, 2025
08456d6
improved resource creation and deletion
okankoAMZ Mar 10, 2025
d50bcaa
added configurable instance type
okankoAMZ Mar 10, 2025
76cc1a1
added application signals test
okankoAMZ Mar 10, 2025
79e69f3
added sample app field to generator
okankoAMZ Mar 10, 2025
edd8f8f
added patch
okankoAMZ Mar 11, 2025
ceb2416
wait for metric propagation
okankoAMZ Mar 12, 2025
3501bd6
cleaned up security test
okankoAMZ Mar 12, 2025
e2cfd2d
added enhanced CI metrics
okankoAMZ Mar 13, 2025
8bd4dc3
removed apiserver_storage_list_duration_seconds from rosa test
okankoAMZ Mar 13, 2025
2acd622
added jitter and increased wait time
okankoAMZ Mar 14, 2025
1f9b296
changed order of tests
okankoAMZ Mar 14, 2025
c08c84b
added json field to helm manager
okankoAMZ Mar 18, 2025
54782f2
fixed jmx w helm wait
okankoAMZ Mar 19, 2025
3314f15
fixed tomcat test w traffic gen
okankoAMZ Mar 19, 2025
c4a2c78
removed openshift version
okankoAMZ Mar 24, 2025
fe376c2
fixed jvm tomcat for rosa
okankoAMZ Mar 24, 2025
09bc823
Merge branch 'main' into e2e-rosa
okankoAMZ Mar 24, 2025
6303d0a
reformatted terraform
okankoAMZ Mar 24, 2025
aa180ea
addressed comments
okankoAMZ Mar 24, 2025
78eef50
removed wait for sample app
okankoAMZ Mar 24, 2025
54f41d6
fixing ci test
okankoAMZ Mar 25, 2025
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 8 additions & 6 deletions environment/computetype/compute_type.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,18 @@ import "strings"
type ComputeType string

const (
EC2 ComputeType = "EC2"
ECS ComputeType = "ECS"
EKS ComputeType = "EKS"
EC2 ComputeType = "EC2"
ECS ComputeType = "ECS"
EKS ComputeType = "EKS"
ROSA ComputeType = "ROSA"
)

var (
computeTypes = map[string]ComputeType{
"EC2": EC2,
"ECS": ECS,
"EKS": EKS,
"EC2": EC2,
"ECS": ECS,
"EKS": EKS,
"ROSA": ROSA,
}
)

Expand Down
2 changes: 1 addition & 1 deletion environment/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ func fillExcludedTests(e *MetaData, data *MetaDataStrings) {
}

func fillEKSData(e *MetaData, data *MetaDataStrings) {
if e.ComputeType != computetype.EKS {
if e.ComputeType != computetype.EKS && e.ComputeType != computetype.ROSA {
return
}

Expand Down
5 changes: 5 additions & 0 deletions generator/resources/rosa_e2e_cluster_test_matrix.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[
{
"nodes": 2
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you can remove this if you don't end up using it -- it's meant to determine the number of nodes on the cluster

}
]
37 changes: 35 additions & 2 deletions generator/test_case_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@ import (
"io"
"log"
"os"
"strings"

"github.com/mitchellh/mapstructure"
"golang.org/x/exp/slices"
)

type matrixRow struct {
TestName string `json:"testName"`
TestDir string `json:"test_dir"`
Os string `json:"os"`
Family string `json:"family"`
Expand All @@ -37,6 +39,7 @@ type matrixRow struct {
ExcludedTests string `json:"excludedTests"`
MetadataEnabled string `json:"metadataEnabled"`
MaxAttempts int `json:"max_attempts"`
SampleAppPath string `json:"sample_app_path"`
}

type testConfig struct {
Expand All @@ -50,7 +53,8 @@ type testConfig struct {
// empty map means a testConfig will be created with a test entry for each entry from *_test_matrix.json
targets map[string]map[string]struct{}
// maxAttempts limits the number of times a test will be run.
maxAttempts int
maxAttempts int
sampleAppPath string
}

const (
Expand Down Expand Up @@ -261,6 +265,15 @@ var testTypeToTestConfigE2E = map[string][]testConfig{
"eks_e2e_jmx": {
{testDir: "../../../test/e2e/jmx"},
},
"rosa_e2e_cluster": {
//{testDir: "../../../test/e2e/jmx"},
{testDir: "test/e2e/container_insights"},
{testDir: "test/e2e/security",
sampleAppPath: "resources/shell.yaml"},
{testDir: "test/e2e/application_signals",
sampleAppPath: "resources/appsignals_sample_app.yaml",
},
},
}

type partition struct {
Expand Down Expand Up @@ -319,7 +332,24 @@ func main() {
}
}
}
func generateTestName(test_directory string) string {
parts := strings.Split(test_directory, "/")

// Remove empty parts caused by leading `../`
var cleaned []string
for _, part := range parts {
if part != "" && part != "." && part != ".." {
cleaned = append(cleaned, part)
}
}

// Reorder: move the first element to the end
if len(cleaned) > 1 {
cleaned = append(cleaned[1:], cleaned[0])
}

return strings.Join(cleaned, "-")
}
func genMatrix(testType string, testConfigs []testConfig, ami []string) []matrixRow {
openTestMatrix, err := os.Open(fmt.Sprintf("generator/resources/%v_test_matrix.json", testType))

Expand All @@ -341,6 +371,7 @@ func genMatrix(testType string, testConfigs []testConfig, ami []string) []matrix
for _, test := range testMatrix {
for _, testConfig := range testConfigs {
row := matrixRow{
TestName: generateTestName(testConfig.testDir),
TestDir: testConfig.testDir,
TestType: testType,
TerraformDir: testConfig.terraformDir,
Expand All @@ -358,7 +389,9 @@ func genMatrix(testType string, testConfigs []testConfig, ami []string) []matrix
if len(ami) != 0 && !slices.Contains(ami, row.Ami) {
continue
}

if testConfig.sampleAppPath != "" {
row.SampleAppPath = testConfig.sampleAppPath
}
if testConfig.targets == nil || shouldAddTest(&row, testConfig.targets) {
testMatrixComplete = append(testMatrixComplete, row)
}
Expand Down
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ require (
github.com/qri-io/jsonschema v0.2.1
github.com/shirou/gopsutil/v3 v3.23.3
github.com/stretchr/testify v1.8.4
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635
go.opentelemetry.io/contrib/propagators/aws v1.21.1
go.opentelemetry.io/otel v1.21.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.21.0
Expand All @@ -40,6 +41,7 @@ require (
golang.org/x/exp v0.0.0-20231127185646-65229373498e
golang.org/x/sys v0.15.0
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.23.0
k8s.io/apimachinery v0.23.0
k8s.io/client-go v0.23.0
)
Expand Down Expand Up @@ -109,7 +111,6 @@ require (
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/api v0.23.0 // indirect
k8s.io/klog/v2 v2.30.0 // indirect
k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
Expand Down
137 changes: 137 additions & 0 deletions terraform/rosa/e2e/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
#
# Copyright (c) 2023 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: verify if we need to include this license

terraform {
required_providers {
aws = {
source = "hashicorp/aws"
version = ">= 4.20.0"
}
rhcs = {
version = ">= 1.6.3"
source = "terraform-redhat/rhcs"
}
}
}

# Export token using the RHCS_TOKEN environment variable
variable "rhcs_token" {
}
provider "rhcs" {
token = var.rhcs_token
}

provider "aws" {
region = var.aws_region
ignore_tags {
key_prefixes = ["kubernetes.io/"]
}
default_tags {
tags = var.default_aws_tags
}
}

data "aws_caller_identity" "current" {}
data "aws_availability_zones" "available" {}

locals {
# Extract availability zone names for the specified region, limit it to 3 if multi-az or 1 if single
region_azs = var.multi_az ? slice([for zone in data.aws_availability_zones.available.names : format("%s", zone)], 0, 3) : slice([for zone in data.aws_availability_zones.available.names : format("%s", zone)], 0, 1)
account_id = data.aws_caller_identity.current.account_id
}

locals {
worker_node_replicas = var.multi_az ? 3 : 2
# If cluster_name is not null, use that, otherwise generate a random cluster name
cluster_name = coalesce(var.cluster_name, "cwa-rosa-test")
}

# The network validator requires an additional 60 seconds to validate Terraform clusters.
resource "time_sleep" "wait_60_seconds" {
count = var.create_vpc ? 1 : 0
depends_on = [module.vpc]
create_duration = "60s"
}

module "hcp" {
source = "terraform-redhat/rosa-hcp/rhcs"
version = "1.6.5"
openshift_version = "4.17.14"

cluster_name = local.cluster_name
replicas = local.worker_node_replicas
aws_availability_zones = local.region_azs
private = var.private_cluster
aws_subnet_ids = var.create_vpc ? var.private_cluster ? module.vpc[0].private_subnets : concat(module.vpc[0].public_subnets, module.vpc[0].private_subnets) : var.aws_subnet_ids

create_oidc = true
create_account_roles = true
account_role_prefix = local.cluster_name
create_operator_roles = true
operator_role_prefix = local.cluster_name
create_admin_user = true

aws_billing_account_id = var.billing_account_id
ec2_metadata_http_tokens = "required"
compute_machine_type = var.instance_type

depends_on = [time_sleep.wait_60_seconds]
}


############################
# HTPASSWD IDP
############################

resource "aws_secretsmanager_secret" "secret" {
name = "${local.cluster_name}-htpasswd"

tags = {
Environment = "Production"
}
}
resource "aws_secretsmanager_secret_version" "secret_version" {
secret_id = aws_secretsmanager_secret.secret.id
secret_string = jsonencode({
"openshift_password": module.hcp.cluster_admin_password
"openshift_username": module.hcp.cluster_admin_username
"openshift_server": module.hcp.cluster_api_url
})
}

############################
# Setup CWA IAM
############################


locals {
cloudwatch_agent_role_arn = lookup(module.hcp.account_roles_arn, "HCP-ROSA-Worker")
}
# Output the ARN of the CloudWatch Agent IAM role
output "cloudwatch_agent_role_arn" {
value = local.cloudwatch_agent_role_arn
description = "ARN of the IAM role created for the CloudWatch agent"
}

# Attach CloudWatchAgentServerPolicy to the role
resource "aws_iam_role_policy_attachment" "cloudwatch_agent_policy" {
role = split("/", local.cloudwatch_agent_role_arn)[1]
policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
}
# Attach CloudWatchAgentAppSignalPolicy to the role
resource "aws_iam_role_policy_attachment" "cloudwatch_agent_appsig_policy" {
role = split("/", local.cloudwatch_agent_role_arn)[1]
policy_arn = "arn:aws:iam::aws:policy/CloudWatchApplicationSignalsFullAccess"
}
8 changes: 8 additions & 0 deletions terraform/rosa/e2e/outputs.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
output "cluster_id" {
value = module.hcp.cluster_id
description = "Unique identifier of the cluster."
}

output "cluster_name" {
value = local.cluster_name
}
Loading