Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -33,10 +33,11 @@ fun BuildConfigurationForServiceSweeper(providerName: String, sweeperName: Strin
return bc
}

fun BuildConfigurationForGlobalSweeper(providerName: String, sweeperName: String, sweeperRun: String, packages: Map<String, Map<String, String>>, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration): BuildType {
fun BuildConfigurationForProjectSweeper(providerName: String, sweeperName: String, packages: Map<String, Map<String, String>>, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration): BuildType {
val sweeperPackage: Map<String, String> = packages.getValue("sweeper")
val sweeperPath: String = sweeperPackage.getValue("path").toString()

val sweeperRun = "GoogleProject" // Name from .google/services/resourcemanager/resource_google_project_sweeper.go
val sweeperRegions = "us-central1" // A value needs to be present, despite projects not being regional resources

val s = SweeperDetails(sweeperName, parentProjectName, providerName, sweeperRun, sweeperRegions)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ const val SharedResourceNameVcr = "ci-test-project-188019 Service Lock"
const val ServiceSweeperName = "Service Sweeper"
const val ServiceSweeperCronName = "$ServiceSweeperName - Cron"
const val ServiceSweeperManualName = "$ServiceSweeperName - Manual"
const val GlobalSweepersProjectName = "Global Sweepers"
const val ProjectSweeperName = "Project Sweeper"
const val NightlyTestsProjectId = "NightlyTests"
const val WeeklyDiffTestsProjectId = "WeeklyDiffTests"
const val MMUpstreamProjectId = "MMUpstreamTests"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

package projects

import GlobalSweepersProjectName
import ProjectSweeperName
import SharedResourceNameBeta
import SharedResourceNameGa
import SharedResourceNameVcr
Expand All @@ -17,11 +17,11 @@ import jetbrains.buildServer.configs.kotlin.Project
import replaceCharsId
import vcs_roots.HashiCorpVCSRootGa

// globalSweepersSubProject returns a subproject that contains sweepers for global resources (projects, folders)
// projectSweeperSubProject returns a subproject that contains a sweeper for project resources
// Sweeping projects is an edge case because it doesn't respect boundaries between different testing projects GA/Beta/PR
fun globalSweepersSubProject(allConfig: AllContextParameters): Project {
fun projectSweeperSubProject(allConfig: AllContextParameters): Project {

val sweeperId = replaceCharsId("GLOBAL_SWEEPER")
val projectId = replaceCharsId("PROJECT_SWEEPER")

// Get config for using the GA identity (arbitrary choice as sweeper isn't confined by GA/Beta etc.)
val gaConfig = getGaAcceptanceTestConfig(allConfig)
Expand All @@ -31,21 +31,17 @@ fun globalSweepersSubProject(allConfig: AllContextParameters): Project {

// Create build config for sweeping project resources
// Uses the HashiCorpVCSRootGa VCS Root so that the latest sweepers in hashicorp/terraform-provider-google are used
val serviceSweeperConfig = BuildConfigurationForGlobalSweeper("N/A", "Project Sweeper", "GoogleProject", SweepersListGa, sweeperId, HashiCorpVCSRootGa, sharedResources, gaConfig)
serviceSweeperConfig.addTrigger(NightlyTriggerConfiguration(startHour=12))

// Create build config for sweeping folder resources
val folderSweeperConfig = BuildConfigurationForGlobalSweeper("N/A", "Folder Sweeper", "GoogleFolder", SweepersListGa, sweeperId, HashiCorpVCSRootGa, sharedResources, gaConfig)
folderSweeperConfig.addTrigger(NightlyTriggerConfiguration(startHour=12))
val serviceSweeperConfig = BuildConfigurationForProjectSweeper("N/A", ProjectSweeperName, SweepersListGa, projectId, HashiCorpVCSRootGa, sharedResources, gaConfig)
val trigger = NightlyTriggerConfiguration(startHour=12)
serviceSweeperConfig.addTrigger(trigger)

return Project{
id(sweeperId)
name = GlobalSweepersProjectName
description = "Subproject containing build configurations for sweeping global resources like projects and folders"
id(projectId)
name = "Project Sweeper"
description = "Subproject containing a build configuration for sweeping project resources"

// Register build configs in the project
buildType(serviceSweeperConfig)
buildType(folderSweeperConfig)

params {
readOnlySettings()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ fun googleCloudRootProject(allConfig: AllContextParameters): Project {
// Projects required for nightly testing, testing MM upstreams, and sweepers
subProject(googleSubProjectGa(allConfig))
subProject(googleSubProjectBeta(allConfig))
subProject(globalSweepersSubProject(allConfig))
subProject(projectSweeperSubProject(allConfig))
subProject(featureBranchResourceIdentitySubProject(allConfig))

// Feature branch-testing projects - these will be added and removed as needed
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,9 @@ class BuildConfigurationFeatureTests {

val gaProject = getSubProject(root, gaProjectName)
val betaProject = getSubProject(root, betaProjectName)
val globalSweepersProject = getSubProject(root, globalSweepersProjectName)
val projectSweeperProject = getSubProject(root, projectSweeperProjectName)

(gaProject.subProjects + betaProject.subProjects + globalSweepersProject.subProjects).forEach{p ->
(gaProject.subProjects + betaProject.subProjects + projectSweeperProject.subProjects).forEach{p ->
p.buildTypes.forEach{bt ->
assertTrue("Build '${bt.id}' should fail on errors!", bt.failureConditions.errorMessage)
}
Expand All @@ -36,9 +36,9 @@ class BuildConfigurationFeatureTests {

val gaProject = getSubProject(root, gaProjectName)
val betaProject = getSubProject(root, betaProjectName)
val globalSweepersProject = getSubProject(root, globalSweepersProjectName)
val projectSweeperProject = getSubProject(root, projectSweeperProjectName)

(gaProject.subProjects + betaProject.subProjects + globalSweepersProject.subProjects).forEach{p ->
(gaProject.subProjects + betaProject.subProjects + projectSweeperProject.subProjects).forEach{p ->
var exists: ArrayList<Boolean> = arrayListOf()
p.buildTypes.forEach{bt ->
bt.features.items.forEach { f ->
Expand Down
75 changes: 29 additions & 46 deletions mmv1/third_party/terraform/.teamcity/tests/sweepers.kt
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

package tests

import GlobalSweepersProjectName
import ProjectSweeperName
import ServiceSweeperCronName
import ServiceSweeperManualName
import ServiceSweeperName
Expand All @@ -21,18 +21,18 @@ import projects.googleCloudRootProject

class SweeperTests {
@Test
fun globalSweepersConfig() {
fun projectSweeperConfig() {
val root = googleCloudRootProject(testContextParameters())

// Find Global sweepers project
val globalSweepersProject = getSubProject(root, globalSweepersProjectName)
// Find Project sweeper project
val projectSweeperProject = getSubProject(root, projectSweeperProjectName)

globalSweepersProject.buildTypes.forEach{bt ->
val skipProjectSweeper = bt.params.findRawParam("env.SKIP_PROJECT_SWEEPER")!!.value
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to an empty value in the ${globalSweepersProject.name} project. Value = `${skipProjectSweeper}` ", skipProjectSweeper == "")
// SKIP_PROJECT_SWEEPER should be empty so project sweepers will be run
// See https://github.com/GoogleCloudPlatform/magic-modules/blob/501429790939717ca6dce76dbf4b1b82aef4e9d9/mmv1/third_party/terraform/services/resourcemanager/resource_google_project_sweeper.go#L18-L26

val skipFolderSweeper = bt.params.findRawParam("env.SKIP_FOLDER_SWEEPER")!!.value
assertTrue("env.SKIP_FOLDER_SWEEPER should be set to an empty value in the ${globalSweepersProject.name} project. Value = `${skipFolderSweeper}` ", skipFolderSweeper == "")
projectSweeperProject.buildTypes.forEach{bt ->
val skipProjectSweeper = bt.params.findRawParam("env.SKIP_PROJECT_SWEEPER")!!.value
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to an empty value, so project sweepers are NOT skipped in the ${projectSweeperProject.name} project. Value = `${skipProjectSweeper}` ", skipProjectSweeper == "")
}
}

Expand All @@ -50,12 +50,10 @@ class SweeperTests {
val value = sweeper.params.findRawParam("PACKAGE_PATH")!!.value
assertEquals("./google/sweeper", value)

// SKIP_PROJECT_SWEEPER and SKIP_FOLDER_SWEEPER should have values so they will be skipped
// SKIP_PROJECT_SWEEPER should have a value so project sweepers will be skipped
// See https://github.com/GoogleCloudPlatform/magic-modules/blob/501429790939717ca6dce76dbf4b1b82aef4e9d9/mmv1/third_party/terraform/services/resourcemanager/resource_google_project_sweeper.go#L18-L26
val skipProjectSweeper = sweeper.params.findRawParam("env.SKIP_PROJECT_SWEEPER")!!.value
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")

val skipFolderSweeper = sweeper.params.findRawParam("env.SKIP_FOLDER_SWEEPER")!!.value
assertTrue("env.SKIP_FOLDER_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipFolderSweeper}` ", skipFolderSweeper != "")
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string so project sweepers are skipped in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")
}

@Test
Expand All @@ -72,12 +70,10 @@ class SweeperTests {
val value = sweeper.params.findRawParam("PACKAGE_PATH")!!.value
assertEquals("./google-beta/sweeper", value)

// SKIP_PROJECT_SWEEPER and SKIP_FOLDER_SWEEPER should have values so they will be skipped
// SKIP_PROJECT_SWEEPER should have a value so project sweepers will be skipped
// See https://github.com/GoogleCloudPlatform/magic-modules/blob/501429790939717ca6dce76dbf4b1b82aef4e9d9/mmv1/third_party/terraform/services/resourcemanager/resource_google_project_sweeper.go#L18-L26
val skipProjectSweeper = sweeper.params.findRawParam("env.SKIP_PROJECT_SWEEPER")!!.value
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")

val skipFolderSweeper = sweeper.params.findRawParam("env.SKIP_FOLDER_SWEEPER")!!.value
assertTrue("env.SKIP_FOLDER_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipFolderSweeper}` ", skipFolderSweeper != "")
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string so project sweepers are skipped in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")
}

@Test
Expand All @@ -96,12 +92,10 @@ class SweeperTests {
val value = sweeper.params.findRawParam("PACKAGE_PATH")!!.value
assertEquals("./google/sweeper", value)

// SKIP_PROJECT_SWEEPER and SKIP_FOLDER_SWEEPER should have values so they will be skipped
// SKIP_PROJECT_SWEEPER should have a value so project sweepers will be skipped
// See https://github.com/GoogleCloudPlatform/magic-modules/blob/501429790939717ca6dce76dbf4b1b82aef4e9d9/mmv1/third_party/terraform/services/resourcemanager/resource_google_project_sweeper.go#L18-L26
val skipProjectSweeper = sweeper.params.findRawParam("env.SKIP_PROJECT_SWEEPER")!!.value
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")

val skipFolderSweeper = sweeper.params.findRawParam("env.SKIP_FOLDER_SWEEPER")!!.value
assertTrue("env.SKIP_FOLDER_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipFolderSweeper}` ", skipFolderSweeper != "")
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string so project sweepers are skipped in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")
}
}

Expand All @@ -121,17 +115,15 @@ class SweeperTests {
val value = sweeper.params.findRawParam("PACKAGE_PATH")!!.value
assertEquals("./google-beta/sweeper", value)

// SKIP_PROJECT_SWEEPER and SKIP_FOLDER_SWEEPER should have values so they will be skipped
// SKIP_PROJECT_SWEEPER should have a value so project sweepers will be skipped
// See https://github.com/GoogleCloudPlatform/magic-modules/blob/501429790939717ca6dce76dbf4b1b82aef4e9d9/mmv1/third_party/terraform/services/resourcemanager/resource_google_project_sweeper.go#L18-L26
val skipProjectSweeper = sweeper.params.findRawParam("env.SKIP_PROJECT_SWEEPER")!!.value
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")

val skipFolderSweeper = sweeper.params.findRawParam("env.SKIP_FOLDER_SWEEPER")!!.value
assertTrue("env.SKIP_FOLDER_SWEEPER should be set to a non-empty string in the ${project.name} project (${sweeper.name}). Value = `${skipFolderSweeper}` ", skipFolderSweeper != "")
assertTrue("env.SKIP_PROJECT_SWEEPER should be set to a non-empty string so project sweepers are skipped in the ${project.name} project (${sweeper.name}). Value = `${skipProjectSweeper}` ", skipProjectSweeper != "")
}
}

@Test
fun globalSweepersRunAfterServiceSweepers() {
fun projectSweepersRunAfterServiceSweepers() {
val root = googleCloudRootProject(testContextParameters())

// Find GA nightly test project's service sweeper
Expand All @@ -142,33 +134,24 @@ class SweeperTests {
val betaNightlyTests : Project = getNestedProjectFromRoot(root, betaProjectName, nightlyTestsProjectName)
val sweeperBeta: BuildType = getBuildFromProject(betaNightlyTests, ServiceSweeperName)

// Find Global sweepers project's builds
val globalSweepersProject = getSubProject(root, globalSweepersProjectName)
val projectSweeper: BuildType = getBuildFromProject(globalSweepersProject, "Project Sweeper")
val folderSweeper: BuildType = getBuildFromProject(globalSweepersProject, "Folder Sweeper")
// Find Project sweeper project's build
val projectSweeperProject = getSubProject(root, projectSweeperProjectName)
val projectSweeper: BuildType = getBuildFromProject(projectSweeperProject, ProjectSweeperName)

// Check only one schedule trigger is on the builds in question
assertTrue(sweeperGa.triggers.items.size == 1)
assertTrue(sweeperBeta.triggers.items.size == 1)
assertTrue(projectSweeper.triggers.items.size == 1)
assertTrue(folderSweeper.triggers.items.size == 1)

// Assert that the hour value that sweeper builds are triggered at is less than the hour value that project/folder sweeper builds are triggered at
// Assert that the hour value that sweeper builds are triggered at is less than the hour value that project sweeper builds are triggered at
// i.e. sweeper builds are triggered first
val stGa = sweeperGa.triggers.items[0] as ScheduleTrigger
val cronGa = stGa.schedulingPolicy as ScheduleTrigger.SchedulingPolicy.Cron
val stBeta = sweeperBeta.triggers.items[0] as ScheduleTrigger
val cronBeta = stBeta.schedulingPolicy as ScheduleTrigger.SchedulingPolicy.Cron

val stProject = projectSweeper.triggers.items[0] as ScheduleTrigger
val cronProject = stProject.schedulingPolicy as ScheduleTrigger.SchedulingPolicy.Cron

val stFolder = folderSweeper.triggers.items[0] as ScheduleTrigger
val cronFolder = stFolder.schedulingPolicy as ScheduleTrigger.SchedulingPolicy.Cron

assertTrue("Service sweeper for the GA Nightly Test project should be triggered at an earlier hour than the project sweeper", cronGa.hours.toString().toInt() < cronProject.hours.toString().toInt())
assertTrue("Service sweeper for the Beta Nightly Test project should be triggered at an earlier hour than the project sweeper", cronBeta.hours.toString().toInt() < cronProject.hours.toString().toInt())

assertTrue("Service sweeper for the GA Nightly Test project should be triggered at an earlier hour than the folder sweeper", cronGa.hours.toString().toInt() < cronFolder.hours.toString().toInt())
assertTrue("Service sweeper for the Beta Nightly Test project should be triggered at an earlier hour than the folder sweeper", cronBeta.hours.toString().toInt() < cronFolder.hours.toString().toInt())
assertTrue("Service sweeper for the GA Nightly Test project should be triggered at an earlier hour than the project sweeper", cronGa.hours.toString().toInt() < cronProject.hours.toString().toInt()) // Converting nullable strings to ints
assertTrue("Service sweeper for the Beta Nightly Test project should be triggered at an earlier hour than the project sweeper", cronBeta.hours.toString().toInt() < cronProject.hours.toString().toInt() )
}
}
2 changes: 1 addition & 1 deletion mmv1/third_party/terraform/.teamcity/tests/test_utils.kt
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ const val betaProjectName = "Google Beta"
const val nightlyTestsProjectName = "Nightly Tests"
const val weeklyDiffTestsProjectName = "Weekly Diff Tests"
const val mmUpstreamProjectName = "Upstream MM Testing"
const val globalSweepersProjectName = "Global Sweepers"
const val projectSweeperProjectName = "Project Sweeper"

fun testContextParameters(): AllContextParameters {
return AllContextParameters(
Expand Down
4 changes: 2 additions & 2 deletions mmv1/third_party/terraform/.teamcity/tests/vcs_roots.kt
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ class VcsTests {

val gaProject = getSubProject(root, gaProjectName)
val betaProject = getSubProject(root, betaProjectName)
val globalSweepersProject = getSubProject(root, globalSweepersProjectName)
val projectSweeperProject = getSubProject(root, betaProjectName)

val allProjects = arrayListOf(gaProject, betaProject, globalSweepersProject)
val allProjects = arrayListOf(gaProject, betaProject, projectSweeperProject)

allProjects.forEach { p ->
p.subProjects.forEach { sp->
Expand Down
Loading