Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 8 additions & 36 deletions common_factories.py
Original file line number Diff line number Diff line change
Expand Up @@ -475,24 +475,12 @@ def addGaleraTests(factory, mtrDbPool):

def addS3Tests(factory, mtrDbPool):
runS3 = lambda props: hasS3(props) and props.hasProperty("compile_step_completed")
factory.addStep(
steps.MasterShellCommand(
name="Create minio S3 bucket",
alwaysRun=True,
command=[
"mc",
"mb",
util.Interpolate("minio/%(prop:buildername)s-%(prop:buildnumber)s"),
],
doStepIf=runS3,
)
)
factory.addStep(
steps.MTR(
name="S3 minio tests",
name="MTR - S3",
alwaysRun=True,
description="testing S3 minio",
descriptionDone="test s3 minio",
description="testing S3",
descriptionDone="test s3",
logfiles={"mysqld*": "./buildbot/mysql_logs.html"},
test_type="s3",
command=[
Expand All @@ -512,34 +500,18 @@ def addS3Tests(factory, mtrDbPool):
dbpool=mtrDbPool,
autoCreateTables=True,
env={
"S3_HOST_NAME": "minio.mariadb.org",
"S3_HOST_NAME": "s3bb.mariadb.org",
"S3_PORT": "443",
"S3_ACCESS_KEY": util.Interpolate("%(secret:minio_access_key)s"),
"S3_SECRET_KEY": util.Interpolate("%(secret:minio_secret_key)s"),
"S3_BUCKET": util.Interpolate(
"%(prop:buildername)s-%(prop:buildnumber)s"
),
"S3_ACCESS_KEY": util.Interpolate("%(secret:s3_access_key)s"),
"S3_SECRET_KEY": util.Interpolate("%(secret:s3_secret_key)s"),
"S3_BUCKET": "mariadb-buildbot",
"S3_USE_HTTP": "OFF",
"S3_PROTOCOL_VERSION": "Path",
"S3_REGION": "garage",
},
doStepIf=runS3,
)
)

factory.addStep(
steps.MasterShellCommand(
name="Delete minio S3 bucket",
alwaysRun=True,
command=[
"mc",
"rb",
"--force",
util.Interpolate("minio/%(prop:buildername)s-%(prop:buildnumber)s"),
],
doStepIf=runS3,
)
)

factory.addStep(
steps.ShellCommand(
name="move mariadb S3 log files",
Expand Down
27 changes: 6 additions & 21 deletions configuration/builders/sequences/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,7 @@
from configuration.steps.base import StepOptions
from configuration.steps.commands.base import URL
from configuration.steps.commands.mtr import MTRReporter, MTRTest
from configuration.steps.commands.util import (
CreateS3Bucket,
DeleteS3Bucket,
SaveCompressedTar,
)
from configuration.steps.commands.util import SaveCompressedTar
from configuration.steps.generators.mtr.generator import MTRGenerator
from configuration.steps.generators.mtr.options import (
MTR,
Expand Down Expand Up @@ -224,12 +220,6 @@ def get_mtr_s3_steps(
env_vars: list[tuple] = [],
):
steps = []
steps.append(
MasterShellStep(
command=CreateS3Bucket(bucket=f"%(prop:buildername)s-%(prop:buildnumber)s")
)
)

steps.append(
step_wrapping_fn(
ShellStep(
Expand All @@ -254,13 +244,14 @@ def get_mtr_s3_steps(
),
),
env_vars=[
("S3_HOST_NAME", "minio.mariadb.org"),
("S3_HOST_NAME", "s3bb.mariadb.org"),
("S3_PORT", "443"),
("S3_ACCESS_KEY", "%(secret:minio_access_key)s"),
("S3_SECRET_KEY", "%(secret:minio_secret_key)s"),
("S3_BUCKET", "%(prop:buildername)s-%(prop:buildnumber)s"),
("S3_ACCESS_KEY", "%(secret:s3_access_key)s"),
("S3_SECRET_KEY", "%(secret:s3_secret_key)s"),
("S3_BUCKET", "mariadb-buildbot"),
("S3_USE_HTTPS", "OFF"),
("S3_PROTOCOL_VERSION", "Path"),
("S3_REGION", "garage"),
]
+ env_vars,
options=StepOptions(
Expand All @@ -269,12 +260,6 @@ def get_mtr_s3_steps(
)
)
)
steps.append(
MasterShellStep(
command=DeleteS3Bucket(bucket=f"%(prop:buildername)s-%(prop:buildnumber)s"),
options=StepOptions(alwaysRun=True),
),
)
return steps


Expand Down
44 changes: 0 additions & 44 deletions configuration/steps/commands/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,50 +4,6 @@
from configuration.steps.commands.base import BashScriptCommand, Command


class CreateS3Bucket(Command):
"""
A command to create an S3 bucket using MinIO client (mc).
This command initializes a new S3 bucket in the MinIO server.
Attributes:
bucket (str): The name of the S3 bucket to create.
workdir (PurePath): The working directory for the command.
"""

def __init__(self, bucket: str, workdir: PurePath = PurePath(".")):
name = "Create S3 bucket"
self.bucket = bucket
super().__init__(name=name, workdir=workdir)

def as_cmd_arg(self) -> list[str]:
return [
"bash",
"-exc",
util.Interpolate(f"mc mb minio/{self.bucket}"),
]


class DeleteS3Bucket(Command):
"""
A command to delete an S3 bucket using MinIO client (mc).
This command removes an existing S3 bucket from the MinIO server.
Attributes:
bucket (str): The name of the S3 bucket to delete.
workdir (PurePath): The working directory for the command.
"""

def __init__(self, bucket: str, workdir: PurePath = PurePath(".")):
name = "Delete S3 bucket"
self.bucket = bucket
super().__init__(name=name, workdir=workdir)

def as_cmd_arg(self) -> list[str]:
return [
"bash",
"-exc",
util.Interpolate(f"mc rb --force minio/{self.bucket}"),
]


class SaveCompressedTar(Command):
"""
A command to create a compressed tar archive of the current working directory.
Expand Down
13 changes: 0 additions & 13 deletions docker-compose/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=9995
- TITLE
Expand Down Expand Up @@ -148,7 +147,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=9996
- TITLE
Expand Down Expand Up @@ -184,7 +182,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=9997
- TITLE
Expand Down Expand Up @@ -220,7 +217,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=9998
- TITLE
Expand Down Expand Up @@ -256,7 +252,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=9999
- TITLE
Expand Down Expand Up @@ -292,7 +287,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10000
- TITLE
Expand Down Expand Up @@ -328,7 +322,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10001
- TITLE
Expand Down Expand Up @@ -364,7 +357,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10002
- TITLE
Expand Down Expand Up @@ -400,7 +392,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10003
- TITLE
Expand Down Expand Up @@ -437,7 +428,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10004
- TITLE
Expand Down Expand Up @@ -473,7 +463,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10005
- TITLE
Expand Down Expand Up @@ -509,7 +498,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10006
- TITLE
Expand Down Expand Up @@ -546,7 +534,6 @@ services:
- MASTER_CREDENTIALS_DIR
- MASTER_NONLATENT_DOCKERLIBRARY_WORKER
- MASTER_PACKAGES_DIR
- MC_HOST_minio=https://access_key:secret_key@minio.mariadb.org
- MQ_ROUTER_URL
- PORT=10007
- TITLE
Expand Down
5 changes: 1 addition & 4 deletions docker-compose/generate-config.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def construct_env_section(env_vars):
for key, value in sorted(env_vars.items()):
if key.startswith("NGINX_"):
continue
elif key not in ["PORT", "MC_HOST_minio"]:
elif key not in ["PORT"]:
env_section += f" - {key}\n"
else:
env_section += f" - {key}={value}\n"
Expand Down Expand Up @@ -186,14 +186,12 @@ def main(args):
f"container_name: master-web\n{construct_env_section(env_vars)}",
)

env_vars["MC_HOST_minio"] = "{mc_host}"
# Modify the docker_compose_template to include the environment variables
docker_compose_template = DOCKER_COMPOSE_TEMPLATE.replace(
"container_name: {master_name}",
f"container_name: {{master_name}}\n{construct_env_section(env_vars)}",
)

mc_host = config["private"]["minio_url"]
starting_port = config["private"]["master-variables"]["starting_port"]
master_web_port = 8010
# Generate startup scripts and Docker Compose pieces for each master directory
Expand Down Expand Up @@ -223,7 +221,6 @@ def main(args):
master_hostname=master_hostname,
master_directory=master_directory,
port=port,
mc_host=mc_host,
volumes=generate_volumes(master_volumes[master_name]),
environment="" if args.env == "prod" else "dev_",
buildbot_stop_grace_period=BUILDBOT_STOP_GRACE_PERIOD,
Expand Down
1 change: 0 additions & 1 deletion master-private.cfg-sample
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ private["db_user"] = "buildmaster"
private["db_password"] = "password"
private["db_mtr_db"] = "buildbot"
private["gh_secret"] = "gh_secret"
private["minio_url"] = "https://access_key:secret_key@minio.mariadb.org"
private["zabbix_server"] = "https://zabbix.server"
private["zabbix_token"] = "zabbix_token"
private["user_pass"]= {
Expand Down
52 changes: 0 additions & 52 deletions minio/README.md

This file was deleted.

Loading