Feature/refactoring for sc cloud ready

.github/workflows/branch.yaml

@@ -11,7 +11,7 @@ permissions:
 jobs:
   build:
     name: Build simple-container in branch
-    runs-on: self-hosted
+    runs-on: blacksmith-8vcpu-ubuntu-2204
     outputs:
       cicd-bot-telegram-token: ${{ steps.prepare-secrets.outputs.cicd-bot-telegram-token }}
       cicd-bot-telegram-chat-id: ${{ steps.prepare-secrets.outputs.cicd-bot-telegram-chat-id }}
@@ -42,7 +42,7 @@ jobs:
 
   finalize:
     name: Finalize build in branch
-    runs-on: self-hosted
+    runs-on: blacksmith-2vcpu-ubuntu-2204
     if: ${{ always() }}
     permissions:
       contents: write

go.mod

@@ -41,7 +41,8 @@ require (
 	github.com/spf13/afero v1.12.0
 	github.com/spf13/cobra v1.8.1
 	github.com/stretchr/testify v1.10.0
-	github.com/vektra/mockery/v2 v2.52.2
+	github.com/tkrajina/typescriptify-golang-structs v0.2.0
+	github.com/vektra/mockery/v2 v2.52.3
 	go.uber.org/atomic v1.11.0
 	golang.org/x/crypto v0.33.0
 	golang.org/x/oauth2 v0.25.0

go.sum

@@ -1572,6 +1572,9 @@ github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3 h1:y4mJRFlM6fUyP
 github.com/timakin/bodyclose v0.0.0-20241017074812-ed6a65f985e3/go.mod h1:mkjARE7Yr8qU23YcGMSALbIxTQ9r9QBVahQOBRfU460=
 github.com/timonwong/loggercheck v0.10.1 h1:uVZYClxQFpw55eh+PIoqM7uAOHMrhVcDoWDery9R8Lg=
 github.com/timonwong/loggercheck v0.10.1/go.mod h1:HEAWU8djynujaAVX7QI65Myb8qgfcZ1uKbdpg3ZzKl8=
+github.com/tkrajina/go-reflector v0.5.5/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4=
+github.com/tkrajina/typescriptify-golang-structs v0.2.0 h1:ZedWk82egydDspGTryAatbX0/1NZDQbdiZLoCbOk4f8=
+github.com/tkrajina/typescriptify-golang-structs v0.2.0/go.mod h1:sjU00nti/PMEOZb07KljFlR+lJ+RotsC0GBQMv9EKls=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tomarrell/wrapcheck/v2 v2.10.0 h1:SzRCryzy4IrAH7bVGG4cK40tNUhmVmMDuJujy4XwYDg=
@@ -1602,8 +1605,8 @@ github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQ
 github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ=
 github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME=
 github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI=
-github.com/vektra/mockery/v2 v2.52.2 h1:8QfPKUIrq8P3Cs7G79Iu4Byd5wdhGCE0quIS27x7rQo=
-github.com/vektra/mockery/v2 v2.52.2/go.mod h1:zGDY/f6bip0Yh13GQ5j7xa43fuEoYBa4ICHEaihisHw=
+github.com/vektra/mockery/v2 v2.52.3 h1:lInrh+OuJu3dY/UPFvdFmJ/lsscEnUFrTmagcRJKoWU=
+github.com/vektra/mockery/v2 v2.52.3/go.mod h1:zGDY/f6bip0Yh13GQ5j7xa43fuEoYBa4ICHEaihisHw=
 github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=

pkg/api/config.go

@@ -8,6 +8,8 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/pkg/errors"
+
+	"github.com/simple-container-com/api/pkg/api/config"
 )
 
 const (
@@ -38,7 +40,7 @@ type InitParams struct {
 	GenerateKeyPair     bool   `json:"generateKeyPair" yaml:"generateKeyPair"`
 }
 
-func ReadConfigFile(workDir, profile string) (*ConfigFile, error) {
+func ReadConfigFile(reader config.Reader, workDir, profile string) (*ConfigFile, error) {
 	configFromEnv := os.Getenv(ScConfigEnvVariable)
 	if configFromEnv != "" {
 		if res, err := UnmarshalDescriptor[ConfigFile]([]byte(configFromEnv)); err != nil {
@@ -47,7 +49,7 @@ func ReadConfigFile(workDir, profile string) (*ConfigFile, error) {
 			return res, nil
 		}
 	}
-	res, err := ReadDescriptor(ConfigFilePath(workDir, profile), &ConfigFile{})
+	res, err := ReadDescriptor(reader, ConfigFilePath(workDir, profile), &ConfigFile{})
 	if err != nil {
 		return nil, errors.Wrapf(err, "profile does not exist: %q", profile)
 	}

pkg/api/config/reader.go

@@ -0,0 +1,34 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type Reader interface {
+	ReadFile(path string) ([]byte, error)
+}
+
+type fileSystemReader struct{}
+
+func (r *fileSystemReader) ReadFile(path string) ([]byte, error) {
+	return os.ReadFile(path)
+}
+
+var FSReader = &fileSystemReader{}
+
+type InlineConfigReader struct {
+	WorkDir string
+	Configs map[string]string
+}
+
+func (r *InlineConfigReader) ReadFile(path string) ([]byte, error) {
+	path = strings.TrimPrefix(path, fmt.Sprintf("%s%c", r.WorkDir, filepath.Separator))
+	if val, ok := r.Configs[path]; ok {
+		return []byte(val), nil
+	} else {
+		return nil, os.ErrNotExist
+	}
+}

pkg/api/read.go

@@ -1,11 +1,11 @@
 package api
 
 import (
-	"os"
-
 	"gopkg.in/yaml.v3"
 
 	"github.com/pkg/errors"
+
+	"github.com/simple-container-com/api/pkg/api/config"
 )
 
 const (
@@ -14,8 +14,8 @@ const (
 	ClientDescriptorFileName  = "client.yaml"
 )
 
-func ReadDescriptor[T any](filePath string, descriptor *T) (*T, error) {
-	fileBytes, err := os.ReadFile(filePath)
+func ReadDescriptor[T any](reader config.Reader, filePath string, descriptor *T) (*T, error) {
+	fileBytes, err := reader.ReadFile(filePath)
 	if err != nil {
 		return nil, errors.Wrapf(err, "failed to read %s", filePath)
 	}
@@ -40,8 +40,8 @@ func UnmarshalDescriptor[T any](bytes []byte) (*T, error) {
 	return &descriptor, nil
 }
 
-func ReadServerDescriptor(path string) (*ServerDescriptor, error) {
-	descriptor, err := ReadDescriptor(path, &ServerDescriptor{})
+func ReadServerDescriptor(reader config.Reader, path string) (*ServerDescriptor, error) {
+	descriptor, err := ReadDescriptor(reader, path, &ServerDescriptor{})
 	if err != nil {
 		return descriptor, errors.Wrapf(err, "failed to read server descriptor from %q", path)
 	}
@@ -53,8 +53,8 @@ func ReadServerDescriptor(path string) (*ServerDescriptor, error) {
 	return res, nil
 }
 
-func ReadSecretsDescriptor(path string) (*SecretsDescriptor, error) {
-	descriptor, err := ReadDescriptor(path, &SecretsDescriptor{})
+func ReadSecretsDescriptor(reader config.Reader, path string) (*SecretsDescriptor, error) {
+	descriptor, err := ReadDescriptor(reader, path, &SecretsDescriptor{})
 	if err != nil {
 		return nil, err
 	}
@@ -66,8 +66,8 @@ func ReadSecretsDescriptor(path string) (*SecretsDescriptor, error) {
 	return res, nil
 }
 
-func ReadClientDescriptor(path string) (*ClientDescriptor, error) {
-	descriptor, err := ReadDescriptor(path, &ClientDescriptor{})
+func ReadClientDescriptor(reader config.Reader, path string) (*ClientDescriptor, error) {
+	descriptor, err := ReadDescriptor(reader, path, &ClientDescriptor{})
 	if err != nil {
 		return descriptor, errors.Wrapf(err, "failed to unmarshal %s", path)
 	}

pkg/api/secrets/cryptor.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/samber/lo"
 
+	"github.com/simple-container-com/api/pkg/api/config"
 	"github.com/simple-container-com/api/pkg/api/git"
 	"github.com/simple-container-com/welder/pkg/util"
 )
@@ -37,9 +38,10 @@ type Cryptor interface {
 }
 
 type cryptor struct {
-	workDir     string
-	projectName string
-	profile     string
+	configReader config.Reader
+	workDir      string
+	projectName  string
+	profile      string
 
 	options []Option
 

pkg/api/secrets/cryptor_test.go

@@ -9,12 +9,24 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/simple-container-com/api/pkg/api"
+	"github.com/simple-container-com/api/pkg/api/config"
 	"github.com/simple-container-com/api/pkg/api/git"
 	"github.com/simple-container-com/api/pkg/api/secrets/ciphers"
 	"github.com/simple-container-com/api/pkg/api/tests/testutil"
 	"github.com/simple-container-com/welder/pkg/util/test"
 )
 
+func withInlineConfigReader(reader *config.InlineConfigReader) Option {
+	return Option{
+		beforeInit: true,
+		f: func(c *cryptor) error {
+			reader.WorkDir = c.workDir
+			opt := WithConfigReader(reader)
+			return opt.f(c)
+		},
+	}
+}
+
 func withGitDir(gitDir string) Option {
 	return Option{
 		beforeInit: true,
@@ -37,6 +49,9 @@ type mocks struct {
 func TestNewCryptor(t *testing.T) {
 	RegisterTestingT(t)
 
+	cfgKeyInline, err := os.ReadFile("testdata/repo/.sc/cfg.local-key-inline.yaml")
+	Expect(err).To(BeNil())
+
 	cases := []struct {
 		name           string
 		testExampleDir string
@@ -86,6 +101,28 @@ func TestNewCryptor(t *testing.T) {
 			prepareMocks: acceptAllChanges,
 			actions:      happyPathScenario,
 		},
+		{
+			name:           "happy path with custom config reader",
+			testExampleDir: "testdata/repo",
+			opts: []Option{
+				withGitDir("gitdir"),
+				withInlineConfigReader(&config.InlineConfigReader{
+					Configs: map[string]string{
+						"stacks/common/secrets.yaml": "blabla",
+						".sc/cfg.default.yaml":       string(cfgKeyInline),
+					},
+				}),
+				WithProfile("default"),
+				WithKeysFromCurrentProfile(),
+			},
+			prepareMocks: acceptAllChanges,
+			actions: func(t *testing.T, c Cryptor, m *mocks, wd string) {
+				Expect(c.AddFile("stacks/common/secrets.yaml")).To(BeNil())
+				Expect(c.EncryptChanged(true, false)).To(BeNil())
+				err := c.DecryptAll(false)
+				Expect(err).To(BeNil())
+			},
+		},
 		{
 			name:           "happy path with invalid passphrase from console",
 			testExampleDir: "testdata/repo",
@@ -140,7 +177,7 @@ func TestNewCryptor(t *testing.T) {
 			prepareMocks: acceptAllChanges,
 			actions: func(t *testing.T, c Cryptor, m *mocks, wd string) {
 				happyPathScenario(t, c, m, wd)
-				cfg, err := api.ReadConfigFile(wd, "test-profile")
+				cfg, err := api.ReadConfigFile(config.FSReader, wd, "test-profile")
 				Expect(err).To(BeNil())
 				Expect(cfg.PrivateKey).To(ContainSubstring(c.PrivateKey()))
 				Expect(cfg.PublicKey).To(ContainSubstring(c.PublicKey()))

pkg/api/secrets/management.go

@@ -16,6 +16,7 @@ import (
 	"github.com/samber/lo"
 
 	"github.com/simple-container-com/api/pkg/api"
+	"github.com/simple-container-com/api/pkg/api/config"
 	"github.com/simple-container-com/api/pkg/api/logger/color"
 	"github.com/simple-container-com/api/pkg/api/secrets/ciphers"
 	"github.com/simple-container-com/welder/pkg/util"
@@ -459,6 +460,9 @@ func (c *cryptor) decryptSecretDataToFile(encryptedData []string, relFilePath st
 }
 
 func (c *cryptor) initData() error {
+	if c.configReader == nil {
+		c.configReader = config.FSReader
+	}
 	if c.secrets.Secrets == nil {
 		c.secrets.Secrets = make(map[string]EncryptedSecrets, 0)
 	}

pkg/api/secrets/opts.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 
 	"github.com/simple-container-com/api/pkg/api"
+	"github.com/simple-container-com/api/pkg/api/config"
 	"github.com/simple-container-com/api/pkg/api/git"
 	"github.com/simple-container-com/welder/pkg/util"
 )
@@ -111,6 +112,16 @@ func WithGeneratedKeys(projectName, profile string) Option {
 	}
 }
 
+func WithConfigReader(reader config.Reader) Option {
+	return Option{
+		beforeInit: true,
+		f: func(c *cryptor) error {
+			c.configReader = reader
+			return nil
+		},
+	}
+}
+
 func WithKeysFromScConfig(profile string) Option {
 	return Option{
 		f: func(c *cryptor) error {
@@ -121,7 +132,11 @@ func WithKeysFromScConfig(profile string) Option {
 				return errors.Errorf("profile is not configured")
 			}
 			c.profile = profile
-			cfg, err := api.ReadConfigFile(c.workDir, c.profile)
+			cfgReader := c.configReader
+			if c.configReader == nil {
+				cfgReader = config.FSReader
+			}
+			cfg, err := api.ReadConfigFile(cfgReader, c.workDir, c.profile)
 			if err != nil {
 				return err
 			}

pkg/api/tests/refapp_test.go

@@ -8,6 +8,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/simple-container-com/api/pkg/api"
+	"github.com/simple-container-com/api/pkg/api/config"
 )
 
 func TestReadServerDescriptor(t *testing.T) {
@@ -45,7 +46,7 @@ func TestReadServerDescriptor(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.path, func(t *testing.T) {
-			got, err := api.ReadServerDescriptor(tt.path)
+			got, err := api.ReadServerDescriptor(config.FSReader, tt.path)
 			Expect(err).To(BeNil())
 			actual := got.ValuesOnly()
 
@@ -77,7 +78,7 @@ func TestReadSecretsDescriptor(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.path, func(t *testing.T) {
-			got, err := api.ReadSecretsDescriptor(tt.path)
+			got, err := api.ReadSecretsDescriptor(config.FSReader, tt.path)
 			Expect(err).To(BeNil())
 
 			assert.EqualValuesf(t, tt.want, got, "%v failed", tt.path)
@@ -109,7 +110,7 @@ func TestReadClientDescriptor(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.path, func(t *testing.T) {
-			got, err := api.ReadClientDescriptor(tt.path)
+			got, err := api.ReadClientDescriptor(config.FSReader, tt.path)
 			Expect(err).To(BeNil())
 
 			assert.EqualValuesf(t, tt.want.Copy(), got.Copy(), "%v failed", tt.path)

pkg/clouds/pulumi/login.go

@@ -44,10 +44,6 @@ func (p *pulumi) login(ctx context.Context, cfg *api.ConfigFile, stack api.Stack
 		organization = provisionerCfg.Organization
 	}
 
-	if err != nil {
-		return errors.Wrapf(err, "failed to init pulumi provisioner context")
-	}
-
 	var pulumiHome string
 	if os.Getenv(workspace.PulumiHomeEnvVar) == "" {
 		if pulumiHome, err = path_util.ReplaceTildeWithHome("~/.pulumi"); err != nil {

pkg/clouds/pulumi/testutil/secrets_test_util.go

@@ -9,6 +9,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/simple-container-com/api/pkg/api"
+	"github.com/simple-container-com/api/pkg/api/config"
 	"github.com/simple-container-com/api/pkg/api/secrets"
 	awsApi "github.com/simple-container-com/api/pkg/clouds/aws"
 	"github.com/simple-container-com/api/pkg/clouds/cloudflare"
@@ -43,7 +44,7 @@ func ReadIntegrationTestConfig() (*api.ConfigFile, secrets.Cryptor) {
 
 	Expect(c.ReadSecretFiles()).To(BeNil())
 
-	cfg, err := api.ReadConfigFile(c.Workdir(), "test")
+	cfg, err := api.ReadConfigFile(config.FSReader, c.Workdir(), "test")
 	Expect(err).To(BeNil())
 
 	return cfg, c