diff --git a/api/types/client.go b/api/types/client.go index 18edc7b3d..b2b4a0037 100644 --- a/api/types/client.go +++ b/api/types/client.go @@ -18,7 +18,11 @@ import ( ) const ( - ENV_PLATFORM = "SKUPPER_PLATFORM" + ENV_PLATFORM = "SKUPPER_PLATFORM" + ENV_SYSTEM_AUTO_RELOAD = "SKUPPER_SYSTEM_RELOAD_TYPE" + + SystemReloadTypeAuto string = "auto" + SystemReloadTypeManual string = "manual" ) type ConnectorCreateOptions struct { diff --git a/cmd/system-controller/main.go b/cmd/system-controller/main.go index 459cbad0a..c663c3eaf 100644 --- a/cmd/system-controller/main.go +++ b/cmd/system-controller/main.go @@ -10,7 +10,9 @@ import ( "syscall" "time" + "github.com/skupperproject/skupper/api/types" "github.com/skupperproject/skupper/internal/nonkube/controller" + "github.com/skupperproject/skupper/internal/utils" "github.com/skupperproject/skupper/internal/version" "github.com/skupperproject/skupper/pkg/nonkube/api" ) @@ -23,6 +25,9 @@ func main() { if api.IsRunningInContainer() { slog.Info("Host path info:", slog.String("path", api.GetHostNamespacesPath())) } + systemReloadType := utils.DefaultStr(os.Getenv(types.ENV_SYSTEM_AUTO_RELOAD), + types.SystemReloadTypeManual) + slog.Info("System Reload:", slog.String("type", systemReloadType)) if err := os.MkdirAll(namespacesPath, 0755); err != nil { slog.Error("Error creating skupper namespaces directory", slog.String("path", namespacesPath), slog.Any("error", err)) os.Exit(1) diff --git a/internal/cmd/skupper/common/constants.go b/internal/cmd/skupper/common/constants.go index 95cfe3945..ad12ba489 100644 --- a/internal/cmd/skupper/common/constants.go +++ b/internal/cmd/skupper/common/constants.go @@ -8,6 +8,7 @@ var ( WorkloadTypes = []string{"deployment", "service", "daemonset", "statefulset"} WaitStatusTypes = []string{"ready", "configured", "none"} BundleTypes = []string{"tarball", "shell-script"} + ReloadTypes = []string{"manual", "auto"} ) const ( diff --git a/internal/cmd/skupper/common/flags.go b/internal/cmd/skupper/common/flags.go index d58b82ac4..fa9487fae 100644 --- a/internal/cmd/skupper/common/flags.go +++ b/internal/cmd/skupper/common/flags.go @@ -89,6 +89,9 @@ for other Kubernetes flavors, loadbalancer is the default.` FlagNameFileName = "filename" FlagDescFileName = "The name of the file with custom resources" + + FlagNameReloadType = "reload-type" + FlagDescReloadType = "Specify the type of reload to perform. Choices: manual, auto" ) type CommandSiteCreateFlags struct { @@ -265,3 +268,7 @@ type CommandSystemApplyFlags struct { type CommandSystemDeleteFlags struct { Filename string } + +type CommandSystemInstallFlags struct { + ReloadType string +} diff --git a/internal/cmd/skupper/system/kube/system_install.go b/internal/cmd/skupper/system/kube/system_install.go index 006f213c8..e2a200f39 100644 --- a/internal/cmd/skupper/system/kube/system_install.go +++ b/internal/cmd/skupper/system/kube/system_install.go @@ -2,6 +2,8 @@ package kube import ( "fmt" + + "github.com/skupperproject/skupper/internal/cmd/skupper/common" skupperv2alpha1 "github.com/skupperproject/skupper/pkg/generated/client/clientset/versioned/typed/skupper/v2alpha1" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" @@ -12,6 +14,7 @@ type CmdSystemInstall struct { KubeClient kubernetes.Interface CobraCmd *cobra.Command Namespace string + Flags *common.CommandSystemInstallFlags } func NewCmdSystemInstall() *CmdSystemInstall { diff --git a/internal/cmd/skupper/system/nonkube/system_apply.go b/internal/cmd/skupper/system/nonkube/system_apply.go index c0934ec4f..eec92bd56 100644 --- a/internal/cmd/skupper/system/nonkube/system_apply.go +++ b/internal/cmd/skupper/system/nonkube/system_apply.go @@ -224,7 +224,7 @@ func (cmd *CmdSystemApply) Run() error { } if crApplied { - fmt.Println("Custom resources are applied. If a site is already running, run `skupper system reload` to make effective the changes.") + fmt.Println("Custom resources are applied.") } return nil diff --git a/internal/cmd/skupper/system/nonkube/system_install.go b/internal/cmd/skupper/system/nonkube/system_install.go index 7be4d8aa3..adbc5944b 100644 --- a/internal/cmd/skupper/system/nonkube/system_install.go +++ b/internal/cmd/skupper/system/nonkube/system_install.go @@ -3,8 +3,11 @@ package nonkube import ( "errors" "fmt" + "github.com/skupperproject/skupper/api/types" + "github.com/skupperproject/skupper/internal/cmd/skupper/common" "github.com/skupperproject/skupper/internal/config" + "github.com/skupperproject/skupper/internal/utils/validator" "github.com/skupperproject/skupper/internal/nonkube/bootstrap" "github.com/spf13/cobra" @@ -13,7 +16,9 @@ import ( type CmdSystemInstall struct { CobraCmd *cobra.Command Namespace string - SystemInstall func(string) error + SystemInstall func(string, string) error + Flags *common.CommandSystemInstallFlags + reloadType string } func NewCmdSystemInstall() *CmdSystemInstall { @@ -30,6 +35,7 @@ func (cmd *CmdSystemInstall) NewClient(cobraCommand *cobra.Command, args []strin func (cmd *CmdSystemInstall) ValidateInput(args []string) error { var validationErrors []error + reloadTypeValidator := validator.NewOptionValidator(common.ReloadTypes) if len(args) > 0 { validationErrors = append(validationErrors, fmt.Errorf("this command does not accept arguments")) @@ -38,13 +44,25 @@ func (cmd *CmdSystemInstall) ValidateInput(args []string) error { if config.GetPlatform() != types.PlatformPodman && config.GetPlatform() != types.PlatformDocker { validationErrors = append(validationErrors, fmt.Errorf("the selected platform is not supported by this command. There is nothing to install")) } + + if cmd.Flags != nil && cmd.Flags.ReloadType != "" { + ok, err := reloadTypeValidator.Evaluate(cmd.Flags.ReloadType) + if !ok { + validationErrors = append(validationErrors, fmt.Errorf("reload type is not valid: %s", err)) + } + } + return errors.Join(validationErrors...) } -func (cmd *CmdSystemInstall) InputToOptions() {} +func (cmd *CmdSystemInstall) InputToOptions() { + if cmd.Flags != nil && cmd.Flags.ReloadType != "" { + cmd.reloadType = cmd.Flags.ReloadType + } +} func (cmd *CmdSystemInstall) Run() error { - err := cmd.SystemInstall(string(config.GetPlatform())) + err := cmd.SystemInstall(string(config.GetPlatform()), cmd.reloadType) if err != nil { return fmt.Errorf("failed to configure the environment : %s", err) diff --git a/internal/cmd/skupper/system/nonkube/system_install_test.go b/internal/cmd/skupper/system/nonkube/system_install_test.go index 11e36f2cd..fde13a1bf 100644 --- a/internal/cmd/skupper/system/nonkube/system_install_test.go +++ b/internal/cmd/skupper/system/nonkube/system_install_test.go @@ -2,10 +2,12 @@ package nonkube import ( "fmt" - "github.com/skupperproject/skupper/internal/config" "os" "testing" + cmd "github.com/skupperproject/skupper/internal/cmd/skupper/common" + "github.com/skupperproject/skupper/internal/config" + "github.com/skupperproject/skupper/internal/cmd/skupper/common/testutils" "gotest.tools/v3/assert" ) @@ -15,6 +17,7 @@ func TestCmdSystemInstall_ValidateInput(t *testing.T) { name string args []string platform string + reloadType string expectedError string } @@ -30,6 +33,12 @@ func TestCmdSystemInstall_ValidateInput(t *testing.T) { platform: "linux", expectedError: "the selected platform is not supported by this command. There is nothing to install", }, + { + name: "reload type not supported", + reloadType: "both", + platform: "podman", + expectedError: "reload type is not valid: value both not allowed. It should be one of this options: [manual auto]", + }, } for _, test := range testTable { @@ -39,7 +48,8 @@ func TestCmdSystemInstall_ValidateInput(t *testing.T) { err := os.Setenv("SKUPPER_PLATFORM", test.platform) assert.Check(t, err == nil) - command := &CmdSystemInstall{} + command := &CmdSystemInstall{Flags: &cmd.CommandSystemInstallFlags{}} + command.Flags.ReloadType = test.reloadType testutils.CheckValidateInput(t, command, test.expectedError, test.args) }) @@ -96,7 +106,7 @@ func newCmdSystemInstallWithMocks(podmanSocketEnablementFails bool) *CmdSystemIn return cmdMock } -func mockCmdSystemInstall(platform string) error { return nil } -func mockCmdSystemInstallSocketEnablementFails(platform string) error { +func mockCmdSystemInstall(platform string, reloadType string) error { return nil } +func mockCmdSystemInstallSocketEnablementFails(platform string, reloadType string) error { return fmt.Errorf("systemd failed to enable podman socket") } diff --git a/internal/cmd/skupper/system/nonkube/system_reload_test.go b/internal/cmd/skupper/system/nonkube/system_reload_test.go index 84bc38cf9..4176d58a8 100644 --- a/internal/cmd/skupper/system/nonkube/system_reload_test.go +++ b/internal/cmd/skupper/system/nonkube/system_reload_test.go @@ -36,7 +36,6 @@ func TestCmdSystemReload_ValidateInput(t *testing.T) { for _, test := range testTable { t.Run(test.name, func(t *testing.T) { - command := &CmdSystemReload{} command.Namespace = test.namespace command.CobraCmd = common.ConfigureCobraCommand(common.PlatformLinux, common.SkupperCmdDescription{}, command, nil) diff --git a/internal/cmd/skupper/system/nonkube/system_start_test.go b/internal/cmd/skupper/system/nonkube/system_start_test.go index 6ecf67158..70edd803a 100644 --- a/internal/cmd/skupper/system/nonkube/system_start_test.go +++ b/internal/cmd/skupper/system/nonkube/system_start_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + "github.com/skupperproject/skupper/api/types" "github.com/skupperproject/skupper/internal/cmd/skupper/common" "github.com/skupperproject/skupper/internal/cmd/skupper/common/testutils" "github.com/skupperproject/skupper/internal/config" @@ -38,7 +39,7 @@ func TestCmdSystemSetup_ValidateInput(t *testing.T) { for _, test := range testTable { t.Run(test.name, func(t *testing.T) { - + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, types.SystemReloadTypeManual) command := &CmdSystemStart{} command.CobraCmd = common.ConfigureCobraCommand(common.PlatformLinux, common.SkupperCmdDescription{}, command, nil) command.Namespace = test.namespace diff --git a/internal/cmd/skupper/system/nonkube/system_stop.go b/internal/cmd/skupper/system/nonkube/system_stop.go index 0d8ccd6e9..ac6eed8e4 100644 --- a/internal/cmd/skupper/system/nonkube/system_stop.go +++ b/internal/cmd/skupper/system/nonkube/system_stop.go @@ -4,8 +4,10 @@ import ( "errors" "fmt" + "github.com/skupperproject/skupper/api/types" "github.com/skupperproject/skupper/internal/config" "github.com/skupperproject/skupper/internal/nonkube/bootstrap" + "github.com/skupperproject/skupper/internal/nonkube/common" "github.com/skupperproject/skupper/internal/utils/validator" "github.com/spf13/cobra" ) @@ -45,6 +47,20 @@ func (cmd *CmdSystemStop) ValidateInput(args []string) error { } } + platformLoader := &common.NamespacePlatformLoader{} + configuredPlatform, err := platformLoader.Load(cmd.Namespace) + if err != nil { + return err + } + + currentPlatform := config.GetPlatform() + if currentPlatform.IsKubernetes() { + currentPlatform = types.PlatformPodman + } + if string(currentPlatform) != configuredPlatform { + validationErrors = append(validationErrors, fmt.Errorf("existing namespace uses %q platform and it cannot change to %q", configuredPlatform, string(currentPlatform))) + } + return errors.Join(validationErrors...) } diff --git a/internal/cmd/skupper/system/nonkube/system_stop_test.go b/internal/cmd/skupper/system/nonkube/system_stop_test.go index 9fa99aaf7..2c4282583 100644 --- a/internal/cmd/skupper/system/nonkube/system_stop_test.go +++ b/internal/cmd/skupper/system/nonkube/system_stop_test.go @@ -2,37 +2,82 @@ package nonkube import ( "fmt" + "os" + "path" "testing" + "github.com/skupperproject/skupper/api/types" "github.com/skupperproject/skupper/internal/cmd/skupper/common" "github.com/skupperproject/skupper/internal/cmd/skupper/common/testutils" + "github.com/skupperproject/skupper/internal/config" + "github.com/skupperproject/skupper/pkg/nonkube/api" "gotest.tools/v3/assert" ) func TestCmdSystemTearDown_ValidateInput(t *testing.T) { type test struct { - name string - namespace string - args []string - expectedError string + name string + namespace string + args []string + envSystemReload string + expectedError string + configPlatform string // platform.yaml content + currentPlatform string // SKUPPER_PLATFORM env var value } testTable := []test{ { - name: "arg-not-accepted", - args: []string{"namespace"}, - expectedError: "this command does not accept arguments", + name: "arg-not-accepted", + args: []string{"namespace"}, + configPlatform: "podman", + currentPlatform: "podman", + expectedError: "this command does not accept arguments", + }, + { + name: "invalid-namespace", + namespace: "Invalid", + configPlatform: "docker", + currentPlatform: "docker", + expectedError: "namespace is not valid: value does not match this regular expression: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", }, { - name: "invalid-namespace", - namespace: "Invalid", - expectedError: "namespace is not valid: value does not match this regular expression: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + name: "platform-mismatch-podman-vs-docker", + namespace: "test-ns", + configPlatform: "podman", + currentPlatform: "docker", + expectedError: `existing namespace uses "podman" platform and it cannot change to "docker"`, }, } for _, test := range testTable { t.Run(test.name, func(t *testing.T) { + // Create a temporary directory to simulate the namespace output path + tmpDir := t.TempDir() + + namespace := test.namespace + if namespace == "" { + namespace = "default" + } + + internalDir := path.Join(tmpDir, "skupper", "namespaces", namespace, string(api.InternalBasePath)) + err := os.MkdirAll(internalDir, 0755) + assert.NilError(t, err) + + //create platform config file + platformYaml := fmt.Sprintf("platform: %s\n", test.configPlatform) + err = os.WriteFile(path.Join(internalDir, "platform.yaml"), []byte(platformYaml), 0644) + assert.NilError(t, err) + + t.Setenv("XDG_DATA_HOME", tmpDir) + + //set the current platform + config.ClearPlatform() + t.Setenv(types.ENV_PLATFORM, test.currentPlatform) + t.Cleanup(func() { + config.ClearPlatform() + }) + command := &CmdSystemStop{} command.Namespace = test.namespace command.CobraCmd = common.ConfigureCobraCommand(common.PlatformLinux, common.SkupperCmdDescription{}, command, nil) diff --git a/internal/cmd/skupper/system/system.go b/internal/cmd/skupper/system/system.go index e74ec9927..c4ef0a0d8 100644 --- a/internal/cmd/skupper/system/system.go +++ b/internal/cmd/skupper/system/system.go @@ -112,8 +112,14 @@ func CmdSystemInstallFactory(configuredPlatform common.Platform) *cobra.Command cmd := common.ConfigureCobraCommand(configuredPlatform, cmdSystemInstallDesc, kubeCommand, nonKubeCommand) + cmdFlags := common.CommandSystemInstallFlags{} + + cmd.Flags().StringVar(&cmdFlags.ReloadType, common.FlagNameReloadType, "manual", common.FlagDescReloadType) + kubeCommand.CobraCmd = cmd + kubeCommand.Flags = &cmdFlags nonKubeCommand.CobraCmd = cmd + nonKubeCommand.Flags = &cmdFlags return cmd } diff --git a/internal/cmd/skupper/system/system_test.go b/internal/cmd/skupper/system/system_test.go index f5df9ecbd..0db8bdac2 100644 --- a/internal/cmd/skupper/system/system_test.go +++ b/internal/cmd/skupper/system/system_test.go @@ -42,9 +42,11 @@ func TestCmdSystemFactory(t *testing.T) { command: CmdSystemGenerateBundleFactory(common.PlatformPodman), }, { - name: "CmdSystemInstallFactory", - expectedFlagsWithDefaultValue: map[string]interface{}{}, - command: CmdSystemInstallFactory(common.PlatformKubernetes), + name: "CmdSystemInstallFactory", + expectedFlagsWithDefaultValue: map[string]interface{}{ + common.FlagNameReloadType: "manual", + }, + command: CmdSystemInstallFactory(common.PlatformKubernetes), }, { name: "CmdSystemUninstallFactory", diff --git a/internal/kube/adaptor/config_sync.go b/internal/kube/adaptor/config_sync.go index d995cefa7..ccbc523e5 100644 --- a/internal/kube/adaptor/config_sync.go +++ b/internal/kube/adaptor/config_sync.go @@ -181,7 +181,7 @@ func syncConnectors(agent *qdr.Agent, desired *qdr.RouterConfig) error { ignorePrefix := "auto-mesh" if differences := qdr.ConnectorsDifference(actual, desired, &ignorePrefix); !differences.Empty() { - if err = agent.UpdateConnectorConfig(differences); err != nil { + if err = agent.UpdateConnectorConfig(differences, true); err != nil { return fmt.Errorf("Error syncing connectors: %s", err) } } diff --git a/internal/nonkube/bootstrap/bootstrap.go b/internal/nonkube/bootstrap/bootstrap.go index eb13329f9..785276a1a 100644 --- a/internal/nonkube/bootstrap/bootstrap.go +++ b/internal/nonkube/bootstrap/bootstrap.go @@ -109,7 +109,7 @@ func Bootstrap(config *Config) (*api.SiteState, error) { if err != nil { return nil, fmt.Errorf("failed to load site state: %v", err) } - // if sources are being consume from namespace sources, they must be properly set + // if sources are being consumed from namespace sources, they must be properly set crNamespace := siteState.GetNamespace() targetNamespace := utils.DefaultStr(config.Namespace, "default") if config.InputPath == sourcesPath { diff --git a/internal/nonkube/bootstrap/install.go b/internal/nonkube/bootstrap/install.go index c516ac399..16aeb2e62 100644 --- a/internal/nonkube/bootstrap/install.go +++ b/internal/nonkube/bootstrap/install.go @@ -14,20 +14,21 @@ import ( "github.com/skupperproject/skupper/internal/nonkube/bootstrap/controller" internalclient "github.com/skupperproject/skupper/internal/nonkube/client/compat" "github.com/skupperproject/skupper/internal/nonkube/common" + "github.com/skupperproject/skupper/internal/utils" "github.com/skupperproject/skupper/pkg/container" "github.com/skupperproject/skupper/pkg/nonkube/api" ) type ControllerConfig struct { containerEngine string - containerEndpointDefault string + containerEndpointDefault string //container endpoint in the local host to create and start the system controller username string hostDataHome string xdgDataHome string - containerEndpoint string + containerEndpoint string // container endpoint mapped to the podman socket inside the container } -func Install(platform string) error { +func Install(platform string, reloadType string) error { systemdGlobal, err := common.NewSystemdGlobal(platform) if err != nil { @@ -60,7 +61,7 @@ func Install(platform string) error { return nil } - cli, err := internalclient.NewCompatClient(config.containerEndpoint, "") + cli, err := internalclient.NewCompatClient(config.containerEndpointDefault, "") if err != nil { return fmt.Errorf("failed to create container client: %v", err) } @@ -71,10 +72,16 @@ func Install(platform string) error { } fmt.Printf("Pulled system-controller image: %s\n", images.GetSystemControllerImageName()) + if reloadType == "" { + reloadType = utils.DefaultStr(os.Getenv(types.ENV_SYSTEM_AUTO_RELOAD), + types.SystemReloadTypeManual) + } + env := map[string]string{ - "CONTAINER_ENDPOINT": config.containerEndpoint, - "SKUPPER_OUTPUT_PATH": config.hostDataHome, - "CONTAINER_ENGINE": config.containerEngine, + "CONTAINER_ENDPOINT": config.containerEndpoint, + "SKUPPER_OUTPUT_PATH": config.hostDataHome, + "CONTAINER_ENGINE": config.containerEngine, + "SKUPPER_SYSTEM_RELOAD_TYPE": reloadType, } //To mount a volume as a bind, the host path must be specified in the Name field @@ -85,18 +92,18 @@ func Install(platform string) error { volumeDestination := fmt.Sprintf("/var/run/%s.sock", platform) - if strings.HasPrefix(config.containerEndpoint, "unix://") { - socketPath := strings.TrimPrefix(config.containerEndpoint, "unix://") + if strings.HasPrefix(config.containerEndpointDefault, "unix://") { + socketPath := strings.TrimPrefix(config.containerEndpointDefault, "unix://") mounts = append(mounts, container.Volume{ Name: socketPath, Destination: volumeDestination, Mode: "z", RW: true, }) - } else if strings.HasPrefix(config.containerEndpoint, "/") { + } else if strings.HasPrefix(config.containerEndpointDefault, "/") { mounts = append(mounts, container.Volume{ - Name: config.containerEndpoint, + Name: config.containerEndpointDefault, Destination: volumeDestination, Mode: "z", RW: true, @@ -182,34 +189,41 @@ func configEnvVariables(platform string) (*ControllerConfig, error) { xdgRuntimeDir = fmt.Sprintf("/run/user/%s", uid) } - containerEndpointDefault := fmt.Sprintf("unix://%s/podman/podman.sock", xdgRuntimeDir) - - if platform == "docker" { - containerEndpointDefault = "unix:///run/docker.sock" - } - - uidInt, _ := strconv.Atoi(uid) xdgDataHome := "/output" + uidInt, _ := strconv.Atoi(uid) hostDataHome := api.GetHostDataHome() if uidInt == 0 { - if platform == "podman" { - containerEndpointDefault = "unix:///run/podman/podman.sock" - } hostDataHome = "/var/lib/skupper" } + containerEndpointDefault := os.Getenv("CONTAINER_ENDPOINT") + + if containerEndpointDefault == "" { + + if platform == "docker" { + containerEndpointDefault = "unix:///run/docker.sock" + } else { + + containerEndpointDefault = fmt.Sprintf("unix://%s/podman/podman.sock", xdgRuntimeDir) + + if uidInt == 0 { + if platform == "podman" { + containerEndpointDefault = "unix:///run/podman/podman.sock" + } + } + } + } + if err := os.MkdirAll(api.GetDefaultOutputNamespacesPath(), 0755); err != nil { return nil, fmt.Errorf("Failed to create directory: %v", err) } - containerEndpoint := os.Getenv("CONTAINER_ENDPOINT") - if containerEndpoint == "" { - containerEndpoint = containerEndpointDefault - } + var containerEndpoint string + if platform == "docker" { + containerEndpoint = "unix:///var/run/docker.sock" + } else { - err = os.Setenv("CONTAINER_ENDPOINT", containerEndpoint) - if err != nil { - return nil, err + containerEndpoint = "unix:///var/run/podman.sock" } controllerConfig.containerEndpoint = containerEndpoint diff --git a/internal/nonkube/bootstrap/teardown.go b/internal/nonkube/bootstrap/teardown.go index a2a2e5874..8ea425fe3 100644 --- a/internal/nonkube/bootstrap/teardown.go +++ b/internal/nonkube/bootstrap/teardown.go @@ -18,15 +18,16 @@ type LocalData struct { func Teardown(namespace string) error { platformLoader := &common.NamespacePlatformLoader{} - platform, err := platformLoader.Load(namespace) + configuredPlatform, err := platformLoader.Load(namespace) if err != nil { return err } - if err := removeRouter(namespace, platform); err != nil { + + if err := removeRouter(namespace, configuredPlatform); err != nil { return err } - if err := removeService(namespace, platform); err != nil { + if err := removeService(namespace, configuredPlatform); err != nil { return err } @@ -41,24 +42,27 @@ func Teardown(namespace string) error { func removeDefinition(namespace string) error { - _, err := os.Stat(api.GetHostNamespaceHome(namespace)) + path := api.GetHostNamespaceHome(namespace) + + if api.IsRunningInContainer() { + path = api.GetDefaultOutputPath(namespace) + } + + _, err := os.Stat(path) if err != nil { return err } - return os.RemoveAll(api.GetHostNamespaceHome(namespace)) + return os.RemoveAll(path) } func removeRouter(namespace string, platform string) error { endpoint := os.Getenv("CONTAINER_ENDPOINT") - if endpoint == "" { - endpoint = fmt.Sprintf("unix://%s/podman/podman.sock", api.GetRuntimeDir()) - if platform == "docker" { - endpoint = "unix:///run/docker.sock" - } - } + if api.IsRunningInContainer() || endpoint == "" { + endpoint = internalclient.GetDefaultContainerEndpoint() + } cli, err := internalclient.NewCompatClient(endpoint, "") if err != nil { return fmt.Errorf("failed to create container client: %v", err) diff --git a/internal/nonkube/client/compat/client.go b/internal/nonkube/client/compat/client.go index c7e854302..02b0eef90 100644 --- a/internal/nonkube/client/compat/client.go +++ b/internal/nonkube/client/compat/client.go @@ -19,7 +19,9 @@ import ( runtimeclient "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" "github.com/skupperproject/skupper/api/types" + "github.com/skupperproject/skupper/internal/config" "github.com/skupperproject/skupper/internal/utils" + "github.com/skupperproject/skupper/pkg/nonkube/api" ) const ( @@ -157,10 +159,29 @@ func NewCompatClient(endpoint, basePath string) (*CompatClient, error) { } func GetDefaultContainerEndpoint() string { - if os.Getenv(types.ENV_PLATFORM) == "docker" { - return "unix:///run/docker.sock" + endpoint := os.Getenv("CONTAINER_ENDPOINT") + + if endpoint != "" { + return endpoint + } + + platform := config.GetPlatform() + if platform == "" { + platform = types.Platform(os.Getenv("CONTAINER_ENGINE")) + } + // the container endpoint is mapped to the podman socket inside the container + if api.IsRunningInContainer() { + if platform == "docker" { + return "unix:///var/run/docker.sock" + } + return "unix:///var/run/podman.sock" + } else { + if platform == "docker" { + return "unix:///run/docker.sock" + } + + return fmt.Sprintf("unix://%s/podman/podman.sock", GetRuntimeDir()) } - return fmt.Sprintf("unix://%s/podman/podman.sock", GetRuntimeDir()) } func GetRuntimeDir() string { diff --git a/internal/nonkube/client/fs/custom_resource_handler.go b/internal/nonkube/client/fs/custom_resource_handler.go index a30a6ce57..ee88bba21 100644 --- a/internal/nonkube/client/fs/custom_resource_handler.go +++ b/internal/nonkube/client/fs/custom_resource_handler.go @@ -17,6 +17,8 @@ type GetOptions struct { RuntimeFirst bool LogWarning bool Attributes map[string]string + InputOnly bool + RuntimeOnly bool } type CustomResourceHandler[T any] interface { diff --git a/internal/nonkube/client/fs/site_handler.go b/internal/nonkube/client/fs/site_handler.go index 42ceb2647..72a40d7b5 100644 --- a/internal/nonkube/client/fs/site_handler.go +++ b/internal/nonkube/client/fs/site_handler.go @@ -93,20 +93,40 @@ func (s *SiteHandler) Delete(name string) error { func (s *SiteHandler) List(opts GetOptions) ([]*v2alpha1.Site, error) { var sites []*v2alpha1.Site + var path string + var files []fs.DirEntry + var err error - // First read from runtime directory, where output is found after bootstrap - // has run. If no runtime sites try and display configured sites - path := s.pathProvider.GetRuntimeNamespace() - err, files := s.ReadDir(path, common.Sites) - if err != nil { - if opts.LogWarning { - os.Stderr.WriteString("Site not initialized yet\n") + inputPath := s.pathProvider.GetNamespace() + runtimePath := s.pathProvider.GetRuntimeNamespace() + + if opts.InputOnly { + path = inputPath + err, files = s.ReadDir(path, common.Sites) + if err != nil { + return nil, err } - path = s.pathProvider.GetNamespace() + } else if opts.RuntimeOnly { + path = runtimePath err, files = s.ReadDir(path, common.Sites) if err != nil { return nil, err } + } else { + // First read from runtime directory, where output is found after bootstrap + // has run. If no runtime sites try and display configured sites + path = runtimePath + err, files = s.ReadDir(path, common.Sites) + if err != nil { + if opts.LogWarning { + os.Stderr.WriteString("Site not initialized yet\n") + } + path = inputPath + err, files = s.ReadDir(path, common.Sites) + if err != nil { + return nil, err + } + } } for _, file := range files { diff --git a/internal/nonkube/common/site_state_loader.go b/internal/nonkube/common/site_state_loader.go index 6073b4565..aa962f22a 100644 --- a/internal/nonkube/common/site_state_loader.go +++ b/internal/nonkube/common/site_state_loader.go @@ -46,7 +46,7 @@ func (f *FileSystemSiteStateLoader) Load() (*api.SiteState, error) { reader := bufio.NewReader(yamlFile) err = LoadIntoSiteState(reader, siteState) if err != nil { - return siteState, fmt.Errorf("error loading %q: %v", yamlFileName, err) + return nil, fmt.Errorf("error loading %q: %v", yamlFileName, err) } } if siteState.Site == nil || siteState.Site.Name == "" { diff --git a/internal/nonkube/common/site_state_renderer_common.go b/internal/nonkube/common/site_state_renderer_common.go index 237d0c8d3..b85ec0afe 100644 --- a/internal/nonkube/common/site_state_renderer_common.go +++ b/internal/nonkube/common/site_state_renderer_common.go @@ -3,6 +3,7 @@ package common import ( "fmt" + "github.com/skupperproject/skupper/internal/qdr" "github.com/skupperproject/skupper/internal/utils" "github.com/skupperproject/skupper/pkg/apis/skupper/v2alpha1" "github.com/skupperproject/skupper/pkg/nonkube/api" @@ -81,3 +82,28 @@ func CreateRouterAccess(siteState *api.SiteState) error { } return nil } + +func RecoverRouterAccess(siteState *api.SiteState, config *qdr.RouterConfig) error { + if !siteState.HasRouterAccess() { + logger := NewLogger() + logger.Debug("Recovering skupper-local RouterAccess") + name := fmt.Sprintf("skupper-local") + if config == nil { + return fmt.Errorf("skupper-local RouterAccess cannot be recovered without a router config") + } + + if config.Listeners == nil { + return fmt.Errorf("skupper-local RouterAccess cannot be recovered without listeners") + } + + var port int + if _, ok := config.Listeners["skupper-local-normal"]; ok { + port = int(config.Listeners["skupper-local-normal"].Port) + siteState.CreateRouterAccess(name, port) + } else { + return fmt.Errorf("skupper-local-normal listener does not exists in the router config.") + } + + } + return nil +} diff --git a/internal/nonkube/compat/site_state_renderer.go b/internal/nonkube/compat/site_state_renderer.go index a61280a01..53f9ba9f7 100644 --- a/internal/nonkube/compat/site_state_renderer.go +++ b/internal/nonkube/compat/site_state_renderer.go @@ -38,12 +38,11 @@ func (s *SiteStateRenderer) Render(loadedSiteState *api.SiteState, reload bool) } s.loadedSiteState = loadedSiteState endpoint := os.Getenv("CONTAINER_ENDPOINT") - if endpoint == "" { - endpoint = fmt.Sprintf("unix://%s/podman/podman.sock", api.GetRuntimeDir()) - if s.Platform == "docker" { - endpoint = "unix:///run/docker.sock" - } + + if api.IsRunningInContainer() || endpoint == "" { + endpoint = internalclient.GetDefaultContainerEndpoint() } + s.cli, err = internalclient.NewCompatClient(endpoint, "") if err != nil { return fmt.Errorf("failed to create container client: %v", err) @@ -374,3 +373,50 @@ func (s *SiteStateRenderer) preventContainersConflict() error { } return nil } + +func (s *SiteStateRenderer) Refresh(loadedSiteState *api.SiteState) error { + var err error + + var validator api.SiteStateValidator = &common.SiteStateValidator{} + err = validator.Validate(loadedSiteState) + if err != nil { + return err + } + s.loadedSiteState = loadedSiteState + + err = s.loadExistingSiteId(loadedSiteState) + if err != nil { + return err + } + + // active (runtime) SiteState + s.siteState = common.CopySiteState(s.loadedSiteState) + routerConfig, err := common.LoadRouterConfig(s.siteState.GetNamespace()) + + err = common.RecoverRouterAccess(s.siteState, routerConfig) + if err != nil { + return err + } + s.siteState.CreateLinkAccessesCertificates() + s.siteState.CreateBridgeCertificates() + + // rendering non-kube configuration files and certificates + platform := types.PlatformPodman + if s.Platform == types.PlatformDocker { + platform = types.PlatformDocker + } + s.configRenderer = &common.FileSystemConfigurationRenderer{ + Platform: string(platform), + } + err = s.configRenderer.Render(s.siteState) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + // Serializing loaded and runtime site states + if err = s.configRenderer.MarshalSiteStates(loadedSiteState, s.siteState); err != nil { + return err + } + + return nil +} diff --git a/internal/nonkube/controller/input_resource_handler.go b/internal/nonkube/controller/input_resource_handler.go new file mode 100644 index 000000000..36a986547 --- /dev/null +++ b/internal/nonkube/controller/input_resource_handler.go @@ -0,0 +1,203 @@ +package controller + +import ( + "fmt" + "log/slog" + "os" + "strings" + "sync" + + "github.com/skupperproject/skupper/api/types" + cmd "github.com/skupperproject/skupper/internal/cmd/skupper/common" + "github.com/skupperproject/skupper/internal/nonkube/bootstrap" + "github.com/skupperproject/skupper/internal/nonkube/client/fs" + "github.com/skupperproject/skupper/internal/nonkube/common" + "github.com/skupperproject/skupper/internal/nonkube/compat" + "github.com/skupperproject/skupper/internal/utils" + "github.com/skupperproject/skupper/pkg/nonkube/api" +) + +// This feature is responsible for handling the creation of input resources and +// execute the start/reload of the site configuration automatically. +type InputResourceHandler struct { + logger *slog.Logger + namespace string + inputPath string + Bootstrap func(config *bootstrap.Config) (*api.SiteState, error) + PostExec func(config *bootstrap.Config, siteState *api.SiteState) + TearDown func(namespace string) error + ConfigBootstrap bootstrap.Config + lock sync.Mutex + siteStateRenderer *compat.SiteStateRenderer + siteStateLoader api.SiteStateLoader + siteHandler *fs.SiteHandler +} + +type Bootstrap func(config *bootstrap.Config) (*api.SiteState, error) +type PostBootstrap func(config *bootstrap.Config, siteState *api.SiteState) +type TearDown func(namespace string) error + +func NewInputResourceHandler(namespace string, inputPath string, bs Bootstrap, pbs PostBootstrap, td TearDown) *InputResourceHandler { + + systemReloadType := utils.DefaultStr(os.Getenv(types.ENV_SYSTEM_AUTO_RELOAD), + types.SystemReloadTypeManual) + + if systemReloadType == types.SystemReloadTypeManual { + slog.Default().Debug("Automatic reloading is not configured.") + return nil + } + + handler := &InputResourceHandler{ + namespace: namespace, + inputPath: inputPath, + } + + handler.logger = slog.Default().With("component", "input.resource.handler", "namespace", namespace) + + handler.Bootstrap = bs + handler.PostExec = pbs + handler.TearDown = td + + var binary string + + platform := types.Platform(utils.DefaultStr(os.Getenv("CONTAINER_ENGINE"), + string(types.PlatformPodman))) + + // TODO: add support for linux platform + switch cmd.Platform(platform) { + case cmd.PlatformDocker: + binary = "docker" + case cmd.PlatformPodman: + binary = "podman" + case cmd.PlatformLinux: + handler.logger.Error("Linux platform is not supported yet") + return nil + default: + handler.logger.Error("This platform value is not supported: ", slog.String("platform", string(platform))) + return nil + } + + handler.ConfigBootstrap = bootstrap.Config{ + Namespace: namespace, + InputPath: inputPath, + Platform: platform, + Binary: binary, + } + + handler.siteStateRenderer = &compat.SiteStateRenderer{ + Platform: platform, + } + + handler.siteStateLoader = &common.FileSystemSiteStateLoader{ + Path: api.GetInternalOutputPath(namespace, api.InputSiteStatePath), + Bundle: false, + } + + handler.siteHandler = fs.NewSiteHandler(namespace) + + return handler +} + +func (h *InputResourceHandler) OnCreate(name string) { + h.lock.Lock() + defer h.lock.Unlock() + + h.logger.Info(fmt.Sprintf("Resource has been created: %s", name)) + err := h.processInputFile() + if err != nil { + h.logger.Error(err.Error()) + } +} + +// This function does not need to be implemented, given that when a file is updated, +// the event OnCreate is triggered anyway. Having it implemented would cause +// the resources to be reloaded multiple times, stopping and starting a router pod. +// (issue: the router pod is still active while going to be deleted, and the controller +// tries to create a new router pod, failing on this) +func (h *InputResourceHandler) OnUpdate(name string) {} +func (h *InputResourceHandler) OnRemove(name string) { + h.lock.Lock() + defer h.lock.Unlock() + + h.logger.Info(fmt.Sprintf("Resource has been deleted: %s", name)) + + sites, err := h.siteHandler.List(fs.GetOptions{InputOnly: true}) + if err != nil { + h.logger.Error(err.Error()) + } + + //If there is no site configured or running, the namespace needs to be removed + if err != nil || len(sites) == 0 { + err = h.tearDownNamespace() + if err != nil { + h.logger.Error(err.Error()) + } + return + } + + err = h.processInputFile() + if err != nil { + h.logger.Error(err.Error()) + } +} +func (h *InputResourceHandler) Filter(name string) bool { + return strings.HasSuffix(name, ".json") || strings.HasSuffix(name, ".yaml") || strings.HasSuffix(name, ".yml") +} + +func (h *InputResourceHandler) OnBasePathAdded(basePath string) {} + +func (h *InputResourceHandler) processInputFile() error { + var siteState *api.SiteState + var inputSiteNames []string + var runtimeSiteNames []string + + inputSites, err := h.siteHandler.List(fs.GetOptions{InputOnly: true}) + if err != nil { + h.logger.Debug("Trying to list input sites:", slog.Any("error", err)) + } + for _, site := range inputSites { + inputSiteNames = append(inputSiteNames, site.Name) + } + + runtimeSites, err := h.siteHandler.List(fs.GetOptions{RuntimeOnly: true}) + if err != nil { + h.logger.Debug("Trying to list runtime sites:", slog.Any("error", err)) + } + for _, site := range runtimeSites { + runtimeSiteNames = append(runtimeSiteNames, site.Name) + } + + inputSitesMatchRuntimeSites := utils.StringSlicesEqual(inputSiteNames, runtimeSiteNames) + + if !inputSitesMatchRuntimeSites { + siteState, err = h.Bootstrap(&h.ConfigBootstrap) + if err != nil { + return fmt.Errorf("Failed to bootstrap: %s", err) + } + h.PostExec(&h.ConfigBootstrap, siteState) + return nil + } + + siteState, err = h.siteStateLoader.Load() + if err != nil || siteState == nil { + return fmt.Errorf("Failed to load site state: %s", err) + } + if !siteState.IsBundle() { + err = h.siteStateRenderer.Refresh(siteState) + if err != nil { + return fmt.Errorf("Failed to refresh site state: %s", err) + } + } + + return nil +} + +func (h *InputResourceHandler) tearDownNamespace() error { + h.logger.Info("No site configured, tearing down namespace") + err := h.TearDown(h.namespace) + if err != nil { + return err + } + + return nil +} diff --git a/internal/nonkube/controller/input_resource_handler_test.go b/internal/nonkube/controller/input_resource_handler_test.go new file mode 100644 index 000000000..c9bf2a54b --- /dev/null +++ b/internal/nonkube/controller/input_resource_handler_test.go @@ -0,0 +1,129 @@ +package controller + +import ( + "fmt" + "log/slog" + "testing" + + "github.com/skupperproject/skupper/api/types" + "github.com/skupperproject/skupper/internal/nonkube/bootstrap" + "github.com/skupperproject/skupper/pkg/nonkube/api" + "gotest.tools/v3/assert" +) + +func TestInputResourceHandler(t *testing.T) { + + t.Run("handler created for docker platform", func(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, "auto") + t.Setenv("CONTAINER_ENGINE", "docker") + inputResourceHandler := NewInputResourceHandler("test_namespace", "test_inputPath", mockBootstrap, mockPostExec, mockTearDown) + expectedConfigBootstrap := bootstrap.Config{ + Namespace: "test_namespace", + InputPath: "test_inputPath", + Platform: "docker", + Binary: "docker", + } + + assert.Assert(t, inputResourceHandler != nil) + assert.Assert(t, inputResourceHandler.inputPath == "test_inputPath") + assert.Assert(t, inputResourceHandler.ConfigBootstrap == expectedConfigBootstrap) + }) + + t.Run("handler created for podman platform", func(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, "auto") + t.Setenv("CONTAINER_ENGINE", "podman") + inputResourceHandler := NewInputResourceHandler("test_namespace", "test_inputPath", mockBootstrap, mockPostExec, mockTearDown) + expectedConfigBootstrap := bootstrap.Config{ + Namespace: "test_namespace", + InputPath: "test_inputPath", + Platform: "podman", + Binary: "podman", + } + + assert.Assert(t, inputResourceHandler != nil) + assert.Assert(t, inputResourceHandler.inputPath == "test_inputPath") + assert.Assert(t, inputResourceHandler.ConfigBootstrap == expectedConfigBootstrap) + }) + + t.Run("handler not created for linux platform", func(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, "auto") + t.Setenv("CONTAINER_ENGINE", "linux") + inputResourceHandler := NewInputResourceHandler("test_namespace", "test_inputPath", mockBootstrap, mockPostExec, mockTearDown) + + assert.Assert(t, inputResourceHandler == nil) + + }) + + t.Run("handler not created because the system reload is configured to be manual", func(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, "manual") + inputResourceHandler := NewInputResourceHandler("test_namespace", "test_inputPath", mockBootstrap, mockPostExec, mockTearDown) + + assert.Assert(t, inputResourceHandler == nil) + + }) + + t.Run("handler not created for unknown platform", func(t *testing.T) { + t.Setenv("CONTAINER_ENGINE", "unknown") + inputResourceHandler := NewInputResourceHandler("test_namespace", "test_inputPath", mockBootstrap, mockPostExec, mockTearDown) + + assert.Assert(t, inputResourceHandler == nil) + + }) + + t.Run("resource file created or updated", func(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, "auto") + namespace := "test-file-created-ns" + inputPath := "test-file-created-input-path" + + handler := NewInputResourceHandler(namespace, inputPath, mockBootstrap, mockPostExec, mockTearDown) + + logSpy := &testLogHandler{ + handler: slog.Default().Handler(), + } + handler.logger = slog.New(logSpy) + + resourceName := "site.yaml" + handler.OnCreate(resourceName) + + expectedMsg := fmt.Sprintf("Resource has been created: %s", resourceName) + if count := logSpy.Count(expectedMsg); count != 1 { + t.Errorf("Expected log '%s' to be present, but found count: %d", expectedMsg, count) + } + + }) + + t.Run("resource file removed", func(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, "auto") + namespace := "test-file-ns" + inputPath := "test-file-input-path" + + handler := NewInputResourceHandler(namespace, inputPath, mockBootstrap, mockPostExec, mockTearDown) + + logSpy := &testLogHandler{ + handler: slog.Default().Handler(), + } + handler.logger = slog.New(logSpy) + + resourceName := "site.yaml" + handler.OnRemove(resourceName) + + expectedMsg := fmt.Sprintf("Resource has been deleted: %s", resourceName) + if count := logSpy.Count(expectedMsg); count != 1 { + t.Errorf("Expected log '%s' to be present, but found count: %d", expectedMsg, count) + } + }) + +} + +func mockBootstrap(config *bootstrap.Config) (*api.SiteState, error) { + return api.NewSiteState(false), nil +} +func mockPostExec(config *bootstrap.Config, siteState *api.SiteState) { + fmt.Println("post bootstrap execution completed") +} + +func mockBootstrapFailed(config *bootstrap.Config) (*api.SiteState, error) { + return nil, fmt.Errorf("failed to bootstrap") +} + +func mockTearDown(namespace string) error { return nil } diff --git a/internal/nonkube/controller/namespace_controller.go b/internal/nonkube/controller/namespace_controller.go index d8c483b0d..a1839ad19 100644 --- a/internal/nonkube/controller/namespace_controller.go +++ b/internal/nonkube/controller/namespace_controller.go @@ -5,21 +5,27 @@ import ( "log/slog" "github.com/skupperproject/skupper/internal/filesystem" + "github.com/skupperproject/skupper/internal/nonkube/bootstrap" + "github.com/skupperproject/skupper/internal/nonkube/client/fs" "github.com/skupperproject/skupper/pkg/nonkube/api" ) type NamespaceController struct { - ns string - stopCh chan struct{} - logger *slog.Logger - watcher *filesystem.FileWatcher - prepare func() + ns string + stopCh chan struct{} + logger *slog.Logger + watcher *filesystem.FileWatcher + prepare func() + pathProvider fs.PathProvider } func NewNamespaceController(namespace string) (*NamespaceController, error) { nsw := &NamespaceController{ ns: namespace, stopCh: make(chan struct{}), + pathProvider: fs.PathProvider{ + Namespace: namespace, + }, } watcher, err := filesystem.NewWatcher(slog.String("namespace", namespace)) if err != nil { @@ -38,9 +44,19 @@ func (w *NamespaceController) Start() { routerStateHandler := NewRouterStateHandler(w.ns) routerConfigHandler.AddCallback(routerStateHandler) collectorLifecycleHandler := NewCollectorLifecycleHandler(w.ns) - routerStateHandler.SetCallback(collectorLifecycleHandler) + routerStateHandler.AddCallback(collectorLifecycleHandler) + inputResourceHandler := NewInputResourceHandler(w.ns, w.pathProvider.GetNamespace(), bootstrap.Bootstrap, bootstrap.PostBootstrap, bootstrap.Teardown) + systemAdaptorHandler := NewSystemAdaptorHandler(w.ns) + if systemAdaptorHandler != nil { + routerStateHandler.AddCallback(systemAdaptorHandler) + } + w.watcher.Add(api.GetInternalOutputPath(w.ns, api.RouterConfigPath), routerConfigHandler) w.watcher.Add(api.GetInternalOutputPath(w.ns, api.RuntimeSiteStatePath), NewNetworkStatusHandler(w.ns)) + + if inputResourceHandler != nil { + w.watcher.Add(w.pathProvider.GetNamespace(), inputResourceHandler) + } } else { w.prepare() } diff --git a/internal/nonkube/controller/router_state_handler.go b/internal/nonkube/controller/router_state_handler.go index f5b5a6366..01b56715e 100644 --- a/internal/nonkube/controller/router_state_handler.go +++ b/internal/nonkube/controller/router_state_handler.go @@ -18,7 +18,7 @@ type RouterStateHandler struct { siteId string logger *slog.Logger mux sync.Mutex - callback ActivationCallback + callbacks []ActivationCallback heartbeat *heartBeatsClient } @@ -33,8 +33,8 @@ func NewRouterStateHandler(namespace string) *RouterStateHandler { return handler } -func (h *RouterStateHandler) SetCallback(callback ActivationCallback) { - h.callback = callback +func (h *RouterStateHandler) AddCallback(callback ActivationCallback) { + h.callbacks = append(h.callbacks, callback) } func (h *RouterStateHandler) Start(stopCh <-chan struct{}) { @@ -45,7 +45,7 @@ func (h *RouterStateHandler) Start(stopCh <-chan struct{}) { } h.logger.Info("Starting") h.running = true - go h.heartbeat.Start(stopCh, h.callback) + go h.heartbeat.Start(stopCh, h.callbacks) go h.handleParentStop(stopCh) } @@ -101,11 +101,12 @@ type heartBeatsClient struct { running bool isRouterUp bool callback ActivationCallback + callbacks []ActivationCallback receiver messaging.Receiver factory func(string, qdr.TlsConfigRetriever) messaging.ConnectionFactory } -func (h *heartBeatsClient) Start(stopCh <-chan struct{}, callback ActivationCallback) { +func (h *heartBeatsClient) Start(stopCh <-chan struct{}, callbacks []ActivationCallback) { h.mutex.Lock() defer h.mutex.Unlock() @@ -115,7 +116,7 @@ func (h *heartBeatsClient) Start(stopCh <-chan struct{}, callback ActivationCall h.logger.Info("Starting") h.running = true - h.callback = callback + h.callbacks = callbacks go h.run(stopCh) } @@ -135,7 +136,9 @@ func (h *heartBeatsClient) routerDown(reason string) { if h.isRouterUp { h.logger.Info("Router is DOWN", slog.Any("reason", reason)) h.isRouterUp = false - h.callback.Stop() + for _, callback := range h.callbacks { + callback.Stop() + } } } @@ -145,7 +148,9 @@ func (h *heartBeatsClient) routerUp(stopCh <-chan struct{}) { if !h.isRouterUp { h.logger.Info("Router is UP") h.isRouterUp = true - h.callback.Start(stopCh) + for _, callback := range h.callbacks { + callback.Start(stopCh) + } } } diff --git a/internal/nonkube/controller/router_state_handler_test.go b/internal/nonkube/controller/router_state_handler_test.go index 754bad446..edeecc00b 100644 --- a/internal/nonkube/controller/router_state_handler_test.go +++ b/internal/nonkube/controller/router_state_handler_test.go @@ -90,7 +90,7 @@ func TestRouterStateHandler(t *testing.T) { return mockFactory } callback := &routerStateCallback{} - routerStateHandler.SetCallback(callback) + routerStateHandler.AddCallback(callback) stopCh := make(chan struct{}) t.Run("start-router-state-handler", func(t *testing.T) { diff --git a/internal/nonkube/controller/system_adaptor.go b/internal/nonkube/controller/system_adaptor.go new file mode 100644 index 000000000..303714a25 --- /dev/null +++ b/internal/nonkube/controller/system_adaptor.go @@ -0,0 +1,93 @@ +package controller + +import ( + "fmt" + "log" + "os" + "path" + + "github.com/skupperproject/skupper/internal/nonkube/client/runtime" + "github.com/skupperproject/skupper/internal/qdr" +) + +type SystemAdaptor struct { + agentPool *qdr.AgentPool + namespace string +} + +func NewSystemAdaptor(namespace string, agentPool *qdr.AgentPool) *SystemAdaptor { + + systemAdaptor := &SystemAdaptor{ + namespace: namespace, + agentPool: agentPool, + } + return systemAdaptor +} + +func (s *SystemAdaptor) syncWithRouter(desired *qdr.RouterConfig) error { + if desired == nil { + return nil + } + + if err := s.syncSslProfileCredentialsToDisk(desired.SslProfiles); err != nil { + return err + } + if err := qdr.SyncSslProfilesToRouter(s.agentPool, s.addSslPathToProfileCredentials(desired.SslProfiles)); err != nil { + return err + } + if err := qdr.SyncBridgeConfig(s.agentPool, &desired.Bridges); err != nil { + log.Printf("sync failed: %s", err) + return err + } + + //Do not double-check that certificates exist; it has been done by previous syncSslProfileCredentialsToDisk + // Also, the paths included in the ssl profiles are relative to the router instead of the runtime directory + if err := qdr.SyncRouterConfig(s.agentPool, desired, false); err != nil { + log.Printf("sync failed: %s", err) + return err + } + return nil +} + +// it should check that the ssl profiles have their respective credentials on disk +// TODO: implement certificate rotation like in kube environments +func (s *SystemAdaptor) syncSslProfileCredentialsToDisk(profiles map[string]qdr.SslProfile) error { + + for certificateName, _ := range profiles { + tlsCert := runtime.GetRuntimeTlsCert(s.namespace, certificateName) + + _, err := os.Stat(tlsCert.CaPath) + if err != nil { + return fmt.Errorf("%s not available for certificate %s: %s\n", tlsCert.CaPath, certificateName, err) + } + _, err = os.Stat(tlsCert.CertPath) + if err != nil { + return fmt.Errorf("%s not available for certificate %s: %s\n", tlsCert.CertPath, certificateName, err) + } + _, err = os.Stat(tlsCert.KeyPath) + if err != nil { + return fmt.Errorf("%s not available for certificate %s: %s\n", tlsCert.KeyPath, certificateName, err) + } + } + + return nil + +} + +func (s *SystemAdaptor) addSslPathToProfileCredentials(profiles map[string]qdr.SslProfile) map[string]qdr.SslProfile { + + completedProfiles := make(map[string]qdr.SslProfile) + sslProfilePath := "/etc/skupper-router/runtime/certs" + + for certificateName, profile := range profiles { + + profile.CaCertFile = path.Join(sslProfilePath, certificateName, "ca.crt") + profile.CertFile = path.Join(sslProfilePath, certificateName, "tls.crt") + profile.PrivateKeyFile = path.Join(sslProfilePath, certificateName, "tls.key") + + completedProfiles[certificateName] = profile + } + + return completedProfiles + +} diff --git a/internal/nonkube/controller/system_adaptor_handler.go b/internal/nonkube/controller/system_adaptor_handler.go new file mode 100644 index 000000000..2d4b30c95 --- /dev/null +++ b/internal/nonkube/controller/system_adaptor_handler.go @@ -0,0 +1,115 @@ +package controller + +import ( + "fmt" + "log/slog" + "os" + "sync" + "time" + + "github.com/skupperproject/skupper/api/types" + "github.com/skupperproject/skupper/internal/nonkube/client/runtime" + "github.com/skupperproject/skupper/internal/nonkube/common" + "github.com/skupperproject/skupper/internal/qdr" + "github.com/skupperproject/skupper/internal/utils" +) + +type SystemAdaptorHandler struct { + running bool + logger *slog.Logger + namespace string + lock sync.Mutex + systemAdaptor *SystemAdaptor + callback ActivationCallback +} + +func NewSystemAdaptorHandler(namespace string) *SystemAdaptorHandler { + + systemReloadType := utils.DefaultStr(os.Getenv(types.ENV_SYSTEM_AUTO_RELOAD), + types.SystemReloadTypeManual) + + if systemReloadType == types.SystemReloadTypeManual { + slog.Default().Debug("Automatic reloading is not configured.") + return nil + } + + handler := &SystemAdaptorHandler{ + namespace: namespace, + } + handler.logger = slog.Default().With("component", "system.adaptor.handler", "namespace", namespace) + return handler +} + +func (s *SystemAdaptorHandler) SetCallback(callback ActivationCallback) { + s.callback = callback +} + +func (s *SystemAdaptorHandler) Start(stopCh <-chan struct{}) { + s.lock.Lock() + defer s.lock.Unlock() + if s.running { + return + } + s.logger.Info("Starting") + s.running = true + + tls := runtime.GetRuntimeTlsCert(s.namespace, "skupper-local-client") + address, err := runtime.GetLocalRouterAddress(s.namespace) + if err != nil { + s.logger.Error(fmt.Sprintf("Error getting local router address: %s", err)) + return + } + + agentPool := qdr.NewAgentPool(address, tls) + + s.systemAdaptor = NewSystemAdaptor(s.namespace, agentPool) + go s.processRouterConfig(stopCh) + +} + +func (s *SystemAdaptorHandler) Stop() { + s.lock.Lock() + defer s.lock.Unlock() + if s.running { + s.logger.Info("Stopping") + s.running = false + } +} + +func (s *SystemAdaptorHandler) Id() string { + return "system.adaptor.handler" +} + +func (s *SystemAdaptorHandler) processRouterConfig(stopCh <-chan struct{}) { + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for { + select { + case <-stopCh: + return + case <-ticker.C: + s.lock.Lock() + if !s.running { + s.lock.Unlock() + return + } + + desired, err := common.LoadRouterConfig(s.namespace) + if err != nil { + s.logger.Error(err.Error()) + s.lock.Unlock() + continue + } + + s.lock.Unlock() + + err = s.systemAdaptor.syncWithRouter(desired) + if err != nil { + s.logger.Debug(err.Error()) + } + + } + + } +} diff --git a/internal/nonkube/controller/system_adaptor_handler_test.go b/internal/nonkube/controller/system_adaptor_handler_test.go new file mode 100644 index 000000000..3d6acd677 --- /dev/null +++ b/internal/nonkube/controller/system_adaptor_handler_test.go @@ -0,0 +1,41 @@ +package controller + +import ( + "testing" + + "github.com/skupperproject/skupper/api/types" + "gotest.tools/v3/assert" +) + +func TestNewSystemAdaptorHandler_ManualReturnsNil(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, types.SystemReloadTypeManual) + h := NewSystemAdaptorHandler("ns") + assert.Assert(t, h == nil) +} + +func TestNewSystemAdaptorHandler_AutoReturnsHandler(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, types.SystemReloadTypeAuto) + h := NewSystemAdaptorHandler("ns") + assert.Assert(t, h != nil) + assert.Equal(t, h.namespace, "ns") + assert.Assert(t, h.logger != nil) +} + +func TestNewSystemAdaptorHandler_ErrorGettingLocalRouterAddress(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, types.SystemReloadTypeAuto) + handler := NewSystemAdaptorHandler("ns") + assert.Assert(t, handler != nil) + handler.Start(nil) + assert.Assert(t, handler.systemAdaptor == nil) + +} + +func TestNewSystemAdaptorHandler_Stop(t *testing.T) { + t.Setenv(types.ENV_SYSTEM_AUTO_RELOAD, types.SystemReloadTypeAuto) + handler := NewSystemAdaptorHandler("ns") + assert.Assert(t, handler != nil) + handler.running = true + handler.Stop() + assert.Assert(t, handler.running == false) + +} diff --git a/internal/qdr/amqp_mgmt.go b/internal/qdr/amqp_mgmt.go index bcccbdca4..b2522d6f9 100644 --- a/internal/qdr/amqp_mgmt.go +++ b/internal/qdr/amqp_mgmt.go @@ -1172,7 +1172,7 @@ func asSslProfile(record Record) SslProfile { } } -func (a *Agent) UpdateConnectorConfig(changes *ConnectorDifference) error { +func (a *Agent) UpdateConnectorConfig(changes *ConnectorDifference, checkCertFilesExist bool) error { for _, deleted := range changes.Deleted { if err := a.Delete("io.skupper.router.connector", deleted.Name); err != nil { return fmt.Errorf("Error deleting connectors: %s", err) @@ -1189,7 +1189,7 @@ func (a *Agent) UpdateConnectorConfig(changes *ConnectorDifference) error { return fmt.Errorf("No port specified while creating a connector") } - if len(added.SslProfile) > 0 { + if len(added.SslProfile) > 0 && checkCertFilesExist { sslProfile, err := a.GetSslProfileByName(added.SslProfile) if err != nil { return err diff --git a/internal/qdr/sync_router_ops.go b/internal/qdr/sync_router_ops.go new file mode 100644 index 000000000..566bc45f8 --- /dev/null +++ b/internal/qdr/sync_router_ops.go @@ -0,0 +1,129 @@ +package qdr + +import ( + "fmt" +) + +//TODO: use this in config-sync + +func SyncSslProfilesToRouter(agentPool *AgentPool, desired map[string]SslProfile) error { + + agent, err := agentPool.Get() + if err != nil { + return err + } + defer agentPool.Put(agent) + + actual, err := agent.GetSslProfiles() + if err != nil { + return err + } + + for _, profile := range desired { + current, ok := actual[profile.Name] + if !ok { + if err := agent.CreateSslProfile(profile); err != nil { + return err + } + } + if current != profile { + if err := agent.UpdateSslProfile(profile); err != nil { + return err + } + } + } + for _, profile := range actual { + if _, ok := desired[profile.Name]; !ok { + if err := agent.Delete("io.skupper.router.sslProfile", profile.Name); err != nil { + return err + } + } + } + return nil +} + +func SyncBridgeConfig(agentPool *AgentPool, desired *BridgeConfig) error { + agent, err := agentPool.Get() + if err != nil { + return fmt.Errorf("Could not get management agent : %s", err) + } + var synced bool + + synced, err = syncBridgeConfig(agent, desired) + + agentPool.Put(agent) + if err != nil { + return fmt.Errorf("Error while syncing bridge config : %s", err) + } + if !synced { + return fmt.Errorf("Bridge config is not synchronised yet") + } + return nil +} + +func SyncRouterConfig(agentPool *AgentPool, desired *RouterConfig, checkCertFilesExist bool) error { + if err := syncConnectors(agentPool, desired, checkCertFilesExist); err != nil { + return err + } + if err := syncListeners(agentPool, desired); err != nil { + return err + } + return nil +} + +func syncConnectors(agentPool *AgentPool, desired *RouterConfig, checkCertFilesExist bool) error { + agent, err := agentPool.Get() + if err != nil { + return err + } + defer agentPool.Put(agent) + + actual, err := agent.GetLocalConnectors() + if err != nil { + return fmt.Errorf("Error retrieving local connectors: %s", err) + } + + ignorePrefix := "auto-mesh" + if differences := ConnectorsDifference(actual, desired, &ignorePrefix); !differences.Empty() { + if err = agent.UpdateConnectorConfig(differences, checkCertFilesExist); err != nil { + return fmt.Errorf("Error syncing connectors: %s", err) + } + } + return nil +} + +func syncListeners(agentPool *AgentPool, desired *RouterConfig) error { + agent, err := agentPool.Get() + if err != nil { + return err + } + defer agentPool.Put(agent) + + actual, err := agent.GetLocalListeners() + if err != nil { + return fmt.Errorf("Error retrieving local listeners: %s", err) + } + + if differences := ListenersDifference(FilterListeners(actual, IsNotProtectedListener), desired.GetMatchingListeners(IsNotProtectedListener)); !differences.Empty() { + if err := agent.UpdateListenerConfig(differences); err != nil { + return fmt.Errorf("Error syncing listeners: %s", err) + } + } + return nil +} + +func syncBridgeConfig(agent *Agent, desired *BridgeConfig) (bool, error) { + actual, err := agent.GetLocalBridgeConfig() + if err != nil { + return false, fmt.Errorf("Error retrieving bridges: %s", err) + } + differences := actual.Difference(desired) + if differences.Empty() { + return true, nil + } else { + if err = agent.UpdateLocalBridgeConfig(differences); err != nil { + return false, fmt.Errorf("Error syncing bridges: %s", err) + } + return false, nil + } +}