Merge pull request #685 from fhke/feature/compose-up-idempotent-remove-orhans

Enhancements for `compose up` command.

Author: AkihiroSuda

cmd/nerdctl/compose_up.go

@@ -41,6 +41,7 @@ func newComposeUpCommand() *cobra.Command {
 	composeUpCommand.Flags().Bool("build", false, "Build images before starting containers.")
 	composeUpCommand.Flags().Bool("ipfs", false, "Allow pulling base images from IPFS during build")
 	composeUpCommand.Flags().Bool("quiet-pull", false, "Pull without printing progress information")
+	composeUpCommand.Flags().Bool("remove-orphans", false, "Remove containers for services not defined in the Compose file.")
 	composeUpCommand.Flags().StringArray("scale", []string{}, "Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present.")
 	return composeUpCommand
 }
@@ -77,6 +78,10 @@ func composeUpAction(cmd *cobra.Command, services []string) error {
 	if err != nil {
 		return err
 	}
+	removeOrphans, err := cmd.Flags().GetBool("remove-orphans")
+	if err != nil {
+		return err
+	}
 	scaleSlice, err := cmd.Flags().GetStringArray("scale")
 	if err != nil {
 		return err
@@ -105,14 +110,15 @@ func composeUpAction(cmd *cobra.Command, services []string) error {
 		return err
 	}
 	uo := composer.UpOptions{
-		Detach:      detach,
-		NoBuild:     noBuild,
-		NoColor:     noColor,
-		NoLogPrefix: noLogPrefix,
-		ForceBuild:  build,
-		IPFS:        enableIPFS,
-		QuietPull:   quietPull,
-		Scale:       scale,
+		Detach:        detach,
+		NoBuild:       noBuild,
+		NoColor:       noColor,
+		NoLogPrefix:   noLogPrefix,
+		ForceBuild:    build,
+		IPFS:          enableIPFS,
+		QuietPull:     quietPull,
+		RemoveOrphans: removeOrphans,
+		Scale:         scale,
 	}
 	return c.Up(ctx, uo, services)
 }

cmd/nerdctl/compose_up_linux_test.go

@@ -329,3 +329,67 @@ networks:
 
 	base.Cmd("inspect", "-f", `{{json .NetworkSettings.Networks }}`, projectName+"_foo_1").AssertOutContains("10.1.100.")
 }
+
+func TestComposeUpRemoveOrphans(t *testing.T) {
+	base := testutil.NewBase(t)
+
+	var (
+		dockerComposeYAMLOrphan = fmt.Sprintf(`
+version: '3.1'
+
+services:
+  test:
+    image: %s
+    command: "sleep infinity"
+`, testutil.AlpineImage)
+
+		dockerComposeYAMLFull = fmt.Sprintf(`
+%s
+  orphan:
+    image: %s
+    command: "sleep infinity"
+`, dockerComposeYAMLOrphan, testutil.AlpineImage)
+	)
+
+	compOrphan := testutil.NewComposeDir(t, dockerComposeYAMLOrphan)
+	defer compOrphan.CleanUp()
+	compFull := testutil.NewComposeDir(t, dockerComposeYAMLFull)
+	defer compFull.CleanUp()
+
+	projectName := fmt.Sprintf("nerdctl-compose-test-%d", time.Now().Unix())
+	t.Logf("projectName=%q", projectName)
+
+	orphanContainer := fmt.Sprintf("%s_orphan_1", projectName)
+
+	base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "up", "-d").AssertOK()
+	defer base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "down", "-v").Run()
+	base.ComposeCmd("-p", projectName, "-f", compOrphan.YAMLFullPath(), "up", "-d").AssertOK()
+	base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "ps").AssertOutContains(orphanContainer)
+	base.ComposeCmd("-p", projectName, "-f", compOrphan.YAMLFullPath(), "up", "-d", "--remove-orphans").AssertOK()
+	base.ComposeCmd("-p", projectName, "-f", compFull.YAMLFullPath(), "ps").AssertOutNotContains(orphanContainer)
+}
+
+func TestComposeUpIdempotent(t *testing.T) {
+	base := testutil.NewBase(t)
+
+	var dockerComposeYAML = fmt.Sprintf(`
+version: '3.1'
+
+services:
+  test:
+    image: %s
+    command: "sleep infinity"
+`, testutil.AlpineImage)
+
+	comp := testutil.NewComposeDir(t, dockerComposeYAML)
+	defer comp.CleanUp()
+
+	projectName := comp.ProjectName()
+	t.Logf("projectName=%q", projectName)
+
+	base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d").AssertOK()
+	defer base.ComposeCmd("-f", comp.YAMLFullPath(), "down", "-v").Run()
+	base.ComposeCmd("-f", comp.YAMLFullPath(), "up", "-d").AssertOK()
+	base.ComposeCmd("-f", comp.YAMLFullPath(), "down").AssertOK()
+
+}

pkg/composer/composer.go

@@ -26,7 +26,6 @@ import (
 	"path/filepath"
 
 	composecli "github.com/compose-spec/compose-go/cli"
-	"github.com/compose-spec/compose-go/types"
 	compose "github.com/compose-spec/compose-go/types"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/identifiers"
@@ -172,7 +171,7 @@ func findComposeYAML(o *Options) (string, error) {
 
 func (c *Composer) Services(ctx context.Context) ([]*serviceparser.Service, error) {
 	var services []*serviceparser.Service
-	if err := c.project.WithServices(nil, func(svc types.ServiceConfig) error {
+	if err := c.project.WithServices(nil, func(svc compose.ServiceConfig) error {
 		parsed, err := serviceparser.Parse(c.project, svc)
 		if err != nil {
 			return err
@@ -187,7 +186,7 @@ func (c *Composer) Services(ctx context.Context) ([]*serviceparser.Service, erro
 
 func (c *Composer) ServiceNames(services ...string) ([]string, error) {
 	var names []string
-	if err := c.project.WithServices(services, func(svc types.ServiceConfig) error {
+	if err := c.project.WithServices(services, func(svc compose.ServiceConfig) error {
 		names = append(names, svc.Name)
 		return nil
 	}); err != nil {

pkg/composer/container.go

@@ -41,3 +41,24 @@ func (c *Composer) Containers(ctx context.Context, services ...string) ([]contai
 	}
 	return containers, nil
 }
+
+func (c *Composer) containerExists(ctx context.Context, name, service string) (bool, error) {
+	// get list of containers for service
+	containers, err := c.Containers(ctx, service)
+	if err != nil {
+		return false, err
+	}
+
+	for _, container := range containers {
+		containerLabels, err := container.Labels(ctx)
+		if err != nil {
+			return false, err
+		}
+		if name == containerLabels[labels.Name] {
+			// container exists
+			return true, nil
+		}
+	}
+	// container doesn't exist
+	return false, nil
+}

pkg/composer/orphans.go

@@ -0,0 +1,59 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package composer
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/nerdctl/pkg/composer/serviceparser"
+	"github.com/containerd/nerdctl/pkg/labels"
+)
+
+func (c *Composer) getOrphanContainers(ctx context.Context, parsedServices []*serviceparser.Service) ([]containerd.Container, error) {
+	// get all running containers for project
+	var filters = []string{fmt.Sprintf("labels.%q==%s", labels.ComposeProject, c.project.Name)}
+	containers, err := c.client.Containers(ctx, filters...)
+	if err != nil {
+		return nil, err
+	}
+
+	var orphanedContainers []containerd.Container
+
+outer:
+	for _, container := range containers {
+		containerLabels, err := container.Labels(ctx)
+		if err != nil {
+			return nil, fmt.Errorf("error getting container labels: %s", err)
+		}
+		containerName := containerLabels[labels.Name]
+
+		for _, parsedService := range parsedServices {
+			for _, serviceContainer := range parsedService.Containers {
+				if containerName == serviceContainer.Name {
+					// container name exists in parsedServices
+					continue outer
+				}
+			}
+		}
+		// container name does not exist in parsedServices
+		orphanedContainers = append(orphanedContainers, container)
+	}
+
+	return orphanedContainers, nil
+}

pkg/composer/up.go

@@ -29,14 +29,15 @@ import (
 )
 
 type UpOptions struct {
-	Detach      bool
-	NoBuild     bool
-	NoColor     bool
-	NoLogPrefix bool
-	ForceBuild  bool
-	IPFS        bool
-	QuietPull   bool
-	Scale       map[string]uint64 // map of service name to replicas
+	Detach        bool
+	NoBuild       bool
+	NoColor       bool
+	NoLogPrefix   bool
+	ForceBuild    bool
+	IPFS          bool
+	QuietPull     bool
+	RemoveOrphans bool
+	Scale         map[string]uint64 // map of service name to replicas
 }
 
 func (c *Composer) Up(ctx context.Context, uo UpOptions, services []string) error {
@@ -86,7 +87,24 @@ func (c *Composer) Up(ctx context.Context, uo UpOptions, services []string) erro
 		return err
 	}
 
-	return c.upServices(ctx, parsedServices, uo)
+	if err := c.upServices(ctx, parsedServices, uo); err != nil {
+		return err
+	}
+
+	if uo.RemoveOrphans {
+		orphans, err := c.getOrphanContainers(ctx, parsedServices)
+		if err != nil {
+			return fmt.Errorf("error getting orphaned containers: %s", err)
+		}
+		if len(orphans) == 0 {
+			return nil
+		}
+		if err := c.downContainers(ctx, orphans, true); err != nil {
+			return fmt.Errorf("error removing orphaned containers: %s", err)
+		}
+	}
+
+	return nil
 }
 
 func validateFileObjectConfig(obj types.FileObjectConfig, shortName, objType string, project *types.Project) error {

pkg/composer/up_service.go

@@ -128,7 +128,23 @@ func (c *Composer) ensureServiceImage(ctx context.Context, ps *serviceparser.Ser
 // upServiceContainer must be called after ensureServiceImage
 // upServiceContainer returns container ID
 func (c *Composer) upServiceContainer(ctx context.Context, service *serviceparser.Service, container serviceparser.Container) (string, error) {
-	logrus.Infof("Creating container %s", container.Name)
+	// check if container already exists
+	exists, err := c.containerExists(ctx, container.Name, service.Unparsed.Name)
+	if err != nil {
+		return "", fmt.Errorf("error while checking for containers with name %q: %s", container.Name, err)
+	}
+
+	// delete container if it already exists
+	if exists {
+		logrus.Debugf("Container %q already exists, deleting", container.Name)
+		delCmd := c.createNerdctlCmd(ctx, "rm", "-f", container.Name)
+		if err = delCmd.Run(); err != nil {
+			return "", fmt.Errorf("could not delete container %q: %s", container.Name, err)
+		}
+		logrus.Infof("Re-creating container %s", container.Name)
+	} else {
+		logrus.Infof("Creating container %s", container.Name)
+	}
 
 	//add metadata labels to container https://github.com/compose-spec/compose-spec/blob/master/spec.md#labels
 	container.RunArgs = append([]string{

pkg/testutil/testutil.go

@@ -290,6 +290,16 @@ func (c *Cmd) AssertOutContains(s string) {
 	c.Assert(expected)
 }
 
+func (c *Cmd) AssertOutNotContains(s string) {
+	c.AssertOutWithFunc(func(stdout string) error {
+		if strings.Contains(stdout, s) {
+			return fmt.Errorf("expected stdout to contain %q", s)
+		} else {
+			return nil
+		}
+	})
+}
+
 func (c *Cmd) AssertOutExactly(s string) {
 	c.Base.T.Helper()
 	fn := func(stdout string) error {