Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 123 additions & 39 deletions cmd/nerdctl/container/container_kill_linux_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,63 +18,147 @@ package container

import (
"fmt"
"strconv"
"strings"
"testing"

"github.com/coreos/go-iptables/iptables"
"gotest.tools/v3/assert"

"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
"github.com/containerd/nerdctl/mod/tigron/expect"
"github.com/containerd/nerdctl/mod/tigron/require"
"github.com/containerd/nerdctl/mod/tigron/test"
"github.com/containerd/nerdctl/mod/tigron/tig"

"github.com/containerd/nerdctl/v2/pkg/testutil"
iptablesutil "github.com/containerd/nerdctl/v2/pkg/testutil/iptables"
"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
"github.com/containerd/nerdctl/v2/pkg/testutil/portlock"
)

// TestKillCleanupForwards runs a container that exposes a port and then kill it.
// The test checks that the kill command effectively clean up
// the iptables forwards creted from the run.
// the iptables forwards created from the run.
func TestKillCleanupForwards(t *testing.T) {
const (
hostPort = 9999
testContainerName = "ngx"
)
base := testutil.NewBase(t)
defer func() {
base.Cmd("rm", "-f", testContainerName).Run()
}()

// skip if rootless
if rootlessutil.IsRootless() {
t.Skip("pkg/testutil/iptables does not support rootless")
}

ipt, err := iptables.New()
assert.NilError(t, err)

containerID := base.Cmd("run", "-d",
"--restart=no",
"--name", testContainerName,
"-p", fmt.Sprintf("127.0.0.1:%d:80", hostPort),
testutil.NginxAlpineImage).Run().Stdout()
containerID = strings.TrimSuffix(containerID, "\n")

containerIP := base.Cmd("inspect",
"-f",
"'{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}'",
testContainerName).Run().Stdout()
containerIP = strings.ReplaceAll(containerIP, "'", "")
containerIP = strings.TrimSuffix(containerIP, "\n")

// define iptables chain name depending on the target (docker/nerdctl)
var chain string
if nerdtest.IsDocker() {
chain = "DOCKER"
} else {
redirectChain := "CNI-HOSTPORT-DNAT"
chain = iptablesutil.GetRedirectedChain(t, ipt, redirectChain, testutil.Namespace, containerID)
testCase := nerdtest.Setup()

testCase.Require = require.Not(nerdtest.Rootless)

testCase.Setup = func(data test.Data, helpers test.Helpers) {
hostPort, err := portlock.Acquire(0)
if err != nil {
t.Logf("Failed to acquire port: %v", err)
t.FailNow()
}

containerName := data.Identifier()

helpers.Ensure(
"run", "-d",
"--restart=no",
"--name", containerName,
"-p", fmt.Sprintf("127.0.0.1:%d:80", hostPort),
testutil.NginxAlpineImage,
)
nerdtest.EnsureContainerStarted(helpers, containerName)

containerID := strings.TrimSpace(
helpers.Capture("inspect", "-f", "{{.Id}}", containerName),
)

containerIP := strings.TrimSpace(
helpers.Capture(
"inspect",
"-f", "{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}",
containerName,
),
)

// define iptables chain name depending on the target (docker/nerdctl)
var chain string
if nerdtest.IsDocker() {
chain = "DOCKER"
} else {
chain = iptablesutil.GetRedirectedChain(
t,
ipt,
"CNI-HOSTPORT-DNAT",
testutil.Namespace,
containerID,
)
}

data.Labels().Set("containerName", containerName)
data.Labels().Set("containerIP", containerIP)
data.Labels().Set("hostPort", strconv.Itoa(hostPort))
data.Labels().Set("chain", chain)
}

testCase.SubTests = []*test.Case{
{
Description: "iptables forwarding rule should exist before container is killed",
NoParallel: true,
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
return helpers.Custom("iptables", "-t", "nat", "-S", data.Labels().Get("chain"))
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
Output: func(stdout string, t tig.T) {
rules := strings.Split(stdout, "\n")
containerIP := data.Labels().Get("containerIP")
hostPort, err := strconv.Atoi(data.Labels().Get("hostPort"))
assert.NilError(t, err)
found, err := iptablesutil.ForwardExistsFromRules(rules, containerIP, hostPort)
assert.NilError(t, err)
assert.Assert(t, found)
},
}
},
},
{
Description: "kill container",
NoParallel: true,
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
return helpers.Command("kill", data.Labels().Get("containerName"))
},
Expected: test.Expects(expect.ExitCodeSuccess, nil, nil),
},
{
Description: "iptables forwarding rule should be removed after container is killed",
NoParallel: true,
Command: func(data test.Data, helpers test.Helpers) test.TestableCommand {
return helpers.Custom("iptables", "-t", "nat", "-S", data.Labels().Get("chain"))
},
Expected: func(data test.Data, helpers test.Helpers) *test.Expected {
return &test.Expected{
ExitCode: expect.ExitCodeGenericFail,
Output: func(stdout string, t tig.T) {
rules := strings.Split(stdout, "\n")
containerIP := data.Labels().Get("containerIP")
hostPort, err := strconv.Atoi(data.Labels().Get("hostPort"))
assert.NilError(t, err)
found, err := iptablesutil.ForwardExistsFromRules(rules, containerIP, hostPort)
assert.NilError(t, err)
assert.Assert(t, !found)
},
}
},
},
}

testCase.Cleanup = func(data test.Data, helpers test.Helpers) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you add logic to release the acquired port number?

		if p := data.Labels().Get("hostPort"); p != "" {
			if port, err := strconv.Atoi(p); err == nil {
				_ = portlock.Release(port)
			}
		}

helpers.Anyhow("rm", "-f", data.Identifier())

if p := data.Labels().Get("hostPort"); p != "" {
if port, err := strconv.Atoi(p); err == nil {
_ = portlock.Release(port)
}
}
}
assert.Equal(t, iptablesutil.ForwardExists(t, ipt, chain, containerIP, hostPort), true)

base.Cmd("kill", testContainerName).AssertOK()
assert.Equal(t, iptablesutil.ForwardExists(t, ipt, chain, containerIP, hostPort), false)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@microness I found the refactoring dropped a check "iptables rule removed after kill". Would you mind adding the post-kill validation?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the comment! Fixed.

testCase.Run(t)
}
23 changes: 23 additions & 0 deletions pkg/testutil/iptables/iptables_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,3 +97,26 @@ func GetRedirectedChain(t *testing.T, ipt *iptables.IPTables, chain, namespace,
}
return redirectedChain
}

// ForwardExistsFromRules checks whether any rule in the given slice
// matches the expected DNAT forwarding pattern.
func ForwardExistsFromRules(rules []string, containerIP string, port int) (bool, error) {
if len(rules) < 1 {
return false, fmt.Errorf("not enough rules: %d", len(rules))
}

// here we check if at least one of the rules in the chain
// matches the required string to identify that the rule was applied
found := false
matchRule := `--dport ` + fmt.Sprintf("%d", port) + ` .+ --to-destination ` + containerIP
for _, rule := range rules {
foundInRule, err := regexp.MatchString(matchRule, rule)
if err != nil {
return false, fmt.Errorf("error in match string: %q", err)
}
if foundInRule {
found = foundInRule
}
}
return found, nil
}
Loading