diff --git a/containerd.spec b/containerd.spec index 5803f05f28e03a64edbc25d503a0af676cccb7d8..70ad3cf78158fae52420a12b15d7544108be634a 100644 --- a/containerd.spec +++ b/containerd.spec @@ -2,7 +2,7 @@ %global debug_package %{nil} Version: 1.2.0 Name: containerd -Release: 100 +Release: 101 Summary: An industry-standard container runtime License: ASL 2.0 URL: https://containerd.io diff --git a/patch/0038-containerd-add-timeout-for-I-O-waitgroups.patch b/patch/0038-containerd-add-timeout-for-I-O-waitgroups.patch new file mode 100644 index 0000000000000000000000000000000000000000..9c7e85774f8993dafc60185834ccf6ebca77daa3 --- /dev/null +++ b/patch/0038-containerd-add-timeout-for-I-O-waitgroups.patch @@ -0,0 +1,89 @@ +From d886f6c03cca051b45fd77cc77d0cc870aed1aed Mon Sep 17 00:00:00 2001 +From: build +Date: Wed, 4 Sep 2019 05:21:06 -0400 +Subject: [PATCH] containerd: add timeout for I/O waitgroups + +reason: This and a combination of a couple Docker changes are needed to fully +resolve the issue on the Docker side. However, this ensures that after +processes exit, we still leave some time for the I/O to fully flush +before closing. Without this timeout, the delete methods would block +forever. + +Cherry-pick from upstream 245052243d +Reference from https://github.com/containerd/containerd/pull/3361 + +Signed-off-by: Michael Crosby +--- + runtime/v1/linux/proc/exec.go | 2 +- + runtime/v1/linux/proc/init.go | 2 +- + runtime/v1/linux/proc/utils.go | 20 ++++++++++++++++++++ + 3 files changed, 22 insertions(+), 2 deletions(-) + +diff --git a/runtime/v1/linux/proc/exec.go b/runtime/v1/linux/proc/exec.go +index 715a977..08c581f 100644 +--- a/runtime/v1/linux/proc/exec.go ++++ b/runtime/v1/linux/proc/exec.go +@@ -94,7 +94,7 @@ func (e *execProcess) setExited(status int) { + } + + func (e *execProcess) delete(ctx context.Context) error { +- e.wg.Wait() ++ waitTimeout(ctx, &e.wg, 2*time.Second) + if e.io != nil { + for _, c := range e.closers { + c.Close() +diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go +index 44d3f58..49fa8ec 100644 +--- a/runtime/v1/linux/proc/init.go ++++ b/runtime/v1/linux/proc/init.go +@@ -263,7 +263,7 @@ func (p *Init) setExited(status int) { + } + + func (p *Init) delete(context context.Context) error { +- p.wg.Wait() ++ waitTimeout(context, &p.wg, 2*time.Second) + err := p.runtime.Delete(context, p.id, nil) + // ignore errors if a runtime has already deleted the process + // but we still hold metadata and pipes +diff --git a/runtime/v1/linux/proc/utils.go b/runtime/v1/linux/proc/utils.go +index ab9f5fa..d6f047c 100644 +--- a/runtime/v1/linux/proc/utils.go ++++ b/runtime/v1/linux/proc/utils.go +@@ -19,10 +19,12 @@ + package proc + + import ( ++ "context" + "encoding/json" + "io" + "os" + "strings" ++ "sync" + "time" + + "github.com/containerd/containerd/errdefs" +@@ -103,3 +105,21 @@ func checkKillError(err error) error { + func hasNoIO(r *CreateConfig) bool { + return r.Stdin == "" && r.Stdout == "" && r.Stderr == "" + } ++ ++// waitTimeout handles waiting on a waitgroup with a specified timeout. ++// this is commonly used for waiting on IO to finish after a process has exited ++func waitTimeout(ctx context.Context, wg *sync.WaitGroup, timeout time.Duration) error { ++ ctx, cancel := context.WithTimeout(ctx, timeout) ++ defer cancel() ++ done := make(chan struct{}, 1) ++ go func() { ++ wg.Wait() ++ close(done) ++ }() ++ select { ++ case <-done: ++ return nil ++ case <-ctx.Done(): ++ return ctx.Err() ++ } ++} +-- +2.20.1 + diff --git a/patch/0039-containerd-fix-shouldKillAllOnExit-check.patch b/patch/0039-containerd-fix-shouldKillAllOnExit-check.patch new file mode 100644 index 0000000000000000000000000000000000000000..1f894bf3b2841d07b20a31ecf70a4901246337c2 --- /dev/null +++ b/patch/0039-containerd-fix-shouldKillAllOnExit-check.patch @@ -0,0 +1,43 @@ +From 7741b1a960799b1724e92d23c6b2d9473ca71fee Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Thu, 31 Oct 2019 23:25:40 -0400 +Subject: [PATCH] containerd: fix shouldKillAllOnExit check + +reason: fix shouldKillAllOnExit check +v1 https://github.com/containerd/containerd/commit/fa5f744a790356472d4649b9ad1d955e36d0c7c0 +v2 https://github.com/containerd/containerd/commit/872296642ac395acbc4344f529fcf4c6fddb5de2 +Signed-off-by: Lifubang +--- + runtime/v1/shim/service.go | 2 +- + runtime/v2/runc/service.go | 2 +- + 2 files changed, 2 insertions(+), 2 deletions(-) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index ac545ea..88f7e0d 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -578,7 +578,7 @@ func shouldKillAllOnExit(bundlePath string) (bool, error) { + + if bundleSpec.Linux != nil { + for _, ns := range bundleSpec.Linux.Namespaces { +- if ns.Type == specs.PIDNamespace { ++ if ns.Type == specs.PIDNamespace && ns.Path == "" { + return false, nil + } + } +diff --git a/runtime/v2/runc/service.go b/runtime/v2/runc/service.go +index e37fb29..82beb8d 100644 +--- a/runtime/v2/runc/service.go ++++ b/runtime/v2/runc/service.go +@@ -680,7 +680,7 @@ func shouldKillAllOnExit(bundlePath string) (bool, error) { + + if bundleSpec.Linux != nil { + for _, ns := range bundleSpec.Linux.Namespaces { +- if ns.Type == specs.PIDNamespace { ++ if ns.Type == specs.PIDNamespace && ns.Path == "" { + return false, nil + } + } +-- +2.20.1 + diff --git a/patch/0044-containerd-change-tmpfile-directory-when-exec.patch b/patch/0044-containerd-change-tmpfile-directory-when-exec.patch new file mode 100644 index 0000000000000000000000000000000000000000..4e1adbe9895a8120f4ca3e7a059d281c32c5ce9e --- /dev/null +++ b/patch/0044-containerd-change-tmpfile-directory-when-exec.patch @@ -0,0 +1,29 @@ +From 8f3291f805c641a6fcf043eb8c4e1a2f4174b579 Mon Sep 17 00:00:00 2001 +From: wujibin +Date: Wed, 14 Aug 2019 17:18:24 +0800 +Subject: [PATCH] containerd: change tmpfile directory when exec + +reason: tmp file stored /tmp before change, if mountain of containers +are runing, the diretory will exist too many tmp file + +Change-Id: I1879ba9d09dca41a7571131d7447bf67356ea79c +--- + vendor/github.com/containerd/go-runc/runc.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index 6323bf21..7a2a8c4d 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -229,7 +229,7 @@ func (o *ExecOpts) args() (out []string, err error) { + // Exec executres and additional process inside the container based on a full + // OCI Process specification + func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts *ExecOpts) error { +- f, err := ioutil.TempFile(os.Getenv("XDG_RUNTIME_DIR"), "runc-process") ++ f, err := ioutil.TempFile(".", "runc-process") + if err != nil { + return err + } +-- +2.19.0 + diff --git a/patch/0045-containerd-shim-disable-fast-gc-on-arm.patch b/patch/0045-containerd-shim-disable-fast-gc-on-arm.patch new file mode 100644 index 0000000000000000000000000000000000000000..ea8fc658dc8f7707970e386b6421a20baa49a206 --- /dev/null +++ b/patch/0045-containerd-shim-disable-fast-gc-on-arm.patch @@ -0,0 +1,53 @@ +From 4a8367ce3a9a8321ceeffbf2934380b139a74f90 Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Fri, 18 Oct 2019 14:49:47 +0800 +Subject: [PATCH] containerd: stw gc sweep for arm64 + +Change-Id: I855c13a21c72bf0e91563db7c11e1348a1a78d55 +Signed-off-by: jingrui +--- + cmd/containerd-shim/main_unix.go | 5 ----- + runtime/v1/shim/client/client.go | 4 ++++ + 2 files changed, 4 insertions(+), 5 deletions(-) + +diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go +index 89f6be91..22283626 100644 +--- a/cmd/containerd-shim/main_unix.go ++++ b/cmd/containerd-shim/main_unix.go +@@ -80,11 +80,6 @@ func init() { + + func main() { + debug.SetGCPercent(40) +- go func() { +- for range time.Tick(30 * time.Second) { +- debug.FreeOSMemory() +- } +- }() + + if debugFlag { + logrus.SetLevel(logrus.DebugLevel) +diff --git a/runtime/v1/shim/client/client.go b/runtime/v1/shim/client/client.go +index a819be6c..a4669d33 100644 +--- a/runtime/v1/shim/client/client.go ++++ b/runtime/v1/shim/client/client.go +@@ -24,6 +24,7 @@ import ( + "net" + "os" + "os/exec" ++ "runtime" + "strings" + "sync" + "syscall" +@@ -161,6 +162,9 @@ func newCommand(binary, daemonAddress string, debug bool, config shim.Config, so + cmd.SysProcAttr = getSysProcAttr() + cmd.ExtraFiles = append(cmd.ExtraFiles, socket) + cmd.Env = append(os.Environ(), "GOMAXPROCS=2") ++ if runtime.GOARCH == "arm64" { ++ cmd.Env = append(cmd.Env, "GODEBUG=gcstoptheworld=2") ++ } + cmd.Stdout = stdout + cmd.Stderr = stderr + return cmd, nil +-- +2.17.1 + diff --git a/patch/0046-containerd-support-hot-upgrade.patch b/patch/0046-containerd-support-hot-upgrade.patch new file mode 100644 index 0000000000000000000000000000000000000000..9aac7294d9a56832e5f2f6b48b90d12d6b514925 --- /dev/null +++ b/patch/0046-containerd-support-hot-upgrade.patch @@ -0,0 +1,630 @@ +From 4656fbac6e4a23cf4e2fcb332777fb17895e67ca Mon Sep 17 00:00:00 2001 +From: jingrui +Date: Wed, 14 Aug 2019 10:51:19 +0800 +Subject: [PATCH] containerd: hot-upgrade support from + containerd-0.2.8 + +This patch support hot-upgrade from containerd-0.2.8. When restore +tasks, it will find containers started by containerd-0.2.8, then start +fake task create, the fake create will run a new shim process, the shim +process will manage the container created by runc. + +After restore legacy created tasks, each task will has 2 shim +process. So it support down-grade to docker-1.11.2 with container still +running. + +Change-Id: I94cd48cbf8ceb408dbc8849fe6916e0ec3d889b0 +Signed-off-by: jingrui +--- + legacy/legacy.go | 145 ++++++++++++++++++++ + runtime/v1/linux/leruntime.go | 243 ++++++++++++++++++++++++++++++++++ + runtime/v1/linux/proc/init.go | 27 +++- + runtime/v1/linux/proc/io.go | 11 +- + runtime/v1/linux/runtime.go | 5 + + runtime/v1/shim/service.go | 10 +- + services/containers/local.go | 19 ++- + 7 files changed, 452 insertions(+), 8 deletions(-) + create mode 100644 legacy/legacy.go + create mode 100644 runtime/v1/linux/leruntime.go + +diff --git a/legacy/legacy.go b/legacy/legacy.go +new file mode 100644 +index 00000000..fde9f709 +--- /dev/null ++++ b/legacy/legacy.go +@@ -0,0 +1,145 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019-2019. All rights reserved. ++Description: support containerd hot-upgrade from 0.2.8 ++Author: jingrui jingrui@huawei.com ++Create: 2019-09-20 ++*/ ++ ++package legacy ++ ++import ( ++ "encoding/json" ++ "fmt" ++ "io" ++ "io/ioutil" ++ "os" ++ "path/filepath" ++ "runtime" ++ "strings" ++ ++ "github.com/sirupsen/logrus" ++ "github.com/opencontainers/runtime-spec/specs-go" ++) ++ ++const ( ++ LegacyFile = "legacy" ++ Config120 = "/var/run/docker/containerd/daemon/io.containerd.runtime.v1.linux/moby/" ++ Stdio120 = "/var/run/docker/containerd/" ++ Config028 = "/var/run/docker/libcontainerd/" ++ State028 = "/var/run/docker/libcontainerd/containerd/" ++ Runtime = "io.containerd.runtime.v1" ++) ++ ++// IsLegacy is used to check if im legacy. ++func IsLegacy(id string) bool { ++ lf := Config120 + id + "/" + LegacyFile ++ if _, err := os.Stat(lf); err == nil { ++ caller := "??" ++ if pc, file, line, ok := runtime.Caller(1); ok { ++ caller = fmt.Sprintf("%s:%d:%s()", file, line, runtime.FuncForPC(pc).Name()) ++ } ++ logrus.Infof("shim pretend to be 0.2.8 in %s", caller) ++ return true ++ } ++ return false ++} ++ ++// IsRunning is used to detect whether legacy container is running. ++func IsRunning(id string) bool { ++ path := State028 + id + "/init/pid" ++ bpid, err := ioutil.ReadFile(path) ++ if err != nil { ++ return false ++ } ++ ++ path = State028 + id + "/init/starttime" ++ btime, err := ioutil.ReadFile(path) ++ if err != nil { ++ return false ++ } ++ ++ path = fmt.Sprintf("/proc/%s/stat", string(bpid)) ++ bstat, err := ioutil.ReadFile(path) ++ if err != nil { ++ return false ++ } ++ ++ if !strings.Contains(string(bstat), string(btime)) { ++ return false ++ } ++ ++ return true ++} ++ ++// CopyFile used to copy a file. ++func CopyFile(dstName, srcName string) (written int64, err error) { ++ src, err := os.Open(srcName) ++ if err != nil { ++ return ++ } ++ defer src.Close() ++ ++ dst, err := os.OpenFile(dstName, os.O_WRONLY|os.O_CREATE, 0644) ++ if err != nil { ++ return ++ } ++ defer dst.Close() ++ ++ return io.Copy(dst, src) ++} ++ ++// InitBundle will copy files from 0.2.8 dirs to 1.2.0 dirs. ++func InitBundle(root string, id string) error { ++ err := os.MkdirAll(Config120+id, 0711) ++ if err != nil { ++ return err ++ } ++ err = os.MkdirAll(Stdio120+id, 0711) ++ if err != nil { ++ return err ++ } ++ err = os.MkdirAll(filepath.Join(root, "moby", id), 0711) ++ if err != nil { ++ return err ++ } ++ ++ err = ioutil.WriteFile(Config120+id+"/"+LegacyFile, []byte{}, 0644) ++ if err != nil { ++ return err ++ } ++ CopyFile(Config120+id+"/config.json", Config028+id+"/config.json") ++ CopyFile(Config120+id+"/init.pid", State028+id+"/init/pid") ++ return nil ++} ++ ++// DeleteBundle will delete unused legacy bundle files. ++func DeleteBundle(id string) error { ++ err1 := os.RemoveAll(Config120 + id) ++ err2 := os.RemoveAll(Stdio120 + id) ++ if err1 != nil { ++ return err1 ++ } ++ if err2 != nil { ++ return err2 ++ } ++ ++ return nil ++} ++ ++// LoadSpec load config.json into spec. ++func LoadSpec(id string) (*specs.Spec, error) { ++ f, err := os.OpenFile(Config120+id+"/config.json", os.O_RDONLY, 0400) ++ if err != nil { ++ return nil, err ++ } ++ defer f.Close() ++ ++ spec := specs.Spec{} ++ dec := json.NewDecoder(f) ++ err = dec.Decode(&spec) ++ if err != nil { ++ return nil, err ++ } ++ ++ return &spec, nil ++} +diff --git a/runtime/v1/linux/leruntime.go b/runtime/v1/linux/leruntime.go +new file mode 100644 +index 00000000..5b887935 +--- /dev/null ++++ b/runtime/v1/linux/leruntime.go +@@ -0,0 +1,243 @@ ++/* ++Copyright (c) Huawei Technologies Co., Ltd. 2019-2019. All rights reserved. ++Description: support containerd hot-upgrade from 0.2.8 ++Author: jingrui jingrui@huawei.com ++Create: 2019-09-20 ++*/ ++ ++package linux ++ ++import ( ++ "context" ++ "fmt" ++ "io/ioutil" ++ goruntime "runtime" ++ ++ "github.com/containerd/containerd/api/types" ++ "github.com/containerd/containerd/containers" ++ "github.com/containerd/containerd/errdefs" ++ "github.com/containerd/containerd/legacy" ++ "github.com/containerd/containerd/log" ++ "github.com/containerd/containerd/namespaces" ++ "github.com/containerd/containerd/runtime" ++ "github.com/containerd/containerd/runtime/linux/runctypes" ++ shim "github.com/containerd/containerd/runtime/v1/shim/v1" ++ scontainers "github.com/containerd/containerd/services/containers" ++ "github.com/containerd/typeurl" ++ "github.com/sirupsen/logrus" ++) ++ ++func taskIsExist(tasks []*Task, id string) bool { ++ for _, t := range tasks { ++ if t.id == id { ++ return true ++ } ++ } ++ return false ++} ++ ++func loadCreateOpts(id string) runtime.CreateOpts { ++ opts := runtime.CreateOpts{ ++ IO: runtime.IO{ ++ Stdin: fmt.Sprintf("/var/run/docker/libcontainerd/%s/init-stdin", id), ++ Stdout: fmt.Sprintf("/var/run/docker/libcontainerd/%s/init-stdout", id), ++ }, ++ } ++ ++ return opts ++} ++ ++func (r *Runtime) legacyCreateMeta(ctx context.Context, id string) { ++ spec, err := legacy.LoadSpec(id) ++ if err != nil { ++ logrus.Errorf("load spec for %s failed %v", id, err) ++ return ++ } ++ ++ s, err := typeurl.MarshalAny(spec) ++ if err != nil { ++ logrus.Errorf("marshal-any for %s failed %v", id, err) ++ return ++ } ++ ++ c := containers.Container{ ++ ID: id, ++ Runtime: containers.RuntimeInfo{ ++ Name: fmt.Sprintf("%s.%s", legacy.Runtime, goruntime.GOOS), ++ }, ++ Spec: s, ++ } ++ ++ err = scontainers.CreateMeta(ctx, c) ++ if err != nil { ++ logrus.Infof("create meta for %s failed %v", c.ID, err) ++ } ++} ++ ++func (r *Runtime) legacyCreate(ctx context.Context, id string, opts runtime.CreateOpts) (*Task, error) { ++ namespace, err := namespaces.NamespaceRequired(ctx) ++ if err != nil { ++ return nil, err ++ } ++ if namespace != "moby" { ++ return nil, fmt.Errorf("legacy not support ns=%s", namespace) ++ } ++ ++ ropts := &runctypes.RuncOptions{} ++ bundle := loadBundle(id, ++ legacy.Config120+id, ++ legacy.Config120+id) ++ ++ defer func() { ++ if err != nil { ++ errd := bundle.Delete() ++ log.G(ctx).WithError(err).Errorf("revert: delete bundle error=%v", errd) ++ } ++ }() ++ ++ shimopt := ShimLocal(r.config, r.events) ++ ++ var cgroup string ++ if opts.TaskOptions != nil { ++ v, err := typeurl.UnmarshalAny(opts.TaskOptions) ++ if err != nil { ++ return nil, err ++ } ++ cgroup = v.(*runctypes.CreateOptions).ShimCgroup ++ } ++ exitHandler := func() { ++ log.G(ctx).WithField("id", id).Info("shim reaped") ++ t, err := r.tasks.Get(ctx, id) ++ if err != nil { ++ // Task was never started or was already successfully deleted ++ return ++ } ++ lc := t.(*Task) ++ ++ log.G(ctx).WithFields(logrus.Fields{ ++ "id": id, ++ "namespace": namespace, ++ }).Warn("cleaning up after killed shim") ++ if err = r.cleanupAfterDeadShim(context.Background(), bundle, namespace, id, lc.pid); err != nil { ++ log.G(ctx).WithError(err).WithFields(logrus.Fields{ ++ "id": id, ++ "namespace": namespace, ++ }).Warn("failed to clean up after killed shim") ++ } ++ } ++ shimopt = ShimRemote(r.config, r.address, cgroup, exitHandler) ++ ++ s, err := bundle.NewShimClient(ctx, namespace, shimopt, ropts) ++ if err != nil { ++ return nil, err ++ } ++ ++ defer func() { ++ if err != nil { ++ kerr := s.KillShim(ctx) ++ log.G(ctx).WithError(err).Errorf("revert: kill shim error=%v", kerr) ++ } ++ }() ++ ++ rt := r.config.Runtime ++ if ropts != nil && ropts.Runtime != "" { ++ rt = ropts.Runtime ++ } ++ sopts := &shim.CreateTaskRequest{ ++ ID: id, ++ Bundle: bundle.path, ++ Runtime: rt, ++ Stdin: opts.IO.Stdin, ++ Stdout: opts.IO.Stdout, ++ Stderr: opts.IO.Stderr, ++ Terminal: opts.IO.Terminal, ++ Checkpoint: opts.Checkpoint, ++ Options: opts.TaskOptions, ++ } ++ for _, m := range opts.Rootfs { ++ sopts.Rootfs = append(sopts.Rootfs, &types.Mount{ ++ Type: m.Type, ++ Source: m.Source, ++ Options: m.Options, ++ }) ++ } ++ cr, err := s.Create(ctx, sopts) ++ if err != nil { ++ return nil, errdefs.FromGRPC(err) ++ } ++ t, err := newTask(id, namespace, int(cr.Pid), s, r.events, r.tasks, bundle) ++ if err != nil { ++ return nil, err ++ } ++ ++ // dont add task to tasklist, restoreTasks() will add it later. ++ ++ return t, nil ++} ++ ++func (r *Runtime) loadLegacyTask(id string) (*Task, error) { ++ logrus.Infof("load-letask id=%s", id) ++ err := legacy.InitBundle(r.root, id) ++ if err != nil { ++ logrus.Errorf("letask %s init bundle failed %s", id, err) ++ return nil, err ++ } ++ ++ defer func() { ++ if err != nil { ++ err1 := legacy.DeleteBundle(id) ++ logrus.Errorf("letask %s failed %v, drop bundle error=%s", id, err, err1) ++ } ++ }() ++ ++ ctx := namespaces.WithNamespace(context.Background(), "moby") ++ r.legacyCreateMeta(ctx, id) ++ task, err := r.legacyCreate(ctx, id, loadCreateOpts(id)) ++ if err != nil { ++ logrus.Errorf("letask %s create failed %v", id, err) ++ return nil, err ++ } ++ ++ return task, nil ++} ++ ++func (r *Runtime) loadLegacyTasks(tasks []*Task, ctx context.Context, ns string) ([]*Task, error) { ++ var o []*Task ++ ++ if ns != "moby" { ++ logrus.Infof("loadLegacyTasks ignore ns=%s", ns) ++ return o, nil ++ } ++ ++ dir, err := ioutil.ReadDir(legacy.State028) ++ if err != nil { ++ logrus.Infof("loadLegacyTasks skipped, no legacy residual") ++ return o, nil ++ } ++ ++ for _, path := range dir { ++ if !path.IsDir() { ++ continue ++ } ++ ++ id := path.Name() ++ if taskIsExist(tasks, id) { ++ logrus.Infof("letask %s already loaded", id) ++ continue ++ } ++ if !legacy.IsRunning(id) { ++ logrus.Infof("letask %s not running", id) ++ continue ++ } ++ ++ task, err := r.loadLegacyTask(id) ++ if err != nil { ++ logrus.Errorf("letask %s load failed %s", err) ++ continue ++ } ++ ++ o = append(o, task) ++ logrus.Infof("letask id=%s load ok", id) ++ } ++ return o, nil ++} +diff --git a/runtime/v1/linux/proc/init.go b/runtime/v1/linux/proc/init.go +index 44d3f58b..ace98621 100644 +--- a/runtime/v1/linux/proc/init.go ++++ b/runtime/v1/linux/proc/init.go +@@ -31,6 +31,7 @@ import ( + "time" + + "github.com/containerd/console" ++ "github.com/containerd/containerd/legacy" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/runtime/proc" +@@ -39,6 +40,7 @@ import ( + google_protobuf "github.com/gogo/protobuf/types" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + // InitPidFile name of the file that contains the init pid +@@ -113,6 +115,19 @@ func New(id string, runtime *runc.Runc, stdio proc.Stdio) *Init { + waitBlock: make(chan struct{}), + } + p.initState = &createdState{p: p} ++ // legacy container is exist, set it running state directly. ++ if legacy.IsLegacy(id) { ++ p.initState = &runningState{p: p} ++ go func(id string) { ++ for { ++ time.Sleep(3 * time.Second) ++ if !legacy.IsRunning(id) { ++ logrus.Infof("legacy container %s exited", id) ++ os.Exit(0) ++ } ++ } ++ }(id) ++ } + return p + } + +@@ -122,6 +137,17 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error { + err error + socket *runc.Socket + ) ++ pidFile := filepath.Join(p.Bundle, InitPidFile) ++ ++ if legacy.IsLegacy(r.ID) { ++ pid, err := runc.ReadPidFile(pidFile) ++ if err != nil { ++ return errors.Wrap(err, "failed to retrieve OCI runtime container pid") ++ } ++ p.pid = pid ++ return nil ++ } ++ + if r.Terminal { + if socket, err = runc.NewTempConsoleSocket(); err != nil { + return errors.Wrap(err, "failed to create OCI runtime console socket") +@@ -136,7 +162,6 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) error { + return errors.Wrap(err, "failed to create OCI runtime io pipes") + } + } +- pidFile := filepath.Join(p.Bundle, InitPidFile) + if r.Checkpoint != "" { + opts := &runc.RestoreOpts{ + CheckpointOpts: runc.CheckpointOpts{ +diff --git a/runtime/v1/linux/proc/io.go b/runtime/v1/linux/proc/io.go +index 71f6ee1b..36066270 100644 +--- a/runtime/v1/linux/proc/io.go ++++ b/runtime/v1/linux/proc/io.go +@@ -79,6 +79,9 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w + }, + }, + } { ++ if i.name == "" { ++ continue ++ } + ok, err := isFifo(i.name) + if err != nil { + return err +@@ -89,10 +92,10 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w + ) + if ok { + if fw, err = fifo.OpenFifo(ctx, i.name, syscall.O_WRONLY, 0); err != nil { +- return fmt.Errorf("containerd-shim: opening %s failed: %s", i.name, err) ++ return fmt.Errorf("containerd-shim syscall.O_WRONLY: opening %s failed: %s", i.name, err) + } + if fr, err = fifo.OpenFifo(ctx, i.name, syscall.O_RDONLY, 0); err != nil { +- return fmt.Errorf("containerd-shim: opening %s failed: %s", i.name, err) ++ return fmt.Errorf("containerd-shim syscall.O_RDONLY: opening %s failed: %s", i.name, err) + } + } else { + if sameFile != nil { +@@ -100,7 +103,7 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w + continue + } + if fw, err = os.OpenFile(i.name, syscall.O_WRONLY|syscall.O_APPEND, 0); err != nil { +- return fmt.Errorf("containerd-shim: opening %s failed: %s", i.name, err) ++ return fmt.Errorf("containerd-shim syscall.O_WRONLY|syscall.O_APPEND: opening %s failed: %s", i.name, err) + } + if stdout == stderr { + sameFile = fw +@@ -113,7 +116,7 @@ func copyPipes(ctx context.Context, rio runc.IO, stdin, stdout, stderr string, w + } + f, err := fifo.OpenFifo(ctx, stdin, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { +- return fmt.Errorf("containerd-shim: opening %s failed: %s", stdin, err) ++ return fmt.Errorf("containerd-shim syscall.O_RDONLY|syscall.O_NONBLOCK: opening %s failed: %s", stdin, err) + } + cwg.Add(1) + go func() { +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index f8e30742..1b763fbc 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -300,6 +300,11 @@ func (r *Runtime) restoreTasks(ctx context.Context) ([]*Task, error) { + } + o = append(o, tasks...) + } ++ lo, err := r.loadLegacyTasks(o, ctx, "moby") ++ if err != nil { ++ logrus.Errorf("load legacy with error %v", err) ++ } ++ o = append(o, lo...) + return o, nil + } + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index ac545ea4..6411fdd9 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -34,6 +34,7 @@ import ( + "github.com/containerd/containerd/api/types/task" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/events" ++ "github.com/containerd/containerd/legacy" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/namespaces" +@@ -381,7 +382,9 @@ func (s *Service) Kill(ctx context.Context, r *shimapi.KillRequest) (*ptypes.Emp + if s.id != p.ID() || r.Signal != uint32(syscall.SIGKILL) { + return + } +- ++ if legacy.IsLegacy(s.id) { ++ return ++ } + for i := 1; i < 5; i++ { + time.Sleep(10 * time.Second) + err := p.Kill(ctx, r.Signal, r.All) +@@ -676,6 +679,11 @@ func newInit(ctx context.Context, path, workDir, runtimeRoot, namespace, criu st + + rootfs := filepath.Join(path, "rootfs") + runtime := proc.NewRunc(runtimeRoot, path, namespace, r.Runtime, criu, systemdCgroup) ++ // legacy container using /run/runc as runc root. ++ if legacy.IsLegacy(r.ID) { ++ runtime.Root = "/run/runc" ++ } ++ + p := proc.New(r.ID, runtime, rproc.Stdio{ + Stdin: r.Stdin, + Stdout: r.Stdout, +diff --git a/services/containers/local.go b/services/containers/local.go +index 95a09872..5934d5ad 100644 +--- a/services/containers/local.go ++++ b/services/containers/local.go +@@ -48,10 +48,11 @@ func init() { + if err != nil { + return nil, err + } +- return &local{ ++ helperLocal = local{ + db: m.(*metadata.DB), + publisher: ic.Events, +- }, nil ++ } ++ return &helperLocal, nil + }, + }) + } +@@ -243,3 +244,17 @@ func (s *localStream) SendMsg(m interface{}) error { + func (s *localStream) RecvMsg(m interface{}) error { + return nil + } ++ ++var helperLocal local // used for create meta only. ++// CreateMeta used only by legacy module to create meta. ++func CreateMeta(ctx context.Context, c containers.Container) error { ++ l := &helperLocal ++ err := l.withStoreUpdate(ctx, func(ctx context.Context, store containers.Store) error { ++ _, err := store.Create(ctx, c) ++ if err != nil { ++ return err ++ } ++ return nil ++ }) ++ return err ++} +-- +2.17.1 + diff --git a/patch/0047-containerd-shim-exit-initiative-after-3s.patch b/patch/0047-containerd-shim-exit-initiative-after-3s.patch new file mode 100644 index 0000000000000000000000000000000000000000..f8e325c643ecebce4c33a16960a0e39a5ce23bdf --- /dev/null +++ b/patch/0047-containerd-shim-exit-initiative-after-3s.patch @@ -0,0 +1,30 @@ +From fe778eb160fc1e3a492b5304890af3843aa91f32 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Tue, 5 Nov 2019 23:07:49 -0500 +Subject: [PATCH] containerd: containerd-shim exit initiative after 3s + +reason: containerd-shim exit initiative after 3s + +Signed-off-by: liuzekun +--- + runtime/v1/shim/service.go | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index 326096c..3abaa99 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -548,6 +548,10 @@ func (s *Service) checkProcesses(e runc.Exit) { + ns := filepath.Base(filepath.Dir(ip.Bundle)) + events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited") + events.InitExitWrite(ip.Bundle, e.Pid) ++ go func() { ++ time.Sleep(3 * time.Second) ++ os.Exit(0) ++ }() + } + if shouldKillAll { + if ip, ok := p.(*proc.Init); ok { +-- +2.20.1 + diff --git a/patch/0048-containerd-modify-shim-initiative-exit-time.patch b/patch/0048-containerd-modify-shim-initiative-exit-time.patch new file mode 100644 index 0000000000000000000000000000000000000000..f76982ceb4fe8cae4eee1a900ae5f2a9f48e1a17 --- /dev/null +++ b/patch/0048-containerd-modify-shim-initiative-exit-time.patch @@ -0,0 +1,32 @@ +From 1735262dfdbc434c3e734c2a4b7e3c5407cd541f Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Sat, 16 Nov 2019 02:28:31 +0800 +Subject: [PATCH] containerd: modify shim initiative exit time + +reason: We set shim exit initiative after 3s of container init process +exiting, but poststop hook will run abnormally if it needs more than 3s. +So we modify the exit time to 120s to avoid this case, as poststop hook +is suggested not more than 120s. + +Change-Id: I3e78b6344fabb0687bc40c3b6da153f403a9f211 +Signed-off-by: xiadanni1 +--- + runtime/v1/shim/service.go | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index 3abaa99..9721660 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -549,7 +549,7 @@ func (s *Service) checkProcesses(e runc.Exit) { + events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited") + events.InitExitWrite(ip.Bundle, e.Pid) + go func() { +- time.Sleep(3 * time.Second) ++ time.Sleep(120 * time.Second) + os.Exit(0) + }() + } +-- +1.8.3.1 + diff --git a/patch/0049-contaienrd-modify-shim-initiative-exit-time-for-post-hook.patch b/patch/0049-contaienrd-modify-shim-initiative-exit-time-for-post-hook.patch new file mode 100644 index 0000000000000000000000000000000000000000..9876c45570ef964e6ded188d939ea4de2563ca21 --- /dev/null +++ b/patch/0049-contaienrd-modify-shim-initiative-exit-time-for-post-hook.patch @@ -0,0 +1,84 @@ +From d2e10b3f23adf3338ee451c926167d18e5ac02e1 Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Thu, 21 Nov 2019 08:23:35 -0500 +Subject: [PATCH] contaienrd: modify shim initiative exit time for post hook + +reason: Modify shim initiative exit time for post hook. In consideration +of each post hook has a execution time with timeout(default 120s), we +should ensure enough time to call all post hook. + +Signed-off-by: liuzekun +--- + runtime/v1/shim/service.go | 29 ++++++++++++++++++++++------- + 1 file changed, 22 insertions(+), 7 deletions(-) + +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index 9721660..cfba225 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -537,7 +537,7 @@ func (s *Service) checkProcesses(e runc.Exit) { + s.mu.Lock() + defer s.mu.Unlock() + +- shouldKillAll, err := shouldKillAllOnExit(s.bundle) ++ shouldKillAll, bundleSpec, err := shouldKillAllOnExit(s.bundle) + if err != nil { + log.G(s.context).WithError(err).Error("failed to check shouldKillAll") + } +@@ -549,8 +549,23 @@ func (s *Service) checkProcesses(e runc.Exit) { + events.ExitAddFile(ns, events.ExitFile(s.id, uint32(e.Pid), uint32(e.Status)), "init exited") + events.InitExitWrite(ip.Bundle, e.Pid) + go func() { +- time.Sleep(120 * time.Second) +- os.Exit(0) ++ t := 30 ++ defer func() { ++ time.Sleep(time.Duration(t) * time.Second) ++ os.Exit(0) ++ }() ++ if bundleSpec.Hooks == nil { ++ return ++ } ++ postStopHooks := bundleSpec.Hooks.Poststop ++ for _, postStopHook := range postStopHooks { ++ hookTimeout := postStopHook.Timeout ++ if hookTimeout == nil { ++ t += 120 ++ } else { ++ t += *hookTimeout ++ } ++ } + }() + } + if shouldKillAll { +@@ -575,23 +590,23 @@ func (s *Service) checkProcesses(e runc.Exit) { + } + } + +-func shouldKillAllOnExit(bundlePath string) (bool, error) { ++func shouldKillAllOnExit(bundlePath string) (bool, specs.Spec, error) { + var bundleSpec specs.Spec + bundleConfigContents, err := ioutil.ReadFile(filepath.Join(bundlePath, "config.json")) + if err != nil { +- return false, err ++ return false, specs.Spec{}, err + } + json.Unmarshal(bundleConfigContents, &bundleSpec) + + if bundleSpec.Linux != nil { + for _, ns := range bundleSpec.Linux.Namespaces { + if ns.Type == specs.PIDNamespace && ns.Path == "" { +- return false, nil ++ return false, bundleSpec, nil + } + } + } + +- return true, nil ++ return true, bundleSpec, nil + } + + func (s *Service) getContainerPids(ctx context.Context, id string) ([]uint32, error) { +-- +2.20.1 + diff --git a/patch/0050-containerd-warp-and-process-return-errors.patch b/patch/0050-containerd-warp-and-process-return-errors.patch new file mode 100644 index 0000000000000000000000000000000000000000..fc20562bf343e556bb1e77e2cbe98eda7556265b --- /dev/null +++ b/patch/0050-containerd-warp-and-process-return-errors.patch @@ -0,0 +1,170 @@ +From 20cb595625dcfdf89fdf766028625a7864674dec Mon Sep 17 00:00:00 2001 +From: liuzekun +Date: Mon, 23 Dec 2019 03:10:49 -0500 +Subject: [PATCH] containerd: wrap and process return errors + +reason: wrap and process return errors + +Signed-off-by: liuzekun +--- + cmd/containerd-shim/main_unix.go | 2 +- + events/exit.go | 4 ++-- + legacy/legacy.go | 8 +++++--- + runtime/v1/linux/leruntime.go | 5 ++++- + runtime/v1/linux/runtime.go | 7 +++++-- + runtime/v1/shim/reaper.go | 4 ++-- + runtime/v1/shim/service.go | 1 + + vendor/github.com/sirupsen/logrus/exported.go | 5 +++++ + 8 files changed, 25 insertions(+), 11 deletions(-) + +diff --git a/cmd/containerd-shim/main_unix.go b/cmd/containerd-shim/main_unix.go +index 2228362..e9c1426 100644 +--- a/cmd/containerd-shim/main_unix.go ++++ b/cmd/containerd-shim/main_unix.go +@@ -259,7 +259,7 @@ func dumpStacks(logger *logrus.Entry) { + bufferLen *= 2 + } + buf = buf[:stackSize] +- ioutil.WriteFile(fmt.Sprintf(stacksLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1)), buf, 0600) ++ logrus.Devour(ioutil.WriteFile(fmt.Sprintf(stacksLogNameTemplate, strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1)), buf, 0600)) + logger.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) + } + +diff --git a/events/exit.go b/events/exit.go +index 772dc24..c0a3583 100644 +--- a/events/exit.go ++++ b/events/exit.go +@@ -48,13 +48,14 @@ func ExitInfo(ef string) (string, uint32, uint32) { + } + + func ExitAddFile(ns string, ef string, reason string) { +- os.MkdirAll(filepath.Join(ExitDir, ns), 0700) ++ logrus.Devour(os.MkdirAll(filepath.Join(ExitDir, ns), 0700)) + err := ioutil.WriteFile(filepath.Join(ExitDir, ns, ef), []byte{}, 0600) + logrus.Infof("exit-add %s/%s [reason: %s] error=%v", ns, ef, reason, err) + } + + func ExitDelFile(ns string, ef string) { + err := os.RemoveAll(filepath.Join(ExitDir, ns, ef)) ++ logrus.Devour(err) + logrus.Infof("exit-del %s/%s error=%v", ns, ef, err) + } + +diff --git a/legacy/legacy.go b/legacy/legacy.go +index fde9f70..219508c 100644 +--- a/legacy/legacy.go ++++ b/legacy/legacy.go +@@ -17,8 +17,8 @@ import ( + "runtime" + "strings" + +- "github.com/sirupsen/logrus" + "github.com/opencontainers/runtime-spec/specs-go" ++ "github.com/sirupsen/logrus" + ) + + const ( +@@ -107,8 +107,10 @@ func InitBundle(root string, id string) error { + if err != nil { + return err + } +- CopyFile(Config120+id+"/config.json", Config028+id+"/config.json") +- CopyFile(Config120+id+"/init.pid", State028+id+"/init/pid") ++ _, err = CopyFile(Config120+id+"/config.json", Config028+id+"/config.json") ++ logrus.Devour(err) ++ _, err = CopyFile(Config120+id+"/init.pid", State028+id+"/init/pid") ++ logrus.Devour(err) + return nil + } + +diff --git a/runtime/v1/linux/leruntime.go b/runtime/v1/linux/leruntime.go +index 9c793a5..e8fbe61 100644 +--- a/runtime/v1/linux/leruntime.go ++++ b/runtime/v1/linux/leruntime.go +@@ -112,7 +112,10 @@ func (r *Runtime) legacyCreate(ctx context.Context, id string, opts runtime.Crea + // Task was never started or was already successfully deleted + return + } +- lc := t.(*Task) ++ lc, ok := t.(*Task) ++ if !ok { ++ log.G(ctx).WithField("id", id).Errorf("task t's type is %T, cannot convert to a *Task value", t) ++ } + + log.G(ctx).WithFields(logrus.Fields{ + "id": id, +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 1b763fb..c334bf4 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -43,7 +43,7 @@ import ( + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/runtime" + "github.com/containerd/containerd/runtime/linux/runctypes" +- "github.com/containerd/containerd/runtime/v1" ++ v1 "github.com/containerd/containerd/runtime/v1" + "github.com/containerd/containerd/runtime/v1/linux/proc" + shim "github.com/containerd/containerd/runtime/v1/shim/v1" + runc "github.com/containerd/go-runc" +@@ -200,7 +200,10 @@ func (r *Runtime) Create(ctx context.Context, id string, opts runtime.CreateOpts + // Task was never started or was already successfully deleted + return + } +- lc := t.(*Task) ++ lc, ok := t.(*Task) ++ if !ok { ++ log.G(ctx).WithField("id", id).Errorf("task t's type is %T, cannot convert to a *Task value", t) ++ } + + log.G(ctx).WithFields(logrus.Fields{ + "id": id, +diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go +index 2846152..c657397 100644 +--- a/runtime/v1/shim/reaper.go ++++ b/runtime/v1/shim/reaper.go +@@ -95,7 +95,7 @@ func (m *Monitor) Wait(c *exec.Cmd, ec chan runc.Exit) (int, error) { + for e := range ec { + if e.Pid == c.Process.Pid { + // make sure we flush all IO +- c.Wait() ++ logrus.Devour(c.Wait()) + m.Unsubscribe(ec) + return e.Status, nil + } +@@ -123,7 +123,7 @@ func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, e + select { + case <-time.After(time.Duration(sec) * time.Second): + if SameProcess(c, c.Process.Pid) { +- syscall.Kill(c.Process.Pid, syscall.SIGKILL) ++ logrus.Devour(syscall.Kill(c.Process.Pid, syscall.SIGKILL)) + } + return 0, errors.Errorf("container did not start before the specified timeout %ds for cmd(pid=%d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) + case status := <-sch: +diff --git a/runtime/v1/shim/service.go b/runtime/v1/shim/service.go +index 4025a72..beb0ed8 100644 +--- a/runtime/v1/shim/service.go ++++ b/runtime/v1/shim/service.go +@@ -146,6 +146,7 @@ func (s *Service) Create(ctx context.Context, r *shimapi.CreateTaskRequest) (_ * + for i := 0; i < 60; i++ { + time.Sleep(time.Second) + _, err := os.Stat(r.Bundle) ++ logrus.Devour(err) + if os.IsNotExist(err) { + logrus.Errorf("bundle dir: %v does not exist, containerd-shim exit", r.Bundle) + os.Exit(0) +diff --git a/vendor/github.com/sirupsen/logrus/exported.go b/vendor/github.com/sirupsen/logrus/exported.go +index 1aeaa90..46fa7f8 100644 +--- a/vendor/github.com/sirupsen/logrus/exported.go ++++ b/vendor/github.com/sirupsen/logrus/exported.go +@@ -191,3 +191,8 @@ func Panicln(args ...interface{}) { + func Fatalln(args ...interface{}) { + std.Fatalln(args...) + } ++ ++// Devour will do nothing and return directly ++func Devour(args ...interface{}) { ++ return ++} +-- +2.20.1 + diff --git a/patch/0051-containerd-add-timeout-for-containerd-shim.patch b/patch/0051-containerd-add-timeout-for-containerd-shim.patch new file mode 100644 index 0000000000000000000000000000000000000000..62125c1cf36fd9a1679b88c074c141150f9959c3 --- /dev/null +++ b/patch/0051-containerd-add-timeout-for-containerd-shim.patch @@ -0,0 +1,134 @@ +From ea6e8c7b10fe1552d14fb9b0337d850a1f4a7178 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Fri, 3 Jan 2020 03:06:00 +0800 +Subject: [PATCH] containerd: add timeout for containerd-shim + +reason:add timeout for containerd-shim to avoid dead lock + +Change-Id: I7886eb9e73dc1a3c8b837687c8ac8361d67f5e4f +Signed-off-by: xiadanni1 +--- + runtime/v1/shim/reaper.go | 2 +- + vendor/github.com/containerd/go-runc/runc.go | 37 ++++++++++++++++++++++------ + 2 files changed, 30 insertions(+), 9 deletions(-) + +diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go +index c657397..d8e8274 100644 +--- a/runtime/v1/shim/reaper.go ++++ b/runtime/v1/shim/reaper.go +@@ -125,7 +125,7 @@ func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, e + if SameProcess(c, c.Process.Pid) { + logrus.Devour(syscall.Kill(c.Process.Pid, syscall.SIGKILL)) + } +- return 0, errors.Errorf("container did not start before the specified timeout %ds for cmd(pid=%d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) ++ return 0, errors.Errorf("timeout %ds for cmd(pid=%d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) + case status := <-sch: + return status, nil + case err := <-ech: +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index 7a2a8c4..430648d 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -53,7 +53,9 @@ const ( + Text Format = "text" + // DefaultCommand is the default command for Runc + DefaultCommand = "runc" +- execTimeout = 30 ++ defaultTimeout = 30 ++ startTimeout = 120 ++ updateTimeout = 60 + ) + + var ( +@@ -99,7 +101,7 @@ func (r *Runc) List(context context.Context) ([]*Container, error) { + + // State returns the state for the container provided by id + func (r *Runc) State(context context.Context, id string) (*Container, error) { +- data, err := cmdOutput(r.command(context, "state", id), true) ++ data, err := cmdOutputTimeout(r.command(context, "state", id), true, defaultTimeout) + if err != nil { + return nil, fmt.Errorf("%s: %s", err, data) + } +@@ -199,7 +201,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp + + // Start will start an already created container + func (r *Runc) Start(context context.Context, id string) error { +- return r.runOrError(r.command(context, "start", id)) ++ return r.runOrErrorTimeout(r.command(context, "start", id), startTimeout) + } + + type ExecOpts struct { +@@ -252,7 +254,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts + opts.Set(cmd) + } + if cmd.Stdout == nil && cmd.Stderr == nil { +- data, err := cmdOutputTimeout(cmd, true, execTimeout) ++ data, err := cmdOutputTimeout(cmd, true, defaultTimeout) + if err != nil { + return fmt.Errorf("%s: %s", err, data) + } +@@ -269,7 +271,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts + } + } + } +- status, err := Monitor.WaitTimeout(cmd, ec, execTimeout) ++ status, err := Monitor.WaitTimeout(cmd, ec, defaultTimeout) + if err == nil && status != 0 { + err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) + } +@@ -338,7 +340,7 @@ func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) + if opts != nil { + args = append(args, opts.args()...) + } +- return r.runOrError(r.command(context, append(args, id, strconv.Itoa(sig))...)) ++ return r.runOrErrorTimeout(r.command(context, append(args, id, strconv.Itoa(sig))...), defaultTimeout) + } + + // Stats return the stats for a container like cpu, memory, and io +@@ -414,7 +416,7 @@ func (r *Runc) Resume(context context.Context, id string) error { + + // Ps lists all the processes inside the container returning their pids + func (r *Runc) Ps(context context.Context, id string) ([]int, error) { +- data, err := cmdOutput(r.command(context, "ps", "--format", "json", id), true) ++ data, err := cmdOutputTimeout(r.command(context, "ps", "--format", "json", id), true, defaultTimeout) + if err != nil { + return nil, fmt.Errorf("%s: %s", err, data) + } +@@ -604,7 +606,7 @@ func (r *Runc) Update(context context.Context, id string, resources *specs.Linux + args := []string{"update", "--resources", "-", id} + cmd := r.command(context, args...) + cmd.Stdin = buf +- return r.runOrError(cmd) ++ return r.runOrErrorTimeout(cmd, updateTimeout) + } + + var ErrParseRuncVersion = errors.New("unable to parse runc version") +@@ -705,6 +707,25 @@ func (r *Runc) runOrError(cmd *exec.Cmd) error { + return nil + } + ++func (r *Runc) runOrErrorTimeout(cmd *exec.Cmd, runTimeout int64) error { ++ if cmd.Stdout != nil || cmd.Stderr != nil { ++ ec, err := Monitor.Start(cmd) ++ if err != nil { ++ return err ++ } ++ status, err := Monitor.WaitTimeout(cmd, ec, runTimeout) ++ if err == nil && status != 0 { ++ err = fmt.Errorf("%s did not terminate sucessfully", cmd.Args[0]) ++ } ++ return err ++ } ++ data, err := cmdOutputTimeout(cmd, true, runTimeout) ++ if err != nil { ++ return fmt.Errorf("%s: %s", err, data) ++ } ++ return nil ++} ++ + func cmdOutput(cmd *exec.Cmd, combined bool) ([]byte, error) { + b := getBuf() + defer putBuf(b) +-- +1.8.3.1 + diff --git a/patch/0052-containerd-modify-runtime-root-if-containe.patch b/patch/0052-containerd-modify-runtime-root-if-containe.patch new file mode 100644 index 0000000000000000000000000000000000000000..92d5a636fcba32930d52c5f0678afcd22ed46f9e --- /dev/null +++ b/patch/0052-containerd-modify-runtime-root-if-containe.patch @@ -0,0 +1,266 @@ +From 3ccf18b7d72ef484093e8a6f578ef9381418bc54 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Fri, 17 Jan 2020 07:07:34 +0800 +Subject: [PATCH] containerd: modify runtime root if container is created by + 1.11.2 + +reason:if container is created by 1.11.2, runtime root is /run/runc, +so we need to modify the root dir when this container stops first time. + +Change-Id: If30e26a719ed61be0a08344860a066ab77b4cb40 +Signed-off-by: xiadanni1 +--- + runtime/v1/linux/runtime.go | 14 ++++--- + .../github.com/containerd/go-runc/command_linux.go | 4 +- + .../github.com/containerd/go-runc/command_other.go | 2 +- + vendor/github.com/containerd/go-runc/runc.go | 45 ++++++++++++---------- + 4 files changed, 37 insertions(+), 28 deletions(-) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index c334bf4..08e563d 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -35,6 +35,7 @@ import ( + "github.com/containerd/containerd/events" + "github.com/containerd/containerd/events/exchange" + "github.com/containerd/containerd/identifiers" ++ "github.com/containerd/containerd/legacy" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/metadata" + "github.com/containerd/containerd/mount" +@@ -521,11 +522,14 @@ func (r *Runtime) terminate(ctx context.Context, bundle *bundle, ns, id string) + }); err != nil { + log.G(ctx).WithError(err).Warnf("delete runtime state %s", id) + } +- if err := mount.Unmount(filepath.Join(bundle.path, "rootfs"), 0); err != nil { +- log.G(ctx).WithError(err).WithFields(logrus.Fields{ +- "path": bundle.path, +- "id": id, +- }).Warnf("unmount task rootfs") ++ ++ if !legacy.IsLegacy(id) { ++ if err := mount.Unmount(filepath.Join(bundle.path, "rootfs"), 0); err != nil { ++ log.G(ctx).WithError(err).WithFields(logrus.Fields{ ++ "path": bundle.path, ++ "id": id, ++ }).Warnf("unmount task rootfs") ++ } + } + return nil + } +diff --git a/vendor/github.com/containerd/go-runc/command_linux.go b/vendor/github.com/containerd/go-runc/command_linux.go +index 6ad27be..0aa6040 100644 +--- a/vendor/github.com/containerd/go-runc/command_linux.go ++++ b/vendor/github.com/containerd/go-runc/command_linux.go +@@ -31,12 +31,12 @@ func (r *Runc) isrunv() bool { + return false + } + +-func (r *Runc) command(context context.Context, args ...string) *exec.Cmd { ++func (r *Runc) command(id string, context context.Context, args ...string) *exec.Cmd { + command := r.Command + if command == "" { + command = DefaultCommand + } +- cmd := exec.CommandContext(context, command, append(r.args(), args...)...) ++ cmd := exec.CommandContext(context, command, append(r.args(id), args...)...) + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: r.Setpgid, + } +diff --git a/vendor/github.com/containerd/go-runc/command_other.go b/vendor/github.com/containerd/go-runc/command_other.go +index b8fd4b8..21bb699 100644 +--- a/vendor/github.com/containerd/go-runc/command_other.go ++++ b/vendor/github.com/containerd/go-runc/command_other.go +@@ -29,7 +29,7 @@ func (r *Runc) command(context context.Context, args ...string) *exec.Cmd { + if command == "" { + command = DefaultCommand + } +- cmd := exec.CommandContext(context, command, append(r.args(), args...)...) ++ cmd := exec.CommandContext(context, command, append(r.args(""), args...)...) + cmd.Env = os.Environ() + return cmd + } +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index 430648d..c1748ff 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -31,6 +31,7 @@ import ( + "syscall" + "time" + ++ "github.com/containerd/containerd/legacy" + specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/sirupsen/logrus" + ) +@@ -88,7 +89,7 @@ func init() { + + // List returns all containers created inside the provided runc root directory + func (r *Runc) List(context context.Context) ([]*Container, error) { +- data, err := cmdOutput(r.command(context, "list", "--format=json"), false) ++ data, err := cmdOutput(r.command("", context, "list", "--format=json"), false) + if err != nil { + return nil, err + } +@@ -101,7 +102,7 @@ func (r *Runc) List(context context.Context) ([]*Container, error) { + + // State returns the state for the container provided by id + func (r *Runc) State(context context.Context, id string) (*Container, error) { +- data, err := cmdOutputTimeout(r.command(context, "state", id), true, defaultTimeout) ++ data, err := cmdOutputTimeout(r.command(id, context, "state", id), true, defaultTimeout) + if err != nil { + return nil, fmt.Errorf("%s: %s", err, data) + } +@@ -168,7 +169,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp + } + args = append(args, oargs...) + } +- cmd := r.command(context, append(args, id)...) ++ cmd := r.command(id, context, append(args, id)...) + if opts != nil && opts.IO != nil { + opts.Set(cmd) + } +@@ -201,7 +202,7 @@ func (r *Runc) Create(context context.Context, id, bundle string, opts *CreateOp + + // Start will start an already created container + func (r *Runc) Start(context context.Context, id string) error { +- return r.runOrErrorTimeout(r.command(context, "start", id), startTimeout) ++ return r.runOrErrorTimeout(r.command(id, context, "start", id), startTimeout) + } + + type ExecOpts struct { +@@ -249,7 +250,7 @@ func (r *Runc) Exec(context context.Context, id string, spec specs.Process, opts + } + args = append(args, oargs...) + } +- cmd := r.command(context, append(args, id)...) ++ cmd := r.command(id, context, append(args, id)...) + if opts != nil && opts.IO != nil { + opts.Set(cmd) + } +@@ -289,7 +290,7 @@ func (r *Runc) Run(context context.Context, id, bundle string, opts *CreateOpts) + } + args = append(args, oargs...) + } +- cmd := r.command(context, append(args, id)...) ++ cmd := r.command(id, context, append(args, id)...) + if opts != nil && opts.IO != nil { + opts.Set(cmd) + } +@@ -317,7 +318,7 @@ func (r *Runc) Delete(context context.Context, id string, opts *DeleteOpts) erro + if opts != nil { + args = append(args, opts.args()...) + } +- return r.runOrError(r.command(context, append(args, id)...)) ++ return r.runOrError(r.command(id, context, append(args, id)...)) + } + + // KillOpts specifies options for killing a container and its processes +@@ -340,12 +341,12 @@ func (r *Runc) Kill(context context.Context, id string, sig int, opts *KillOpts) + if opts != nil { + args = append(args, opts.args()...) + } +- return r.runOrErrorTimeout(r.command(context, append(args, id, strconv.Itoa(sig))...), defaultTimeout) ++ return r.runOrErrorTimeout(r.command(id, context, append(args, id, strconv.Itoa(sig))...), defaultTimeout) + } + + // Stats return the stats for a container like cpu, memory, and io + func (r *Runc) Stats(context context.Context, id string) (*Stats, error) { +- cmd := r.command(context, "events", "--stats", id) ++ cmd := r.command(id, context, "events", "--stats", id) + rd, err := cmd.StdoutPipe() + if err != nil { + return nil, err +@@ -367,7 +368,7 @@ func (r *Runc) Stats(context context.Context, id string) (*Stats, error) { + + // Events returns an event stream from runc for a container with stats and OOM notifications + func (r *Runc) Events(context context.Context, id string, interval time.Duration) (chan *Event, error) { +- cmd := r.command(context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id) ++ cmd := r.command(id, context, "events", fmt.Sprintf("--interval=%ds", int(interval.Seconds())), id) + rd, err := cmd.StdoutPipe() + if err != nil { + return nil, err +@@ -406,17 +407,17 @@ func (r *Runc) Events(context context.Context, id string, interval time.Duration + + // Pause the container with the provided id + func (r *Runc) Pause(context context.Context, id string) error { +- return r.runOrError(r.command(context, "pause", id)) ++ return r.runOrError(r.command(id, context, "pause", id)) + } + + // Resume the container with the provided id + func (r *Runc) Resume(context context.Context, id string) error { +- return r.runOrError(r.command(context, "resume", id)) ++ return r.runOrError(r.command(id, context, "resume", id)) + } + + // Ps lists all the processes inside the container returning their pids + func (r *Runc) Ps(context context.Context, id string) ([]int, error) { +- data, err := cmdOutputTimeout(r.command(context, "ps", "--format", "json", id), true, defaultTimeout) ++ data, err := cmdOutputTimeout(r.command(id, context, "ps", "--format", "json", id), true, defaultTimeout) + if err != nil { + return nil, fmt.Errorf("%s: %s", err, data) + } +@@ -429,7 +430,7 @@ func (r *Runc) Ps(context context.Context, id string) ([]int, error) { + + // Top lists all the processes inside the container returning the full ps data + func (r *Runc) Top(context context.Context, id string, psOptions string) (*TopResults, error) { +- data, err := cmdOutput(r.command(context, "ps", "--format", "table", id, psOptions), true) ++ data, err := cmdOutput(r.command(id, context, "ps", "--format", "table", id, psOptions), true) + if err != nil { + return nil, fmt.Errorf("%s: %s", err, data) + } +@@ -528,7 +529,7 @@ func (r *Runc) Checkpoint(context context.Context, id string, opts *CheckpointOp + for _, a := range actions { + args = a(args) + } +- return r.runOrError(r.command(context, append(args, id)...)) ++ return r.runOrError(r.command(id, context, append(args, id)...)) + } + + type RestoreOpts struct { +@@ -577,7 +578,7 @@ func (r *Runc) Restore(context context.Context, id, bundle string, opts *Restore + args = append(args, oargs...) + } + args = append(args, "--bundle", bundle) +- cmd := r.command(context, append(args, id)...) ++ cmd := r.command(id, context, append(args, id)...) + if opts != nil && opts.IO != nil { + opts.Set(cmd) + } +@@ -604,7 +605,7 @@ func (r *Runc) Update(context context.Context, id string, resources *specs.Linux + return err + } + args := []string{"update", "--resources", "-", id} +- cmd := r.command(context, args...) ++ cmd := r.command(id, context, args...) + cmd.Stdin = buf + return r.runOrErrorTimeout(cmd, updateTimeout) + } +@@ -619,7 +620,7 @@ type Version struct { + + // Version returns the runc and runtime-spec versions + func (r *Runc) Version(context context.Context) (Version, error) { +- data, err := cmdOutput(r.command(context, "--version"), false) ++ data, err := cmdOutput(r.command("", context, "--version"), false) + if err != nil { + return Version{}, err + } +@@ -658,9 +659,13 @@ func parseVersion(data []byte) (Version, error) { + return v, nil + } + +-func (r *Runc) args() (out []string) { ++func (r *Runc) args(id string) (out []string) { + if r.Root != "" { +- out = append(out, "--root", r.Root) ++ if id != "" && legacy.IsLegacy(id) { ++ out = append(out, "--root", "/run/runc") ++ } else { ++ out = append(out, "--root", r.Root) ++ } + } + if r.Debug { + out = append(out, "--debug") +-- +1.8.3.1 + diff --git a/patch/0053-containerd-add-pid-check-to-avoid-poststop-ho.patch b/patch/0053-containerd-add-pid-check-to-avoid-poststop-ho.patch new file mode 100644 index 0000000000000000000000000000000000000000..fddaf5679877df373593fc5bbc3d88aa9323c2d8 --- /dev/null +++ b/patch/0053-containerd-add-pid-check-to-avoid-poststop-ho.patch @@ -0,0 +1,81 @@ +From 489f69209650aa743ffd6e53571b822ad0b63c2d Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Sat, 18 Jan 2020 04:18:22 +0800 +Subject: [PATCH] containerd: add pid check to avoid poststop hook + execute twice + +reason:If start a container at docker 1.11.2, upgrade docker to 18.09, +downgrade to 1.11.2, stop/restart container, upgrade to 18.09 again, +poststop hook will execute again when containerd load task. +So we add pid check to avoid poststop hook execute twice. + +Change-Id: I8b88b69bfa0a4141bd9595da8ad4e786666e114b +Signed-off-by: xiadanni1 +--- + legacy/legacy.go | 21 +++++++++++++++++++++ + runtime/v1/linux/runtime.go | 10 ++++++---- + 2 files changed, 27 insertions(+), 4 deletions(-) + +diff --git a/legacy/legacy.go b/legacy/legacy.go +index 219508c..644f94a 100644 +--- a/legacy/legacy.go ++++ b/legacy/legacy.go +@@ -44,6 +44,25 @@ func IsLegacy(id string) bool { + return false + } + ++func IsSamePid(id string) bool { ++ pid120, err := ioutil.ReadFile(filepath.Join(Config120, id, "init.pid")) ++ if err != nil { ++ logrus.Infof("read 1.2.0 init.pid file error: %v", err) ++ return false ++ } ++ pid028, err := ioutil.ReadFile(filepath.Join(State028, id, "init", "pid")) ++ if err != nil { ++ logrus.Infof("read 0.2.8 pid file error: %v", err) ++ return false ++ } ++ logrus.Infof("pid1.2.0: %v, pid0.2.8: %v", string(pid120), string(pid028)) ++ if string(pid120) != string(pid028) { ++ return false ++ } ++ ++ return true ++} ++ + // IsRunning is used to detect whether legacy container is running. + func IsRunning(id string) bool { + path := State028 + id + "/init/pid" +@@ -111,6 +130,8 @@ func InitBundle(root string, id string) error { + logrus.Devour(err) + _, err = CopyFile(Config120+id+"/init.pid", State028+id+"/init/pid") + logrus.Devour(err) ++ _, err = CopyFile(Config120+id+"/starttime", State028+id+"/init/starttime") ++ logrus.Devour(err) + return nil + } + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 08e563d..96ad815 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -517,10 +517,12 @@ func (r *Runtime) terminate(ctx context.Context, bundle *bundle, ns, id string) + return err + } + +- if err := rt.Delete(ctx, id, &runc.DeleteOpts{ +- Force: true, +- }); err != nil { +- log.G(ctx).WithError(err).Warnf("delete runtime state %s", id) ++ if !legacy.IsLegacy(id) || legacy.IsSamePid(id) { ++ if err := rt.Delete(ctx, id, &runc.DeleteOpts{ ++ Force: true, ++ }); err != nil { ++ log.G(ctx).WithError(err).Warnf("delete runtime state %s", id) ++ } + } + + if !legacy.IsLegacy(id) { +-- +1.8.3.1 + diff --git a/patch/0054-containerd-clean-up-residual-container.patch b/patch/0054-containerd-clean-up-residual-container.patch new file mode 100644 index 0000000000000000000000000000000000000000..645b90f7bd849939344c878f9850c7a59b6814ee --- /dev/null +++ b/patch/0054-containerd-clean-up-residual-container.patch @@ -0,0 +1,100 @@ +From a2310cbcff07f660b8d17584f687561b64bf27ad Mon Sep 17 00:00:00 2001 +From: zhangtianyang +Date: Thu, 27 Feb 2020 16:51:59 +0800 +Subject: [PATCH] containerd: clean up residual container after + shim abnormal exit + +reason:from update/revert test an occasional failure has been found that +shim process has exited but container is still running, then following exec +call all report ttrpc close error. +the triggering condition is uncertain. this patch will make up the clean +work of the residual container after such failure occurred to avoid +subsequent call errors. + +Change-Id: I0da9d4e46010cbe58f2fda21895caeb301936c47 +Signed-off-by: zhangtianyang +--- + runtime/v1/linux/runtime.go | 11 +++++++++++ + services/tasks/local.go | 25 +++++++++++++++++++++++++ + 2 files changed, 36 insertions(+) + +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 96ad815..47a0cb6 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -511,6 +511,17 @@ func (r *Runtime) cleanupAfterDeadShim(ctx context.Context, bundle *bundle, ns, + return nil + } + ++func (r *Runtime) CleanupAfterDeadShim(ctx context.Context, ns, id string) error { ++ bund := &bundle{id: id, ++ path: filepath.Join(r.state, ns, id), ++ workDir: filepath.Join(r.root, ns, id)} ++ pid, err := runc.ReadPidFile(filepath.Join(bund.path, proc.InitPidFile)) ++ if err != nil { ++ return fmt.Errorf("failed to read pid from %s", proc.InitPidFile) ++ } ++ return r.cleanupAfterDeadShim(ctx, bund, ns, id, pid) ++} ++ + func (r *Runtime) terminate(ctx context.Context, bundle *bundle, ns, id string) error { + rt, err := r.getRuntime(ctx, ns, id) + if err != nil { +diff --git a/services/tasks/local.go b/services/tasks/local.go +index 990e841..9818971 100644 +--- a/services/tasks/local.go ++++ b/services/tasks/local.go +@@ -24,6 +24,7 @@ import ( + "io/ioutil" + "os" + "path/filepath" ++ "strings" + "time" + + api "github.com/containerd/containerd/api/services/tasks/v1" +@@ -41,6 +42,7 @@ import ( + "github.com/containerd/containerd/mount" + "github.com/containerd/containerd/plugin" + "github.com/containerd/containerd/runtime" ++ "github.com/containerd/containerd/runtime/v1/linux" + "github.com/containerd/containerd/runtime/v2" + "github.com/containerd/containerd/services" + "github.com/containerd/typeurl" +@@ -383,11 +385,34 @@ func (l *local) Kill(ctx context.Context, r *api.KillRequest, _ ...grpc.CallOpti + } + } + if err := p.Kill(ctx, r.Signal, r.All); err != nil { ++ if (r.Signal == 9 || r.Signal == 15) && strings.Contains(err.Error(), "ttrpc: client shutting down") { ++ // not sure under what conditions will cause such ttrpc error. since the error has ++ // happened, we have to make up the clean up work to avoid container residue. ++ cleanErr := l.cleanupResidualContainer(ctx, r, t.Namespace()) ++ log.G(ctx).WithField("clean error", cleanErr).Warnf( ++ "previous actions might encounter failure, try clean up the dead container.") ++ } + return nil, errdefs.ToGRPC(err) + } + return empty, nil + } + ++func (l *local) cleanupResidualContainer(ctx context.Context, r *api.KillRequest, namespace string) error { ++ container, err := l.getContainer(ctx, r.ContainerID) ++ if err != nil { ++ return fmt.Errorf("failed to get container %s, %v", r.ContainerID, err) ++ } ++ rt, err := l.getRuntime(container.Runtime.Name) ++ if err != nil { ++ return fmt.Errorf("failed to get runtime %s, %v", container.Runtime.Name, err) ++ } ++ lRuntime, ok := rt.(*linux.Runtime) ++ if !ok { ++ return fmt.Errorf("no clean work for runtime other than linux ones") ++ } ++ return lRuntime.CleanupAfterDeadShim(ctx, namespace, r.ContainerID) ++} ++ + func (l *local) ListPids(ctx context.Context, r *api.ListPidsRequest, _ ...grpc.CallOption) (*api.ListPidsResponse, error) { + t, err := l.getTask(ctx, r.ContainerID) + if err != nil { +-- +1.8.3.1 + diff --git a/patch/0055-containerd-add-LLT-for-containerd-shim-timeou.patch b/patch/0055-containerd-add-LLT-for-containerd-shim-timeou.patch new file mode 100644 index 0000000000000000000000000000000000000000..0daafda48fc514f6680e3008659324dddf77bb8c --- /dev/null +++ b/patch/0055-containerd-add-LLT-for-containerd-shim-timeou.patch @@ -0,0 +1,115 @@ +From 47e981ebb8996e432968ed68f08e3fc108210cd4 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Tue, 3 Mar 2020 06:29:56 +0800 +Subject: [PATCH 1/2] containerd:add LLT for containerd-shim timeout + requirement + +reason:add LLT testcases for containerd-shim timeout requirement. + +Change-Id: If422542b72f3550d86a6eba6b19d0cdea2d2a660 +Signed-off-by: xiadanni1 + +--- + vendor/github.com/containerd/go-runc/runc_test.go | 90 +++++++++++++++++++++++ + 1 file changed, 90 insertions(+) + create mode 100644 vendor/github.com/containerd/go-runc/runc_test.go + +diff --git a/vendor/github.com/containerd/go-runc/runc_test.go b/vendor/github.com/containerd/go-runc/runc_test.go +new file mode 100644 +index 0000000..8f9212d +--- /dev/null ++++ b/vendor/github.com/containerd/go-runc/runc_test.go +@@ -0,0 +1,90 @@ ++package runc ++ ++import ( ++ "context" ++ "os" ++ "os/exec" ++ "testing" ++ ++ specs "github.com/opencontainers/runtime-spec/specs-go" ++) ++ ++func TestRuncCommandInvoke(t *testing.T) { ++ rc := &Runc{ ++ Command: "/bin/true", ++ } ++ ctx := context.Background() ++ id := "containerid" ++ bundle := "bundlepath" ++ ++ createOpts := CreateOpts{} ++ err := rc.Create(ctx, id, bundle, &createOpts) ++ if err != nil { ++ t.Errorf("Create command invoke error, %v", err) ++ } ++ ++ err = rc.Start(ctx, id) ++ if err != nil { ++ t.Errorf("Start command invoke error, %v", err) ++ } ++ ++ execSpec := specs.Process{} ++ nullIO, _ := NewNullIO() ++ execOpts := ExecOpts{IO: nullIO} ++ err = rc.Exec(ctx, id, execSpec, &execOpts) ++ if err != nil { ++ t.Errorf("Exec command invoke error, %v", err) ++ } ++ ++ execOptsnil := ExecOpts{} ++ err = rc.Exec(ctx, id, execSpec, &execOptsnil) ++ if err != nil { ++ t.Errorf("Exec command invoke error, %v", err) ++ } ++ ++ killOpts := KillOpts{} ++ err = rc.Kill(ctx, id, 9, &killOpts) ++ if err != nil { ++ t.Errorf("Kill command invoke error, %v", err) ++ } ++ ++ resource := specs.LinuxResources{} ++ err = rc.Update(ctx, id, &resource) ++ if err != nil { ++ t.Errorf("Update command invoke error, %v", err) ++ } ++ ++ _, err = rc.State(ctx, id) ++ if err == nil { ++ t.Errorf("State command invoke should return error") ++ } ++ ++ _, err = rc.Ps(ctx, id) ++ if err == nil { ++ t.Errorf("Ps command invoke should return error") ++ } ++} ++ ++func TestRunOrErrorTimeout(t *testing.T) { ++ rc := &Runc{} ++ ++ cmd := exec.Cmd{Path: "/bin/bash2"} ++ cmd.Stdout = os.Stdout ++ err := rc.runOrErrorTimeout(&cmd, 10) ++ if err == nil { ++ t.Errorf("runOrErrorTimeout should return error") ++ } ++ ++ cmd = exec.Cmd{Path: "/usr/bin/sleep", Args: []string{"2"}} ++ cmd.Stdout = os.Stdout ++ rc.runOrErrorTimeout(&cmd, 1) ++ if err == nil { ++ t.Errorf("runOrErrorTimeout should return error") ++ } ++ ++ cmd = exec.Cmd{Path: "/usr/bin/sleep", Args: []string{"2"}} ++ rc.runOrErrorTimeout(&cmd, 1) ++ if err == nil { ++ t.Errorf("runOrErrorTimeout should return error") ++ } ++} +-- +1.8.3.1 + diff --git a/patch/0056-containerd-save-dumpstack-to-file.patch b/patch/0056-containerd-save-dumpstack-to-file.patch new file mode 100644 index 0000000000000000000000000000000000000000..2a67217c00cbaaec3f946c6bf7c7866043bb60b4 --- /dev/null +++ b/patch/0056-containerd-save-dumpstack-to-file.patch @@ -0,0 +1,51 @@ +From 7db93cf813023f2a5ac209617aaae5c3f5c202d5 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Tue, 3 Mar 2020 09:01:22 +0800 +Subject: [PATCH] containerd:save dumpstack to file + +Change-Id: I54a41a13b4523de279337a9ff208347859c0fb4d +Signed-off-by: xiadanni1 +--- + cmd/containerd/command/main_unix.go | 5 +++++ + runtime/v1/linux/runtime.go | 1 + + 2 files changed, 6 insertions(+) + +diff --git a/cmd/containerd/command/main_unix.go b/cmd/containerd/command/main_unix.go +index 12c1426..2f9398f 100644 +--- a/cmd/containerd/command/main_unix.go ++++ b/cmd/containerd/command/main_unix.go +@@ -20,8 +20,12 @@ package command + + import ( + "context" ++ "fmt" ++ "io/ioutil" + "os" + "runtime" ++ "strings" ++ "time" + + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/services/server" +@@ -79,5 +83,6 @@ func dumpStacks() { + bufferLen *= 2 + } + buf = buf[:stackSize] ++ logrus.Devour(ioutil.WriteFile(fmt.Sprintf("/var/run/docker/containerd/containerd-stacks-%s.log", strings.Replace(time.Now().Format(time.RFC3339), ":", "", -1)), buf, 0600)) + logrus.Infof("=== BEGIN goroutine stack dump ===\n%s\n=== END goroutine stack dump ===", buf) + } +diff --git a/runtime/v1/linux/runtime.go b/runtime/v1/linux/runtime.go +index 47a0cb6..5be785d 100644 +--- a/runtime/v1/linux/runtime.go ++++ b/runtime/v1/linux/runtime.go +@@ -481,6 +481,7 @@ func (r *Runtime) loadTasks(ctx context.Context, ns string) ([]*Task, error) { + } + + func (r *Runtime) cleanupAfterDeadShim(ctx context.Context, bundle *bundle, ns, id string, pid int) error { ++ logrus.Infof("cleanup dead shim(legacy=%t): %s %d", legacy.IsLegacy(id), id, pid) + ctx = namespaces.WithNamespace(ctx, ns) + if err := r.terminate(ctx, bundle, ns, id); err != nil { + log.G(ctx).WithError(err).Warn("failed to terminate task") +-- +1.8.3.1 + diff --git a/patch/0057-containerd-add-timeout-for-delete-command.patch b/patch/0057-containerd-add-timeout-for-delete-command.patch new file mode 100644 index 0000000000000000000000000000000000000000..7a0cb9eae2e4956970b1bffdd99eed37cba0ee54 --- /dev/null +++ b/patch/0057-containerd-add-timeout-for-delete-command.patch @@ -0,0 +1,150 @@ +From 313e7f972e887c715b8feaad332ffe505653c496 Mon Sep 17 00:00:00 2001 +From: xiadanni1 +Date: Tue, 3 Mar 2020 06:31:18 +0800 +Subject: [PATCH] containerd:add timeout for delete command + +Change-Id: I620d2f19a8ac9086b5c83792a6fe49b0389da87d +Signed-off-by: xiadanni1 +--- + runtime/v1/linux/task.go | 2 +- + runtime/v1/shim/reaper.go | 23 +-------------- + vendor/github.com/containerd/go-runc/monitor.go | 37 +++++++++++++++++++++++-- + vendor/github.com/containerd/go-runc/runc.go | 3 +- + 4 files changed, 38 insertions(+), 27 deletions(-) + +diff --git a/runtime/v1/linux/task.go b/runtime/v1/linux/task.go +index d2bbb76..d200e9d 100644 +--- a/runtime/v1/linux/task.go ++++ b/runtime/v1/linux/task.go +@@ -91,7 +91,7 @@ func (t *Task) Namespace() string { + func (t *Task) delete(ctx context.Context, force bool, pid uint32) (*runtime.Exit, error) { + rsp, err := t.shim.Delete(ctx, empty) + if err != nil { +- log.G(ctx).WithError(err).Error("failed to delete container, force=%t", force) ++ log.G(ctx).WithError(err).Errorf("failed to delete container, force=%t", force) + } + t.tasks.Delete(ctx, t.id) + if err := t.shim.KillShim(ctx); err != nil { +diff --git a/runtime/v1/shim/reaper.go b/runtime/v1/shim/reaper.go +index d8e8274..f5f8096 100644 +--- a/runtime/v1/shim/reaper.go ++++ b/runtime/v1/shim/reaper.go +@@ -19,11 +19,7 @@ + package shim + + import ( +- "io/ioutil" + "os/exec" +- "path/filepath" +- "strconv" +- "strings" + "sync" + "syscall" + "time" +@@ -122,7 +118,7 @@ func (m *Monitor) WaitTimeout(c *exec.Cmd, ec chan runc.Exit, sec int64) (int, e + }() + select { + case <-time.After(time.Duration(sec) * time.Second): +- if SameProcess(c, c.Process.Pid) { ++ if runc.SameProcess(c, c.Process.Pid) { + logrus.Devour(syscall.Kill(c.Process.Pid, syscall.SIGKILL)) + } + return 0, errors.Errorf("timeout %ds for cmd(pid=%d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) +@@ -149,20 +145,3 @@ func (m *Monitor) Unsubscribe(c chan runc.Exit) { + close(c) + m.Unlock() + } +- +-func SameProcess(cmd *exec.Cmd, pid int) bool { +- bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline")) +- if err != nil { +- return false +- } +- for i := range bytes { +- if bytes[i] == 0 { +- bytes[i] = 32 +- } +- } +- cmdline := string(bytes) +- if strings.EqualFold(cmdline, strings.Join(cmd.Args, " ")+" ") { +- return true +- } +- return false +-} +diff --git a/vendor/github.com/containerd/go-runc/monitor.go b/vendor/github.com/containerd/go-runc/monitor.go +index 2c184d2..bb8bbab 100644 +--- a/vendor/github.com/containerd/go-runc/monitor.go ++++ b/vendor/github.com/containerd/go-runc/monitor.go +@@ -20,6 +20,13 @@ import ( + "os/exec" + "syscall" + "time" ++ "io/ioutil" ++ "path/filepath" ++ "strconv" ++ "strings" ++ ++ "github.com/pkg/errors" ++ "github.com/sirupsen/logrus" + ) + + var Monitor ProcessMonitor = &defaultMonitor{} +@@ -77,6 +84,30 @@ func (m *defaultMonitor) Wait(c *exec.Cmd, ec chan Exit) (int, error) { + } + + func (m *defaultMonitor) WaitTimeout(c *exec.Cmd, ec chan Exit, sec int64) (int, error) { +- e := <-ec +- return e.Status, nil +-} +\ No newline at end of file ++ select { ++ case <-time.After(time.Duration(sec) * time.Second): ++ if SameProcess(c, c.Process.Pid) { ++ logrus.Devour(syscall.Kill(c.Process.Pid, syscall.SIGKILL)) ++ } ++ return 0, errors.Errorf("timeout %ds for cmd(pid=%d): %s, %s", sec, c.Process.Pid, c.Path, c.Args) ++ case e := <-ec: ++ return e.Status, nil ++ } ++} ++ ++func SameProcess(cmd *exec.Cmd, pid int) bool { ++ bytes, err := ioutil.ReadFile(filepath.Join("/proc", strconv.Itoa(pid), "cmdline")) ++ if err != nil { ++ return false ++ } ++ for i := range bytes { ++ if bytes[i] == 0 { ++ bytes[i] = 32 ++ } ++ } ++ cmdline := string(bytes) ++ if strings.EqualFold(cmdline, strings.Join(cmd.Args, " ")+" ") { ++ return true ++ } ++ return false ++} +diff --git a/vendor/github.com/containerd/go-runc/runc.go b/vendor/github.com/containerd/go-runc/runc.go +index c1748ff..1c96317 100644 +--- a/vendor/github.com/containerd/go-runc/runc.go ++++ b/vendor/github.com/containerd/go-runc/runc.go +@@ -57,6 +57,7 @@ const ( + defaultTimeout = 30 + startTimeout = 120 + updateTimeout = 60 ++ deleteTimeout = 120 + ) + + var ( +@@ -318,7 +319,7 @@ func (r *Runc) Delete(context context.Context, id string, opts *DeleteOpts) erro + if opts != nil { + args = append(args, opts.args()...) + } +- return r.runOrError(r.command(id, context, append(args, id)...)) ++ return r.runOrErrorTimeout(r.command(id, context, append(args, id)...), deleteTimeout) + } + + // KillOpts specifies options for killing a container and its processes +-- +1.8.3.1 + diff --git a/series.conf b/series.conf index b268de2c4df846590679825462783a6c7f6629b4..05c09a330a82ac67161d26fd7abc1bda712ff66c 100644 --- a/series.conf +++ b/series.conf @@ -44,3 +44,19 @@ patch/0041-containerd-fix-containerd-call-runv-delete-directly.patch patch/0042-containerd-close-inherit-shim.sock-fd-to-adap.patch patch/0043-containerd-run-state-with-timeout-10s.patch patch/0044-containerd-add-copyright.patch +patch/0038-containerd-add-timeout-for-I-O-waitgroups.patch +patch/0039-containerd-fix-shouldKillAllOnExit-check.patch +patch/0044-containerd-change-tmpfile-directory-when-exec.patch +patch/0045-containerd-shim-disable-fast-gc-on-arm.patch +patch/0046-containerd-support-hot-upgrade.patch +patch/0047-containerd-shim-exit-initiative-after-3s.patch +patch/0048-containerd-modify-shim-initiative-exit-time.patch +patch/0049-contaienrd-modify-shim-initiative-exit-time-for-post-hook.patch +patch/0050-containerd-warp-and-process-return-errors.patch +patch/0051-containerd-add-timeout-for-containerd-shim.patch +patch/0052-containerd-modify-runtime-root-if-containe.patch +patch/0053-containerd-add-pid-check-to-avoid-poststop-ho.patch +patch/0054-containerd-clean-up-residual-container.patch +patch/0055-containerd-add-LLT-for-containerd-shim-timeou.patch +patch/0056-containerd-save-dumpstack-to-file.patch +patch/0057-containerd-add-timeout-for-delete-command.patch