runc-shim-v2 hold lock cause cri can't delete container

[ningmingxiao]

Description

pod have a pause container (53fd29e212f3) and nginx container (509e39bd68c0)

[root@cimcluster-minion-0-0:~]$ nerdctl  -n k8s.io ps  -a|grep test-continue001-64c6f49df9-qzswz
509e39bd68c0    swr:2512/admin/nginx:v1                                                              "nginx -g daemon off;"    2 days ago    Created             k8s://ns-sunq1/test-continue001-64c6f49df9-qzswz/nginx-991
53fd29e212f3    swr-plat:2524/admin/image/op-containers-pause:v7.23.30.03.15402830                   "/pause"                  2 days ago    Unknown             k8s://ns-sunq1/test-continue001-64c6f49df9-qzswz

53fd29 containerd log

53fd29e212f3    swr-plat:2524/admin/image/op-containers-pause:v7.23.30.03.15402830                   "/pause"                  2 days ago    Unknown             k8s://ns-sunq1/test-continue001-64c6f49df9-qzswz
53fd29e212f3    swr-plat:2524/admin/image/op-containers-pause:v7.23.30.03.15402830                   "/pause"                  2 days ago    Unknown             k8s://ns-sunq1/test-continue001-64c6f49df9-qzswz
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:01+08:00 [verbose] ConvertNet ContainerID: [53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d], IfName: [eth0], Netns: [/var/run/netns/cni-07d1552b-11dd-3a7b-6084-7a6362971603], needn't to convert network
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:01+08:00 [verbose] ConvertGateway ContainerID: [53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d], IfName: [eth0], Netns: [/var/run/netns/cni-07d1552b-11dd-3a7b-6084-7a6362971603], Args: [K8S_POD_UID=020f90cd-d766-42e2-a40e-6c65b5ff7cb1;IgnoreUnknown=1;K8S_POD_NAMESPACE=ns-sunq1;K8S_POD_NAME=test-continue001-64c6f49df9-qzswz;K8S_POD_INFRA_CONTAINER_ID=53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d], Path: [/opt/cni/bin], StdinData: [{"capabilities":{"bandwidth":true,"portMappings":true},"cniDir":"/paasdata/op-data/op-nw-multus/cni","cniVersion":"0.3.1","delegates":[{"cniVersion":"0.3.1","name":"cilium","plugins":[{"enable-debug":false,"type":"cilium-cni"},{"capabilities":{"bandwidth":true},"type":"bandwidth"}]}],"kubeconfig":"/etc/cni/net.d/multus.d/multus.kubeconfig","logFile":"/paasdata/op-log/op-nw-multus/multus.log","logLevel":"verbose","name":"multus-cni-network","type":"multus"}]
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:01+08:00 [verbose] ConvertGateway ContainerID: [53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d], IfName: [eth0], Netns: [/var/run/netns/cni-07d1552b-11dd-3a7b-6084-7a6362971603], needn't to convert gateway
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:01+08:00 [verbose] cmdAdd ContainerID: [53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d], IfName: [eth0], Netns: [/var/run/netns/cni-07d1552b-11dd-3a7b-6084-7a6362971603], Args: [K8S_POD_UID=020f90cd-d766-42e2-a40e-6c65b5ff7cb1;IgnoreUnknown=1;K8S_POD_NAMESPACE=ns-sunq1;K8S_POD_NAME=test-continue001-64c6f49df9-qzswz;K8S_POD_INFRA_CONTAINER_ID=53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d], Path: [/opt/cni/bin], StdinData: [{"capabilities":{"bandwidth":true,"portMappings":true},"cniDir":"/paasdata/op-data/op-nw-multus/cni","cniVersion":"0.3.1","delegates":[{"cniVersion":"0.3.1","name":"cilium","plugins":[{"enable-debug":false,"type":"cilium-cni"},{"capabilities":{"bandwidth":true},"type":"bandwidth"}]}],"kubeconfig":"/etc/cni/net.d/multus.d/multus.kubeconfig","logFile":"/paasdata/op-log/op-nw-multus/multus.log","logLevel":"verbose","name":"multus-cni-network","type":"multus"}]
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:01+08:00 [verbose] saveScratchNetConf: 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d, /paasdata/op-data/op-nw-multus/cni, [{"Conf":{"cniVersion":"0.3.1","name":"cilium","ipam":{},"dns":{}},"ConfList":{"cniVersion":"0.3.1","name":"cilium","plugins":[{"type":"cilium-cni","ipam":{},"dns":{}},{"type":"bandwidth","capabilities":{"bandwidth":true},"ipam":{},"dns":{}}]},"Name":"cilium","IsFilterV4Gateway":false,"IsFilterV6Gateway":false,"deviceID":"cilium:53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d.eth0","Bytes":"eyJjbmlWZXJzaW9uIjoiMC4zLjEiLCJuYW1lIjoiY2lsaXVtIiwicGx1Z2lucyI6W3siZW5hYmxlLWRlYnVnIjpmYWxzZSwidHlwZSI6ImNpbGl1bS1jbmkifSx7ImNhcGFiaWxpdGllcyI6eyJiYW5kd2lkdGgiOnRydWV9LCJ0eXBlIjoiYmFuZHdpZHRoIn1dfQ=="}]
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:01+08:00 [verbose] saveScratchNetConf for container[53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d] successfully
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:01+08:00 [verbose] rt.Args[[[IgnoreUnknown true] [K8S_POD_NAMESPACE ns-sunq1] [K8S_POD_NAME test-continue001-64c6f49df9-qzswz] [K8S_POD_INFRA_CONTAINER_ID 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d] [K8S_POD_UID 020f90cd-d766-42e2-a40e-6c65b5ff7cb1]]]
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: 2024-04-16T14:10:03+08:00 [verbose] Add all interfaces for container[53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d] successfully
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:03.556420749+08:00" level=info msg="RunPodSandbox for &PodSandboxMetadata{Name:test-continue001-64c6f49df9-qzswz,Uid:020f90cd-d766-42e2-a40e-6c65b5ff7cb1,Namespace:ns-sunq1,Attempt:0,} returns sandbox id \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:03.559347439+08:00" level=info msg="CreateContainer within sandbox \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" for container &ContainerMetadata{Name:nginx-991,Attempt:0,}"
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:03.605487629+08:00" level=info msg="CreateContainer within sandbox \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" for &ContainerMetadata{Name:nginx-991,Attempt:0,} returns container id \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\""
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:03.832894320+08:00" level=info msg="TaskOOM event container_id:\"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:12:04 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:04.467373147+08:00" level=info msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:12:06 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:06.470021895+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:14:04 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:14:04.467854934+08:00" level=error msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" failed" error="failed to stop sandbox container \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" in \"SANDBOX_READY\" state: failed to kill sandbox container: context canceled: unknown"
Apr 16 14:14:05 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:14:05.069569968+08:00" level=info msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:14:07 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:14:07.071507057+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:14:07 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:14:07.217783692+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:14:43 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:14:43.237044411+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:15:09 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:15:09.084643897+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:15:50 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:15:50.402905346+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:16:05 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:16:05.069935475+08:00" level=error msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" failed" error="failed to stop sandbox container \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" in \"SANDBOX_READY\" state: failed to kill sandbox container: context canceled: unknown"
Apr 16 14:16:05 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:16:05.681945676+08:00" level=info msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:16:07 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:16:07.683382512+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:16:11 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:16:11.155052181+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:16:38 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:16:38.559847703+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:17:17 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:17:17.841240032+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:17:38 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:17:38.578633490+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:18:05 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:18:05.682246821+08:00" level=error msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" failed" error="failed to stop sandbox container \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" in \"SANDBOX_READY\" state: failed to kill sandbox container: context deadline exceeded: unknown"
Apr 16 14:18:06 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:18:06.271745446+08:00" level=info msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:18:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:18:08.273564442+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:18:10 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:18:10.109630279+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:18:42 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:18:42.679494187+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:19:14 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:19:14.229106303+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:19:47 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:19:47.312310507+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:20:06 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:20:06.272244364+08:00" level=error msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" failed" error="failed to stop sandbox container \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" in \"SANDBOX_READY\" state: failed to kill sandbox container: context deadline exceeded: unknown"
Apr 16 14:20:06 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:20:06.843394740+08:00" level=info msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:20:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:20:08.845807980+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:20:16 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:20:16.811647241+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:20:40 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:20:40.613315464+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:21:13 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:21:13.200043291+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:21:42 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:21:42.695610889+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:22:06 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:06.844028341+08:00" level=error msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" failed" error="failed to stop sandbox container \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" in \"SANDBOX_READY\" state: failed to kill sandbox container: context deadline exceeded: unknown"
Apr 16 14:22:07 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:07.437848479+08:00" level=info msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""
Apr 16 14:22:09 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:09.439491522+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:22:14 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:14.239711931+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:22:19 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:19.626694149+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:22:21 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:21.629791900+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:22:24 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:24.721011409+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:22:26 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:26.724107813+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:22:45 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:45.245888140+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:23:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:23:08.562427788+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:23:42 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:23:42.675599846+08:00" level=error msg="get state for 53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d" error="context deadline exceeded: unknown"
Apr 16 14:24:07 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:24:07.438183339+08:00" level=error msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" failed" error="failed to stop sandbox container \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" in \"SANDBOX_READY\" state: failed to kill sandbox container: context canceled: unknown"
Apr 16 14:24:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:24:08.048468365+08:00" level=info msg="StopPodSandbox for \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\""

509e39bd68c0 containerd log

Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:03.590770982+08:00" level=info msg="Container 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0: CDI devices from CRI Config.CDIDevices: []"
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:03.605487629+08:00" level=info msg="CreateContainer within sandbox \"53fd29e212f38b95e1bb0b926220b11f7902aaaa877ec75d6705db9d5786196d\" for &ContainerMetadata{Name:nginx-991,Attempt:0,} returns container id \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\""
Apr 16 14:10:03 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:03.606631682+08:00" level=info msg="StartContainer for \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\""
Apr 16 14:10:05 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:05.741860411+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:07 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:07.843725736+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:10 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:10.045515445+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:12 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:12.448357393+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:15 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:15.250773159+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:18 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:18.853803561+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:54 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:54.966239108+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:57 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:57.070397847+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:10:59 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:10:59.272597055+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:11:01 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:11:01.674325903+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:11:04 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:11:04.477503735+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:11:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:11:08.079989075+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:11:50 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:11:50.294898212+08:00" level=error msg="collecting metrics for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context canceled: unknown"
Apr 16 14:11:54 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:11:54.967851517+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:11:57 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:11:57.070399463+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:11:59 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:11:59.271898890+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:12:01 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:01.674836313+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:12:04 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:04.468243321+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_CREATED\""
Apr 16 14:12:04 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:04.477760358+08:00" level=error msg="get state for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="context deadline exceeded: unknown"
Apr 16 14:12:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:08.606672329+08:00" level=error msg="collecting metrics for 509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0" error="ttrpc: closed: unknown"
Apr 16 14:12:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:08.606751136+08:00" level=info msg="shim disconnected" id=509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0 namespace=k8s.io
Apr 16 14:12:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:08.606804172+08:00" level=warning msg="cleaning up after shim disconnected" id=509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0 namespace=k8s.io
Apr 16 14:12:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:12:08.636182454+08:00" level=error msg="StartContainer for \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" failed" error="failed to create containerd task: failed to create shim task: context canceled: unknown"
Apr 16 14:14:05 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:14:05.069887334+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:16:05 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:16:05.682469616+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:18:06 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:18:06.271870686+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:20:06 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:20:06.843531011+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:22:07 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:22:07.437998337+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:24:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:24:08.049385929+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:26:08 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:26:08.674373414+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:28:09 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:28:09.248057723+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:30:09 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:30:09.793766519+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:32:10 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:32:10.369664186+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:34:10 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:34:10.962933113+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:36:11 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:36:11.571082230+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:38:12 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:38:12.209840409+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:40:12 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:40:12.797026613+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:42:13 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:42:13.408888826+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:44:13 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:44:13.978794244+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:46:14 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:46:14.549013474+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:48:15 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:48:15.120903120+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:50:15 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:50:15.769887010+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:52:16 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:52:16.369858583+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:54:16 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:54:16.962257869+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:56:17 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:56:17.530817214+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 14:58:18 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T14:58:18.155994444+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 15:00:18 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T15:00:18.873707639+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 15:02:19 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T15:02:19.513430289+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 15:04:20 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T15:04:20.101360901+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 15:06:20 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T15:06:20.731229228+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""
Apr 16 15:08:21 cimcluster-minion-0-0 containerd[8563]: time="2024-04-16T15:08:21.343629309+08:00" level=info msg="Container to stop \"509e39bd68c06259ebcbb976e13460031ebbd7e2b9291a704b2e7e49a6fb46a0\" must be in running or unknown state, current state \"CONTAINER_EXITED\""

shim-v2 stack
shim_stack.log

Steps to reproduce the issue

It is defficlut to reproduce

Describe the results you received and expected

shim can release lock

What version of containerd are you using?

1.7.6

Any other relevant information

No response

Show configuration if it is related to CRI plugin.

No response

[ningmingxiao]

I can't find out who hold the lock in containerd/runtime/v2/runc/task/service.go
some similar issue #9446 @fuweid

[fuweid]

@ningmingxiao you can use https://github.com/linuxerwang/goroutine-inspect to de-duplicate the goroutines and find which goroutine hold the lock.