Skip to content

Commit 4c18f9d

Browse files
prezhaprezha
committed
fix cri-o: work with selected cni and detected cgroup driver
1 parent e2f2bc9 commit 4c18f9d

File tree

5 files changed

+82
-55
lines changed

5 files changed

+82
-55
lines changed

‎pkg/minikube/cni/cni.go‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,6 @@ const (
4848
DefaultConfDir = "/etc/cni/net.d"
4949
)
5050

51-
var (
52-
// Network is the network name that CNI should use (eg, "kindnet").
53-
// Currently, only crio (and podman) can use it, so that setting custom ConfDir is not necessary.
54-
// ref: https://github.com/cri-o/cri-o/issues/2121 (and https://github.com/containers/podman/issues/2370)
55-
// ref: https://github.com/cri-o/cri-o/blob/master/docs/crio.conf.5.md#crionetwork-table
56-
Network = ""
57-
)
58-
5951
// Runner is the subset of command.Runner this package consumes
6052
type Runner interface {
6153
RunCmd(cmd *exec.Cmd) (*command.RunResult, error)
@@ -200,25 +192,41 @@ func applyManifest(cc config.ClusterConfig, r Runner, f assets.CopyableFile) err
200192
return nil
201193
}
202194

203-
// ConfigureLoopback ensures loopback has expected version ("1.0.0") and valid name ("loopback") in its config file in /etc/cni/net.d
204-
// cri-o is leaving name out atm (https://github.com/cri-o/cri-o/pull/6273)
205-
// avoid errors like:
195+
// ConfigureLoopbackCNI configures loopback cni.
196+
// If disable is true, sets extension of its config file in /etc/cni/net.d to "mk_disabled".
197+
// Otherwise, ensures loopback cni has expected version ("1.0.0") and valid name ("loopback") in its config file in /etc/cni/net.d.
198+
// Note: cri-o is leaving out name atm (https://github.com/cri-o/cri-o/pull/6273).
199+
// Avoid errors like:
206200
// - Failed to create pod sandbox: rpc error: code = Unknown desc = [failed to set up sandbox container "..." network for pod "...": networkPlugin cni failed to set up pod "..." network: missing network name:,
207201
// - failed to clean up sandbox container "..." network for pod "...": networkPlugin cni failed to teardown pod "..." network: missing network name]
208202
// It is caller's responsibility to restart container runtime for these changes to take effect.
209-
func ConfigureLoopback(r Runner) error {
203+
func ConfigureLoopbackCNI(r Runner, disable bool) error {
210204
loopback := "/etc/cni/net.d/*loopback.conf*" // usually: 200-loopback.conf
211205
// turn { "cniVersion": "0.3.1", "type": "loopback" }
212206
// into { "cniVersion": "0.3.1", "name": "loopback", "type": "loopback" }
213207
if _, err := r.RunCmd(exec.Command("sh", "-c", fmt.Sprintf("stat %s", loopback))); err != nil {
214-
klog.Warningf("%q not found, skipping patching loopback config step", loopback)
208+
klog.Warningf("loopback cni configuration skipped: %q not found", loopback)
209+
return nil
210+
}
211+
212+
findExec := []string{"find", filepath.Dir(loopback), "-maxdepth", "1", "-type", "f", "-name", filepath.Base(loopback), "-exec", "sh", "-c"}
213+
214+
if disable {
215+
if _, err := r.RunCmd(exec.Command(
216+
"sudo", append(findExec,
217+
`sudo mv {} {}.mk_disabled`, ";")...)); err != nil {
218+
return fmt.Errorf("unable to disable loopback cni %q: %v", loopback, err)
219+
}
220+
klog.Infof("loopback cni configuration disabled: %q found", loopback)
215221
return nil
216222
}
223+
217224
if _, err := r.RunCmd(exec.Command(
218-
"sudo", "find", filepath.Dir(loopback), "-maxdepth", "1", "-type", "f", "-name", filepath.Base(loopback), "-exec", "sh", "-c",
219-
`grep -q loopback {} && ( grep -q name {} || sudo sed -i '/"type": "loopback"/i \ \ \ \ "name": "loopback",' {} ) && sudo sed -i 's|"cniVersion": ".*"|"cniVersion": "1.0.0"|g' {}`, ";")); err != nil {
220-
return fmt.Errorf("unable to patch loopback config %q: %v", loopback, err)
225+
"sudo", append(findExec,
226+
`grep -q loopback {} && ( grep -q name {} || sudo sed -i '/"type": "loopback"/i \ \ \ \ "name": "loopback",' {} ) && sudo sed -i 's|"cniVersion": ".*"|"cniVersion": "1.0.0"|g' {}`, ";")...)); err != nil {
227+
return fmt.Errorf("unable to patch loopback cni config %q: %v", loopback, err)
221228
}
229+
klog.Infof("loopback cni configuration patched: %q found", loopback)
222230
return nil
223231
}
224232

@@ -239,7 +247,8 @@ func disableAllBridgeCNIs(r Runner) error {
239247
path := "/etc/cni/net.d"
240248

241249
out, err := r.RunCmd(exec.Command(
242-
"sudo", "find", path, "-maxdepth", "1", "-type", "f", "-name", "*bridge*", "-not", "-name", "*.mk_disabled", "-printf", "%p, ", "-exec", "sh", "-c",
250+
// for cri-o, we also disable 87-podman.conflist (that does not have 'bridge' in its name)
251+
"sudo", "find", path, "-maxdepth", "1", "-type", "f", "(", "-name", "*bridge*", "-or", "-name", "*podman*", "-and", "-not", "-name", "*.mk_disabled", ")", "-printf", "%p, ", "-exec", "sh", "-c",
243252
`sudo mv {} {}.mk_disabled`, ";"))
244253
if err != nil {
245254
return fmt.Errorf("failed to disable all bridge cni configs in %q: %v", path, err)
@@ -257,7 +266,7 @@ func disableAllBridgeCNIs(r Runner) error {
257266
// ref: https://github.com/containernetworking/cni/blob/main/libcni/conf.go
258267
// ref: https://kubernetes.io/docs/tasks/administer-cluster/migrating-from-dockershim/troubleshooting-cni-plugin-related-errors/
259268
func configureAllBridgeCNIs(r Runner, cidr string) error {
260-
// non-podman configs:
269+
// non-podman bridge configs:
261270
out, err := r.RunCmd(exec.Command(
262271
"sudo", "find", DefaultConfDir, "-maxdepth", "1", "-type", "f", "-name", "*bridge*", "-not", "-name", "*podman*", "-not", "-name", "*.mk_disabled", "-printf", "%p, ", "-exec", "sh", "-c",
263272
// remove ipv6 entries to avoid "failed to set bridge addr: could not add IP address to \"cni0\": permission denied"
@@ -273,7 +282,8 @@ func configureAllBridgeCNIs(r Runner, cidr string) error {
273282
}
274283
configs := out.Stdout.String()
275284

276-
// podman config(s):
285+
// podman bridge config(s):
286+
// could be eg, 87-podman-bridge.conflist or 87-podman.conflist
277287
// ref: https://github.com/containers/podman/blob/main/cni/87-podman-bridge.conflist
278288
ip, ipnet, err := net.ParseCIDR(cidr)
279289
if err != nil || ip.To4() == nil {
@@ -282,7 +292,7 @@ func configureAllBridgeCNIs(r Runner, cidr string) error {
282292
gateway := ip.Mask(ipnet.Mask)
283293
gateway[3]++
284294
out, err = r.RunCmd(exec.Command(
285-
"sudo", "find", DefaultConfDir, "-maxdepth", "1", "-type", "f", "-name", "*bridge*", "-name", "*podman*", "-not", "-name", "*.mk_disabled", "-printf", "%p, ", "-exec", "sh", "-c",
295+
"sudo", "find", DefaultConfDir, "-maxdepth", "1", "-type", "f", "-name", "*podman*", "-not", "-name", "*.mk_disabled", "-printf", "%p, ", "-exec", "sh", "-c",
286296
fmt.Sprintf(`sudo sed -i -r -e 's|^(.*)"subnet": ".*"(.*)$|\1"subnet": "%s"\2|g' -e 's|^(.*)"gateway": ".*"(.*)$|\1"gateway": "%s"\2|g' {}`, cidr, gateway), ";"))
287297
if err != nil {
288298
return fmt.Errorf("failed to configure podman bridge cni configs in %q: %v", DefaultConfDir, err)

‎pkg/minikube/cruntime/containerd.go‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -137,34 +137,41 @@ func generateContainerdConfig(cr CommandRunner, imageRepository string, kv semve
137137
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)restrict_oom_score_adj = .*$|\1restrict_oom_score_adj = %t|' %s`, inUserNamespace, containerdConfigFile))); err != nil {
138138
return errors.Wrap(err, "update restrict_oom_score_adj")
139139
}
140+
140141
// configure cgroup driver
141-
if cgroupDriver != constants.UnknownCgroupDriver {
142-
klog.Infof("configuring containerd to use %q as cgroup driver...", cgroupDriver)
143-
useSystemd := cgroupDriver == constants.SystemdCgroupDriver
144-
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = %t|g' %s`, useSystemd, containerdConfigFile))); err != nil {
145-
return errors.Wrap(err, "configuring SystemdCgroup")
146-
}
142+
if cgroupDriver == constants.UnknownCgroupDriver {
143+
klog.Warningf("unable to configure containerd to use unknown cgroup driver, will use default %q instead", constants.DefaultCgroupDriver)
144+
cgroupDriver = constants.DefaultCgroupDriver
147145
}
146+
klog.Infof("configuring containerd to use %q as cgroup driver...", cgroupDriver)
147+
useSystemd := cgroupDriver == constants.SystemdCgroupDriver
148+
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)SystemdCgroup = .*$|\1SystemdCgroup = %t|g' %s`, useSystemd, containerdConfigFile))); err != nil {
149+
return errors.Wrap(err, "configuring SystemdCgroup")
150+
}
151+
148152
// handle deprecated/removed features
149153
// ref: https://github.com/containerd/containerd/blob/main/RELEASES.md#deprecated-features
150154
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i 's|"io.containerd.runtime.v1.linux"|"io.containerd.runc.v2"|g' %s`, containerdConfigFile))); err != nil {
151155
return errors.Wrap(err, "configuring io.containerd.runtime version")
152156
}
157+
153158
// avoid containerd v1.6.14+ "failed to load plugin io.containerd.grpc.v1.cri" error="invalid plugin config: `systemd_cgroup` only works for runtime io.containerd.runtime.v1.linux" error
154159
// that then leads to crictl "getting the runtime version: rpc error: code = Unimplemented desc = unknown service runtime.v1alpha2.RuntimeService" error
155160
// ref: https://github.com/containerd/containerd/issues/4203
156161
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/systemd_cgroup/d' %s`, containerdConfigFile))); err != nil {
157162
return errors.Wrap(err, "removing deprecated systemd_cgroup param")
158163
}
164+
159165
// "runtime_type" has to be specified and it should be "io.containerd.runc.v2"
160166
// ref: https://github.com/containerd/containerd/issues/6964#issuecomment-1132378279
161167
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i 's|"io.containerd.runc.v1"|"io.containerd.runc.v2"|g' %s`, containerdConfigFile))); err != nil {
162168
return errors.Wrap(err, "configuring io.containerd.runc version")
163169
}
170+
164171
// ensure conf_dir is using '/etc/cni/net.d'
165172
// we might still want to try removing '/etc/cni/net.mk' in case of upgrade from previous minikube version that had/used it
166173
if _, err := cr.RunCmd(exec.Command("sh", "-c", `sudo rm -rf /etc/cni/net.mk`)); err != nil {
167-
return fmt.Errorf("unable to remove /etc/cni/net.mk directory: %v", err)
174+
klog.Warningf("unable to remove /etc/cni/net.mk directory: %v", err)
168175
}
169176
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i -r 's|^( *)conf_dir = .*$|\1conf_dir = %q|g' %s`, cni.DefaultConfDir, containerdConfigFile))); err != nil {
170177
return errors.Wrap(err, "update conf_dir")

‎pkg/minikube/cruntime/crio.go‎

Lines changed: 25 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ import (
3131
"k8s.io/klog/v2"
3232
"k8s.io/minikube/pkg/minikube/assets"
3333
"k8s.io/minikube/pkg/minikube/bootstrapper/images"
34-
"k8s.io/minikube/pkg/minikube/cni"
3534
"k8s.io/minikube/pkg/minikube/command"
3635
"k8s.io/minikube/pkg/minikube/config"
3736
"k8s.io/minikube/pkg/minikube/constants"
@@ -41,7 +40,7 @@ import (
4140
)
4241

4342
const (
44-
// CRIOConfFile is the path to the CRI-O configuration
43+
// crioConfigFile is the path to the CRI-O configuration
4544
crioConfigFile = "/etc/crio/crio.conf.d/02-crio.conf"
4645
)
4746

@@ -54,34 +53,36 @@ type CRIO struct {
5453
Init sysinit.Manager
5554
}
5655

57-
// generateCRIOConfig sets up /etc/crio/crio.conf
58-
func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Version) error {
56+
// generateCRIOConfig sets up pause image and cgroup manager for cri-o in crioConfigFile
57+
func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Version, cgroupDriver string) error {
5958
pauseImage := images.Pause(kv, imageRepository)
60-
61-
c := exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*pause_image = .*$|pause_image = \"%s\"|' -i %s", pauseImage, crioConfigFile))
59+
klog.Infof("configure cri-o to use %q pause image...", pauseImage)
60+
c := exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i 's|^.*pause_image = .*$|pause_image = %q|' %s`, pauseImage, crioConfigFile))
6261
if _, err := cr.RunCmd(c); err != nil {
63-
return errors.Wrap(err, "generateCRIOConfig")
62+
return errors.Wrap(err, "update pause_image")
6463
}
6564

66-
if cni.Network != "" {
67-
klog.Infof("Updating CRIO to use the custom CNI network %q", cni.Network)
68-
if _, err := cr.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("sudo sed -e 's|^.*cni_default_network = .*$|cni_default_network = \"%s\"|' -i %s", cni.Network, crioConfigFile))); err != nil {
69-
return errors.Wrap(err, "update network_dir")
70-
}
65+
// configure cgroup driver
66+
if cgroupDriver == constants.UnknownCgroupDriver {
67+
klog.Warningf("unable to configure cri-o to use unknown cgroup driver, will use default %q instead", constants.DefaultCgroupDriver)
68+
cgroupDriver = constants.DefaultCgroupDriver
7169
}
72-
73-
return nil
74-
}
75-
76-
func (r *CRIO) setCGroup(driver string) error {
77-
if driver == constants.UnknownCgroupDriver {
78-
return fmt.Errorf("unable to configure cri-o to use unknown cgroup driver")
70+
klog.Infof("configuring cri-o to use %q as cgroup driver...", cgroupDriver)
71+
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i 's|^.*cgroup_manager = .*$|cgroup_manager = %q|' %s`, cgroupDriver, crioConfigFile))); err != nil {
72+
return errors.Wrap(err, "configuring cgroup_manager")
73+
}
74+
// avoid errors like:
75+
// - level=fatal msg="Validating runtime config: conmon cgroup should be 'pod' or a systemd slice"
76+
// - level=fatal msg="Validating runtime config: cgroupfs manager conmon cgroup should be 'pod' or empty"
77+
// ref: https://github.com/cri-o/cri-o/pull/3940
78+
// ref: https://github.com/cri-o/cri-o/issues/6047
79+
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/cgroup_manager = .*/a conmon_cgroup = %q' %s`, "pod", crioConfigFile))); err != nil {
80+
return errors.Wrap(err, "configuring conmon_cgroup")
7981
}
8082

81-
klog.Infof("configuring cri-o to use %q as cgroup driver...", driver)
82-
c := exec.Command("/bin/bash", "-c", fmt.Sprintf(`sudo sed -i 's|^.*cgroup_manager = .*$|cgroup_manager = %q|' %s`, driver, crioConfigFile))
83-
if _, err := r.Runner.RunCmd(c); err != nil {
84-
return errors.Wrap(err, "configuring cgroup_manager")
83+
// we might still want to try removing '/etc/cni/net.mk' in case of upgrade from previous minikube version that had/used it
84+
if _, err := cr.RunCmd(exec.Command("sh", "-c", `sudo rm -rf /etc/cni/net.mk`)); err != nil {
85+
klog.Warningf("unable to remove /etc/cni/net.mk directory: %v", err)
8586
}
8687

8788
return nil
@@ -200,15 +201,12 @@ func (r *CRIO) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool)
200201
if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
201202
return err
202203
}
203-
if err := generateCRIOConfig(r.Runner, r.ImageRepository, r.KubernetesVersion); err != nil {
204+
if err := generateCRIOConfig(r.Runner, r.ImageRepository, r.KubernetesVersion, cgroupDriver); err != nil {
204205
return err
205206
}
206207
if err := enableIPForwarding(r.Runner); err != nil {
207208
return err
208209
}
209-
if err := r.setCGroup(cgroupDriver); err != nil {
210-
return err
211-
}
212210
if inUserNamespace {
213211
if err := CheckKernelCompatibility(r.Runner, 5, 11); err != nil {
214212
// For using overlayfs

‎pkg/minikube/node/start.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -381,9 +381,16 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
381381
exit.Error(reason.InternalRuntime, "Failed runtime", err)
382382
}
383383

384+
// 87-podman.conflist cni conf potentially conflicts with others and is created by podman on its first invocation,
385+
// so we "provoke" it here to ensure it's generated and that we can disable it
386+
// note: using 'help' or '--help' would be cheaper, but does not trigger that; 'version' seems to be next best option
387+
if co.Type == constants.CRIO {
388+
_, _ = runner.RunCmd(exec.Command("sudo", "sh", "-c", `podman version >/dev/null`))
389+
}
384390
// ensure loopback is properly configured
385391
// make sure container runtime is restarted afterwards for these changes to take effect
386-
if err := cni.ConfigureLoopback(runner); err != nil {
392+
disableLoopback := co.Type == constants.CRIO
393+
if err := cni.ConfigureLoopbackCNI(runner, disableLoopback); err != nil {
387394
klog.Warningf("unable to name loopback interface in dockerConfigureNetworkPlugin: %v", err)
388395
}
389396
if kv.GTE(semver.MustParse("1.24.0-alpha.2")) {

‎test/integration/net_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,11 @@ func TestNetworkPlugins(t *testing.T) {
218218
func validateFalseCNI(ctx context.Context, t *testing.T, profile string) {
219219
cr := ContainerRuntime()
220220

221+
// override cri-o name
222+
if cr == "cri-o" {
223+
cr = "crio"
224+
}
225+
221226
startArgs := []string{"start", "-p", profile, "--memory=2048", "--alsologtostderr", "--cni=false"}
222227
startArgs = append(startArgs, StartArgs()...)
223228

0 commit comments

Comments
 (0)