Merge pull request #14848 from minikube-bot/auto_bump_k8s_versions

spowelljr · web-flow · commit 30f99f47834a · 2022-09-14T15:05:39.000-07:00
Bump Kubernetes version to v1.25.0 and resolve `/etc/resolv.conf` regression
diff --git a/deploy/addons/gvisor/gvisor-runtimeclass.yaml.tmpl b/deploy/addons/gvisor/gvisor-runtimeclass.yaml.tmpl
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-apiVersion: node.k8s.io/v1beta1
+apiVersion: node.k8s.io/v1{{if .LegacyRuntimeClass}}beta1{{end}}
 kind: RuntimeClass
 metadata:
   name: gvisor
diff --git a/hack/update/kubernetes_version/templates/v1beta3/containerd-api-port.yaml b/hack/update/kubernetes_version/templates/v1beta3/containerd-api-port.yaml
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/containerd-pod-network-cidr.yaml b/hack/update/kubernetes_version/templates/v1beta3/containerd-pod-network-cidr.yaml
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/containerd.yaml b/hack/update/kubernetes_version/templates/v1beta3/containerd.yaml
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/crio-options-gates.yaml b/hack/update/kubernetes_version/templates/v1beta3/crio-options-gates.yaml
@@ -65,6 +65,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/crio.yaml b/hack/update/kubernetes_version/templates/v1beta3/crio.yaml
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/default.yaml b/hack/update/kubernetes_version/templates/v1beta3/default.yaml
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/dns.yaml b/hack/update/kubernetes_version/templates/v1beta3/dns.yaml
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/image-repository.yaml b/hack/update/kubernetes_version/templates/v1beta3/image-repository.yaml
@@ -60,6 +60,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/hack/update/kubernetes_version/templates/v1beta3/options.yaml b/hack/update/kubernetes_version/templates/v1beta3/options.yaml
@@ -62,6 +62,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/assets/addons.go b/pkg/minikube/assets/addons.go
@@ -878,6 +878,7 @@ func GenerateTemplateData(addon *Addon, cc *config.ClusterConfig, netInfo Networ
 		CustomRegistries        map[string]string
 		NetworkInfo             map[string]string
 		LegacyPodSecurityPolicy bool
+		LegacyRuntimeClass      bool
 	}{
 		KubernetesVersion:       make(map[string]uint64),
 		PreOneTwentyKubernetes:  false,
@@ -895,6 +896,7 @@ func GenerateTemplateData(addon *Addon, cc *config.ClusterConfig, netInfo Networ
 		CustomRegistries:        customRegistries,
 		NetworkInfo:             make(map[string]string),
 		LegacyPodSecurityPolicy: v.LT(semver.Version{Major: 1, Minor: 25}),
+		LegacyRuntimeClass:      v.LT(semver.Version{Major: 1, Minor: 25}),
 	}
 	if opts.ImageRepository != "" && !strings.HasSuffix(opts.ImageRepository, "/") {
 		opts.ImageRepository += "/"
diff --git a/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go b/pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta3.go
@@ -86,7 +86,8 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
-staticPodPath: {{.StaticPodPath}}
+staticPodPath: {{.StaticPodPath}}{{if .ResolvConfSearchRegression}}
+resolvConf: /etc/kubelet-resolv.conf{{end}}
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/kubeadm.go b/pkg/minikube/bootstrapper/bsutil/kubeadm.go
@@ -87,27 +87,28 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 	klog.Infof("Using pod CIDR: %s", podCIDR)
 
 	opts := struct {
-		CertDir             string
-		ServiceCIDR         string
-		PodSubnet           string
-		AdvertiseAddress    string
-		APIServerPort       int
-		KubernetesVersion   string
-		EtcdDataDir         string
-		EtcdExtraArgs       map[string]string
-		ClusterName         string
-		NodeName            string
-		DNSDomain           string
-		CRISocket           string
-		ImageRepository     string
-		ComponentOptions    []componentOptions
-		FeatureArgs         map[string]bool
-		NodeIP              string
-		CgroupDriver        string
-		ClientCAFile        string
-		StaticPodPath       string
-		ControlPlaneAddress string
-		KubeProxyOptions    map[string]string
+		CertDir                    string
+		ServiceCIDR                string
+		PodSubnet                  string
+		AdvertiseAddress           string
+		APIServerPort              int
+		KubernetesVersion          string
+		EtcdDataDir                string
+		EtcdExtraArgs              map[string]string
+		ClusterName                string
+		NodeName                   string
+		DNSDomain                  string
+		CRISocket                  string
+		ImageRepository            string
+		ComponentOptions           []componentOptions
+		FeatureArgs                map[string]bool
+		NodeIP                     string
+		CgroupDriver               string
+		ClientCAFile               string
+		StaticPodPath              string
+		ControlPlaneAddress        string
+		KubeProxyOptions           map[string]string
+		ResolvConfSearchRegression bool
 	}{
 		CertDir:           vmpath.GuestKubernetesCertsDir,
 		ServiceCIDR:       constants.DefaultServiceCIDR,
@@ -119,18 +120,19 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		EtcdExtraArgs:     etcdExtraArgs(k8s.ExtraOptions),
 		ClusterName:       cc.Name,
 		// kubeadm uses NodeName as the --hostname-override parameter, so this needs to be the name of the machine
-		NodeName:            KubeNodeName(cc, n),
-		CRISocket:           r.SocketPath(),
-		ImageRepository:     k8s.ImageRepository,
-		ComponentOptions:    componentOpts,
-		FeatureArgs:         kubeadmFeatureArgs,
-		DNSDomain:           k8s.DNSDomain,
-		NodeIP:              n.IP,
-		CgroupDriver:        cgroupDriver,
-		ClientCAFile:        path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt"),
-		StaticPodPath:       vmpath.GuestManifestsDir,
-		ControlPlaneAddress: constants.ControlPlaneAlias,
-		KubeProxyOptions:    createKubeProxyOptions(k8s.ExtraOptions),
+		NodeName:                   KubeNodeName(cc, n),
+		CRISocket:                  r.SocketPath(),
+		ImageRepository:            k8s.ImageRepository,
+		ComponentOptions:           componentOpts,
+		FeatureArgs:                kubeadmFeatureArgs,
+		DNSDomain:                  k8s.DNSDomain,
+		NodeIP:                     n.IP,
+		CgroupDriver:               cgroupDriver,
+		ClientCAFile:               path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt"),
+		StaticPodPath:              vmpath.GuestManifestsDir,
+		ControlPlaneAddress:        constants.ControlPlaneAlias,
+		KubeProxyOptions:           createKubeProxyOptions(k8s.ExtraOptions),
+		ResolvConfSearchRegression: HasResolvConfSearchRegression(k8s.KubernetesVersion),
 	}
 
 	if k8s.ServiceCIDR != "" {
@@ -203,3 +205,13 @@ func etcdExtraArgs(extraOpts config.ExtraOptionSlice) map[string]string {
 	}
 	return args
 }
+
+// HasResolvConfSearchRegression returns if the k8s version includes https://github.com/kubernetes/kubernetes/pull/109441
+func HasResolvConfSearchRegression(k8sVersion string) bool {
+	versionSemver, err := util.ParseKubernetesVersion(k8sVersion)
+	if err != nil {
+		klog.Warningf("was unable to parse Kubernetes version %q: %v", k8sVersion, err)
+		return false
+	}
+	return versionSemver.EQ(semver.Version{Major: 1, Minor: 25})
+}
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/containerd-api-port.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/containerd-api-port.yaml
@@ -38,7 +38,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "10.244.0.0/16"
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/containerd-pod-network-cidr.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/containerd-pod-network-cidr.yaml
@@ -38,7 +38,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "192.168.32.0/20"
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/containerd.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/containerd.yaml
@@ -38,7 +38,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "10.244.0.0/16"
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/crio-options-gates.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/crio-options-gates.yaml
@@ -44,7 +44,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "10.244.0.0/16"
@@ -65,6 +65,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/crio.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/crio.yaml
@@ -38,7 +38,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "10.244.0.0/16"
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/default.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/default.yaml
@@ -38,7 +38,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "10.244.0.0/16"
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/dns.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/dns.yaml
@@ -38,7 +38,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: minikube.local
   podSubnet: "10.244.0.0/16"
@@ -59,6 +59,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/image-repository.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/image-repository.yaml
@@ -39,7 +39,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "10.244.0.0/16"
@@ -60,6 +60,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/options.yaml b/pkg/minikube/bootstrapper/bsutil/testdata/v1.25/options.yaml
@@ -41,7 +41,7 @@ etcd:
     dataDir: /var/lib/minikube/etcd
     extraArgs:
       proxy-refresh-interval: "70000"
-kubernetesVersion: v1.25.0-rc.1
+kubernetesVersion: v1.25.0
 networking:
   dnsDomain: cluster.local
   podSubnet: "10.244.0.0/16"
@@ -62,6 +62,7 @@ evictionHard:
   imagefs.available: "0%"
 failSwapOn: false
 staticPodPath: /etc/kubernetes/manifests
+resolvConf: /etc/kubelet-resolv.conf
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go
@@ -988,6 +988,10 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru
 		return errors.Wrap(err, "copy")
 	}
 
+	if err := k.copyResolvConf(cfg); err != nil {
+		return errors.Wrap(err, "resolv.conf")
+	}
+
 	cp, err := config.PrimaryControlPlane(&cfg)
 	if err != nil {
 		return errors.Wrap(err, "control plane")
@@ -1000,6 +1004,23 @@ func (k *Bootstrapper) UpdateNode(cfg config.ClusterConfig, n config.Node, r cru
 	return nil
 }
 
+// copyResolvConf is a workaround for a regression introduced with https://github.com/kubernetes/kubernetes/pull/109441
+// The regression is resolved by making a copy of /etc/resolv.conf, removing the line "search ." from the copy, and setting kubelet to use the copy
+// Only Kubernetes v1.25.0 is affected by this regression
+func (k *Bootstrapper) copyResolvConf(cfg config.ClusterConfig) error {
+	if !bsutil.HasResolvConfSearchRegression(cfg.KubernetesConfig.KubernetesVersion) {
+		return nil
+	}
+	if _, err := k.c.RunCmd(exec.Command("sudo", "cp", "/etc/resolv.conf", "/etc/kubelet-resolv.conf")); err != nil {
+		return errors.Wrap(err, "copy")
+	}
+	if _, err := k.c.RunCmd(exec.Command("sudo", "sed", "-i", "-e", "s/^search .$//", "/etc/kubelet-resolv.conf")); err != nil {
+		return errors.Wrap(err, "sed")
+	}
+
+	return nil
+}
+
 // kubectlPath returns the path to the kubelet
 func kubectlPath(cfg config.ClusterConfig) string {
 	return path.Join(vmpath.GuestPersistentDir, "binaries", cfg.KubernetesConfig.KubernetesVersion, "kubectl")
diff --git a/pkg/minikube/cni/calico.go b/pkg/minikube/cni/calico.go
diff --git a/pkg/minikube/cni/calico.yaml b/pkg/minikube/cni/calico.yaml
diff --git a/pkg/minikube/cni/flannel.go b/pkg/minikube/cni/flannel.go
diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go
diff --git a/pkg/minikube/reason/k8s.go b/pkg/minikube/reason/k8s.go
diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md
diff --git a/test/integration/error_spam_test.go b/test/integration/error_spam_test.go
diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go
diff --git a/test/integration/testdata/kube-flannel.yaml b/test/integration/testdata/kube-flannel.yaml
