Merge pull request #335 from aojea/tunnel_ingress

Tunnel ingress for gateway should be idempotent

Author: k8s-ci-robot

.github/workflows/bats.yml

@@ -52,13 +52,18 @@ jobs:
             ./_artifacts-custom-network
  
   bats_mac_tests:
-    runs-on: macos-13
+    runs-on: macos-15-intel
     name: Bats e2e tests on Mac
-    # Skip until port forwarding works
+    # Skip until runner does not timeout on kind create cluster
     if: false
     steps:
       - name: Checkout
         uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
+      # c.f. https://github.com/actions/runner-images/issues/13358
+      - name: Work around x86 macOS bug for multiple CPU cores
+        run: |
+          sudo defaults -currentHost write /Library/Preferences/com.apple.powerlogd SMCMonitorCadence 0
+          sudo killall PerfPowerServices || true
       - name: Set up environment (download dependencies)
         run: |
           brew install docker colima
@@ -71,7 +76,6 @@ jobs:
         env:
           TERM: linux
         run: |
-          bash --version
           bash -c "time bats -o _artifacts --trace --formatter tap tests/"
       - name: Debug info
         if: always()
@@ -84,5 +88,5 @@ jobs:
         if: always()
         uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5
         with:
-          name: kind-logs-${{ env.JOB_NAME }}-${{ github.run_id }}
+          name: kind-logs-mac-${{ github.run_id }}
           path: ./_artifacts

pkg/gateway/envoy.go

@@ -193,7 +193,9 @@ func createGateway(clusterName string, nameserver string, localAddress string, l
 		"--sysctl=net.ipv6.conf.all.forwarding=1",
 	}...)
 
-	if enableTunnel {
+	if enableTunnel ||
+		config.DefaultConfig.LoadBalancerConnectivity == config.Portmap {
+		// Forward the Listener Ports to the host so they are accessible on Mac and Windows
 		for _, listener := range gateway.Spec.Listeners {
 			if listener.Protocol == gatewayv1.UDPProtocolType {
 				args = append(args, fmt.Sprintf("--publish=%d/%s", listener.Port, "udp"))
@@ -202,7 +204,6 @@ func createGateway(clusterName string, nameserver string, localAddress string, l
 			}
 		}
 	}
-	args = append(args, fmt.Sprintf("--publish=%d/tcp", envoyAdminPort))
 	args = append(args, "--publish-all")
 
 	// Construct the multi-step command

pkg/tunnels/tunnel.go

@@ -41,31 +41,49 @@ func (t *TunnelManager) SetupTunnels(containerName string) error {
 	}
 
 	klog.V(0).Infof("setting IPv4 address %s associated to container %s", ipv4, containerName)
-	output, err := AddIPToLocalInterface(ipv4)
-	if err != nil {
-		return fmt.Errorf("error adding IP to local interface: %w - %s", err, output)
+	// check if the IP is already assigned to the local interface
+	if !ipOnHost(ipv4) {
+		output, err := AddIPToLocalInterface(ipv4)
+		if err != nil {
+			return fmt.Errorf("error adding IP to local interface: %w - %s", err, output)
+		}
 	}
 
 	// create tunnel from the ip:svcport to the localhost:portmap
 	t.mu.Lock()
 	defer t.mu.Unlock()
+	_, ok := t.tunnels[containerName]
+	if !ok {
+		t.tunnels[containerName] = map[string]*tunnel{}
+	}
+
+	// Reconcile: Remove tunnels that are not in portmaps
+	for containerPort, tun := range t.tunnels[containerName] {
+		if _, ok := portmaps[containerPort]; !ok {
+			klog.V(0).Infof("removing tunnel for %s %s as it is no longer in portmaps", containerName, containerPort)
+			tun.Stop() // nolint: errcheck
+			delete(t.tunnels[containerName], containerPort)
+		}
+	}
+
 	// There is one IP per Service and a tunnel per Service Port
 	for containerPort, hostPort := range portmaps {
 		parts := strings.Split(containerPort, "/")
 		if len(parts) != 2 {
 			return fmt.Errorf("expected format port/protocol for container port, got %s", containerPort)
 		}
 
+		if _, ok := t.tunnels[containerName][containerPort]; ok {
+			klog.V(2).Infof("tunnel for %s %s already exists", containerName, containerPort)
+			continue
+		}
+
 		tun := NewTunnel(ipv4, parts[0], parts[1], "localhost", hostPort)
 		// TODO check if we can leak tunnels
 		err = tun.Start()
 		if err != nil {
 			return err
 		}
-		_, ok := t.tunnels[containerName]
-		if !ok {
-			t.tunnels[containerName] = map[string]*tunnel{}
-		}
 		t.tunnels[containerName][containerPort] = tun
 	}
 	return nil
@@ -88,6 +106,7 @@ func (t *TunnelManager) RemoveTunnels(containerName string) error {
 		}
 		tunnel.Stop() // nolint: errcheck
 	}
+	delete(t.tunnels, containerName)
 
 	klog.V(0).Infof("Removing IPv4 address %s associated to local interface", tunnelIP)
 	output, err := RemoveIPFromLocalInterface(tunnelIP)
@@ -275,3 +294,29 @@ func (t *tunnel) handleUDPConnection(conn *net.UDPConn) error {
 
 	return err
 }
+
+func ipOnHost(ip string) bool {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		return false
+	}
+	for _, i := range ifaces {
+		addrs, err := i.Addrs()
+		if err != nil {
+			continue
+		}
+		for _, addr := range addrs {
+			var ipAddr net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ipAddr = v.IP
+			case *net.IPAddr:
+				ipAddr = v.IP
+			}
+			if ipAddr != nil && ipAddr.String() == ip {
+				return true
+			}
+		}
+	}
+	return false
+}

tests/custom-network/setup_suite.bash

@@ -18,16 +18,9 @@ function setup_suite {
   --subnet=172.20.0.0/16 $KIND_EXPERIMENTAL_DOCKER_NETWORK
 
   # create cluster
-  cat <<EOF | kind create cluster \
+  kind create cluster \
   --name $CLUSTER_NAME           \
-  -v7 --wait 1m --retain --config=-
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-- role: worker
-- role: worker
-EOF
+  -v7 --wait 1m --retain --config="$BATS_TEST_DIRNAME/../kind.yaml"
 
   # build & run cloud-provider-kind
   cd "$BATS_TEST_DIRNAME"/../.. && make

tests/kind.yaml

@@ -0,0 +1,6 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
+- role: worker

tests/setup_suite.bash

@@ -12,16 +12,7 @@ function setup_suite {
   rm -rf "$ARTIFACTS_DIR"/*
 
   # create cluster
-  cat <<EOF | kind create cluster \
-  --name $CLUSTER_NAME           \
-  -v7 --wait 1m --retain --config=-
-kind: Cluster
-apiVersion: kind.x-k8s.io/v1alpha4
-nodes:
-- role: control-plane
-- role: worker
-- role: worker
-EOF
+  kind create cluster --name $CLUSTER_NAME -v7 --wait 1m --retain --config="$BATS_TEST_DIRNAME/kind.yaml"
 
   cd "$BATS_TEST_DIRNAME"/.. && make
   nohup "$BATS_TEST_DIRNAME"/../bin/cloud-provider-kind -v 2 --gateway-channel=standard --enable-log-dumping --logs-dir "$ARTIFACTS_DIR" > "$ARTIFACTS_DIR"/ccm-kind.log 2>&1 &