diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f956a1c27e..4ddd22b356 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -13,7 +13,7 @@ This requires that `kind` is installed and that either `podman` or `docker` is a ### Terminology - `apps-flavor` - one of `prod`, `dev` or `air-gapped` is a global switch for configuring the clusters and deployed applications in different ways. The recommended value when working with local clusters is `dev`. -- `local-cluster-profile` - a reference to a preconfigured `Cluster` config passed to `kind` when creating or updating local clusters. We use profiles to differentiate between single-/multi-node clusters and/or to enable special features such as container image caching. Use `./scripts/local-clusters.sh list profiles` to see a list of built-in profiles. +- `local-cluster-profile` - a reference to a preconfigured `Cluster` config passed to `kind` when creating or updating local clusters. We use profiles to differentiate between single-/multi-node clusters and/or to enable special features such as container image caching. Use `./scripts/local-cluster.sh list profiles` to see a list of built-in profiles. - `domain` - a local domain name. This can be arbitrary, but using a real domain (or subdomain) for which we have authority allows for setting up DNS based challenges for certificates issued by `cert-manager`. ### Setup diff --git a/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml b/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml index 83dc504810..9d0230531f 100644 --- a/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml +++ b/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml @@ -189,4 +189,15 @@ spec: acme.cert-manager.io/http01-solver: "true" ports: - port: 8089 + {{- if .Values.objectStorage.localEnabled }} + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: minio-system + podSelector: + matchLabels: + app: minio + ports: + - port: 9000 + {{- end }} {{- end }} diff --git a/helmfile.d/charts/networkpolicy/service-cluster/values.yaml b/helmfile.d/charts/networkpolicy/service-cluster/values.yaml index d094f0edd8..c7fc209732 100644 --- a/helmfile.d/charts/networkpolicy/service-cluster/values.yaml +++ b/helmfile.d/charts/networkpolicy/service-cluster/values.yaml @@ -132,3 +132,6 @@ dex: enabled: true ips: - "0.0.0.0/0" + +objectStorage: + localEnabled: false diff --git a/helmfile.d/values/ingress-nginx.yaml.gotmpl b/helmfile.d/values/ingress-nginx.yaml.gotmpl index e3d68c21b3..5f3390e199 100644 --- a/helmfile.d/values/ingress-nginx.yaml.gotmpl +++ b/helmfile.d/values/ingress-nginx.yaml.gotmpl @@ -87,12 +87,18 @@ controller: {{- if .Values | get "ingressNginx.controller.service.type" "" | eq "LoadBalancer" }} allocateLoadBalancerNodePorts: {{ .Values.ingressNginx.controller.service.allocateLoadBalancerNodePorts }} + {{- with .Values | get "ingressNginx.controller.service.externalIPs" list }} + externalIPs: {{- toYaml . | nindent 6 }} + {{- end }} {{- with .Values | get "ingressNginx.controller.service.loadBalancerSourceRanges" list }} loadBalancerSourceRanges: {{- toYaml . | nindent 6 }} {{- end }} {{- with .Values | get "ingressNginx.controller.service.loadBalancerIP" "" }} loadBalancerIP: {{ . }} {{- end }} + {{- if .Values.ingressNginx.controller.service.allocateLoadBalancerNodePorts }} + nodePorts: {{- toYaml .Values.ingressNginx.controller.service.nodePorts | nindent 6 }} + {{- end }} {{- else if .Values | get "ingressNginx.controller.service.type" "" | eq "NodePort" }} diff --git a/helmfile.d/values/minio.yaml.gotmpl b/helmfile.d/values/minio.yaml.gotmpl index d054681e64..ce6638b817 100644 --- a/helmfile.d/values/minio.yaml.gotmpl +++ b/helmfile.d/values/minio.yaml.gotmpl @@ -14,3 +14,9 @@ buckets: {{- range $key, $value := .Values.objectStorage.buckets }} - name: {{ $value }} {{- end }} + +ingress: + enabled: true + ingressClassName: nginx + hosts: + - minio.{{ .Values.global.baseDomain }} diff --git a/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl b/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl index 45530a97d7..96463b7263 100644 --- a/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl +++ b/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl @@ -81,3 +81,14 @@ dex: connectors: ips: {{- toYaml .Values.networkPolicies.dex.connectors.ips | nindent 6 }} ports: {{- toYaml .Values.networkPolicies.dex.connectors.ports | nindent 6 }} + +{{- with .Values.objectStorage }} +{{- if and + (eq .type "s3") + (.s3.regionEndpoint | hasPrefix "http://minio.") + (eq .s3.region "local") +}} +objectStorage: + localEnabled: true +{{- end }} +{{- end }} diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 78704a7443..f535ef11df 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -428,6 +428,11 @@ create() { kubectl get configmap -n kube-system coredns -oyaml | sed '/forward/a \ prefer_udp' | kubectl apply -f - fi + declare workers + workers="$(kubectl get no -oyaml | yq -I0 -oj '[.items[] | select(.metadata.labels."node-role.kubernetes.io/control-plane" != "") | .status.addresses[] | select(.type == "InternalIP") | .address] | sort')" + + yq -i ".ingressNginx.controller.service.externalIPs = ${workers}" "${CK8S_CONFIG_PATH}/${affix}-config.yaml" + kubectl label namespace local-path-storage owner=operator # install calico @@ -443,6 +448,10 @@ create() { helmfile -e local_cluster -f "${ROOT}/helmfile.d" -lapp=tigera apply --output simple fi + #install ingress-nginx + log.info "Installing ingress-nginx in SC" + "${ROOT}/bin/ck8s" ops helmfile sc -lapp=ingress-nginx apply --include-transitive-needs --output simple + # install s3 if ! [[ "${*}" =~ --skip-minio ]]; then log.info "installing minio" diff --git a/scripts/local-clusters/configs/common-config.yaml b/scripts/local-clusters/configs/common-config.yaml index 71e0807f59..358a0a1670 100644 --- a/scripts/local-clusters/configs/common-config.yaml +++ b/scripts/local-clusters/configs/common-config.yaml @@ -13,7 +13,7 @@ objectStorage: type: s3 s3: region: local - regionEndpoint: http://minio.minio-system.svc.cluster.local:9000 + regionEndpoint: http://minio.${domain} forcePathStyle: true calicoAccountant: backend: nftables @@ -35,10 +35,7 @@ ingressNginx: annotations: elastisys.io/local-cluster: tests clusterIP: 10.96.0.20 - type: NodePort - nodePorts: - http: 30080 - https: 30443 + type: LoadBalancer allocateLoadBalancerNodePorts: false config: useProxyProtocol: false @@ -53,7 +50,7 @@ networkPolicies: ips: - 0.0.0.0/0 ports: - - 9000 + - 80 scIngress: ips: - 0.0.0.0/0 @@ -90,4 +87,6 @@ networkPolicies: - 0.0.0.0/0 ingressNginx: ingressOverride: - enabled: false + enabled: true + ips: + - 0.0.0.0/0 diff --git a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml index 150d8b1a58..9e2df5c94d 100644 --- a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml +++ b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml @@ -4,7 +4,7 @@ nodeLocalDns: errors bind 169.254.20.10 10.96.0.10 template IN A $domain { - match "(^dex|^grafana|^harbor|^opensearch|\.ops)\.$domain\.$" + match "(^dex|^grafana|^harbor|^opensearch|^minio|\.ops)\.$domain\.$" answer "{{ .Name }} 60 IN A 10.96.0.20" fallthrough } diff --git a/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml b/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml index 38bb984e5d..0984e97c88 100644 --- a/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml +++ b/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml @@ -4,7 +4,7 @@ nodeLocalDns: errors bind 169.254.20.10 10.96.0.10 template IN A $domain { - match "(^dex|^grafana|^harbor|^opensearch|\.ops)\.$domain\.$" + match "(^dex|^grafana|^harbor|^minio|^opensearch|\.ops)\.$domain\.$" answer "{{ .Name }} 60 IN A $sc_node_ip" fallthrough } diff --git a/scripts/local-clusters/profiles/multi-node-cache.yaml b/scripts/local-clusters/profiles/multi-node-cache.yaml index c1d34c9348..6b5a8f86b9 100644 --- a/scripts/local-clusters/profiles/multi-node-cache.yaml +++ b/scripts/local-clusters/profiles/multi-node-cache.yaml @@ -47,11 +47,11 @@ nodes: hostPath: ${ROOT}/scripts/local-clusters/registries readOnly: true extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP diff --git a/scripts/local-clusters/profiles/multi-node.yaml b/scripts/local-clusters/profiles/multi-node.yaml index 289c05636a..7ef1e21191 100644 --- a/scripts/local-clusters/profiles/multi-node.yaml +++ b/scripts/local-clusters/profiles/multi-node.yaml @@ -36,11 +36,11 @@ nodes: readOnly: true - role: worker extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP diff --git a/scripts/local-clusters/profiles/single-node-cache.yaml b/scripts/local-clusters/profiles/single-node-cache.yaml index b96d6ebeb9..a58eb3f29b 100644 --- a/scripts/local-clusters/profiles/single-node-cache.yaml +++ b/scripts/local-clusters/profiles/single-node-cache.yaml @@ -47,11 +47,11 @@ nodes: hostPath: ${ROOT}/scripts/local-clusters/registries readOnly: true extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP diff --git a/scripts/local-clusters/profiles/single-node.yaml b/scripts/local-clusters/profiles/single-node.yaml index 0c99889c4f..f6968497d7 100644 --- a/scripts/local-clusters/profiles/single-node.yaml +++ b/scripts/local-clusters/profiles/single-node.yaml @@ -36,11 +36,11 @@ nodes: readOnly: true - role: worker extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP