From 1fa4c9a84dad7035d3b570a37c1ef452b0b231cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Arnqvist?= Date: Fri, 12 Dec 2025 13:49:05 +0100 Subject: [PATCH 1/6] scripts: Enable mock direct routed load balancer for local clusters --- helmfile.d/values/ingress-nginx.yaml.gotmpl | 6 ++++++ scripts/local-cluster.sh | 5 +++++ scripts/local-clusters/configs/common-config.yaml | 9 ++++----- scripts/local-clusters/profiles/multi-node-cache.yaml | 4 ++-- scripts/local-clusters/profiles/multi-node.yaml | 4 ++-- scripts/local-clusters/profiles/single-node-cache.yaml | 4 ++-- scripts/local-clusters/profiles/single-node.yaml | 4 ++-- 7 files changed, 23 insertions(+), 13 deletions(-) diff --git a/helmfile.d/values/ingress-nginx.yaml.gotmpl b/helmfile.d/values/ingress-nginx.yaml.gotmpl index e3d68c21b3..5f3390e199 100644 --- a/helmfile.d/values/ingress-nginx.yaml.gotmpl +++ b/helmfile.d/values/ingress-nginx.yaml.gotmpl @@ -87,12 +87,18 @@ controller: {{- if .Values | get "ingressNginx.controller.service.type" "" | eq "LoadBalancer" }} allocateLoadBalancerNodePorts: {{ .Values.ingressNginx.controller.service.allocateLoadBalancerNodePorts }} + {{- with .Values | get "ingressNginx.controller.service.externalIPs" list }} + externalIPs: {{- toYaml . | nindent 6 }} + {{- end }} {{- with .Values | get "ingressNginx.controller.service.loadBalancerSourceRanges" list }} loadBalancerSourceRanges: {{- toYaml . | nindent 6 }} {{- end }} {{- with .Values | get "ingressNginx.controller.service.loadBalancerIP" "" }} loadBalancerIP: {{ . }} {{- end }} + {{- if .Values.ingressNginx.controller.service.allocateLoadBalancerNodePorts }} + nodePorts: {{- toYaml .Values.ingressNginx.controller.service.nodePorts | nindent 6 }} + {{- end }} {{- else if .Values | get "ingressNginx.controller.service.type" "" | eq "NodePort" }} diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 78704a7443..edef64382d 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -428,6 +428,11 @@ create() { kubectl get configmap -n kube-system coredns -oyaml | sed '/forward/a \ prefer_udp' | kubectl apply -f - fi + declare workers + workers="$(kubectl get no -oyaml | yq -I0 -oj '[.items[] | select(.metadata.labels."node-role.kubernetes.io/control-plane" != "") | .status.addresses[] | select(.type == "InternalIP") | .address] | sort')" + + yq -i ".ingressNginx.controller.service.externalIPs = ${workers}" "${CK8S_CONFIG_PATH}/${affix}-config.yaml" + kubectl label namespace local-path-storage owner=operator # install calico diff --git a/scripts/local-clusters/configs/common-config.yaml b/scripts/local-clusters/configs/common-config.yaml index 71e0807f59..67f625cd77 100644 --- a/scripts/local-clusters/configs/common-config.yaml +++ b/scripts/local-clusters/configs/common-config.yaml @@ -35,10 +35,7 @@ ingressNginx: annotations: elastisys.io/local-cluster: tests clusterIP: 10.96.0.20 - type: NodePort - nodePorts: - http: 30080 - https: 30443 + type: LoadBalancer allocateLoadBalancerNodePorts: false config: useProxyProtocol: false @@ -90,4 +87,6 @@ networkPolicies: - 0.0.0.0/0 ingressNginx: ingressOverride: - enabled: false + enabled: true + ips: + - 0.0.0.0/0 diff --git a/scripts/local-clusters/profiles/multi-node-cache.yaml b/scripts/local-clusters/profiles/multi-node-cache.yaml index c1d34c9348..6b5a8f86b9 100644 --- a/scripts/local-clusters/profiles/multi-node-cache.yaml +++ b/scripts/local-clusters/profiles/multi-node-cache.yaml @@ -47,11 +47,11 @@ nodes: hostPath: ${ROOT}/scripts/local-clusters/registries readOnly: true extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP diff --git a/scripts/local-clusters/profiles/multi-node.yaml b/scripts/local-clusters/profiles/multi-node.yaml index 289c05636a..7ef1e21191 100644 --- a/scripts/local-clusters/profiles/multi-node.yaml +++ b/scripts/local-clusters/profiles/multi-node.yaml @@ -36,11 +36,11 @@ nodes: readOnly: true - role: worker extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP diff --git a/scripts/local-clusters/profiles/single-node-cache.yaml b/scripts/local-clusters/profiles/single-node-cache.yaml index b96d6ebeb9..a58eb3f29b 100644 --- a/scripts/local-clusters/profiles/single-node-cache.yaml +++ b/scripts/local-clusters/profiles/single-node-cache.yaml @@ -47,11 +47,11 @@ nodes: hostPath: ${ROOT}/scripts/local-clusters/registries readOnly: true extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP diff --git a/scripts/local-clusters/profiles/single-node.yaml b/scripts/local-clusters/profiles/single-node.yaml index 0c99889c4f..f6968497d7 100644 --- a/scripts/local-clusters/profiles/single-node.yaml +++ b/scripts/local-clusters/profiles/single-node.yaml @@ -36,11 +36,11 @@ nodes: readOnly: true - role: worker extraPortMappings: - - containerPort: 30080 + - containerPort: 80 hostPort: 80 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP - - containerPort: 30443 + - containerPort: 443 hostPort: 443 listenAddress: ${CK8S_LOCAL_LISTEN_ADDRESS} protocol: TCP From 8fd16a10bbc8d78386d493c5e3befa596c8238c0 Mon Sep 17 00:00:00 2001 From: Mohamed Shafiq Date: Fri, 21 Nov 2025 10:33:29 +0100 Subject: [PATCH 2/6] added sharing s3 with ingress --- DEVELOPMENT.md | 2 +- scripts/local-cluster.sh | 27 +++++++++++++++++++ .../configs/partial/sc-node-local-dns.yaml | 2 +- .../configs/partial/wc-node-local-dns.yaml | 2 +- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index f956a1c27e..4ddd22b356 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -13,7 +13,7 @@ This requires that `kind` is installed and that either `podman` or `docker` is a ### Terminology - `apps-flavor` - one of `prod`, `dev` or `air-gapped` is a global switch for configuring the clusters and deployed applications in different ways. The recommended value when working with local clusters is `dev`. -- `local-cluster-profile` - a reference to a preconfigured `Cluster` config passed to `kind` when creating or updating local clusters. We use profiles to differentiate between single-/multi-node clusters and/or to enable special features such as container image caching. Use `./scripts/local-clusters.sh list profiles` to see a list of built-in profiles. +- `local-cluster-profile` - a reference to a preconfigured `Cluster` config passed to `kind` when creating or updating local clusters. We use profiles to differentiate between single-/multi-node clusters and/or to enable special features such as container image caching. Use `./scripts/local-cluster.sh list profiles` to see a list of built-in profiles. - `domain` - a local domain name. This can be arbitrary, but using a real domain (or subdomain) for which we have authority allows for setting up DNS based challenges for certificates issued by `cert-manager`. ### Setup diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index edef64382d..3976b81289 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -515,6 +515,30 @@ setup_node_local_dns() { fi } +setup_sc_s3_sharable() { + log.info "Setting up s3 as sharable" + local domain CK8S_CLUSTER CK8S_DRY_RUN_INSTALL + domain="$(yq ".global.baseDomain" <"${CK8S_CONFIG_PATH}/common-config.yaml")" + # shellcheck source=scripts/migration/lib.sh + source "${ROOT}/scripts/migration/lib.sh" + CK8S_CLUSTER=both + CK8S_DRY_RUN_INSTALL=false + + yq_add common '.objectStorage.s3.regionEndpoint' "\"http://minio.${domain}\"" + yq_add common '.ingressNginx.controller.useHostPort' 'true' + yq_add common '.networkPolicies.global.objectStorage.ports[0]' '80' + yq_add common '.networkPolicies.ingressNginx.ingressOverride.enabled' 'false' + + log.info "Installing Ingress in SC" + helmfile_do sc apply -lapp=ingress-nginx --include-transitive-needs --output simple + log.info "Upgrading Minio Chart with Ingress" + helm_do sc upgrade -n minio-system minio "${ROOT}/helmfile.d/upstream/minio/minio" \ + --reuse-values \ + --set ingress.enabled=true \ + --set ingress.ingressClassName=nginx \ + --set ingress.hosts[0]=minio."${domain}" +} + delete() { local cluster cluster="${1:-}" @@ -557,6 +581,9 @@ main() { node-local-dns) setup_node_local_dns ;; + sharable-s3) + setup_sc_s3_sharable + ;; *) log.usage ;; diff --git a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml index 150d8b1a58..db68608e04 100644 --- a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml +++ b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml @@ -4,7 +4,7 @@ nodeLocalDns: errors bind 169.254.20.10 10.96.0.10 template IN A $domain { - match "(^dex|^grafana|^harbor|^opensearch|\.ops)\.$domain\.$" + match "(^dex|^grafana|^harbor|^minio|^opensearch|\.ops)\.$domain\.$" answer "{{ .Name }} 60 IN A 10.96.0.20" fallthrough } diff --git a/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml b/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml index 38bb984e5d..0984e97c88 100644 --- a/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml +++ b/scripts/local-clusters/configs/partial/wc-node-local-dns.yaml @@ -4,7 +4,7 @@ nodeLocalDns: errors bind 169.254.20.10 10.96.0.10 template IN A $domain { - match "(^dex|^grafana|^harbor|^opensearch|\.ops)\.$domain\.$" + match "(^dex|^grafana|^harbor|^minio|^opensearch|\.ops)\.$domain\.$" answer "{{ .Name }} 60 IN A $sc_node_ip" fallthrough } From 92c8233891c3e597e5b1a77f45ac277c929ec676 Mon Sep 17 00:00:00 2001 From: Mohamed Shafiq Date: Thu, 27 Nov 2025 11:42:34 +0100 Subject: [PATCH 3/6] added ingress setup in configuration --- .../templates/ingress-nginx/controller.yaml | 9 +++ scripts/local-cluster.sh | 69 ++++++++++--------- .../configs/partial/sc-node-local-dns.yaml | 7 +- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml b/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml index 83dc504810..6b31c473bb 100644 --- a/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml +++ b/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml @@ -189,4 +189,13 @@ spec: acme.cert-manager.io/http01-solver: "true" ports: - port: 8089 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: minio-system + podSelector: + matchLabels: + app: minio + ports: + - port: 9000 {{- end }} diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 3976b81289..51c6fb5264 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -280,16 +280,22 @@ resolve() { } config() { - local name flavor domain ops_prefix self_signed="false" + local name flavor domain ops_prefix self_signed="false" share_object_storage="false" name="${1:-}" flavor="${2:-}" domain="${3:-}" if [[ "${4:-}" != "--self-signed" ]]; then ops_prefix="${4:-"ops"}" fi - if [[ "${4:-}" == "--self-signed" ]] || [[ "${5:-}" == "--self-signed" ]]; then - self_signed="true" - fi + + for arg in "${@:4}"; do + if [[ "$arg" == "--self-signed" ]]; then + self_signed="true" + fi + if [[ "$arg" == "--share-object-storage" ]]; then + share_object_storage="true" + fi + done export name export domain @@ -353,13 +359,27 @@ config() { fi "${ROOT}/bin/ck8s" init both + + if [[ "${share_object_storage}" == "true" ]]; then + log.info "Configuring shared object storage endpoint for Minio" + yq -Pi ".objectStorage.s3.regionEndpoint = \"http://minio.${domain}:30080\"" "${CK8S_CONFIG_PATH}/common-config.yaml" + yq -Pi '.networkPolicies.global.objectStorage.ports[0] = 30080' "${CK8S_CONFIG_PATH}/common-config.yaml" + yq -Pi '.networkPolicies.global.objectStorage.ports[1] = 80' "${CK8S_CONFIG_PATH}/common-config.yaml" + yq -Pi '.networkPolicies.ingressNginx.ingressOverride.enabled = false' "${CK8S_CONFIG_PATH}/common-config.yaml" + fi } create() { - local cluster config affix + local cluster config affix share_object_storage=false cluster="${1:-}" config="${2:-}" + for arg in "${@:3}"; do + if [[ "$arg" == "--share-object-storage" ]]; then + share_object_storage="true" + fi + done + if [[ -z "${cluster}" ]]; then log.usage fi @@ -459,6 +479,18 @@ create() { helmfile -e local_cluster -f "${ROOT}/helmfile.d" -lapp=minio apply --output simple fi + if [[ "${share_object_storage}" == "true" ]]; then + log.info "Installing ingress-nginx in service cluster for shared object storage" + "${ROOT}/bin/ck8s" ops helmfile sc -lapp=ingress-nginx apply --include-transitive-needs --output simple + log.info "Enabling Ingress in Minio" + domain="$(yq ".global.baseDomain" <"${CK8S_CONFIG_PATH}/common-config.yaml")" + "${ROOT}/bin/ck8s" ops helm sc upgrade -n minio-system minio "${ROOT}/helmfile.d/upstream/minio/minio" \ + --reuse-values \ + --set ingress.enabled=true \ + --set ingress.ingressClassName=nginx \ + --set ingress.hosts[0]=minio."${domain}" + fi + index.state "${cluster}" "ready" log.info "cluster ${cluster} is ready" } @@ -515,30 +547,6 @@ setup_node_local_dns() { fi } -setup_sc_s3_sharable() { - log.info "Setting up s3 as sharable" - local domain CK8S_CLUSTER CK8S_DRY_RUN_INSTALL - domain="$(yq ".global.baseDomain" <"${CK8S_CONFIG_PATH}/common-config.yaml")" - # shellcheck source=scripts/migration/lib.sh - source "${ROOT}/scripts/migration/lib.sh" - CK8S_CLUSTER=both - CK8S_DRY_RUN_INSTALL=false - - yq_add common '.objectStorage.s3.regionEndpoint' "\"http://minio.${domain}\"" - yq_add common '.ingressNginx.controller.useHostPort' 'true' - yq_add common '.networkPolicies.global.objectStorage.ports[0]' '80' - yq_add common '.networkPolicies.ingressNginx.ingressOverride.enabled' 'false' - - log.info "Installing Ingress in SC" - helmfile_do sc apply -lapp=ingress-nginx --include-transitive-needs --output simple - log.info "Upgrading Minio Chart with Ingress" - helm_do sc upgrade -n minio-system minio "${ROOT}/helmfile.d/upstream/minio/minio" \ - --reuse-values \ - --set ingress.enabled=true \ - --set ingress.ingressClassName=nginx \ - --set ingress.hosts[0]=minio."${domain}" -} - delete() { local cluster cluster="${1:-}" @@ -581,9 +589,6 @@ main() { node-local-dns) setup_node_local_dns ;; - sharable-s3) - setup_sc_s3_sharable - ;; *) log.usage ;; diff --git a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml index db68608e04..38df0a432c 100644 --- a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml +++ b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml @@ -4,10 +4,15 @@ nodeLocalDns: errors bind 169.254.20.10 10.96.0.10 template IN A $domain { - match "(^dex|^grafana|^harbor|^minio|^opensearch|\.ops)\.$domain\.$" + match "(^dex|^grafana|^harbor|^opensearch|\.ops)\.$domain\.$" answer "{{ .Name }} 60 IN A 10.96.0.20" fallthrough } + template IN A $domain { + match "^minio\.$domain\.$" + answer "{{ .Name }} 60 IN A $sc_node_ip" + fallthrough + } template IN A $domain { match "\.$domain\.$" answer "{{ .Name }} 60 IN A $wc_node_ip" From 66f2e5428b215335fcae005df6b5c1665d98b2f6 Mon Sep 17 00:00:00 2001 From: Mohamed Shafiq Date: Thu, 27 Nov 2025 11:43:58 +0100 Subject: [PATCH 4/6] added log info --- scripts/local-cluster.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 51c6fb5264..6a093cf67b 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -359,7 +359,7 @@ config() { fi "${ROOT}/bin/ck8s" init both - + log.info "Adding Shared object storage configs" if [[ "${share_object_storage}" == "true" ]]; then log.info "Configuring shared object storage endpoint for Minio" yq -Pi ".objectStorage.s3.regionEndpoint = \"http://minio.${domain}:30080\"" "${CK8S_CONFIG_PATH}/common-config.yaml" From 642b1134a8833ab78acdc7506799cffe161a3bc4 Mon Sep 17 00:00:00 2001 From: Mohamed Shafiq Date: Thu, 18 Dec 2025 13:42:20 +0100 Subject: [PATCH 5/6] resolved conflict --- .../templates/ingress-nginx/controller.yaml | 2 + .../networkpolicy/service-cluster/values.yaml | 3 ++ helmfile.d/values/minio.yaml.gotmpl | 6 +++ .../networkpolicy/service-cluster.yaml.gotmpl | 11 +++++ scripts/local-cluster.sh | 46 ++++--------------- .../local-clusters/configs/common-config.yaml | 5 +- 6 files changed, 34 insertions(+), 39 deletions(-) diff --git a/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml b/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml index 6b31c473bb..9d0230531f 100644 --- a/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml +++ b/helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml @@ -189,6 +189,7 @@ spec: acme.cert-manager.io/http01-solver: "true" ports: - port: 8089 + {{- if .Values.objectStorage.localEnabled }} - to: - namespaceSelector: matchLabels: @@ -198,4 +199,5 @@ spec: app: minio ports: - port: 9000 + {{- end }} {{- end }} diff --git a/helmfile.d/charts/networkpolicy/service-cluster/values.yaml b/helmfile.d/charts/networkpolicy/service-cluster/values.yaml index d094f0edd8..c7fc209732 100644 --- a/helmfile.d/charts/networkpolicy/service-cluster/values.yaml +++ b/helmfile.d/charts/networkpolicy/service-cluster/values.yaml @@ -132,3 +132,6 @@ dex: enabled: true ips: - "0.0.0.0/0" + +objectStorage: + localEnabled: false diff --git a/helmfile.d/values/minio.yaml.gotmpl b/helmfile.d/values/minio.yaml.gotmpl index d054681e64..ce6638b817 100644 --- a/helmfile.d/values/minio.yaml.gotmpl +++ b/helmfile.d/values/minio.yaml.gotmpl @@ -14,3 +14,9 @@ buckets: {{- range $key, $value := .Values.objectStorage.buckets }} - name: {{ $value }} {{- end }} + +ingress: + enabled: true + ingressClassName: nginx + hosts: + - minio.{{ .Values.global.baseDomain }} diff --git a/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl b/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl index 45530a97d7..96463b7263 100644 --- a/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl +++ b/helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl @@ -81,3 +81,14 @@ dex: connectors: ips: {{- toYaml .Values.networkPolicies.dex.connectors.ips | nindent 6 }} ports: {{- toYaml .Values.networkPolicies.dex.connectors.ports | nindent 6 }} + +{{- with .Values.objectStorage }} +{{- if and + (eq .type "s3") + (.s3.regionEndpoint | hasPrefix "http://minio.") + (eq .s3.region "local") +}} +objectStorage: + localEnabled: true +{{- end }} +{{- end }} diff --git a/scripts/local-cluster.sh b/scripts/local-cluster.sh index 6a093cf67b..f535ef11df 100755 --- a/scripts/local-cluster.sh +++ b/scripts/local-cluster.sh @@ -280,22 +280,16 @@ resolve() { } config() { - local name flavor domain ops_prefix self_signed="false" share_object_storage="false" + local name flavor domain ops_prefix self_signed="false" name="${1:-}" flavor="${2:-}" domain="${3:-}" if [[ "${4:-}" != "--self-signed" ]]; then ops_prefix="${4:-"ops"}" fi - - for arg in "${@:4}"; do - if [[ "$arg" == "--self-signed" ]]; then - self_signed="true" - fi - if [[ "$arg" == "--share-object-storage" ]]; then - share_object_storage="true" - fi - done + if [[ "${4:-}" == "--self-signed" ]] || [[ "${5:-}" == "--self-signed" ]]; then + self_signed="true" + fi export name export domain @@ -359,27 +353,13 @@ config() { fi "${ROOT}/bin/ck8s" init both - log.info "Adding Shared object storage configs" - if [[ "${share_object_storage}" == "true" ]]; then - log.info "Configuring shared object storage endpoint for Minio" - yq -Pi ".objectStorage.s3.regionEndpoint = \"http://minio.${domain}:30080\"" "${CK8S_CONFIG_PATH}/common-config.yaml" - yq -Pi '.networkPolicies.global.objectStorage.ports[0] = 30080' "${CK8S_CONFIG_PATH}/common-config.yaml" - yq -Pi '.networkPolicies.global.objectStorage.ports[1] = 80' "${CK8S_CONFIG_PATH}/common-config.yaml" - yq -Pi '.networkPolicies.ingressNginx.ingressOverride.enabled = false' "${CK8S_CONFIG_PATH}/common-config.yaml" - fi } create() { - local cluster config affix share_object_storage=false + local cluster config affix cluster="${1:-}" config="${2:-}" - for arg in "${@:3}"; do - if [[ "$arg" == "--share-object-storage" ]]; then - share_object_storage="true" - fi - done - if [[ -z "${cluster}" ]]; then log.usage fi @@ -468,6 +448,10 @@ create() { helmfile -e local_cluster -f "${ROOT}/helmfile.d" -lapp=tigera apply --output simple fi + #install ingress-nginx + log.info "Installing ingress-nginx in SC" + "${ROOT}/bin/ck8s" ops helmfile sc -lapp=ingress-nginx apply --include-transitive-needs --output simple + # install s3 if ! [[ "${*}" =~ --skip-minio ]]; then log.info "installing minio" @@ -479,18 +463,6 @@ create() { helmfile -e local_cluster -f "${ROOT}/helmfile.d" -lapp=minio apply --output simple fi - if [[ "${share_object_storage}" == "true" ]]; then - log.info "Installing ingress-nginx in service cluster for shared object storage" - "${ROOT}/bin/ck8s" ops helmfile sc -lapp=ingress-nginx apply --include-transitive-needs --output simple - log.info "Enabling Ingress in Minio" - domain="$(yq ".global.baseDomain" <"${CK8S_CONFIG_PATH}/common-config.yaml")" - "${ROOT}/bin/ck8s" ops helm sc upgrade -n minio-system minio "${ROOT}/helmfile.d/upstream/minio/minio" \ - --reuse-values \ - --set ingress.enabled=true \ - --set ingress.ingressClassName=nginx \ - --set ingress.hosts[0]=minio."${domain}" - fi - index.state "${cluster}" "ready" log.info "cluster ${cluster} is ready" } diff --git a/scripts/local-clusters/configs/common-config.yaml b/scripts/local-clusters/configs/common-config.yaml index 67f625cd77..e0252a8356 100644 --- a/scripts/local-clusters/configs/common-config.yaml +++ b/scripts/local-clusters/configs/common-config.yaml @@ -13,7 +13,7 @@ objectStorage: type: s3 s3: region: local - regionEndpoint: http://minio.minio-system.svc.cluster.local:9000 + regionEndpoint: http://minio.${domain}:30080 forcePathStyle: true calicoAccountant: backend: nftables @@ -50,7 +50,8 @@ networkPolicies: ips: - 0.0.0.0/0 ports: - - 9000 + - 30080 + - 80 scIngress: ips: - 0.0.0.0/0 From afcb5fd9174e492ea68476d348b8f6b4f0849006 Mon Sep 17 00:00:00 2001 From: Mohamed Shafiq Date: Thu, 18 Dec 2025 22:54:30 +0100 Subject: [PATCH 6/6] changed according to latest mock lb change --- scripts/local-clusters/configs/common-config.yaml | 3 +-- .../local-clusters/configs/partial/sc-node-local-dns.yaml | 7 +------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/scripts/local-clusters/configs/common-config.yaml b/scripts/local-clusters/configs/common-config.yaml index e0252a8356..358a0a1670 100644 --- a/scripts/local-clusters/configs/common-config.yaml +++ b/scripts/local-clusters/configs/common-config.yaml @@ -13,7 +13,7 @@ objectStorage: type: s3 s3: region: local - regionEndpoint: http://minio.${domain}:30080 + regionEndpoint: http://minio.${domain} forcePathStyle: true calicoAccountant: backend: nftables @@ -50,7 +50,6 @@ networkPolicies: ips: - 0.0.0.0/0 ports: - - 30080 - 80 scIngress: ips: diff --git a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml index 38df0a432c..9e2df5c94d 100644 --- a/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml +++ b/scripts/local-clusters/configs/partial/sc-node-local-dns.yaml @@ -4,15 +4,10 @@ nodeLocalDns: errors bind 169.254.20.10 10.96.0.10 template IN A $domain { - match "(^dex|^grafana|^harbor|^opensearch|\.ops)\.$domain\.$" + match "(^dex|^grafana|^harbor|^opensearch|^minio|\.ops)\.$domain\.$" answer "{{ .Name }} 60 IN A 10.96.0.20" fallthrough } - template IN A $domain { - match "^minio\.$domain\.$" - answer "{{ .Name }} 60 IN A $sc_node_ip" - fallthrough - } template IN A $domain { match "\.$domain\.$" answer "{{ .Name }} 60 IN A $wc_node_ip"