scripts: added sharing objectstorage for local cluster with ingress

DEVELOPMENT.md

@@ -13,7 +13,7 @@ This requires that `kind` is installed and that either `podman` or `docker` is a
 ### Terminology
 
 - `apps-flavor` - one of `prod`, `dev` or `air-gapped` is a global switch for configuring the clusters and deployed applications in different ways. The recommended value when working with local clusters is `dev`.
-- `local-cluster-profile` - a reference to a preconfigured `Cluster` config passed to `kind` when creating or updating local clusters. We use profiles to differentiate between single-/multi-node clusters and/or to enable special features such as container image caching. Use `./scripts/local-clusters.sh list profiles` to see a list of built-in profiles.
+- `local-cluster-profile` - a reference to a preconfigured `Cluster` config passed to `kind` when creating or updating local clusters. We use profiles to differentiate between single-/multi-node clusters and/or to enable special features such as container image caching. Use `./scripts/local-cluster.sh list profiles` to see a list of built-in profiles.
 - `domain` - a local domain name. This can be arbitrary, but using a real domain (or subdomain) for which we have authority allows for setting up DNS based challenges for certificates issued by `cert-manager`.
 
 ### Setup

helmfile.d/charts/networkpolicy/service-cluster/templates/ingress-nginx/controller.yaml

@@ -189,4 +189,15 @@ spec:
               acme.cert-manager.io/http01-solver: "true"
       ports:
         - port: 8089
+    {{- if .Values.objectStorage.localEnabled }}
+    - to:
+       - namespaceSelector:
+           matchLabels:
+             kubernetes.io/metadata.name: minio-system
+         podSelector:
+           matchLabels:
+             app: minio
+      ports:
+        - port: 9000
+    {{- end }}
 {{- end }}

helmfile.d/charts/networkpolicy/service-cluster/values.yaml

@@ -132,3 +132,6 @@ dex:
   enabled: true
   ips:
     - "0.0.0.0/0"
+
+objectStorage:
+  localEnabled: false

helmfile.d/values/minio.yaml.gotmpl

@@ -14,3 +14,9 @@ buckets:
   {{- range $key, $value := .Values.objectStorage.buckets }}
   - name: {{ $value }}
   {{- end }}
+
+ingress:
+  enabled: true
+  ingressClassName: nginx
+  hosts:
+    - minio.{{ .Values.global.baseDomain }}

helmfile.d/values/networkpolicy/service-cluster.yaml.gotmpl

@@ -81,3 +81,14 @@ dex:
   connectors:
     ips: {{- toYaml .Values.networkPolicies.dex.connectors.ips | nindent 6 }}
     ports: {{- toYaml .Values.networkPolicies.dex.connectors.ports | nindent 6 }}
+
+{{- with .Values.objectStorage }}
+{{- if and
+  (eq .type "s3")
+  (.s3.regionEndpoint | hasPrefix "http://minio.")
+  (eq .s3.region "local")
+}}
+objectStorage:
+  localEnabled: true
+{{- end }}
+{{- end }}

scripts/local-cluster.sh

@@ -443,6 +443,10 @@ create() {
     helmfile -e local_cluster -f "${ROOT}/helmfile.d" -lapp=tigera apply --output simple
   fi
 
+  #install ingress-nginx
+  log.info "Installing ingress-nginx in SC"
+  "${ROOT}/bin/ck8s" ops helmfile sc -lapp=ingress-nginx apply --include-transitive-needs --output simple
+
   # install s3
   if ! [[ "${*}" =~ --skip-minio ]]; then
     log.info "installing minio"

scripts/local-clusters/configs/common-config.yaml

@@ -13,7 +13,7 @@ objectStorage:
   type: s3
   s3:
     region: local
-    regionEndpoint: http://minio.minio-system.svc.cluster.local:9000
+    regionEndpoint: http://minio.${domain}:30080
     forcePathStyle: true
 calicoAccountant:
   backend: nftables
@@ -53,7 +53,8 @@ networkPolicies:
       ips:
         - 0.0.0.0/0
       ports:
-        - 9000
+        - 30080
+        - 80
     scIngress:
       ips:
         - 0.0.0.0/0
@@ -88,3 +89,6 @@ networkPolicies:
     packageRegistry:
       ips:
         - 0.0.0.0/0
+  ingressNginx:
+    ingressOverride:
+      enabled: false

scripts/local-clusters/configs/partial/sc-node-local-dns.yaml

@@ -8,6 +8,11 @@ nodeLocalDns:
         answer "{{ .Name }} 60 IN A 10.96.0.20"
         fallthrough
       }
+      template IN A $domain {
+         match "^minio\.$domain\.$"
+         answer "{{ .Name }} 60 IN A $sc_node_ip"
+         fallthrough
+      }
       template IN A $domain {
         match "\.$domain\.$"
         answer "{{ .Name }} 60 IN A $wc_node_ip"

scripts/local-clusters/configs/partial/wc-node-local-dns.yaml

@@ -4,7 +4,7 @@ nodeLocalDns:
       errors
       bind 169.254.20.10 10.96.0.10
       template IN A $domain {
-        match "(^dex|^grafana|^harbor|^opensearch|\.ops)\.$domain\.$"
+        match "(^dex|^grafana|^harbor|^minio|^opensearch|\.ops)\.$domain\.$"
         answer "{{ .Name }} 60 IN A $sc_node_ip"
         fallthrough
       }