Integration test for hotrod on k8s

scripts/build-hotrod-image.sh

@@ -5,41 +5,22 @@
 
 set -euf -o pipefail
 
+JAEGER_QUERY_URL="http://localhost:16686"
+EXPECTED_SPANS=35
+MAX_RETRIES=30
+SLEEP_INTERVAL=3
+
 print_help() {
-  echo "Usage: $0 [-h] [-l] [-o] [-p platforms]"
+  echo "Usage: $0 [-h] [-l] [-o] [-p platforms] [-d | -k]"
   echo "-h: Print help"
   echo "-l: Enable local-only mode that only pushes images to local registry"
-  echo "-o: overwrite image in the target remote repository even if the semver tag already exists"
+  echo "-o: Overwrite image in the target remote repository even if the semver tag already exists"
   echo "-p: Comma-separated list of platforms to build for (default: all supported)"
+  echo "-d: Run Docker tests only"
+  echo "-k: Run Kubernetes tests only"
   exit 1
 }
 
-docker_compose_file="./examples/hotrod/docker-compose.yml"
-platforms="$(make echo-linux-platforms)"
-current_platform="$(go env GOOS)/$(go env GOARCH)"
-FLAGS=()
-success="false"
-
-while getopts "hlop:" opt; do
-	case "${opt}" in
-	l)
-		# in the local-only mode the images will only be pushed to local registry
-    FLAGS=("${FLAGS[@]}" -l)
-		;;
-	o)
-		FLAGS=("${FLAGS[@]}" -o)
-		;;
-	p)
-		platforms=${OPTARG}
-		;;
-	*)
-		print_help
-		;;
-	esac
-done
-
-set -x
-
 dump_logs() {
   local compose_file=$1
   echo "::group:: Hotrod logs"
@@ -48,97 +29,194 @@ dump_logs() {
 }
 
 teardown() {
-  echo "Tearing down..."
+  if [[ "${TEST_MODE:-}" == "docker" ]]; then
+    teardown_docker
+  elif [[ "${TEST_MODE:-}" == "k8s" ]]; then
+    teardown_k8s
+  else
+    teardown_docker
+    teardown_k8s
+  fi
+}
+
+teardown_docker() {
+  echo "Tearing down Docker..."
   if [[ "$success" == "false" ]]; then
     dump_logs "${docker_compose_file}"
   fi
   docker compose -f "$docker_compose_file" down
 }
-trap teardown EXIT
-
-make prepare-docker-buildx
-make create-baseimg LINUX_PLATFORMS="$platforms"
-
-# Build hotrod binary for each target platform (separated by commas)
-for platform in $(echo "$platforms" | tr ',' ' '); do
-  # Extract the operating system from the platform string
-  os=${platform%%/*}  #remove everything after the last slash
-  # Extract the architecture from the platform string
-  arch=${platform##*/}  # Remove everything before the last slash
-  make build-examples GOOS="${os}" GOARCH="${arch}"
-done
-
-# Build hotrod image locally (-l) for integration test.
-# Note: hotrod's Dockerfile is different from main binaries,
-# so we do not pass flags like -b and -t.
-bash scripts/build-upload-a-docker-image.sh -l -c example-hotrod -d examples/hotrod -p "${current_platform}"
-
-# Build all-in-one image locally (-l) for integration test
-make build-all-in-one
-bash scripts/build-upload-a-docker-image.sh -l -b -c all-in-one -d cmd/all-in-one -p "${current_platform}" -t release
-
-echo '::group:: docker compose'
-JAEGER_VERSION=$GITHUB_SHA REGISTRY="localhost:5000/" docker compose -f "$docker_compose_file" up -d
-echo '::endgroup::'
-
-i=0
-while [[ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8080)" != "200" && $i -lt 30 ]]; do
-  sleep 1
-  i=$((i+1))
-done
-
-echo '::group:: check HTML'
-echo 'Check that home page contains text Rides On Demand'
-body=$(curl localhost:8080)
-if [[ $body != *"Rides On Demand"* ]]; then
-  echo "String \"Rides On Demand\" is not present on the index page"
-  exit 1
-fi
-echo '::endgroup::'
-
-response=$(curl -i -X POST "http://localhost:8080/dispatch?customer=123")
-TRACE_ID=$(echo "$response" | grep -Fi "Traceresponse:" | awk '{print $2}' | cut -d '-' -f 2)
 
-if [ -n "$TRACE_ID" ]; then
-  echo "TRACE_ID is not empty: $TRACE_ID"
-else
-  echo "TRACE_ID is empty"
-  exit 1
-fi
-
-JAEGER_QUERY_URL="http://localhost:16686"
-EXPECTED_SPANS=35
-MAX_RETRIES=30
-SLEEP_INTERVAL=3
+teardown_k8s() {
+  echo "Cleaning up Kubernetes resources..."
+  if [[ -n "${HOTROD_PORT_FWD_PID:-}" ]]; then
+    kill $HOTROD_PORT_FWD_PID || true
+  fi
+  if [[ -n "${JAEGER_PORT_FWD_PID:-}" ]]; then
+    kill $JAEGER_PORT_FWD_PID || true
+  fi
+  kubectl delete namespace example-hotrod --ignore-not-found=true
+}
 
-poll_jaeger() {
-  local trace_id=$1
-  local url="${JAEGER_QUERY_URL}/api/traces/${trace_id}"
+build_setup() {
+  local platforms=$1
+
+  make prepare-docker-buildx
+  make create-baseimg LINUX_PLATFORMS="$platforms"
+
+  # Build hotrod binary for each target platform
+  for platform in $(echo "$platforms" | tr ',' ' '); do
+    local os=${platform%%/*}
+    local arch=${platform##*/}
+    make build-examples GOOS="${os}" GOARCH="${arch}"
+  done
+
+  # Build local images for testing
+  bash scripts/build-upload-a-docker-image.sh -l -c example-hotrod -d examples/hotrod -p "${current_platform}"
+  make build-all-in-one
+  bash scripts/build-upload-a-docker-image.sh -l -b -c all-in-one -d cmd/all-in-one -p "${current_platform}" -t release
+}
 
-  curl -s "${url}" | jq '.data[0].spans | length' || echo "0"
+check_web_app() {
+  local i=0
+  while [[ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8080)" != "200" && $i -lt 30 ]]; do
+    sleep 1
+    i=$((i+1))
+  done
+
+  if [[ $i -eq 30 ]]; then
+    echo "Timeout waiting for web app to become available"
+    exit 1
+  fi
 }
 
-# Poll Jaeger until trace with desired number of spans is loaded or we timeout.
-span_count=0
-for ((i=1; i<=MAX_RETRIES; i++)); do
-  span_count=$(poll_jaeger "${TRACE_ID}")
+verify_home_page() {
+  echo '::group:: check HTML'
+  echo 'Check that home page contains text Rides On Demand'
+  local body
+  body=$(curl -s localhost:8080)
+  if [[ $body != *"Rides On Demand"* ]]; then
+    echo "String \"Rides On Demand\" is not present on the index page"
+    exit 1
+  fi
+  echo '::endgroup::'
+}
 
-  if [[ "$span_count" -ge "$EXPECTED_SPANS" ]]; then
-    echo "Trace found with $span_count spans."
-    break
+get_trace_id() {
+  local response
+  local trace_id
+
+  response=$(curl -s -i -X POST "http://localhost:8080/dispatch?customer=123")
+  trace_id=$(echo "$response" | grep -Fi "Traceresponse:" | awk '{print $2}' | cut -d '-' -f 2)
+
+  if [ -z "$trace_id" ]; then
+    echo "Failed to get trace ID" >&2
+    exit 1
   fi
+
+  echo "$trace_id"
+}
 
-  echo "Retry $i/$MAX_RETRIES: Trace not found or insufficient spans ($span_count/$EXPECTED_SPANS). Retrying in $SLEEP_INTERVAL seconds..."
-  sleep $SLEEP_INTERVAL
-done
+verify_trace() {
+  local trace_id=$1
+  local span_count=0
+
+  for ((i=1; i<=MAX_RETRIES; i++)); do
+    span_count=$(curl -s "${JAEGER_QUERY_URL}/api/traces/${trace_id}" | jq '.data[0].spans | length' || echo "0")
+
+    if [[ "$span_count" -ge "$EXPECTED_SPANS" ]]; then
+      echo "Trace found with $span_count spans."
+      return 0
+    fi
+
+    echo "Retry $i/$MAX_RETRIES: Found $span_count/$EXPECTED_SPANS spans. Retrying in $SLEEP_INTERVAL seconds..."
+    sleep $SLEEP_INTERVAL
+  done
+
+  echo "Failed to find trace with expected number of spans within timeout period"
+  return 1
+}
 
-if [[ "$span_count" -lt "$EXPECTED_SPANS" ]]; then
-  echo "Failed to find the trace with the expected number of spans within the timeout period."
-  exit 1
-fi
+run_docker_tests() {
+  TEST_MODE="docker"
+  echo "Running Docker tests..."
+
+  JAEGER_VERSION=$GITHUB_SHA REGISTRY="localhost:5000/" docker compose -f "$docker_compose_file" up -d
+
+  check_web_app
+  verify_home_page
+
+  local trace_id
+  trace_id=$(get_trace_id)
+  verify_trace "$trace_id"
+}
+
+run_k8s_tests() {
+  TEST_MODE="k8s"
+  echo "Running Kubernetes tests..."
+
+  # Create namespace and deploy resources
+  kubectl create namespace example-hotrod --dry-run=client -o yaml | kubectl apply -f -
+  kustomize build ./examples/hotrod/kubernetes | kubectl apply -n example-hotrod -f -
+
+  # Wait for deployments
+  kubectl wait --for=condition=available --timeout=180s -n example-hotrod deployment/example-hotrod
+  kubectl wait --for=condition=available --timeout=180s -n example-hotrod deployment/jaeger
+
+  # Setup port forwarding
+  kubectl port-forward -n example-hotrod svc/example-hotrod 8080:8080 &
+  HOTROD_PORT_FWD_PID=$!
+  kubectl port-forward -n example-hotrod svc/jaeger 16686:16686 &
+  JAEGER_PORT_FWD_PID=$!
+
+  # Wait for port-forward to be ready
+  sleep 5
+
+  check_web_app
+  verify_home_page
+
+  local trace_id
+  trace_id=$(get_trace_id)
+  verify_trace "$trace_id"
+}
 
-success="true"
+main() {
+  docker_compose_file="./examples/hotrod/docker-compose.yml"
+  platforms="$(make echo-linux-platforms)"
+  current_platform="$(go env GOOS)/$(go env GOARCH)"
+  FLAGS=()
+  success="false"
+
+  trap teardown EXIT
+
+  local run_docker=true
+  local run_kubernetes=true
+
+  while getopts "hlop:dk" opt; do
+    case "${opt}" in
+      l) FLAGS=("${FLAGS[@]}" -l) ;;
+      o) FLAGS=("${FLAGS[@]}" -o) ;;
+      p) platforms=${OPTARG} ;;
+      d) run_kubernetes=false ;;
+      k) run_docker=false ;;
+      *) print_help ;;
+    esac
+  done
+
+  build_setup "$platforms"
+
+  if [[ "$run_docker" == "true" ]]; then
+    run_docker_tests
+  fi
+
+  if [[ "$run_kubernetes" == "true" ]]; then
+    run_k8s_tests
+  fi
+
+  success="true"
+
+  # Build and upload the final image
+  bash scripts/build-upload-a-docker-image.sh "${FLAGS[@]}" -c example-hotrod -d examples/hotrod -p "${platforms}"
+}
 
-# Ensure the image is published after successful test (maybe with -l flag if on a pull request).
-# This is where all those multi-platform binaries we built earlier are utilized.
-bash scripts/build-upload-a-docker-image.sh "${FLAGS[@]}" -c example-hotrod -d examples/hotrod -p "${platforms}"
+main "$@"