#!/usr/bin/env bash
# scripts/dns-churn-stages.sh — STRICT_DNS dead-IP 3-stage 실험 하네스 (단일 스크립트로 순차 실행 —
# 배경 디스패치 타이밍 함정을 피하려고 top-level 백그라운드 호출을 쓰지 않는다).
#   stage1: DR46(churn만)          + VS42(no-retry)
#   stage2: DR47(churn+outlier)    + VS42(no-retry)
#   stage3: DR47(churn+outlier)    + VS45(retry)
#
# 전제: scripts/dns-lab-setup.sh 로 랩이 떠 있고, SE=gslb-strict(resolution:DNS) 적용 상태.
# scenarios/50-dns-resolution/46-destinationrule-tls-churn.yaml, 47-destinationrule-tls-churn-outlier.yaml
# (44 교정판+46 합본)이 있어야 한다. maxRequestsPerConnection:1 로 매 요청 upstream 연결을 강제
# churn시켜야 dead-IP를 실제로 뽑는다 — client 쪽 keepalive=false 만으로는 upstream pool이 재사용돼
# 유실이 안 만들어진다(§2 실측 함정).
#
# 출처: docs/test-reports/2026-07-02_121151_dns-mode2-churn.md
#       ("유실"이 아니라 "지연세" — Envoy 내장 connect-failure 재시도가 VS retries 정책과 무관하게 동작)
#
# usage: bash scripts/dns-churn-stages.sh
set -uo pipefail
CTX="${CTX:-homelab}"; NS="${NS:-dns-lab}"
DIR="$(cd "$(dirname "$0")/.." && pwd)"
SC="$DIR/scenarios/50-dns-resolution"
WORK="$DIR/tmp/dns-churn"; mkdir -p "$WORK"
K="kubectl --context=$CTX -n $NS"

IP_A="$($K get svc backend-a -o jsonpath='{.spec.clusterIP}')"
IP_B="$($K get svc backend-b -o jsonpath='{.spec.clusterIP}')"
LOADSEC=40

snap(){ # $1 = label
  echo "=== $1 @ $(date +%H:%M:%S) ==="
  echo "-- dig --"
  $K exec deploy/netshoot -- dig +short gslb.lab.internal 2>/dev/null
  echo "-- endpoints (netshoot) --"
  istioctl --context="$CTX" proxy-config endpoints "deploy/netshoot.$NS" --cluster "outbound|443||gslb.lab.internal" 2>/dev/null
  echo "-- cluster stats (gslb, netshoot sidecar) --"
  $K exec deploy/netshoot -c istio-proxy -- pilot-agent request GET "stats?filter=gslb" 2>/dev/null \
    | grep -E 'cluster\.outbound\|443.*(upstream_cx_total|upstream_cx_destroy|upstream_cx_active|upstream_cx_connect_fail|upstream_rq_total|upstream_rq_retry|upstream_rq_pending_failure_eject|membership_change|outlier_detection)' \
    | sed 's#cluster.outbound|443||gslb.lab.internal.##'
  echo "-- endpoint cx (clusters, netshoot sidecar) --"
  $K exec deploy/netshoot -c istio-proxy -- pilot-agent request GET clusters 2>/dev/null \
    | grep -E 'outbound\|443\|\|gslb\.lab\.internal::[0-9]' \
    | sed 's#outbound|443||gslb.lab.internal::##'
  echo
}

loop(){ # $1=sec $2=outfile -- 매 요청 신규 curl 프로세스, 타임스탬프+순번+코드+소요시간 기록
  $K exec deploy/netshoot -- sh -c '
    end=$(( $(date +%s) + '"$1"' ))
    n=0
    while [ $(date +%s) -lt $end ]; do
      n=$((n+1))
      r=$(curl -s -m 3 -o /dev/null -w "%{http_code} %{time_total}" http://gslb.lab.internal/ 2>/dev/null)
      printf "%s n=%d %s\n" "$(date +%H:%M:%S)" "$n" "$r"
    done' | tee "$2"
}

run_stage(){
  STAGE="$1"; DRFILE="$2"; VSFILE="$3"; LABEL="$4"
  echo "########## STAGE $STAGE ($LABEL) ##########"
  echo ">> apply DR=$DRFILE VS=$VSFILE"
  kubectl --context="$CTX" apply -f "$DRFILE" -f "$VSFILE"
  sleep 3
  echo ">> flip_both A,B (dead-IP 조건 준비)"
  $K exec deploy/lab-dns -c writer -- sh -c "printf '%s gslb.lab.internal\n%s gslb.lab.internal\n' '$IP_A' '$IP_B' > /hosts/addn"
  sleep 6
  echo "-- pre-kill dig (both A,B 등록 확인) --"
  $K exec deploy/netshoot -- dig +short gslb.lab.internal
  echo ">> KILL backend-a (scale 0) @ $(date +%H:%M:%S) — DNS엔 A 잔존(dead-IP)"
  $K scale deploy/backend-a --replicas=0
  sleep 4
  echo
  echo "## STAGE $STAGE BEFORE snapshot"
  snap "stage${STAGE}_BEFORE" | tee "$WORK/stage${STAGE}_before.txt"
  echo
  echo "## STAGE $STAGE LOAD (${LOADSEC}s churn loop) start=$(date +%H:%M:%S)"
  loop "$LOADSEC" "$WORK/stage${STAGE}_loop.txt"
  echo "## STAGE $STAGE LOAD end=$(date +%H:%M:%S)"
  echo
  echo "## STAGE $STAGE AFTER snapshot"
  snap "stage${STAGE}_AFTER" | tee "$WORK/stage${STAGE}_after.txt"
  echo
  echo ">> restore backend-a (scale 1) + wait READY"
  $K scale deploy/backend-a --replicas=1
  $K rollout status deploy/backend-a --timeout=90s
  sleep 5
  echo "-- post-stage sanity dig/curl --"
  $K exec deploy/lab-dns -c writer -- sh -c "printf '%s gslb.lab.internal\n' '$IP_A' > /hosts/addn"
  sleep 6
  $K exec deploy/netshoot -- dig +short gslb.lab.internal
  $K exec deploy/netshoot -- curl -s -m 5 http://gslb.lab.internal/
  echo "########## STAGE $STAGE ($LABEL) DONE ##########"
  echo
}

{
echo "# 작업B churn 실험 전체 raw 로그 — $(date +%Y-%m-%d_%H%M%S)"
run_stage 1 "$SC/46-destinationrule-tls-churn.yaml"          "$SC/42-virtualservice-80to443.yaml"        "baseline+churn(no outlier/no retry)"
run_stage 2 "$SC/47-destinationrule-tls-churn-outlier.yaml"  "$SC/42-virtualservice-80to443.yaml"        "+outlier+churn(no retry)"
run_stage 3 "$SC/47-destinationrule-tls-churn-outlier.yaml"  "$SC/45-virtualservice-80to443-retry.yaml"  "+outlier+churn+retry"
} 2>&1 | tee "$WORK/full_run.txt"

echo ">> ALL STAGES DONE. raw log: $WORK/full_run.txt"