#!/usr/bin/env bash
# Researched starting configuration, checked 2026-10-22. No GPU execution claimed.
# Run inside an SSH-capable Linux Runpod PyTorch Pod, not on the Windows client.
set -Eeuo pipefail

MODEL='dealignai/GLM-5.3-UNCENSORED-FP8'
REVISION='12170112d07e50d7dbe8e5ca1691f4fd8c1179e1'
STATE_DIR="${GLM_STATE_DIR:-/workspace/glm53}"
VENV="$STATE_DIR/venv-cu129-v0320"
READY_MARKER="$STATE_DIR/runtime-ready-v0320-cu129"
WHEEL='https://github.com/vllm-project/vllm/releases/download/v0.32.0/vllm-0.32.0%2Bcu129-cp38-abi3-manylinux_2_28_x86_64.whl#sha256=9918be806fb2808c406547ff4f9f43870ac6ff28dc4b179291fcf532145bb50b'
ARGS=(
  "$MODEL" --revision "$REVISION"
  --host 127.0.0.1 --port 8000 --served-model-name glm-5.3-uncensored
  --tensor-parallel-size 8 --gpu-memory-utilization 0.90
  --enforce-eager --disable-custom-all-reduce --enable-prefix-caching
  --max-num-seqs 1 --max-model-len 65536 --kv-cache-dtype fp8
  --reasoning-parser glm45 --tool-call-parser glm47 --enable-auto-tool-choice
  --default-chat-template-kwargs '{"reasoning_effort":"low","clear_thinking":true}'
  --sse-keep-alive-interval 15
)

case "${1:-}" in
  --dry-run)
    printf 'DRY RUN: no installation, GPU access, or model download.\n'
    printf 'Runtime: Python 3.12, official vLLM 0.32.0 CUDA 12.9 wheel.\n'
    printf 'Persistent state: %s\n' "$STATE_DIR"
    printf 'Command: '; printf '%q ' "$VENV/bin/vllm" serve "${ARGS[@]}"; printf '\n'
    exit 0 ;;
  --help|-h)
    printf 'Usage: bash run-server.sh [--dry-run]\n'
    printf 'Requires one Linux x86_64 Pod with 8 H200 GPUs and a CUDA 12.9-capable driver.\n'
    printf 'Keep the API private; connect with a Full SSH tunnel to localhost:8000.\n'
    printf 'Optional: MODEL_API_KEY enables API-key authentication inside the tunnel.\n'
    exit 0 ;;
  '') ;;
  *) printf 'Unknown option: %s\n' "$1" >&2; exit 2 ;;
esac

fail() { printf '\nSetup stopped: %s\n' "$*" >&2; exit 1; }
trap 'printf "\nCommand failed at line %s. The model stays cached under %s.\n" "$LINENO" "$STATE_DIR" >&2' ERR
[[ "$(uname -s)" == Linux && "$(uname -m)" == x86_64 ]] || fail 'Use this installer inside the Linux x86_64 GPU Pod.'
command -v python3 >/dev/null || fail 'Choose an official Runpod PyTorch template with Python and pip.'
command -v nvidia-smi >/dev/null || fail 'No NVIDIA driver is visible. Check the Pod GPU allocation.'
python3 -m pip --version >/dev/null || fail 'This template needs Python pip for the installer.'
command -v "${CXX:-c++}" >/dev/null || fail 'Choose runpod/pytorch:1.2.0-cu1290-torch2130-ubuntu2204; its development toolchain includes a C++ compiler.'
NVCC="${CUDA_HOME:-/usr/local/cuda}/bin/nvcc"
if [[ ! -x "$NVCC" ]]; then
  NVCC="$(command -v nvcc || true)"
fi
[[ -n "$NVCC" && -x "$NVCC" ]] || fail 'The CUDA compiler is missing. Use runpod/pytorch:1.2.0-cu1290-torch2130-ubuntu2204, not a runtime-only image.'
export CUDA_HOME
CUDA_HOME="$(dirname "$(dirname "$(readlink -f "$NVCC")")")"
export GLM_NVCC="$NVCC"

# Check the host before downloading the large runtime or checkpoint.
python3 - <<'PY'
import re
import os
import subprocess
import sys

nvcc = subprocess.check_output([os.environ["GLM_NVCC"], "--version"], text=True)
toolkit = re.search(r"release\s+(\d+)\.(\d+)", nvcc)
if not toolkit or tuple(map(int, toolkit.groups())) != (12, 9):
    sys.exit("Expected the CUDA 12.9 development toolkit. Select runpod/pytorch:1.2.0-cu1290-torch2130-ubuntu2204 instead of mixing CUDA toolkits.")
text = subprocess.check_output(["nvidia-smi"], text=True)
cuda = re.search(r"CUDA Version:\s*(\d+)\.(\d+)", text)
if not cuda or tuple(map(int, cuda.groups())) < (12, 9):
    sys.exit("Choose a host whose nvidia-smi reports CUDA Version 12.9 or newer; this installer uses the cu129 wheel.")
rows = subprocess.check_output(
    ["nvidia-smi", "--query-gpu=name,memory.total", "--format=csv,noheader,nounits"], text=True
).strip().splitlines()
if len(rows) != 8:
    sys.exit(f"Expected 8 GPUs on ONE Pod; found {len(rows)}. Stop and check the rental configuration.")
for row in rows:
    name, memory = row.rsplit(",", 1)
    if float(memory.strip()) < 130000:
        sys.exit(f"{name.strip()} has insufficient memory for this 8xH200 configuration.")
print("GPU/toolchain check passed: 8 GPUs with H200-class memory, CUDA 12.9 toolkit, compatible host driver.")
PY

mkdir -p "$STATE_DIR"
export HF_HOME="$STATE_DIR/huggingface"
export VLLM_CACHE_ROOT="$STATE_DIR/vllm-cache"
export UV_CACHE_DIR="$STATE_DIR/uv-cache"
export UV_PYTHON_INSTALL_DIR="$STATE_DIR/python"
export XDG_CACHE_HOME="$STATE_DIR/cache"
export GLM_STATE_DIR="$STATE_DIR"
mkdir -p "$HF_HOME" "$VLLM_CACHE_ROOT"

python3 - <<'PY'
import os
import shutil
import subprocess
import sys

root = os.environ["GLM_STATE_DIR"]
cache = os.path.join(os.environ["HF_HOME"], "hub", "models--dealignai--GLM-5.3-UNCENSORED-FP8")
cached = 0
if os.path.isdir(cache):
    cached = int(subprocess.check_output(["du", "-sk", cache], text=True).split()[0]) * 1024
free = shutil.disk_usage(root).free
if free < 30 * 1024**3 or free + cached < 850 * 1024**3:
    sys.exit("Not enough persistent storage. Allocate about 1.2 TB to /workspace; the weights alone are approximately 756 GB.")
print(f"Persistent storage check passed: {free / 1024**3:.0f} GiB free; existing checkpoint cache {cached / 1024**3:.0f} GiB.")
PY

# The system PyTorch environment stays untouched. uv manages a separate Python 3.12.
TOOLS="$STATE_DIR/installer-tools"
if [[ ! -x "$TOOLS/bin/uv" ]]; then
  printf '\nInstalling the small environment manager into persistent storage...\n'
  python3 -m pip install --disable-pip-version-check --target "$TOOLS" 'uv>=0.8,<1'
fi
UV="$TOOLS/bin/uv"
[[ -x "$UV" ]] || fail 'The uv installer did not create its executable.'
if [[ ! -x "$VENV/bin/python" ]]; then
  "$UV" venv --python 3.12 --managed-python --seed "$VENV"
fi

REUSE_RUNTIME=false
if [[ -f "$READY_MARKER" ]] && "$VENV/bin/python" - <<'PY'
from importlib.metadata import PackageNotFoundError, version
try:
    assert version("vllm") == "0.32.0+cu129"
except (PackageNotFoundError, AssertionError):
    raise SystemExit(1)
PY
then
  if "$UV" pip check --python "$VENV/bin/python"; then
    REUSE_RUNTIME=true
  fi
fi
if [[ "$REUSE_RUNTIME" != true ]]; then
  # A missing marker means installation or kernel download may have been
  # interrupted, even if package metadata already exists. Reinstall to repair it.
  rm -f -- "$READY_MARKER"
  printf '\nInstalling the pinned vLLM CUDA 12.9 wheel and matching PyTorch dependencies...\n'
  "$UV" pip install --python "$VENV/bin/python" --torch-backend=cu129 --reinstall "$WHEEL"
  "$UV" pip check --python "$VENV/bin/python"
  "$VENV/bin/vllm" download-kernels
fi

"$VENV/bin/python" - <<'PY'
from importlib.metadata import version
import sys
import torch

if version("vllm") != "0.32.0+cu129" or torch.version.cuda != "12.9":
    sys.exit("Runtime mismatch: expected vLLM 0.32.0+cu129 and a CUDA 12.9 PyTorch build. Do not replace these with system PyTorch.")
if not torch.cuda.is_available() or torch.cuda.device_count() != 8:
    sys.exit("The isolated runtime cannot access all 8 GPUs. Check Pod CUDA/driver compatibility before continuing.")
print(f"Runtime check passed: vLLM {version('vllm')}, PyTorch {torch.__version__}, CUDA {torch.version.cuda}.")
PY
# Write readiness only after install, dependency check, kernel download, and
# runtime validation succeeded. Atomic replacement avoids a partial marker.
printf 'vllm=0.32.0+cu129\npython=3.12\n' > "$READY_MARKER.tmp"
mv -f -- "$READY_MARKER.tmp" "$READY_MARKER"

if [[ -n "${MODEL_API_KEY:-}" ]]; then
  export VLLM_API_KEY="$MODEL_API_KEY"
  printf 'Model API key enabled. Use the same MODEL_API_KEY on your PC.\n'
else
  unset VLLM_API_KEY || true
  printf 'Tunnel-only mode: no API key required; keep port 8000 unexposed.\n'
fi
printf '\nStarting the private model API at http://127.0.0.1:8000/v1 .\n'
printf 'First startup downloads about 756 GB and loads the model. Wait for application startup complete.\n'
printf 'Stop the model with Ctrl+C. Stop the Pod in Runpod when finished to end GPU billing.\n'
exec "$VENV/bin/vllm" serve "${ARGS[@]}"