下記はスクリプト内でip統一できていない（統一できないとrayエラーが出る）

2nodeを10.255.255.78/79で統一するところから再開

#!/bin/bash
#SBATCH --job-name=deepseek_r1_eval
#SBATCH --partition=P10
#SBATCH --nodes=2
#SBATCH --gpus-per-node=8
#SBATCH --cpus-per-task=120
#SBATCH --time=06:00:00
#SBATCH --exclusive
#SBATCH --mem=0
#SBATCH --output=%x-%j.out
#SBATCH --error=%x-%j.err

set -Eeo pipefail
trap 'echo "ERR $(date +%F\ %T) line=$LINENO cmd=${BASH_COMMAND}" >&2' ERR
PS4='+ $(date "+%F %T") [$$] '; set -x
log(){ echo "===== $(date '+%F %T') | $*"; }

# 変数
LOGDIR="$HOME/logs"
MODEL_PATH="/home/Competition2025/P10/shareP10/llm_project/models/deepseek/DeepSeek-R1-0528"
PORT=8000
MAXLEN=8192
TP=16
BATCH_TOKENS=4096   # 保守
MAX_SEQS=8          # 保守
VLLM_LOG="$LOGDIR/vllm_server-$SLURM_JOB_ID.log"

# 事前セットアップ
mkdir -p "$LOGDIR" "$HOME/.hf_cache"
module reset
module load nccl/2.24.3
module load hpcx/2.18.1-gcc-cuda12/hpcx-mt
module load cudnn/9.4.0
module load miniconda/24.7.1-py311
source /home/appli/miniconda3/24.7.1-py311/etc/profile.d/conda.sh
conda activate "$HOME/conda_env_deepseek"

# （既に挿入済みの環境混在対策を再掲）
export PYTHONNOUSERSITE=1
unset PYTHONPATH
export PATH="$CONDA_PREFIX/bin:$PATH"

# ulimitは失敗無視で（落下回避）
set +e
ulimit -n 1048576 2>/dev/null || true
set -e

# キャッシュ/通信
export HF_HOME=${SLURM_TMPDIR:-$HOME}/.hf_cache
export TRANSFORMERS_CACHE=$HF_HOME
export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-enp25s0np0}
export NCCL_IB_DISABLE=0
export NCCL_DEBUG=WARN
export RAY_USAGE_STATS_ENABLED=0
export RAY_USAGE_STATS_OOM_ENABLED=0

# 全ノードCLEANUP
log "CLEANUP(all nodes)"
HOSTS=($(scontrol show hostnames "$SLURM_JOB_NODELIST"))
for node in "${HOSTS[@]}"; do
  srun -N1 -n1 -w "$node" --exclusive --kill-on-bad-exit=0 /bin/bash -lc '
    set +e
    "$CONDA_PREFIX/bin/ray" stop --force || true
    pkill -9 -f vllm || true
    pkill -9 -f ray || true
    pkill -9 gcs_server || true
    rm -rf /tmp/ray{,_tmp_*} /dev/shm/ray* ~/.ray || true
    exit 0
  ' || true
done

# Ray起動（環境Pythonに固定・安定化オプション）
cleanup_exit(){ log "CLEANUP(exit): ray stop"; "$CONDA_PREFIX/bin/ray" stop --force || true; }
trap cleanup_exit EXIT ERR SIGINT SIGTERM
HEAD=$(hostname -s); HEAD_IP=$(hostname -i | awk '{print $1}')
export RAY_ADDRESS=$HEAD_IP:6379
export VLLM_HOST_IP=$HEAD_IP

log "Ray HEAD start on $HEAD ($HEAD_IP)"
"$CONDA_PREFIX/bin/python" -s -m ray.scripts.scripts start --head --node-ip-address="$HEAD_IP" --port=6379 --dashboard-host=0.0.0.0 --dashboard-port=8265 --include-dashboard=false --disable-usage-stats --object-store-memory=150000000000

if [[ ${#HOSTS[@]} -gt 1 ]]; then
  for W in "${HOSTS[@]:1}"; do
    log "Ray WORKER start on $W"
    srun -N1 -n1 -w "$W" --exclusive /bin/bash -lc '
      source /home/appli/miniconda3/24.7.1-py311/etc/profile.d/conda.sh
      conda activate "$HOME/conda_env_deepseek"
      export PYTHONNOUSERSITE=1; unset PYTHONPATH; export PATH="$CONDA_PREFIX/bin:$PATH"
      IP=$(hostname -i | awk "{print \$1}")
      "$CONDA_PREFIX/bin/python" -s -m ray.scripts.scripts start --address="'"$RAY_ADDRESS"'" --node-ip-address="$IP" --block
    ' &
  done
fi

# ノード合流待ちを延長
log "Wait Ray nodes"
for i in $(seq 1 120); do
  N=$("$CONDA_PREFIX/bin/ray" status 2>/dev/null | awk "/Nodes:/ {print \$2}" | tail -n1)
  [[ "$N" == "${SLURM_NNODES}" ]] && break
  sleep 1
done
"$CONDA_PREFIX/bin/ray" status | sed -n '1,120p'

# vLLM起動
sleep 3
log "vLLM serve start (LEN=$MAXLEN TP=$TP SEQS=$MAX_SEQS TOKENS=$BATCH_TOKENS)"
"$CONDA_PREFIX/bin/python" -s -m vllm.entrypoints.openai.api_server \
  --model "$MODEL_PATH" \
  --served-model-name deepseek-reasoner \
  --tensor-parallel-size $TP \
  --enable-expert-parallel \
  --distributed-executor-backend ray \
  --trust-remote-code \
  --dtype bfloat16 \
  --gpu-memory-utilization 0.90 \
  --max-model-len $MAXLEN \
  --max-num-seqs $MAX_SEQS \
  --max-num-batched-tokens $BATCH_TOKENS \
  --host 0.0.0.0 --port $PORT \
  --disable-log-requests \
  > "$VLLM_LOG" 2>&1 & pid_vllm=$!
log "vLLM PID=$pid_vllm"

# 起動待ち（最大10分）
log "Waiting vLLM on :$PORT ..."
READY=0
for i in $(seq 1 120); do
  /usr/bin/curl -s http://127.0.0.1:$PORT/v1/models >/dev/null 2>&1 && { READY=1; break; }
  sleep 5
done
if [[ $READY -ne 1 ]]; then
  log "[FATAL] vLLM not ready"; tail -n 200 "$VLLM_LOG" || true; exit 1
fi
log "vLLM is UP"

# メモリチェック
log "Memory AFTER"; nvidia-smi --query-gpu=index,name,memory.used,memory.total --format=csv,noheader,nounits; df -h /dev/shm || true

# eval_hle
cd "$HOME/llm_bridge_prod/eval_hle"
mkdir -p predictions judged leaderboard
log "RUN predict.py"; "$CONDA_PREFIX/bin/python" predict.py; RC_P=$?; log "predict.py exit=$RC_P"
log "RUN judge.py"  ; "$CONDA_PREFIX/bin/python" judge.py  ; RC_J=$?; log "judge.py exit=$RC_J"
if [[ $RC_P -ne 0 || $RC_J -ne 0 ]]; then
  log "[ERROR] predict/judge failed: predict=$RC_P judge=$RC_J"; exit 1
fi
log "DONE"