#!/bin/bash
TM=/workspace/Eval-Reasoning-Consistency/training/trained_models
for ARM in A B C; do
  echo "### ARM $ARM START $(date -u +%H:%M:%S) ###"
  bash /tmp/launch_arm.sh $ARM
  for i in $(seq 1 120); do
    sleep 30
    [ -f $TM/epoch_0/adapter_model.safetensors ] && break
  done
  sleep 15
  if [ -f $TM/epoch_0/adapter_model.safetensors ]; then
    rm -rf /data/jobs/abc_$ARM; cp -r $TM/epoch_0 /data/jobs/abc_$ARM
    echo "### ARM $ARM DONE -> /data/jobs/abc_$ARM ($(grep -oE 'TRAINING STEP [0-9]+' /tmp/train_arm_$ARM.log|tail -1)) ###"
  else
    echo "### ARM $ARM FAILED (no adapter) ###"
  fi
  pkill -9 -f sft_product_based 2>/dev/null; pkill -9 -f sglang 2>/dev/null; sleep 5
done
echo ALLARMSDONE