45 lines
1017 B
Bash
45 lines
1017 B
Bash
# Base model (HF id or local path)
|
|
BASE_MODEL="Qwen/Qwen2.5-Coder-7B-Instruct"
|
|
|
|
# Input splits
|
|
TRAIN_FILE="training_data/lora/splits/mcp_lora.train.jsonl"
|
|
VAL_FILE="training_data/lora/splits/mcp_lora.val.jsonl"
|
|
TEST_FILE="training_data/lora/splits/mcp_lora.test.jsonl"
|
|
|
|
# Prepared SFT data output
|
|
SFT_DIR="training_data/lora/sft_qwen7b_quality"
|
|
|
|
# Adapter output
|
|
OUTPUT_DIR="training_data/lora/adapters/qwen2.5-coder-7b-quality-lora"
|
|
|
|
# LoRA hyperparameters (smaller rank for 7B)
|
|
LORA_R=32
|
|
LORA_ALPHA=64
|
|
LORA_DROPOUT=0.05
|
|
LORA_TARGET_MODULES="q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj"
|
|
|
|
# Training hyperparameters (memory-safe for 12GB)
|
|
NUM_EPOCHS=2
|
|
LEARNING_RATE=0.0002
|
|
WEIGHT_DECAY=0.01
|
|
MAX_SEQ_LEN=1536
|
|
PER_DEVICE_TRAIN_BATCH=1
|
|
PER_DEVICE_EVAL_BATCH=1
|
|
GRAD_ACCUM=16
|
|
WARMUP_RATIO=0.03
|
|
LOGGING_STEPS=20
|
|
EVAL_STEPS=200
|
|
SAVE_STEPS=200
|
|
MAX_STEPS=0
|
|
|
|
# Runtime
|
|
USE_4BIT=1
|
|
USE_8BIT=0
|
|
BF16=0
|
|
SEED=42
|
|
LOCAL_FILES_ONLY=1
|
|
CPU_OFFLOAD=0
|
|
GPU_MAX_MEMORY_GIB=0
|
|
CPU_MAX_MEMORY_GIB=0
|
|
OFFLOAD_DIR="/tmp/qwen7b_offload"
|