From 7a30e06ab0c4ea3e82fddc73bfeb2ea5287cb8bc Mon Sep 17 00:00:00 2001
From: marcuspaico <marcus@thepai.co>
Date: Mon, 17 Aug 2026 13:43:32 -0700
Subject: [PATCH] feat: QLoRA fine-tune v1 on M5 + fused model (M2)

---
 configs/lora.yaml        | 16 ++++++++++++++++
 scripts/prepare_train.py | 12 ++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 configs/lora.yaml
 create mode 100644 scripts/prepare_train.py

diff --git a/configs/lora.yaml b/configs/lora.yaml
new file mode 100644
index 0000000..4bcd312
--- /dev/null
+++ b/configs/lora.yaml
@@ -0,0 +1,16 @@
+model: mlx-community/Qwen2.5-7B-Instruct-4bit
+train: true
+data: data/train
+adapter_path: adapters/buddhagpt-v1
+batch_size: 1
+grad_accumulation_steps: 8
+iters: 1200
+learning_rate: 1e-5
+num_layers: 16
+lora_parameters:
+  rank: 16
+  scale: 20.0
+  dropout: 0.05
+max_seq_length: 1024
+steps_per_eval: 200
+save_every: 200
diff --git a/scripts/prepare_train.py b/scripts/prepare_train.py
new file mode 100644
index 0000000..b3de208
--- /dev/null
+++ b/scripts/prepare_train.py
@@ -0,0 +1,12 @@
+import json, random
+from pathlib import Path
+
+rows = [json.loads(l) for l in Path("data/instructions.jsonl").read_text().splitlines()]
+random.seed(7); random.shuffle(rows)
+n_val = max(200, len(rows) // 20)
+Path("data/train").mkdir(parents=True, exist_ok=True)
+for name, part in [("valid", rows[:n_val]), ("train", rows[n_val:])]:
+    with Path(f"data/train/{name}.jsonl").open("w") as f:
+        for r in part:
+            f.write(json.dumps(r) + "\n")
+print(len(rows) - n_val, "train /", n_val, "valid")