feat: synthetic instruction generation via OpenRouter deepseek-v4-flash

This commit is contained in:
marcuspaico
2026-08-15 14:18:37 -07:00
parent 1b2d84094d
commit 7b97992b1b
6 changed files with 267 additions and 0 deletions

16
tests/test_datagen.py Normal file
View File

@@ -0,0 +1,16 @@
from buddhagpt.datagen import build_messages, parse_pairs, dedupe
def test_build_messages_varies_templates():
sutta = {"uid": "mn21", "title": "T", "text": "x" * 900}
prompts = {build_messages(sutta, v)[1]["content"] for v in range(6)}
assert len(prompts) == 6 # rotating templates, not one fixed prompt
def test_parse_pairs_extracts_json_lines():
out = '{"question": "Q1?", "answer": "A1"}\n{"question": "Q2?", "answer": "A2"}'
assert len(parse_pairs(out, uid="mn21")) == 2
def test_dedupe_drops_near_duplicates():
pairs = [{"question": "What is craving?", "answer": "a"},
{"question": "What is craving?", "answer": "b"},
{"question": "How does one practice metta?", "answer": "c"}]
assert len(dedupe(pairs)) == 2