13 lines
505 B
Python
13 lines
505 B
Python
import json, random
|
|
from pathlib import Path
|
|
|
|
rows = [json.loads(l) for l in Path("data/instructions.jsonl").read_text().splitlines()]
|
|
random.seed(7); random.shuffle(rows)
|
|
n_val = max(200, len(rows) // 20)
|
|
Path("data/train").mkdir(parents=True, exist_ok=True)
|
|
for name, part in [("valid", rows[:n_val]), ("train", rows[n_val:])]:
|
|
with Path(f"data/train/{name}.jsonl").open("w") as f:
|
|
for r in part:
|
|
f.write(json.dumps(r) + "\n")
|
|
print(len(rows) - n_val, "train /", n_val, "valid")
|