15 lines
449 B
Python
15 lines
449 B
Python
from buddhagpt.index import chunk_text
|
|
|
|
|
|
def test_chunk_text_respects_max_words():
|
|
text = " ".join(f"w{i}" for i in range(700))
|
|
chunks = chunk_text(text, max_words=300, overlap=50)
|
|
assert all(len(c.split()) <= 300 for c in chunks)
|
|
assert len(chunks) == 3
|
|
|
|
|
|
def test_chunks_overlap():
|
|
text = " ".join(f"w{i}" for i in range(400))
|
|
a, b = chunk_text(text, max_words=300, overlap=50)
|
|
assert a.split()[-50:] == b.split()[:50]
|