35 lines
1.1 KiB
Python
35 lines
1.1 KiB
Python
|
import re
|
|||
|
|
|
|||
|
|
def chunk_text_paragraphs(text: str, max_length: int = 500) -> list[str]:
|
|||
|
|
# Abs<62>tze trennen
|
|||
|
|
paragraphs = re.split(r'\n\s*\n', text.strip())
|
|||
|
|
chunks: list[str] = []
|
|||
|
|
current_chunk = ""
|
|||
|
|
|
|||
|
|
for para in paragraphs:
|
|||
|
|
para = para.strip()
|
|||
|
|
if not para:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# Passt der Absatz noch zum aktuellen Chunk? +2 f<>r die sp<73>ter hinzugef<65>gten "\n\n"
|
|||
|
|
if len(current_chunk) + len(para) + 2 <= max_length:
|
|||
|
|
current_chunk = (current_chunk + "\n\n" + para) if current_chunk else para
|
|||
|
|
else:
|
|||
|
|
# Aktuellen Chunk abschlie<69>en
|
|||
|
|
if current_chunk:
|
|||
|
|
chunks.append(current_chunk)
|
|||
|
|
|
|||
|
|
# Ist der Absatz selbst zu gro<72>? Dann hart splitten
|
|||
|
|
if len(para) > max_length:
|
|||
|
|
for i in range(0, len(para), max_length):
|
|||
|
|
chunks.append(para[i:i + max_length])
|
|||
|
|
current_chunk = ""
|
|||
|
|
else:
|
|||
|
|
current_chunk = para
|
|||
|
|
|
|||
|
|
# Letzten Chunk nicht vergessen
|
|||
|
|
if current_chunk:
|
|||
|
|
chunks.append(current_chunk)
|
|||
|
|
|
|||
|
|
return chunks
|