From 5f7990cbbd181d7293b970c83d4a3e29ba276082 Mon Sep 17 00:00:00 2001
From: Axy <gilliardmarthey.axel@gmail.com>
Date: Wed, 26 Aug 2026 19:29:52 +0200
Subject: [PATCH] Stopword splitting

---
 src/rag/__init__.py  | 3 +--
 src/rag/retrieval.py | 2 ++
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/rag/__init__.py b/src/rag/__init__.py
index b2e4e39..7e66bb0 100644
--- a/src/rag/__init__.py
+++ b/src/rag/__init__.py
@@ -5,7 +5,6 @@ from sys import stderr
 import fire
 import pydantic
 from pydantic_core import ValidationError
-from stopwordsiso import stopwords
 from tqdm import tqdm
 
 from rag.chunking import FileType, chunk_file
@@ -15,7 +14,7 @@ from rag.models import (
     RetrievedQuestion,
     SearchResults,
 )
-from rag.retrieval import BM25, BagOfWords
+from rag.retrieval import BM25, BagOfWords, stopwords
 
 
 def main() -> None:
diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py
index b616a58..9240324 100644
--- a/src/rag/retrieval.py
+++ b/src/rag/retrieval.py
@@ -27,6 +27,8 @@ def bag_of_words(s: str, stopwords: set[str]) -> BagOfWords:
         res[word] = res.get(word, 0) + 1
     return res
 
+def stopwords(lang: str | Iterable[str]) -> set[str]:
+    return {word for e in stopwords(lang) for word in words_normalize(e)}
 
 @dataclass
 class BM25[T]:
-- 
2.53.0