Stopword splitting

diff --git a/src/rag/__init__.py b/src/rag/__init__.py
index b2e4e39f1e38b729d2f6599344547771a47e6ece..7e66bb04393ccd4a3ccce0c1e00aa6ffa6ce8132 100644 (file)
--- a/src/rag/__init__.py
+++ b/src/rag/__init__.py
@@ -5,7 +5,6 @@ from sys import stderr
 import fire
 import pydantic
 from pydantic_core import ValidationError
-from stopwordsiso import stopwords
 from tqdm import tqdm
 
 from rag.chunking import FileType, chunk_file
@@ -15,7 +14,7 @@ from rag.models import (
     RetrievedQuestion,
     SearchResults,
 )
-from rag.retrieval import BM25, BagOfWords
+from rag.retrieval import BM25, BagOfWords, stopwords
 
 
 def main() -> None:

diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py
index b616a587e681e318a008318ef2c0e152f249d59b..92403240f90d847ad3eeea426683148288f29789 100644 (file)
--- a/src/rag/retrieval.py
+++ b/src/rag/retrieval.py
@@ -27,6 +27,8 @@ def bag_of_words(s: str, stopwords: set[str]) -> BagOfWords:
         res[word] = res.get(word, 0) + 1
     return res
 
+def stopwords(lang: str | Iterable[str]) -> set[str]:
+    return {word for e in stopwords(lang) for word in words_normalize(e)}
 
 @dataclass
 class BM25[T]: