From: Axy <gilliardmarthey.axel@gmail.com>
Date: Wed, 26 Aug 2026 17:29:52 +0000 (+0200)
Subject: Stopword splitting
X-Git-Url: https://git.uwuaxy.net/sitemap.xml?a=commitdiff_plain;h=5f7990cbbd181d7293b970c83d4a3e29ba276082;p=axy%2Fft%2Frag.git

Stopword splitting
---

diff --git a/src/rag/__init__.py b/src/rag/__init__.py
index b2e4e39..7e66bb0 100644
--- a/src/rag/__init__.py
+++ b/src/rag/__init__.py
@@ -5,7 +5,6 @@ from sys import stderr
 import fire
 import pydantic
 from pydantic_core import ValidationError
-from stopwordsiso import stopwords
 from tqdm import tqdm
 
 from rag.chunking import FileType, chunk_file
@@ -15,7 +14,7 @@ from rag.models import (
     RetrievedQuestion,
     SearchResults,
 )
-from rag.retrieval import BM25, BagOfWords
+from rag.retrieval import BM25, BagOfWords, stopwords
 
 
 def main() -> None:
diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py
index b616a58..9240324 100644
--- a/src/rag/retrieval.py
+++ b/src/rag/retrieval.py
@@ -27,6 +27,8 @@ def bag_of_words(s: str, stopwords: set[str]) -> BagOfWords:
         res[word] = res.get(word, 0) + 1
     return res
 
+def stopwords(lang: str | Iterable[str]) -> set[str]:
+    return {word for e in stopwords(lang) for word in words_normalize(e)}
 
 @dataclass
 class BM25[T]: