stopwordsiso

diff --git a/pyproject.toml b/pyproject.toml
index 943bcc067065887c230e56aeb3e58e1000c3b584..cfca57488fd0e1a461147aae61c439316a0abf8f 100644 (file)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,6 +7,7 @@ requires-python = ">=3.13"
 dependencies = [
     "fire>=0.7.1",
     "pydantic>=2.13.4",
+    "stopwordsiso>=0.7.1",
     "torch>=2.13.0",
     "tqdm>=4.66.5",
     "transformers>=5.15.1",

diff --git a/src/rag/__init__.py b/src/rag/__init__.py
index f07f984cc325ce80e8c55713072e359dca466bbe..b2e4e39f1e38b729d2f6599344547771a47e6ece 100644 (file)
--- a/src/rag/__init__.py
+++ b/src/rag/__init__.py
@@ -5,6 +5,7 @@ from sys import stderr
 import fire
 import pydantic
 from pydantic_core import ValidationError
+from stopwordsiso import stopwords
 from tqdm import tqdm
 
 from rag.chunking import FileType, chunk_file
@@ -14,7 +15,7 @@ from rag.models import (
     RetrievedQuestion,
     SearchResults,
 )
-from rag.retrieval import BM25, BagOfWords, fetch_stopwords
+from rag.retrieval import BM25, BagOfWords
 
 
 def main() -> None:
@@ -54,7 +55,7 @@ class RAG:
                     continue
                 chunks.update(chunk_file(path, max_chunk_size, filetype))
         bm25 = BM25.from_corpus(
-            tqdm(chunks.items(), desc="Indexing files"), fetch_stopwords()
+            tqdm(chunks.items(), desc="Indexing files"), stopwords("en")
         )
         try:
             os.makedirs(index, exist_ok=True)

diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py
index b7b1932131eb946f1af2b888de5fdb897c06ba23..b616a587e681e318a008318ef2c0e152f249d59b 100644 (file)
--- a/src/rag/retrieval.py
+++ b/src/rag/retrieval.py
@@ -2,8 +2,6 @@ import math
 from collections.abc import Iterable
 from dataclasses import dataclass
 
-from rag import stopwords
-
 type Multiset[T] = dict[T, int]
 
 type BagOfWords = Multiset[str]
@@ -30,10 +28,6 @@ def bag_of_words(s: str, stopwords: set[str]) -> BagOfWords:
     return res
 
 
-def fetch_stopwords() -> set[str]:
-    return {word for word in words_normalize(stopwords.stopwords)}
-
-
 @dataclass
 class BM25[T]:
     corpus_words: int

diff --git a/src/rag/stopwords.py b/src/rag/stopwords.py
deleted file mode 100644 (file)
index 4005961..0000000
--- a/src/rag/stopwords.py
+++ /dev/null
@@ -1,72 +0,0 @@
-stopwords = """ able about above abroad according accordingly across actually
-adj after afterwards again against ago ahead ain't all allow allows almost
-alone along alongside already also although always am amid amidst among amongst
-an and another any anybody anyhow anyone anything anyway anyways anywhere apart
-appear appreciate appropriate are aren't around as a's aside ask asking
-associated at available away awfully back backward backwards be became because
-become becomes becoming been before beforehand begin behind being believe below
-beside besides best better between beyond both brief but by came can cannot
-cant can't caption cause causes certain certainly changes clearly c'mon co co.
-com come comes concerning consequently consider considering contain containing
-contains corresponding could couldn't course c's currently dare daren't
-definitely described despite did didn't different directly do does doesn't
-doing done don't down downwards during each edu eg eight eighty either else
-elsewhere end ending enough entirely especially et etc even ever evermore every
-everybody everyone everything everywhere ex exactly example except fairly far
-farther few fewer fifth first five followed following follows for forever
-former formerly forth forward found four from further furthermore get gets
-getting given gives go goes going gone got gotten greetings had hadn't half
-happens hardly has hasn't have haven't having he he'd he'll hello help hence
-her here hereafter hereby herein here's hereupon hers herself he's hi him
-himself his hither hopefully how howbeit however hundred i'd ie if ignored
-i'll i'm immediate in inasmuch inc inc. indeed indicate indicated indicates
-inner inside insofar instead into inward is isn't it it'd it'll its it's itself
-i've just k keep keeps kept know known knows last lately later latter latterly
-least less lest let let's like liked likely likewise little look looking looks
-low lower ltd made mainly make makes many may maybe mayn't me mean meantime
-meanwhile merely might mightn't mine minus miss more moreover most mostly mr
-mrs much must mustn't my myself name namely nd near nearly necessary need
-needn't needs neither never neverf neverless nevertheless new next nine ninety
-no nobody non none nonetheless noone no-one nor normally not nothing
-notwithstanding novel now nowhere obviously of off often oh ok okay old on
-once one ones one's only onto opposite or other others otherwise ought
-oughtn't our ours ourselves out outside over overall own particular
-particularly past per perhaps placed please plus possible presumably probably
-provided provides que quite qv rather rd re really reasonably recent recently
-regarding regardless regards relatively respectively right round said same saw
-say saying says second secondly see seeing seem seemed seeming seems seen self
-selves sensible sent serious seriously seven several shall shan't she she'd
-she'll she's should shouldn't since six so some somebody someday somehow
-someone something sometime sometimes somewhat somewhere soon sorry specified
-specify specifying still sub such sup sure take taken taking tell tends th than
-thank thanks thanx that that'll thats that's that've the their theirs them
-themselves then thence there thereafter thereby there'd therefore therein
-there'll there're theres there's thereupon there've these they they'd they'll
-they're they've thing things think third thirty this thorough thoroughly those
-though three through throughout thru thus till to together too took toward
-towards tried tries truly try trying t's twice two un under underneath undoing
-unfortunately unless unlike unlikely until unto up upon upwards us use used
-useful uses using usually v value various versus very via viz vs want wants was
-wasn't way we we'd welcome well we'll went were we're weren't we've what
-whatever what'll what's what've when whence whenever where whereafter whereas
-whereby wherein where's whereupon wherever whether which whichever while whilst
-whither who who'd whoever whole who'll whom whomever who's whose why will
-willing wish with within without wonder won't would wouldn't yes yet you you'd
-you'll your you're yours yourself yourselves you've zero a how's i when's why's
-b c d e f g h j l m n o p q r s t u uucp w x y z I www amount bill bottom call
-computer con couldnt cry de describe detail due eleven empty fifteen fifty fill
-find fire forty front full give hasnt herse himse interest itse” mill move
-myse” part put show side sincere sixty system ten thick thin top twelve twenty
-abst accordance act added adopted affected affecting affects ah announce
-anymore apparently approximately aren arent arise auth beginning beginnings
-begins biol briefly ca date ed effect et-al ff fix gave giving heres hes hid
-home id im immediately importance important index information invention itd
-keys kg km largely lets line 'll means mg million ml mug na nay necessarily nos
-noted obtain obtained omitted ord owing page pages poorly possibly potentially
-pp predominantly present previously primarily promptly proud quickly ran
-readily ref refs related research resulted resulting results run sec section
-shed shes showed shown showns shows significant significantly similar similarly
-slightly somethan specifically state states stop strongly substantially
-successfully sufficiently suggest thered thereof therere thereto theyd theyre
-thou thoughh thousand throug til tip ts ups usefully usefulness 've vol vols
-wed whats wheres whim whod whos widely words world youd youre"""

diff --git a/uv.lock b/uv.lock
index 4d6ce6b3b30c12692003f3ff117093fca528ea51..46d2b45a133bfa1c90ea740f4cb9dc25685bd314 100644 (file)
--- a/uv.lock
+++ b/uv.lock
@@ -715,6 +715,7 @@ source = { editable = "." }
 dependencies = [
     { name = "fire" },
     { name = "pydantic" },
+    { name = "stopwordsiso" },
     { name = "torch", version = "2.13.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and sys_platform == 'darwin'" },
     { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version >= '3.15' or sys_platform != 'darwin'" },
     { name = "tqdm" },
@@ -733,6 +734,7 @@ dev = [
 requires-dist = [
     { name = "fire", specifier = ">=0.7.1" },
     { name = "pydantic", specifier = ">=2.13.4" },
+    { name = "stopwordsiso", specifier = ">=0.7.1" },
     { name = "torch", specifier = ">=2.13.0", index = "https://download.pytorch.org/whl/cpu" },
     { name = "tqdm", specifier = ">=4.66.5" },
     { name = "transformers", specifier = ">=5.15.1" },
@@ -898,6 +900,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" },
 ]
 
+[[package]]
+name = "stopwordsiso"
+version = "0.7.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/66/f6/19709db9bff8c20a8d2c99dd657dbd51efcd76a2d43608c468307d503eea/stopwordsiso-0.7.1.tar.gz", hash = "sha256:10c1cf5df1c31646175178b0b13f0bd2e622384f4a3554986ed3acf85d73c9f0", size = 73352, upload-time = "2026-07-07T11:22:02.644Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/97/5f/2d2a6013c7afe7c06d39e208749cc669dd404ed521f7f603265e13758b04/stopwordsiso-0.7.1-py3-none-any.whl", hash = "sha256:e23edcffca952f98cd26f54a5c56f93b2fa435c88d3ee60f3beffb4c44279380", size = 74890, upload-time = "2026-07-07T11:22:01.435Z" },
+]
+
 [[package]]
 name = "sympy"
 version = "1.14.0"