From: Axy Date: Wed, 26 Aug 2026 17:08:16 +0000 (+0200) Subject: stopwordsiso X-Git-Url: https://git.uwuaxy.net/sitemap.xml?a=commitdiff_plain;h=48beb81ddf53ed388ce9ca26488b899a3d7a684f;p=axy%2Fft%2Frag.git stopwordsiso --- diff --git a/pyproject.toml b/pyproject.toml index 943bcc0..cfca574 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.13" dependencies = [ "fire>=0.7.1", "pydantic>=2.13.4", + "stopwordsiso>=0.7.1", "torch>=2.13.0", "tqdm>=4.66.5", "transformers>=5.15.1", diff --git a/src/rag/__init__.py b/src/rag/__init__.py index f07f984..b2e4e39 100644 --- a/src/rag/__init__.py +++ b/src/rag/__init__.py @@ -5,6 +5,7 @@ from sys import stderr import fire import pydantic from pydantic_core import ValidationError +from stopwordsiso import stopwords from tqdm import tqdm from rag.chunking import FileType, chunk_file @@ -14,7 +15,7 @@ from rag.models import ( RetrievedQuestion, SearchResults, ) -from rag.retrieval import BM25, BagOfWords, fetch_stopwords +from rag.retrieval import BM25, BagOfWords def main() -> None: @@ -54,7 +55,7 @@ class RAG: continue chunks.update(chunk_file(path, max_chunk_size, filetype)) bm25 = BM25.from_corpus( - tqdm(chunks.items(), desc="Indexing files"), fetch_stopwords() + tqdm(chunks.items(), desc="Indexing files"), stopwords("en") ) try: os.makedirs(index, exist_ok=True) diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py index b7b1932..b616a58 100644 --- a/src/rag/retrieval.py +++ b/src/rag/retrieval.py @@ -2,8 +2,6 @@ import math from collections.abc import Iterable from dataclasses import dataclass -from rag import stopwords - type Multiset[T] = dict[T, int] type BagOfWords = Multiset[str] @@ -30,10 +28,6 @@ def bag_of_words(s: str, stopwords: set[str]) -> BagOfWords: return res -def fetch_stopwords() -> set[str]: - return {word for word in words_normalize(stopwords.stopwords)} - - @dataclass class BM25[T]: corpus_words: int diff --git a/src/rag/stopwords.py b/src/rag/stopwords.py deleted file mode 100644 index 4005961..0000000 --- a/src/rag/stopwords.py +++ /dev/null @@ -1,72 +0,0 @@ -stopwords = """ able about above abroad according accordingly across actually -adj after afterwards again against ago ahead ain't all allow allows almost -alone along alongside already also although always am amid amidst among amongst -an and another any anybody anyhow anyone anything anyway anyways anywhere apart -appear appreciate appropriate are aren't around as a's aside ask asking -associated at available away awfully back backward backwards be became because -become becomes becoming been before beforehand begin behind being believe below -beside besides best better between beyond both brief but by came can cannot -cant can't caption cause causes certain certainly changes clearly c'mon co co. -com come comes concerning consequently consider considering contain containing -contains corresponding could couldn't course c's currently dare daren't -definitely described despite did didn't different directly do does doesn't -doing done don't down downwards during each edu eg eight eighty either else -elsewhere end ending enough entirely especially et etc even ever evermore every -everybody everyone everything everywhere ex exactly example except fairly far -farther few fewer fifth first five followed following follows for forever -former formerly forth forward found four from further furthermore get gets -getting given gives go goes going gone got gotten greetings had hadn't half -happens hardly has hasn't have haven't having he he'd he'll hello help hence -her here hereafter hereby herein here's hereupon hers herself he's hi him -himself his hither hopefully how howbeit however hundred i'd ie if ignored -i'll i'm immediate in inasmuch inc inc. indeed indicate indicated indicates -inner inside insofar instead into inward is isn't it it'd it'll its it's itself -i've just k keep keeps kept know known knows last lately later latter latterly -least less lest let let's like liked likely likewise little look looking looks -low lower ltd made mainly make makes many may maybe mayn't me mean meantime -meanwhile merely might mightn't mine minus miss more moreover most mostly mr -mrs much must mustn't my myself name namely nd near nearly necessary need -needn't needs neither never neverf neverless nevertheless new next nine ninety -no nobody non none nonetheless noone no-one nor normally not nothing -notwithstanding novel now nowhere obviously of off often oh ok okay old on -once one ones one's only onto opposite or other others otherwise ought -oughtn't our ours ourselves out outside over overall own particular -particularly past per perhaps placed please plus possible presumably probably -provided provides que quite qv rather rd re really reasonably recent recently -regarding regardless regards relatively respectively right round said same saw -say saying says second secondly see seeing seem seemed seeming seems seen self -selves sensible sent serious seriously seven several shall shan't she she'd -she'll she's should shouldn't since six so some somebody someday somehow -someone something sometime sometimes somewhat somewhere soon sorry specified -specify specifying still sub such sup sure take taken taking tell tends th than -thank thanks thanx that that'll thats that's that've the their theirs them -themselves then thence there thereafter thereby there'd therefore therein -there'll there're theres there's thereupon there've these they they'd they'll -they're they've thing things think third thirty this thorough thoroughly those -though three through throughout thru thus till to together too took toward -towards tried tries truly try trying t's twice two un under underneath undoing -unfortunately unless unlike unlikely until unto up upon upwards us use used -useful uses using usually v value various versus very via viz vs want wants was -wasn't way we we'd welcome well we'll went were we're weren't we've what -whatever what'll what's what've when whence whenever where whereafter whereas -whereby wherein where's whereupon wherever whether which whichever while whilst -whither who who'd whoever whole who'll whom whomever who's whose why will -willing wish with within without wonder won't would wouldn't yes yet you you'd -you'll your you're yours yourself yourselves you've zero a how's i when's why's -b c d e f g h j l m n o p q r s t u uucp w x y z I www amount bill bottom call -computer con couldnt cry de describe detail due eleven empty fifteen fifty fill -find fire forty front full give hasnt herse himse interest itse” mill move -myse” part put show side sincere sixty system ten thick thin top twelve twenty -abst accordance act added adopted affected affecting affects ah announce -anymore apparently approximately aren arent arise auth beginning beginnings -begins biol briefly ca date ed effect et-al ff fix gave giving heres hes hid -home id im immediately importance important index information invention itd -keys kg km largely lets line 'll means mg million ml mug na nay necessarily nos -noted obtain obtained omitted ord owing page pages poorly possibly potentially -pp predominantly present previously primarily promptly proud quickly ran -readily ref refs related research resulted resulting results run sec section -shed shes showed shown showns shows significant significantly similar similarly -slightly somethan specifically state states stop strongly substantially -successfully sufficiently suggest thered thereof therere thereto theyd theyre -thou thoughh thousand throug til tip ts ups usefully usefulness 've vol vols -wed whats wheres whim whod whos widely words world youd youre""" diff --git a/uv.lock b/uv.lock index 4d6ce6b..46d2b45 100644 --- a/uv.lock +++ b/uv.lock @@ -715,6 +715,7 @@ source = { editable = "." } dependencies = [ { name = "fire" }, { name = "pydantic" }, + { name = "stopwordsiso" }, { name = "torch", version = "2.13.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version < '3.15' and sys_platform == 'darwin'" }, { name = "torch", version = "2.13.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "python_full_version >= '3.15' or sys_platform != 'darwin'" }, { name = "tqdm" }, @@ -733,6 +734,7 @@ dev = [ requires-dist = [ { name = "fire", specifier = ">=0.7.1" }, { name = "pydantic", specifier = ">=2.13.4" }, + { name = "stopwordsiso", specifier = ">=0.7.1" }, { name = "torch", specifier = ">=2.13.0", index = "https://download.pytorch.org/whl/cpu" }, { name = "tqdm", specifier = ">=4.66.5" }, { name = "transformers", specifier = ">=5.15.1" }, @@ -898,6 +900,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] +[[package]] +name = "stopwordsiso" +version = "0.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/f6/19709db9bff8c20a8d2c99dd657dbd51efcd76a2d43608c468307d503eea/stopwordsiso-0.7.1.tar.gz", hash = "sha256:10c1cf5df1c31646175178b0b13f0bd2e622384f4a3554986ed3acf85d73c9f0", size = 73352, upload-time = "2026-07-07T11:22:02.644Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/5f/2d2a6013c7afe7c06d39e208749cc669dd404ed521f7f603265e13758b04/stopwordsiso-0.7.1-py3-none-any.whl", hash = "sha256:e23edcffca952f98cd26f54a5c56f93b2fa435c88d3ee60f3beffb4c44279380", size = 74890, upload-time = "2026-07-07T11:22:01.435Z" }, +] + [[package]] name = "sympy" version = "1.14.0"