From: Axy Date: Wed, 26 Aug 2026 16:47:06 +0000 (+0200) Subject: Cleanup X-Git-Url: https://git.uwuaxy.net/sitemap.xml?a=commitdiff_plain;h=8045918ca2b2a112b889813fbdd1d486dfae1fa8;p=axy%2Fft%2Frag.git Cleanup --- diff --git a/pyproject.toml b/pyproject.toml index 1362157..943bcc0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ dev = [ "flake8>=7.3.0", "mypy>=2.3.1", "ruff>=0.16.4", + "types-tqdm>=4.70.0.20260805", ] [project.scripts] diff --git a/src/rag/__init__.py b/src/rag/__init__.py index fe805a6..f07f984 100644 --- a/src/rag/__init__.py +++ b/src/rag/__init__.py @@ -4,8 +4,8 @@ from sys import stderr import fire import pydantic -from tqdm import tqdm from pydantic_core import ValidationError +from tqdm import tqdm from rag.chunking import FileType, chunk_file from rag.models import ( @@ -93,7 +93,7 @@ class RAG: logging.getLogger(__name__).error(f"Failed to open index {e}") exit(1) - def _search(self, query: str, k: int, index) -> list[MinimalSource]: + def _search(self, query: str, k: int, index: str) -> list[MinimalSource]: bm25 = self._bm25(index) return bm25.best_k(query, k) diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py index 8e4dee3..b7b1932 100644 --- a/src/rag/retrieval.py +++ b/src/rag/retrieval.py @@ -107,7 +107,7 @@ class BM25[T]: def scores(self, querry: str) -> dict[T, float]: # Old slow impl: # return {k: self.score(querry, k) for k in self.corpus} - res = {} + res: dict[T, float] = {} for word in words_normalize(querry): for k, v in self.word_scores(word).items(): res[k] = res.get(k, 0.0) + v diff --git a/src/rag/stopwords.py b/src/rag/stopwords.py index 8102339..4005961 100644 --- a/src/rag/stopwords.py +++ b/src/rag/stopwords.py @@ -1,853 +1,72 @@ -stopwords = """ -able -about -above -abroad -according -accordingly -across -actually -adj -after -afterwards -again -against -ago -ahead -ain't -all -allow -allows -almost -alone -along -alongside -already -also -although -always -am -amid -amidst -among -amongst -an -and -another -any -anybody -anyhow -anyone -anything -anyway -anyways -anywhere -apart -appear -appreciate -appropriate -are -aren't -around -as -a's -aside -ask -asking -associated -at -available -away -awfully -back -backward -backwards -be -became -because -become -becomes -becoming -been -before -beforehand -begin -behind -being -believe -below -beside -besides -best -better -between -beyond -both -brief -but -by -came -can -cannot -cant -can't -caption -cause -causes -certain -certainly -changes -clearly -c'mon -co -co. -com -come -comes -concerning -consequently -consider -considering -contain -containing -contains -corresponding -could -couldn't -course -c's -currently -dare -daren't -definitely -described -despite -did -didn't -different -directly -do -does -doesn't -doing -done -don't -down -downwards -during -each -edu -eg -eight -eighty -either -else -elsewhere -end -ending -enough -entirely -especially -et -etc -even -ever -evermore -every -everybody -everyone -everything -everywhere -ex -exactly -example -except -fairly -far -farther -few -fewer -fifth -first -five -followed -following -follows -for -forever -former -formerly -forth -forward -found -four -from -further -furthermore -get -gets -getting -given -gives -go -goes -going -gone -got -gotten -greetings -had -hadn't -half -happens -hardly -has -hasn't -have -haven't -having -he -he'd -he'll -hello -help -hence -her -here -hereafter -hereby -herein -here's -hereupon -hers -herself -he's -hi -him -himself -his -hither -hopefully -how -howbeit -however -hundred -i'd -ie -if -ignored -i'll -i'm -immediate -in -inasmuch -inc -inc. -indeed -indicate -indicated -indicates -inner -inside -insofar -instead -into -inward -is -isn't -it -it'd -it'll -its -it's -itself -i've -just -k -keep -keeps -kept -know -known -knows -last -lately -later -latter -latterly -least -less -lest -let -let's -like -liked -likely -likewise -little -look -looking -looks -low -lower -ltd -made -mainly -make -makes -many -may -maybe -mayn't -me -mean -meantime -meanwhile -merely -might -mightn't -mine -minus -miss -more -moreover -most -mostly -mr -mrs -much -must -mustn't -my -myself -name -namely -nd -near -nearly -necessary -need -needn't -needs -neither -never -neverf -neverless -nevertheless -new -next -nine -ninety -no -nobody -non -none -nonetheless -noone -no-one -nor -normally -not -nothing -notwithstanding -novel -now -nowhere -obviously -of -off -often -oh -ok -okay -old -on -once -one -ones -one's -only -onto -opposite -or -other -others -otherwise -ought -oughtn't -our -ours -ourselves -out -outside -over -overall -own -particular -particularly -past -per -perhaps -placed -please -plus -possible -presumably -probably -provided -provides -que -quite -qv -rather -rd -re -really -reasonably -recent -recently -regarding -regardless -regards -relatively -respectively -right -round -said -same -saw -say -saying -says -second -secondly -see -seeing -seem -seemed -seeming -seems -seen -self -selves -sensible -sent -serious -seriously -seven -several -shall -shan't -she -she'd -she'll -she's -should -shouldn't -since -six -so -some -somebody -someday -somehow -someone -something -sometime -sometimes -somewhat -somewhere -soon -sorry -specified -specify -specifying -still -sub -such -sup -sure -take -taken -taking -tell -tends -th -than -thank -thanks -thanx -that -that'll -thats -that's -that've -the -their -theirs -them -themselves -then -thence -there -thereafter -thereby -there'd -therefore -therein -there'll -there're -theres -there's -thereupon -there've -these -they -they'd -they'll -they're -they've -thing -things -think -third -thirty -this -thorough -thoroughly -those -though -three -through -throughout -thru -thus -till -to -together -too -took -toward -towards -tried -tries -truly -try -trying -t's -twice -two -un -under -underneath -undoing -unfortunately -unless -unlike -unlikely -until -unto -up -upon -upwards -us -use -used -useful -uses -using -usually -v -value -various -versus -very -via -viz -vs -want -wants -was -wasn't -way -we -we'd -welcome -well -we'll -went -were -we're -weren't -we've -what -whatever -what'll -what's -what've -when -whence -whenever -where -whereafter -whereas -whereby -wherein -where's -whereupon -wherever -whether -which -whichever -while -whilst -whither -who -who'd -whoever -whole -who'll -whom -whomever -who's -whose -why -will -willing -wish -with -within -without -wonder -won't -would -wouldn't -yes -yet -you -you'd -you'll -your -you're -yours -yourself -yourselves -you've -zero -a -how's -i -when's -why's -b -c -d -e -f -g -h -j -l -m -n -o -p -q -r -s -t -u -uucp -w -x -y -z -I -www -amount -bill -bottom -call -computer -con -couldnt -cry -de -describe -detail -due -eleven -empty -fifteen -fifty -fill -find -fire -forty -front -full -give -hasnt -herse -himse -interest -itse” -mill -move -myse” -part -put -show -side -sincere -sixty -system -ten -thick -thin -top -twelve -twenty -abst -accordance -act -added -adopted -affected -affecting -affects -ah -announce -anymore -apparently -approximately -aren -arent -arise -auth -beginning -beginnings -begins -biol -briefly -ca -date -ed -effect -et-al -ff -fix -gave -giving -heres -hes -hid -home -id -im -immediately -importance -important -index -information -invention -itd -keys -kg -km -largely -lets -line -'ll -means -mg -million -ml -mug -na -nay -necessarily -nos -noted -obtain -obtained -omitted -ord -owing -page -pages -poorly -possibly -potentially -pp -predominantly -present -previously -primarily -promptly -proud -quickly -ran -readily -ref -refs -related -research -resulted -resulting -results -run -sec -section -shed -shes -showed -shown -showns -shows -significant -significantly -similar -similarly -slightly -somethan -specifically -state -states -stop -strongly -substantially -successfully -sufficiently -suggest -thered -thereof -therere -thereto -theyd -theyre -thou -thoughh -thousand -throug -til -tip -ts -ups -usefully -usefulness -'ve -vol -vols -wed -whats -wheres -whim -whod -whos -widely -words -world -youd -youre -""" +stopwords = """ able about above abroad according accordingly across actually +adj after afterwards again against ago ahead ain't all allow allows almost +alone along alongside already also although always am amid amidst among amongst +an and another any anybody anyhow anyone anything anyway anyways anywhere apart +appear appreciate appropriate are aren't around as a's aside ask asking +associated at available away awfully back backward backwards be became because +become becomes becoming been before beforehand begin behind being believe below +beside besides best better between beyond both brief but by came can cannot +cant can't caption cause causes certain certainly changes clearly c'mon co co. +com come comes concerning consequently consider considering contain containing +contains corresponding could couldn't course c's currently dare daren't +definitely described despite did didn't different directly do does doesn't +doing done don't down downwards during each edu eg eight eighty either else +elsewhere end ending enough entirely especially et etc even ever evermore every +everybody everyone everything everywhere ex exactly example except fairly far +farther few fewer fifth first five followed following follows for forever +former formerly forth forward found four from further furthermore get gets +getting given gives go goes going gone got gotten greetings had hadn't half +happens hardly has hasn't have haven't having he he'd he'll hello help hence +her here hereafter hereby herein here's hereupon hers herself he's hi him +himself his hither hopefully how howbeit however hundred i'd ie if ignored +i'll i'm immediate in inasmuch inc inc. indeed indicate indicated indicates +inner inside insofar instead into inward is isn't it it'd it'll its it's itself +i've just k keep keeps kept know known knows last lately later latter latterly +least less lest let let's like liked likely likewise little look looking looks +low lower ltd made mainly make makes many may maybe mayn't me mean meantime +meanwhile merely might mightn't mine minus miss more moreover most mostly mr +mrs much must mustn't my myself name namely nd near nearly necessary need +needn't needs neither never neverf neverless nevertheless new next nine ninety +no nobody non none nonetheless noone no-one nor normally not nothing +notwithstanding novel now nowhere obviously of off often oh ok okay old on +once one ones one's only onto opposite or other others otherwise ought +oughtn't our ours ourselves out outside over overall own particular +particularly past per perhaps placed please plus possible presumably probably +provided provides que quite qv rather rd re really reasonably recent recently +regarding regardless regards relatively respectively right round said same saw +say saying says second secondly see seeing seem seemed seeming seems seen self +selves sensible sent serious seriously seven several shall shan't she she'd +she'll she's should shouldn't since six so some somebody someday somehow +someone something sometime sometimes somewhat somewhere soon sorry specified +specify specifying still sub such sup sure take taken taking tell tends th than +thank thanks thanx that that'll thats that's that've the their theirs them +themselves then thence there thereafter thereby there'd therefore therein +there'll there're theres there's thereupon there've these they they'd they'll +they're they've thing things think third thirty this thorough thoroughly those +though three through throughout thru thus till to together too took toward +towards tried tries truly try trying t's twice two un under underneath undoing +unfortunately unless unlike unlikely until unto up upon upwards us use used +useful uses using usually v value various versus very via viz vs want wants was +wasn't way we we'd welcome well we'll went were we're weren't we've what +whatever what'll what's what've when whence whenever where whereafter whereas +whereby wherein where's whereupon wherever whether which whichever while whilst +whither who who'd whoever whole who'll whom whomever who's whose why will +willing wish with within without wonder won't would wouldn't yes yet you you'd +you'll your you're yours yourself yourselves you've zero a how's i when's why's +b c d e f g h j l m n o p q r s t u uucp w x y z I www amount bill bottom call +computer con couldnt cry de describe detail due eleven empty fifteen fifty fill +find fire forty front full give hasnt herse himse interest itse” mill move +myse” part put show side sincere sixty system ten thick thin top twelve twenty +abst accordance act added adopted affected affecting affects ah announce +anymore apparently approximately aren arent arise auth beginning beginnings +begins biol briefly ca date ed effect et-al ff fix gave giving heres hes hid +home id im immediately importance important index information invention itd +keys kg km largely lets line 'll means mg million ml mug na nay necessarily nos +noted obtain obtained omitted ord owing page pages poorly possibly potentially +pp predominantly present previously primarily promptly proud quickly ran +readily ref refs related research resulted resulting results run sec section +shed shes showed shown showns shows significant significantly similar similarly +slightly somethan specifically state states stop strongly substantially +successfully sufficiently suggest thered thereof therere thereto theyd theyre +thou thoughh thousand throug til tip ts ups usefully usefulness 've vol vols +wed whats wheres whim whod whos widely words world youd youre""" diff --git a/stopwords-en.txt b/stopwords-en.txt deleted file mode 100644 index f9d6929..0000000 --- a/stopwords-en.txt +++ /dev/null @@ -1,851 +0,0 @@ -able -about -above -abroad -according -accordingly -across -actually -adj -after -afterwards -again -against -ago -ahead -ain't -all -allow -allows -almost -alone -along -alongside -already -also -although -always -am -amid -amidst -among -amongst -an -and -another -any -anybody -anyhow -anyone -anything -anyway -anyways -anywhere -apart -appear -appreciate -appropriate -are -aren't -around -as -a's -aside -ask -asking -associated -at -available -away -awfully -back -backward -backwards -be -became -because -become -becomes -becoming -been -before -beforehand -begin -behind -being -believe -below -beside -besides -best -better -between -beyond -both -brief -but -by -came -can -cannot -cant -can't -caption -cause -causes -certain -certainly -changes -clearly -c'mon -co -co. -com -come -comes -concerning -consequently -consider -considering -contain -containing -contains -corresponding -could -couldn't -course -c's -currently -dare -daren't -definitely -described -despite -did -didn't -different -directly -do -does -doesn't -doing -done -don't -down -downwards -during -each -edu -eg -eight -eighty -either -else -elsewhere -end -ending -enough -entirely -especially -et -etc -even -ever -evermore -every -everybody -everyone -everything -everywhere -ex -exactly -example -except -fairly -far -farther -few -fewer -fifth -first -five -followed -following -follows -for -forever -former -formerly -forth -forward -found -four -from -further -furthermore -get -gets -getting -given -gives -go -goes -going -gone -got -gotten -greetings -had -hadn't -half -happens -hardly -has -hasn't -have -haven't -having -he -he'd -he'll -hello -help -hence -her -here -hereafter -hereby -herein -here's -hereupon -hers -herself -he's -hi -him -himself -his -hither -hopefully -how -howbeit -however -hundred -i'd -ie -if -ignored -i'll -i'm -immediate -in -inasmuch -inc -inc. -indeed -indicate -indicated -indicates -inner -inside -insofar -instead -into -inward -is -isn't -it -it'd -it'll -its -it's -itself -i've -just -k -keep -keeps -kept -know -known -knows -last -lately -later -latter -latterly -least -less -lest -let -let's -like -liked -likely -likewise -little -look -looking -looks -low -lower -ltd -made -mainly -make -makes -many -may -maybe -mayn't -me -mean -meantime -meanwhile -merely -might -mightn't -mine -minus -miss -more -moreover -most -mostly -mr -mrs -much -must -mustn't -my -myself -name -namely -nd -near -nearly -necessary -need -needn't -needs -neither -never -neverf -neverless -nevertheless -new -next -nine -ninety -no -nobody -non -none -nonetheless -noone -no-one -nor -normally -not -nothing -notwithstanding -novel -now -nowhere -obviously -of -off -often -oh -ok -okay -old -on -once -one -ones -one's -only -onto -opposite -or -other -others -otherwise -ought -oughtn't -our -ours -ourselves -out -outside -over -overall -own -particular -particularly -past -per -perhaps -placed -please -plus -possible -presumably -probably -provided -provides -que -quite -qv -rather -rd -re -really -reasonably -recent -recently -regarding -regardless -regards -relatively -respectively -right -round -said -same -saw -say -saying -says -second -secondly -see -seeing -seem -seemed -seeming -seems -seen -self -selves -sensible -sent -serious -seriously -seven -several -shall -shan't -she -she'd -she'll -she's -should -shouldn't -since -six -so -some -somebody -someday -somehow -someone -something -sometime -sometimes -somewhat -somewhere -soon -sorry -specified -specify -specifying -still -sub -such -sup -sure -take -taken -taking -tell -tends -th -than -thank -thanks -thanx -that -that'll -thats -that's -that've -the -their -theirs -them -themselves -then -thence -there -thereafter -thereby -there'd -therefore -therein -there'll -there're -theres -there's -thereupon -there've -these -they -they'd -they'll -they're -they've -thing -things -think -third -thirty -this -thorough -thoroughly -those -though -three -through -throughout -thru -thus -till -to -together -too -took -toward -towards -tried -tries -truly -try -trying -t's -twice -two -un -under -underneath -undoing -unfortunately -unless -unlike -unlikely -until -unto -up -upon -upwards -us -use -used -useful -uses -using -usually -v -value -various -versus -very -via -viz -vs -want -wants -was -wasn't -way -we -we'd -welcome -well -we'll -went -were -we're -weren't -we've -what -whatever -what'll -what's -what've -when -whence -whenever -where -whereafter -whereas -whereby -wherein -where's -whereupon -wherever -whether -which -whichever -while -whilst -whither -who -who'd -whoever -whole -who'll -whom -whomever -who's -whose -why -will -willing -wish -with -within -without -wonder -won't -would -wouldn't -yes -yet -you -you'd -you'll -your -you're -yours -yourself -yourselves -you've -zero -a -how's -i -when's -why's -b -c -d -e -f -g -h -j -l -m -n -o -p -q -r -s -t -u -uucp -w -x -y -z -I -www -amount -bill -bottom -call -computer -con -couldnt -cry -de -describe -detail -due -eleven -empty -fifteen -fifty -fill -find -fire -forty -front -full -give -hasnt -herse -himse -interest -itse” -mill -move -myse” -part -put -show -side -sincere -sixty -system -ten -thick -thin -top -twelve -twenty -abst -accordance -act -added -adopted -affected -affecting -affects -ah -announce -anymore -apparently -approximately -aren -arent -arise -auth -beginning -beginnings -begins -biol -briefly -ca -date -ed -effect -et-al -ff -fix -gave -giving -heres -hes -hid -home -id -im -immediately -importance -important -index -information -invention -itd -keys -kg -km -largely -lets -line -'ll -means -mg -million -ml -mug -na -nay -necessarily -nos -noted -obtain -obtained -omitted -ord -owing -page -pages -poorly -possibly -potentially -pp -predominantly -present -previously -primarily -promptly -proud -quickly -ran -readily -ref -refs -related -research -resulted -resulting -results -run -sec -section -shed -shes -showed -shown -showns -shows -significant -significantly -similar -similarly -slightly -somethan -specifically -state -states -stop -strongly -substantially -successfully -sufficiently -suggest -thered -thereof -therere -thereto -theyd -theyre -thou -thoughh -thousand -throug -til -tip -ts -ups -usefully -usefulness -'ve -vol -vols -wed -whats -wheres -whim -whod -whos -widely -words -world -youd -youre \ No newline at end of file diff --git a/uv.lock b/uv.lock index 9b79cb5..4d6ce6b 100644 --- a/uv.lock +++ b/uv.lock @@ -726,6 +726,7 @@ dev = [ { name = "flake8" }, { name = "mypy" }, { name = "ruff" }, + { name = "types-tqdm" }, ] [package.metadata] @@ -742,6 +743,7 @@ dev = [ { name = "flake8", specifier = ">=7.3.0" }, { name = "mypy", specifier = ">=2.3.1" }, { name = "ruff", specifier = ">=0.16.4" }, + { name = "types-tqdm", specifier = ">=4.70.0.20260805" }, ] [[package]] @@ -1057,6 +1059,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/43/89/9518bc0c3929bee36b3a4a8e3daddd6e03f92f9961c66d4983b837160543/typer-0.27.1-py3-none-any.whl", hash = "sha256:53150287edd11baeb4e4722c8e394fcdf8181c0ae89485cba8d25c778d5edd56", size = 122874, upload-time = "2026-08-03T14:41:04.391Z" }, ] +[[package]] +name = "types-tqdm" +version = "4.70.0.20260805" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/41/284465d9bb5e657a1a4b17a4addf8800bbfad3d2e903f5a1a4d5bd10dd53/types_tqdm-4.70.0.20260805.tar.gz", hash = "sha256:569efd0fd8b2a460e6e6548634ec253c1362ed8907f8b0daacf0a07eab204a26", size = 19115, upload-time = "2026-08-05T04:52:04.435Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/92/acbd53e79df923bca3881192558a3bc73a47b8b7bd6c94b3649ae45e8094/types_tqdm-4.70.0.20260805-py3-none-any.whl", hash = "sha256:c83113470a46caee2d045428554714f3c314be94eb4ed4030aaa8a8ab21b830c", size = 25135, upload-time = "2026-08-05T04:52:03.166Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0"