From f5e356b5988c185a6b6e37b6a679d629087ddff2 Mon Sep 17 00:00:00 2001 From: Axy Date: Wed, 26 Aug 2026 16:01:03 +0200 Subject: [PATCH] Maybe working without llm --- .gitignore | 1 + pyproject.toml | 4 + src/rag/__init__.py | 11 +- src/rag/chunking.py | 5 - src/rag/models.py | 22 +- src/rag/retrieval.py | 10 +- src/rag/stopwords.py | 853 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 880 insertions(+), 26 deletions(-) create mode 100644 src/rag/stopwords.py diff --git a/.gitignore b/.gitignore index f49777d..f8555d9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ **/__pycache__ vllm-0.10.1 data +moulinette* diff --git a/pyproject.toml b/pyproject.toml index 32a6f3c..1362157 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,3 +40,7 @@ lint.select = ["E", "F", "UP", "B", "SIM", "I", "ARG", "N"] [tool.uv] preview-features = ["format", "check"] + +[[tool.mypy.overrides]] +module = ["fire"] +follow_untyped_imports = true diff --git a/src/rag/__init__.py b/src/rag/__init__.py index c4587ba..79e5b4c 100644 --- a/src/rag/__init__.py +++ b/src/rag/__init__.py @@ -1,4 +1,3 @@ -import json import logging import os @@ -6,14 +5,13 @@ import fire import pydantic from pydantic_core import ValidationError +from rag.chunking import FileType, chunk_file from rag.models import ( - AnsweredQuestion, MinimalSource, RagDataset, RetrievedQuestion, SearchResults, ) -from rag.chunking import FileType, chunk_file from rag.retrieval import BM25, BagOfWords, fetch_stopwords @@ -30,7 +28,7 @@ key_adapter: pydantic.TypeAdapter[MinimalSource] = pydantic.TypeAdapter( class RAG: """A retrieval augmented generation CLI""" - def __init__(self): + def __init__(self) -> None: self._bm25_store: BM25[MinimalSource] | None = None def index( @@ -92,7 +90,6 @@ class RAG: def search( self, query: str, /, k: int = 5, index: str = "data/processed" ) -> None: - bm25 = self._bm25(index) for source in self._search(query, k, index): print(source) @@ -123,7 +120,9 @@ class RAG: k=k, ) try: - with open(save_directory +"/"+ os.path.basename(dataset_path), "w") as f: + with open( + save_directory + "/" + os.path.basename(dataset_path), "w" + ) as f: f.write(result.model_dump_json()) except (OSError, ValidationError) as e: logging.getLogger(__name__).error(f"Failed to open dataset {e}") diff --git a/src/rag/chunking.py b/src/rag/chunking.py index 5e0c80f..24e8fec 100644 --- a/src/rag/chunking.py +++ b/src/rag/chunking.py @@ -2,8 +2,6 @@ import logging from enum import Enum, auto from itertools import count -import pydantic - from rag.models import MinimalSource @@ -32,9 +30,6 @@ class FileType(Enum): ) -# type FileChunkID = tuple[int, pydantic.NonNegativeInt, str] - - def chunk_by_depth( s: str, max_chunk_size: int, by: FileType, layer: int = 0 ) -> list[str]: diff --git a/src/rag/models.py b/src/rag/models.py index 71d4321..dada28d 100644 --- a/src/rag/models.py +++ b/src/rag/models.py @@ -1,7 +1,7 @@ -from pydantic import BaseModel, Field -from typing import List import uuid +from pydantic import BaseModel, Field + class MinimalSource(BaseModel, frozen=True): file_path: str @@ -9,10 +9,10 @@ class MinimalSource(BaseModel, frozen=True): last_character_index: int def __str__(self) -> str: - return "{} [{}:{}]".format( - self.file_path, - self.first_character_index, - self.last_character_index, + return ( + f"{self.file_path}" + + f" [{self.first_character_index}:" + + f"{self.last_character_index}]" ) @@ -22,11 +22,11 @@ class Question(BaseModel): class RagDataset(BaseModel): - rag_questions: List[Question] + rag_questions: list[Question] class RetrievedQuestion(Question): - retrieved_sources: List[MinimalSource] + retrieved_sources: list[MinimalSource] class AnsweredQuestion(RetrievedQuestion): @@ -34,10 +34,10 @@ class AnsweredQuestion(RetrievedQuestion): class SearchResults(BaseModel): - search_results: List[RetrievedQuestion] + search_results: list[RetrievedQuestion] k: int -class SearchAnswer(BaseModel): - search_results: List[AnsweredQuestion] +class SearchAnswers(BaseModel): + search_results: list[AnsweredQuestion] k: int diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py index 80ee944..842a263 100644 --- a/src/rag/retrieval.py +++ b/src/rag/retrieval.py @@ -2,6 +2,8 @@ import math from collections.abc import Iterable from dataclasses import dataclass +from rag import stopwords + type Multiset[T] = dict[T, int] type BagOfWords = Multiset[str] @@ -28,8 +30,8 @@ def bag_of_words(s: str, stopwords: set[str]) -> BagOfWords: return res -def fetch_stopwords(source: str = "stopwords-en.txt") -> set[str]: - return {word for word in words_normalize(open(source).read())} +def fetch_stopwords() -> set[str]: + return {word for word in words_normalize(stopwords.stopwords)} @dataclass @@ -57,12 +59,12 @@ class BM25[T]: @staticmethod def from_corpus( - raw_corpus: dict[T, str], stopwords: set[str] = set() + raw_corpus: dict[T, str], stopwords: set[str] | None = None ) -> "BM25[T]": return BM25.from_storage( (k, bow) for k, s in raw_corpus.items() - if (bow := bag_of_words(s, stopwords)) + if (bow := bag_of_words(s, stopwords if stopwords else set())) ) def idf(self, q: str) -> float: diff --git a/src/rag/stopwords.py b/src/rag/stopwords.py new file mode 100644 index 0000000..8102339 --- /dev/null +++ b/src/rag/stopwords.py @@ -0,0 +1,853 @@ +stopwords = """ +able +about +above +abroad +according +accordingly +across +actually +adj +after +afterwards +again +against +ago +ahead +ain't +all +allow +allows +almost +alone +along +alongside +already +also +although +always +am +amid +amidst +among +amongst +an +and +another +any +anybody +anyhow +anyone +anything +anyway +anyways +anywhere +apart +appear +appreciate +appropriate +are +aren't +around +as +a's +aside +ask +asking +associated +at +available +away +awfully +back +backward +backwards +be +became +because +become +becomes +becoming +been +before +beforehand +begin +behind +being +believe +below +beside +besides +best +better +between +beyond +both +brief +but +by +came +can +cannot +cant +can't +caption +cause +causes +certain +certainly +changes +clearly +c'mon +co +co. +com +come +comes +concerning +consequently +consider +considering +contain +containing +contains +corresponding +could +couldn't +course +c's +currently +dare +daren't +definitely +described +despite +did +didn't +different +directly +do +does +doesn't +doing +done +don't +down +downwards +during +each +edu +eg +eight +eighty +either +else +elsewhere +end +ending +enough +entirely +especially +et +etc +even +ever +evermore +every +everybody +everyone +everything +everywhere +ex +exactly +example +except +fairly +far +farther +few +fewer +fifth +first +five +followed +following +follows +for +forever +former +formerly +forth +forward +found +four +from +further +furthermore +get +gets +getting +given +gives +go +goes +going +gone +got +gotten +greetings +had +hadn't +half +happens +hardly +has +hasn't +have +haven't +having +he +he'd +he'll +hello +help +hence +her +here +hereafter +hereby +herein +here's +hereupon +hers +herself +he's +hi +him +himself +his +hither +hopefully +how +howbeit +however +hundred +i'd +ie +if +ignored +i'll +i'm +immediate +in +inasmuch +inc +inc. +indeed +indicate +indicated +indicates +inner +inside +insofar +instead +into +inward +is +isn't +it +it'd +it'll +its +it's +itself +i've +just +k +keep +keeps +kept +know +known +knows +last +lately +later +latter +latterly +least +less +lest +let +let's +like +liked +likely +likewise +little +look +looking +looks +low +lower +ltd +made +mainly +make +makes +many +may +maybe +mayn't +me +mean +meantime +meanwhile +merely +might +mightn't +mine +minus +miss +more +moreover +most +mostly +mr +mrs +much +must +mustn't +my +myself +name +namely +nd +near +nearly +necessary +need +needn't +needs +neither +never +neverf +neverless +nevertheless +new +next +nine +ninety +no +nobody +non +none +nonetheless +noone +no-one +nor +normally +not +nothing +notwithstanding +novel +now +nowhere +obviously +of +off +often +oh +ok +okay +old +on +once +one +ones +one's +only +onto +opposite +or +other +others +otherwise +ought +oughtn't +our +ours +ourselves +out +outside +over +overall +own +particular +particularly +past +per +perhaps +placed +please +plus +possible +presumably +probably +provided +provides +que +quite +qv +rather +rd +re +really +reasonably +recent +recently +regarding +regardless +regards +relatively +respectively +right +round +said +same +saw +say +saying +says +second +secondly +see +seeing +seem +seemed +seeming +seems +seen +self +selves +sensible +sent +serious +seriously +seven +several +shall +shan't +she +she'd +she'll +she's +should +shouldn't +since +six +so +some +somebody +someday +somehow +someone +something +sometime +sometimes +somewhat +somewhere +soon +sorry +specified +specify +specifying +still +sub +such +sup +sure +take +taken +taking +tell +tends +th +than +thank +thanks +thanx +that +that'll +thats +that's +that've +the +their +theirs +them +themselves +then +thence +there +thereafter +thereby +there'd +therefore +therein +there'll +there're +theres +there's +thereupon +there've +these +they +they'd +they'll +they're +they've +thing +things +think +third +thirty +this +thorough +thoroughly +those +though +three +through +throughout +thru +thus +till +to +together +too +took +toward +towards +tried +tries +truly +try +trying +t's +twice +two +un +under +underneath +undoing +unfortunately +unless +unlike +unlikely +until +unto +up +upon +upwards +us +use +used +useful +uses +using +usually +v +value +various +versus +very +via +viz +vs +want +wants +was +wasn't +way +we +we'd +welcome +well +we'll +went +were +we're +weren't +we've +what +whatever +what'll +what's +what've +when +whence +whenever +where +whereafter +whereas +whereby +wherein +where's +whereupon +wherever +whether +which +whichever +while +whilst +whither +who +who'd +whoever +whole +who'll +whom +whomever +who's +whose +why +will +willing +wish +with +within +without +wonder +won't +would +wouldn't +yes +yet +you +you'd +you'll +your +you're +yours +yourself +yourselves +you've +zero +a +how's +i +when's +why's +b +c +d +e +f +g +h +j +l +m +n +o +p +q +r +s +t +u +uucp +w +x +y +z +I +www +amount +bill +bottom +call +computer +con +couldnt +cry +de +describe +detail +due +eleven +empty +fifteen +fifty +fill +find +fire +forty +front +full +give +hasnt +herse +himse +interest +itse” +mill +move +myse” +part +put +show +side +sincere +sixty +system +ten +thick +thin +top +twelve +twenty +abst +accordance +act +added +adopted +affected +affecting +affects +ah +announce +anymore +apparently +approximately +aren +arent +arise +auth +beginning +beginnings +begins +biol +briefly +ca +date +ed +effect +et-al +ff +fix +gave +giving +heres +hes +hid +home +id +im +immediately +importance +important +index +information +invention +itd +keys +kg +km +largely +lets +line +'ll +means +mg +million +ml +mug +na +nay +necessarily +nos +noted +obtain +obtained +omitted +ord +owing +page +pages +poorly +possibly +potentially +pp +predominantly +present +previously +primarily +promptly +proud +quickly +ran +readily +ref +refs +related +research +resulted +resulting +results +run +sec +section +shed +shes +showed +shown +showns +shows +significant +significantly +similar +similarly +slightly +somethan +specifically +state +states +stop +strongly +substantially +successfully +sufficiently +suggest +thered +thereof +therere +thereto +theyd +theyre +thou +thoughh +thousand +throug +til +tip +ts +ups +usefully +usefulness +'ve +vol +vols +wed +whats +wheres +whim +whod +whos +widely +words +world +youd +youre +""" -- 2.53.0