Maybe working without llm

diff --git a/.gitignore b/.gitignore
index f49777db513583d0ca4d2a3a99362fa0bfd99f0c..f8555d9337a7896fad5d7981753b8029f52d7f68 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 **/__pycache__
 vllm-0.10.1
 data
+moulinette*

diff --git a/pyproject.toml b/pyproject.toml
index 32a6f3cf10a6334aca4c66c89ec3e4719712baef..13621571562d2ebc22ccab5cf3b23576ca725fca 100644 (file)
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,3 +40,7 @@ lint.select = ["E", "F", "UP", "B", "SIM", "I", "ARG", "N"]
 
 [tool.uv]
 preview-features = ["format", "check"]
+
+[[tool.mypy.overrides]]
+module = ["fire"]
+follow_untyped_imports = true

diff --git a/src/rag/__init__.py b/src/rag/__init__.py
index c4587baec9861bb51c4d9a697b0f14f52063bc61..79e5b4ca5e9e8f7919112fc8a3419405eff9f0ca 100644 (file)
--- a/src/rag/__init__.py
+++ b/src/rag/__init__.py
@@ -1,4 +1,3 @@
-import json
 import logging
 import os
 
@@ -6,14 +5,13 @@ import fire
 import pydantic
 from pydantic_core import ValidationError
 
+from rag.chunking import FileType, chunk_file
 from rag.models import (
-    AnsweredQuestion,
     MinimalSource,
     RagDataset,
     RetrievedQuestion,
     SearchResults,
 )
-from rag.chunking import FileType, chunk_file
 from rag.retrieval import BM25, BagOfWords, fetch_stopwords
 
 
@@ -30,7 +28,7 @@ key_adapter: pydantic.TypeAdapter[MinimalSource] = pydantic.TypeAdapter(
 class RAG:
     """A retrieval augmented generation CLI"""
 
-    def __init__(self):
+    def __init__(self) -> None:
         self._bm25_store: BM25[MinimalSource] | None = None
 
     def index(
@@ -92,7 +90,6 @@ class RAG:
     def search(
         self, query: str, /, k: int = 5, index: str = "data/processed"
     ) -> None:
-        bm25 = self._bm25(index)
         for source in self._search(query, k, index):
             print(source)
 
@@ -123,7 +120,9 @@ class RAG:
             k=k,
         )
         try:
-            with open(save_directory +"/"+ os.path.basename(dataset_path), "w") as f:
+            with open(
+                save_directory + "/" + os.path.basename(dataset_path), "w"
+            ) as f:
                 f.write(result.model_dump_json())
         except (OSError, ValidationError) as e:
             logging.getLogger(__name__).error(f"Failed to open dataset {e}")

diff --git a/src/rag/chunking.py b/src/rag/chunking.py
index 5e0c80f2f8cf5b7489eff261161cf95a1021409d..24e8fec7c9fac23ddd5ed456abf8fe4cb018eb93 100644 (file)
--- a/src/rag/chunking.py
+++ b/src/rag/chunking.py
@@ -2,8 +2,6 @@ import logging
 from enum import Enum, auto
 from itertools import count
 
-import pydantic
-
 from rag.models import MinimalSource
 
 
@@ -32,9 +30,6 @@ class FileType(Enum):
                 )
 
 
-# type FileChunkID = tuple[int, pydantic.NonNegativeInt, str]
-
-
 def chunk_by_depth(
     s: str, max_chunk_size: int, by: FileType, layer: int = 0
 ) -> list[str]:

diff --git a/src/rag/models.py b/src/rag/models.py
index 71d4321fcda404613746c8b15141558c3e14f0c4..dada28dbd763f751532d5c0d462a4dc0af511b16 100644 (file)
--- a/src/rag/models.py
+++ b/src/rag/models.py
@@ -1,7 +1,7 @@
-from pydantic import BaseModel, Field
-from typing import List
 import uuid
 
+from pydantic import BaseModel, Field
+
 
 class MinimalSource(BaseModel, frozen=True):
     file_path: str
@@ -9,10 +9,10 @@ class MinimalSource(BaseModel, frozen=True):
     last_character_index: int
 
     def __str__(self) -> str:
-        return "{} [{}:{}]".format(
-            self.file_path,
-            self.first_character_index,
-            self.last_character_index,
+        return (
+            f"{self.file_path}"
+            + f" [{self.first_character_index}:"
+            + f"{self.last_character_index}]"
         )
 
 
@@ -22,11 +22,11 @@ class Question(BaseModel):
 
 
 class RagDataset(BaseModel):
-    rag_questions: List[Question]
+    rag_questions: list[Question]
 
 
 class RetrievedQuestion(Question):
-    retrieved_sources: List[MinimalSource]
+    retrieved_sources: list[MinimalSource]
 
 
 class AnsweredQuestion(RetrievedQuestion):
@@ -34,10 +34,10 @@ class AnsweredQuestion(RetrievedQuestion):
 
 
 class SearchResults(BaseModel):
-    search_results: List[RetrievedQuestion]
+    search_results: list[RetrievedQuestion]
     k: int
 
 
-class SearchAnswer(BaseModel):
-    search_results: List[AnsweredQuestion]
+class SearchAnswers(BaseModel):
+    search_results: list[AnsweredQuestion]
     k: int

diff --git a/src/rag/retrieval.py b/src/rag/retrieval.py
index 80ee9446e92bc8e3b773b0ed734025a6eb439e25..842a263c55e638360f7a62735337f0068a6bdc71 100644 (file)
--- a/src/rag/retrieval.py
+++ b/src/rag/retrieval.py
@@ -2,6 +2,8 @@ import math
 from collections.abc import Iterable
 from dataclasses import dataclass
 
+from rag import stopwords
+
 type Multiset[T] = dict[T, int]
 
 type BagOfWords = Multiset[str]
@@ -28,8 +30,8 @@ def bag_of_words(s: str, stopwords: set[str]) -> BagOfWords:
     return res
 
 
-def fetch_stopwords(source: str = "stopwords-en.txt") -> set[str]:
-    return {word for word in words_normalize(open(source).read())}
+def fetch_stopwords() -> set[str]:
+    return {word for word in words_normalize(stopwords.stopwords)}
 
 
 @dataclass
@@ -57,12 +59,12 @@ class BM25[T]:
 
     @staticmethod
     def from_corpus(
-        raw_corpus: dict[T, str], stopwords: set[str] = set()
+        raw_corpus: dict[T, str], stopwords: set[str] | None = None
     ) -> "BM25[T]":
         return BM25.from_storage(
             (k, bow)
             for k, s in raw_corpus.items()
-            if (bow := bag_of_words(s, stopwords))
+            if (bow := bag_of_words(s, stopwords if stopwords else set()))
         )
 
     def idf(self, q: str) -> float:

diff --git a/src/rag/stopwords.py b/src/rag/stopwords.py
new file mode 100644 (file)
index 0000000..8102339
--- /dev/null
+++ b/src/rag/stopwords.py
@@ -0,0 +1,853 @@
+stopwords = """
+able
+about
+above
+abroad
+according
+accordingly
+across
+actually
+adj
+after
+afterwards
+again
+against
+ago
+ahead
+ain't
+all
+allow
+allows
+almost
+alone
+along
+alongside
+already
+also
+although
+always
+am
+amid
+amidst
+among
+amongst
+an
+and
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+apart
+appear
+appreciate
+appropriate
+are
+aren't
+around
+as
+a's
+aside
+ask
+asking
+associated
+at
+available
+away
+awfully
+back
+backward
+backwards
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+begin
+behind
+being
+believe
+below
+beside
+besides
+best
+better
+between
+beyond
+both
+brief
+but
+by
+came
+can
+cannot
+cant
+can't
+caption
+cause
+causes
+certain
+certainly
+changes
+clearly
+c'mon
+co
+co.
+com
+come
+comes
+concerning
+consequently
+consider
+considering
+contain
+containing
+contains
+corresponding
+could
+couldn't
+course
+c's
+currently
+dare
+daren't
+definitely
+described
+despite
+did
+didn't
+different
+directly
+do
+does
+doesn't
+doing
+done
+don't
+down
+downwards
+during
+each
+edu
+eg
+eight
+eighty
+either
+else
+elsewhere
+end
+ending
+enough
+entirely
+especially
+et
+etc
+even
+ever
+evermore
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+fairly
+far
+farther
+few
+fewer
+fifth
+first
+five
+followed
+following
+follows
+for
+forever
+former
+formerly
+forth
+forward
+found
+four
+from
+further
+furthermore
+get
+gets
+getting
+given
+gives
+go
+goes
+going
+gone
+got
+gotten
+greetings
+had
+hadn't
+half
+happens
+hardly
+has
+hasn't
+have
+haven't
+having
+he
+he'd
+he'll
+hello
+help
+hence
+her
+here
+hereafter
+hereby
+herein
+here's
+hereupon
+hers
+herself
+he's
+hi
+him
+himself
+his
+hither
+hopefully
+how
+howbeit
+however
+hundred
+i'd
+ie
+if
+ignored
+i'll
+i'm
+immediate
+in
+inasmuch
+inc
+inc.
+indeed
+indicate
+indicated
+indicates
+inner
+inside
+insofar
+instead
+into
+inward
+is
+isn't
+it
+it'd
+it'll
+its
+it's
+itself
+i've
+just
+k
+keep
+keeps
+kept
+know
+known
+knows
+last
+lately
+later
+latter
+latterly
+least
+less
+lest
+let
+let's
+like
+liked
+likely
+likewise
+little
+look
+looking
+looks
+low
+lower
+ltd
+made
+mainly
+make
+makes
+many
+may
+maybe
+mayn't
+me
+mean
+meantime
+meanwhile
+merely
+might
+mightn't
+mine
+minus
+miss
+more
+moreover
+most
+mostly
+mr
+mrs
+much
+must
+mustn't
+my
+myself
+name
+namely
+nd
+near
+nearly
+necessary
+need
+needn't
+needs
+neither
+never
+neverf
+neverless
+nevertheless
+new
+next
+nine
+ninety
+no
+nobody
+non
+none
+nonetheless
+noone
+no-one
+nor
+normally
+not
+nothing
+notwithstanding
+novel
+now
+nowhere
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+one's
+only
+onto
+opposite
+or
+other
+others
+otherwise
+ought
+oughtn't
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+particular
+particularly
+past
+per
+perhaps
+placed
+please
+plus
+possible
+presumably
+probably
+provided
+provides
+que
+quite
+qv
+rather
+rd
+re
+really
+reasonably
+recent
+recently
+regarding
+regardless
+regards
+relatively
+respectively
+right
+round
+said
+same
+saw
+say
+saying
+says
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+shan't
+she
+she'd
+she'll
+she's
+should
+shouldn't
+since
+six
+so
+some
+somebody
+someday
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+sorry
+specified
+specify
+specifying
+still
+sub
+such
+sup
+sure
+take
+taken
+taking
+tell
+tends
+th
+than
+thank
+thanks
+thanx
+that
+that'll
+thats
+that's
+that've
+the
+their
+theirs
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+there'd
+therefore
+therein
+there'll
+there're
+theres
+there's
+thereupon
+there've
+these
+they
+they'd
+they'll
+they're
+they've
+thing
+things
+think
+third
+thirty
+this
+thorough
+thoroughly
+those
+though
+three
+through
+throughout
+thru
+thus
+till
+to
+together
+too
+took
+toward
+towards
+tried
+tries
+truly
+try
+trying
+t's
+twice
+two
+un
+under
+underneath
+undoing
+unfortunately
+unless
+unlike
+unlikely
+until
+unto
+up
+upon
+upwards
+us
+use
+used
+useful
+uses
+using
+usually
+v
+value
+various
+versus
+very
+via
+viz
+vs
+want
+wants
+was
+wasn't
+way
+we
+we'd
+welcome
+well
+we'll
+went
+were
+we're
+weren't
+we've
+what
+whatever
+what'll
+what's
+what've
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+where's
+whereupon
+wherever
+whether
+which
+whichever
+while
+whilst
+whither
+who
+who'd
+whoever
+whole
+who'll
+whom
+whomever
+who's
+whose
+why
+will
+willing
+wish
+with
+within
+without
+wonder
+won't
+would
+wouldn't
+yes
+yet
+you
+you'd
+you'll
+your
+you're
+yours
+yourself
+yourselves
+you've
+zero
+a
+how's
+i
+when's
+why's
+b
+c
+d
+e
+f
+g
+h
+j
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+uucp
+w
+x
+y
+z
+I
+www
+amount
+bill
+bottom
+call
+computer
+con
+couldnt
+cry
+de
+describe
+detail
+due
+eleven
+empty
+fifteen
+fifty
+fill
+find
+fire
+forty
+front
+full
+give
+hasnt
+herse
+himse
+interest
+itse”
+mill
+move
+myse”
+part
+put
+show
+side
+sincere
+sixty
+system
+ten
+thick
+thin
+top
+twelve
+twenty
+abst
+accordance
+act
+added
+adopted
+affected
+affecting
+affects
+ah
+announce
+anymore
+apparently
+approximately
+aren
+arent
+arise
+auth
+beginning
+beginnings
+begins
+biol
+briefly
+ca
+date
+ed
+effect
+et-al
+ff
+fix
+gave
+giving
+heres
+hes
+hid
+home
+id
+im
+immediately
+importance
+important
+index
+information
+invention
+itd
+keys
+kg
+km
+largely
+lets
+line
+'ll
+means
+mg
+million
+ml
+mug
+na
+nay
+necessarily
+nos
+noted
+obtain
+obtained
+omitted
+ord
+owing
+page
+pages
+poorly
+possibly
+potentially
+pp
+predominantly
+present
+previously
+primarily
+promptly
+proud
+quickly
+ran
+readily
+ref
+refs
+related
+research
+resulted
+resulting
+results
+run
+sec
+section
+shed
+shes
+showed
+shown
+showns
+shows
+significant
+significantly
+similar
+similarly
+slightly
+somethan
+specifically
+state
+states
+stop
+strongly
+substantially
+successfully
+sufficiently
+suggest
+thered
+thereof
+therere
+thereto
+theyd
+theyre
+thou
+thoughh
+thousand
+throug
+til
+tip
+ts
+ups
+usefully
+usefulness
+'ve
+vol
+vols
+wed
+whats
+wheres
+whim
+whod
+whos
+widely
+words
+world
+youd
+youre
+"""