From ef61b926a1d885a84f8b7c7ba121c3d1e5562882 Mon Sep 17 00:00:00 2001
From: Firq <firelp42@gmail.com>
Date: Fri, 6 Dec 2024 23:40:17 +0100
Subject: [PATCH] improved doc import and fixed duplication glitch

---
 data/init.sql                                 |   4 +-
 .../__pycache__/backend.cpython-310.pyc       | Bin 5869 -> 5536 bytes
 server-implementation/backend.py              |  43 ++++++++----------
 server-implementation/config_backend.py       |   2 -
 4 files changed, 21 insertions(+), 28 deletions(-)
 mode change 100644 => 100755 data/init.sql
 delete mode 100644 server-implementation/config_backend.py

diff --git a/data/init.sql b/data/init.sql
old mode 100644
new mode 100755
index 76571af..e4dc0ab
--- a/data/init.sql
+++ b/data/init.sql
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:458fca4a56476e81656f8e486ee37dee786dc3e6269f11819f38d44ea1ca687e
-size 224905425
+oid sha256:9152cd9c36d5b6883f2171d612b378dc88f972bff27d83c786f45cd102b96a21
+size 43633981
diff --git a/server-implementation/__pycache__/backend.cpython-310.pyc b/server-implementation/__pycache__/backend.cpython-310.pyc
index bc486c319bfd45771de7ff1a10f9a395c63ee921..aaba6667f89a5b4a9a92df760a27a750d5c36562 100644
GIT binary patch
delta 2210
zcmaJ>O^g&p6z;12tDfoU`C(_+AA|+Mu9F2sK_UduW%<#ds|ffrF%AROyUfx(GgkM2
zklCzeH-Q*81}a|E!|cU`i!qXT(u0XdV@UL3CwlUtk+T;PeXnLSyY9j6)Yo5Ey{dZe
ztM}^Rp6-O#%w$p;JU<2JW`Ey&$J;`_zB4hhz%@<-Jv1r?(SYlL8Cn%faRd78imkXA
zu+XVEid#V{Ojpv1+kqQqDjCID;Dy;rHq2FW>feDm`AS~#R8R;vRW>P}4hF(vrAV~X
z8sF~n%#y}4i?mYW9?vdmmCZcI^Y9(y<eXM6+$C?x7I{EEkdySdJ;)h%mWaGfPm&?|
zBfY*rF{sr?hpz=~3R-j<Al4Q&qTOUM=~7NsNJp#cuWJqP9bc~De{68NNQm~P)*<4F
z*z8)dy+Y;~*M~H2jA+~((dH?)xV@ysP6upHXp6W~Do%5DhC4jP)7)J)pv&!O-OLIu
z>jBFumg5@4k(s_}U2gN*;e9s;r<<cw7fwu$&bHd4+`m@8?2pbg+YK%s>VlNyMPq;r
z%eHZvJTAXAKB48j{LtKGnJrOo#PUmXXUPegHD2YlxF!k^rC<no`M0^v$^lJEcQd6~
zkp)`T6C?72nZ)f+iTpURaV`)sj58IC;6zrf)*JP>THOkN(GCEMP?94SyeVSPK{7xn
zrO<~jdz7+0xorKJ^219$=e`gH`M&+>b7BmaRj%lw-B3;@R#o^lexppqUffC-ezb9W
z>v~Ou?bZSLHrwwTV1-Q`=Bq$rjT1bm7{s5W&~#3h={$)+`8kv8%VeI$)}l7Acl55!
zjSg9%L{r@C&{fX3!Ob3rQ?cYeXZzSanwVje`$1y$j^iE?dDu`qg|Hu>?1(Y>i!+km
zg*`SxUjE~3pEgu5Ylca_-r)Yb^~P+qRd4x0z2R>NQtXD=N1#Qh4+C*sVo?{+rE)$s
zx$q5i|9`0c&>>vlwhCR3S)UuR12UL#3ef6U-E?0J<Yq@#0wA|KMxWaq6lO|O6WFR^
zR@dz-h8)UZb%M*yb=>Bx*AJ>XvYRejhae7dME;yUzmVxAL>haJA}AR%z!FypUlp~+
zRY)HP2bJ`AR|yxvL^y`<EI`@rDL#oEClQV#V3LRv07>r3jZ31=A=zS4_pj9gF+7fg
z{UjHA;P(Y+D!=gn;0WLVEGmC<zgsAx@$&$G;Kq=x$P+VewyIalx={P7-Ei|V5d-)!
z#6h%5pSuyoet1%drcmW4p2pE$#;b)kCIc>BgQk+fB8<qb-ba>-t03=tWw;K0^|o!@
zP^OugrU+{>#2B@MIF!TLV}!|<yuI>Pc5tCX=Lomp^0GfOR-j(qVI90l*>q1Ybz{5;
zdx~j<mjM!|9@b|4YFh*l%74eiYiRWf07Q>vnAp8-BXJ%_Hn`@^?C$vnE>>%;`gO30
zN&p7UQAQ{+AjsaEdSpQT<+f0AN}kD$r&VU0!Zi=d`P}e=y4BFz#0ct9EOZ=t65$yH
ztRr=ni4KFkV{)Uo7S}I_etf0L#Rar@17Xd{wKVWx01r>4K}s{gGw=<3#FR);mM<~J
zGTtC{Q-nGm5uO~+e@~|6U-=z-P_>>is1}IU*K%Di6)KrZa<Z`d&{?!I6L<CIt+s0M
z6_q>*nNdn6WNw|GcpZxgfLrqO!Y(p#|A&GkWK3Qhc<<<1G|FlxSbYpF!iX5z0s$1@
zI@t$J;8=M``0x>`3_>|xJS)E{K1OW$pm+e%#w;C9tGZJqmzG}?hn%@4UR;rI@$XWv
yI3s}tD{vF#m|A<C$9-iJb_9yL9r(ux*hZrO4E$mgs|>F>Y`vOuiAP!FdjA49Jis6T

delta 2617
zcmZWrO>7%Q6yBNr@!DRm?b!JbKdBn1`3V)}w?HMJP|8nBDM)B(ST>$*;&j(`XD1B-
z7gtITAQ7Q57oZ#*;L<C-A|xa(aN@#c#SICmNC<HU#Cx;GDaEV(=FPmf^JeCI?~Mm{
z{&u(9DU}Ku{Jwnt<jNN_m)tq>!=*cC`ypM^d5vj|M22TTOCvM3Jd0=`>rpPYJzL2J
zjOV?)lFg_P7rmmAt;mT>UP;Nh$c@WhS;=-(iK|{!$@yq3u6eb%?$uSl0N;;$;~+nx
zv0^k4PkNI|;Y3sMv^Pz(BN{6)_oBw!E$Yp%GOJwFyjfOdW6;hqa#Cy5F8y+<Tshon
zuE$|poa{y|A>?D^NAj+$(I-rs<u7XT6n%<J$<OFUpWx<)KMJh{@00K*7XXB|Mz|yB
zmPRz84-7_jNKf-kVGVM^-XUvwMyKICW;JHaYUc^q7x)4*nRQVU#U9xrhqW#I!Vx8w
zW8j+=SaI8c5x1ue$~#&E#0rSj9Sx+hp0+kNs4-`U;J#rurDJbd&2<(W+%JFD`NSkF
zisLdHG^LCCQ^Kxjd{&+@j*#2sRpXQU->R=}F7p<<n}>q8!gE2i*ln*g>Z#l8v`@EI
z{N<o|CTz2`*bYOM_@cv`tMa(HAm29k^qr^^FrNh?NJ5e3M?GFczj=f)fOOh#@-Psg
z-)gh)%~pHG@3y*O)M|&kiu#Fv^=H>ZzPYQcpweJG*nssLNf@0@b3<=@R?{AVy`Yad
zG_!q}y(N7AR_G-+16af&4w<4BsS`SS>^}>+0;$MXtX~U>Vmsu#CXePWKarYU-fD|%
zO*&t1t9_)F4~u7;4a)CDM>-Fa%<ryqYa`(CdiQ|bYai&FV1=&=i}yhgkQQ6OkRXgQ
zof*vB)`5IXnMncUu9@&!%-W{sY1_(1SZ<q~ry>U$o7o`iBEO}bH+sgP!16t^Ly4y3
zLXVE*Vo%?pw<<*uV`r!+ViaXEgoq=*vgiA$vC-k|+67qNH3Oibr?xr{e|a-?GARn%
zD`GV*WxdY1kVirH3Qf(p!@?-FSa`1046hJ=7$$%Qx#sr+q*XS|CqImrL&o5UA3!G#
ze+X@I+C6}KQCvX4BuXo4*5L;bBMIM!IvZh3HuCdJrXoOj;LqSN&Y4#KA5@B%5jz~5
zL99Sj%mCQLg}4|a4t0nHuX>=BALSqK{{#lRY^_l~C3Ijwal|O_Lxcr=WW`i{+IBJv
zrUOSqt&*v*!79%T#a&?}8=?q2n!*9Jde)%C%psfBpjl!TQV0y9BG)rUvfVRBa(+m;
zHE^+Z498up7$e!bPKG*?jRX4ydAZOiVrcTG5st|33orM}885qxIH)^{ph$2*N}Y@#
z9<<K@pLh#oj2BTVAz;+=XAzzQXyh`!UciA92*(hPBfJQZR<hdT^9TPeJ~!kQ-w$T^
zQG%<mfw#(#Ib_Wu^4sD!{RvR<mjSM!d&p9hQ&V)h{+Wi(m3QS6m(U=592<ilM62S-
z%|wLpQ=E4=$7;hLMs1eqYN8uggp;qro1nEtY$A=)2bS7hO@3Tz9Kw<d$z(y5SdT>N
zWUViP<tR+an>WmtmQHtg90({eXf8f<58aZ38C31S7+-;Fq6hT@t_`?44r@!(a>?D-
zXV#}W7X7n!4==IYvFw0V-F+24;~v6m0B{8V9pI<3(bu6#bD7_SzX81)*X9n_28|zd
zTkn8LQU@^TScTez5(D;+PvI}al>5tb@_6~SqKcZAaJqx?LV1R%YXBFP8c{0|Tu0S?
z2#+FQaw?uv9h4b)P9g%)YQ~{h?LhTMizdQuUhgJ%859%p@A6dLrgqJ??V3A7odThG
zm+-Q@qw+2B<nNU|2k<b`d@BxCLVuk{>Qo>OMhC^)=mO6Q_*X$SDGycedUy#(%+wiP
zJbpJ2t15_6a#|f*O4hDh8D2(w5nx+>UR@xISHG)DLLQP!wF~v#^((8}!0NL6qc%B*
zQsX9m7QbyE>}KMooT;C9;J|7CkqUS5VzU!>J8k%@k}P(4*yWvOm?RyZERIU<=vK?g
zmHIrf<$Lw}NM3$jKUh>rtMb*6`uIF4$%XOxH(d0H=cv}raWU6zXmr%6{U}s`^(Z`q
be}WP;vmq%VEg=}NoQ@t^8TMUjZ#eEhQhrOm

diff --git a/server-implementation/backend.py b/server-implementation/backend.py
index ffdbd6d..8032108 100644
--- a/server-implementation/backend.py
+++ b/server-implementation/backend.py
@@ -1,15 +1,13 @@
 import json
 import pathlib
-import config_backend
 
-if config_backend.needs_torch:
-    import torch
+import torch
 
 from haystack import Document
 from haystack.utils import ComponentDevice
 from haystack import Pipeline
+
 from haystack.components.embedders import SentenceTransformersDocumentEmbedder
-from haystack.components.preprocessors.document_splitter import DocumentSplitter
 from haystack.components.writers import DocumentWriter
 
 from haystack_integrations.document_stores.pgvector import PgvectorDocumentStore
@@ -29,14 +27,18 @@ class AIBackend:
     document_store: PgvectorDocumentStore
     documents: list[Document] = []
 
-    def __init__(self):
-        if config_backend.needs_torch:
-            get_torch_info()
+    def __init__(self, load_dataset = False):
+        get_torch_info()
+        try:
             self.gpu = ComponentDevice.from_str("cuda:0")
+        except:
+            self.gpu = None
+            print("No CUDA gpu device found")
 
-        dataset = pathlib.Path(__file__).parents[1] / "data" / "dataset.jsonl"
-        if config_backend.load_dataset:
+        if load_dataset:
+            dataset = pathlib.Path(__file__).parents[1] / "data" / "dataset.jsonl"
             self.documents = [ Document(content=d["text"], meta=d["meta"]) for d in load_data(dataset) ]
+
         self.document_store = PgvectorDocumentStore(
             embedding_dimension=768,
             vector_function="cosine_similarity",
@@ -50,40 +52,32 @@ class AIBackend:
     def warmup(self):
         print("Running warmup routine ...")
         print("Launching indexing pipeline to generate document embeddings")
-        res = self.index_pipeline.run({"document_splitter": {"documents": self.documents}})
+        res = self.index_pipeline.run({"document_embedder": {"documents": self.documents}})
         print(f"Finished running indexing pipeline\nDocument Store: Wrote {res['document_writer']['documents_written']} documents")
         self._ready = True
         print("'.query(\"text\")' is now ready to be used")
 
     def _create_indexing_pipeline(self):
         print("Creating indexing pipeline ...")
-        document_splitter = DocumentSplitter(split_by="word", split_length=128, split_overlap=4)
-        if config_backend.needs_torch:
-            document_embedder = SentenceTransformersDocumentEmbedder(model=self.model_embeddings, device=self.gpu)
-        else:
-            document_embedder = SentenceTransformersDocumentEmbedder(model=self.model_embeddings)
+        document_embedder = SentenceTransformersDocumentEmbedder(model=self.model_embeddings, device=self.gpu)
         document_writer = DocumentWriter(document_store=self.document_store)
 
         indexing_pipeline = Pipeline()
-        indexing_pipeline.add_component("document_splitter", document_splitter)
         indexing_pipeline.add_component("document_embedder", document_embedder)
         indexing_pipeline.add_component("document_writer", document_writer)
 
-        indexing_pipeline.connect("document_splitter", "document_embedder")
         indexing_pipeline.connect("document_embedder", "document_writer")
 
         return indexing_pipeline
     
     def _create_query_pipeline(self):
         print("Creating hybrid retrival pipeline ...")
-        if config_backend.needs_torch:
-            text_embedder = SentenceTransformersTextEmbedder(model=self.model_embeddings, device=self.gpu)
-            ranker = TransformersSimilarityRanker(model=self.model_ranker, device=self.gpu)
-        else:
-            text_embedder = SentenceTransformersTextEmbedder(model=self.model_embeddings)
-            ranker = TransformersSimilarityRanker(model=self.model_ranker)
+        text_embedder = SentenceTransformersTextEmbedder(model=self.model_embeddings, device=self.gpu)
+        ranker = TransformersSimilarityRanker(model=self.model_ranker, device=self.gpu)
+
         embedding_retriever = PgvectorEmbeddingRetriever(document_store=self.document_store)
         keyword_retriever = PgvectorKeywordRetriever(document_store=self.document_store)
+
         document_joiner = DocumentJoiner()
 
         hybrid_retrieval = Pipeline()
@@ -132,7 +126,8 @@ class AIBackend:
             results.append({
                 "id": x.meta["id"],
                 "title": x.meta["title"], 
-                "url": x.meta["url"]
+                "url": x.meta["url"],
+                "image_url": x.meta["image_url"]
             })
         return results
 
diff --git a/server-implementation/config_backend.py b/server-implementation/config_backend.py
deleted file mode 100644
index 7966785..0000000
--- a/server-implementation/config_backend.py
+++ /dev/null
@@ -1,2 +0,0 @@
-needs_torch = True
-load_dataset = True
\ No newline at end of file