From 8b2aad77aa0d626daaf95300c1b720f5ae882c43 Mon Sep 17 00:00:00 2001
From: oleh <opoiasnik@gmail.com>
Date: Tue, 22 Oct 2024 15:39:09 +0200
Subject: [PATCH] implemented cloud requesting elasticsearch and more through
 langchain

---
 Backend/__pycache__/model.cpython-311.pyc | Bin 5180 -> 6702 bytes
 Backend/config.json                       |   3 ++
 Backend/indexCloud.py                     |  41 ++++++++++++++++
 Backend/index_JSON.py                     |   2 -
 Backend/model.py                          |  56 ++++++++++++++++------
 5 files changed, 85 insertions(+), 17 deletions(-)
 create mode 100644 Backend/config.json
 create mode 100644 Backend/indexCloud.py

diff --git a/Backend/__pycache__/model.cpython-311.pyc b/Backend/__pycache__/model.cpython-311.pyc
index 7e4d1d4766481c1094496ea66b252c963c66e216..78faa395149f7bf1e4b0f9d049adfea087013482 100644
GIT binary patch
delta 2911
zcmaJ?YitwQ6~1>o9*@Uy{K~_SkW5IR7}6vV9=j=Y3C^QwVpFgelIgaQ;~8R`@ni3d
zfux~H=~hJB)po1-u`BIXiz=0NwQ0pJMOy6xweY8vs%pHAwAM%~p{o6X)JjI866&A!
zTpI(XsCQ<*d+vGNJ7>;0^XabN>~H?1+btj%cRutkzT^FAb8z*q3_@=r8Oc~hSu9~p
zX+~uvraIds8_YJ9&Dte<RmK9%N!+8gU3Fx6iLc5y)tMC}p-MYcSGGxNdg#wBxf!Im
zGtG2&I)L`Tv|3$k@K-TH@556Y7*&<M^|j(_luLfuDFu{(<^sJIS!k4A!VH>3vg;Kj
zH(iC-dmo<KkXjZ0mA3kd&c+eev5I7OqlfDLyxhFuY2-`ovR4Yio?Dcl>}&LW07+Xx
z!elz?{*|pyS$E0|y&CN>>dd)%(O#sn=a~z(IdmSr!5qSHDV<lg8GG5l79{)SK%*Ji
zMPG$Vk080FaU+O%tJDRD5tJ^uO=*|g<>1Dat9A$lJhdTp=X&cNhLQZ#QL6zE<PK`R
z(RtMdmhjXq>vj$#Z>ygXg808@WaO?Kmb>3ZviLLG(`SV9wJHI*XJdQgn@OU)L+*P8
zOFc3ZL!q5_>9!$kxyRLnrl*sdk|4=c6gZxk@^Vqh>FI<TZcQc9xe#OVGc%Der3njW
zMgi~Z?mDR?i$qJGQ$RoxNnIgv<$@k7sA(N&<6Yd3yNy5L?ZyOiJ?I8ALen&4vsg`0
z&yX6&@Y!C%!RtBOADP{qAYOJw^CD5SLO!P{A!k*~;t4&GrfU{k$ZI-jHWFOh;C4{A
zI5oexs1S=y=g#CU+oGa}c;cg*wNvP(AX4Ztu5$g@Z~TSZhdT|<@nv%hu%jV{v{G@K
zanCU*1}HfI@E#&mO#Bp@>d<Z+<rlfCpvm|(|LUqMna`a`FAik1e9pqe;)|LRR`W%9
zV%Z&@IsW39I6gBPi%*4@f2`{x@@zzxPERb$;i09O?8I<#>43U03F}EUGjlqsEexMJ
zJClu`TZrd}qGOS>$0sAh$Hz_#NvBVxr1<z!baHMunw`i*Gbfg!V-u<9q0H3&X!cZU
zZaH#RicbyCWz@6L_=%z8V{$4wc`6;vjxR;x$qSK8<U%z2%~a&{)Q~j!;t?a_+}q?*
zwD}}G>wH=^e&ig8yJwDvM`s?LxHYBgg^|HQHJ?nVDG2+>^ZSN|;PY`{SR^X>8V$=~
zfZ|>N^Ih|%`CId2^OMr+rOW0G^E2}!^M*0Nw2j{s&5y+*4JD|2V18m=1BK6k{<QSN
zB8?`nZi3}CuzjsASZ)>@GSm3b`Bm&S?l`*`p82_ecj`2O9uB!#_H1|?6aJ!O#c{<!
zOZhpp!Znu3MSg`}VP)SUmhJD@VyGACkGdj4eTYn7bn1}juQ!PvzfQB_;aG7RaaV63
zun2Ye6#rFMcf0fZ&KPw1i!h(Z=TWWigj%lu&ee&7yTQkMjKvHLS+-PK*J%F~MdCN^
zH@$$rZR~ds;}#?3&bCj1fJ_5CQlX5f@ej~=$q<{VUcC9wqqW2ZS;+QOw%`2&aPQL4
zLhTC4R55pUUP~`4mb*sh&nZMr6pX{3mwAEgqV7Y+4Nqv5=04#8EJ34%Jg-AZTh8$d
zNu{8t^EtAQ`V=fX)FW9U$EbQgRd>qyWHAd}LnC8!?SnEcCiNmw<oN>0XA3$BQ$AmH
z3;n@z*EZ0K*=&L=Sy<5o1PwxbG$1lBc*Rfx%wq6pKY<_F8v2qIE>B;YHofB|c4CvA
zFxiO;({4Y9|8A^%KbZO#hg$Z16f?hd(hS5(f!Jmsw$7J>;upa^p9lB+YO)mEzZu-W
z?!MEp>)HWxYPEvU6dq+@@*aDP&SQ)b@%><-AN?MW?mpI!ZuJXe*mmpSC<-68-9Ct6
zar>|xFoelbI9zDE<yM|-q=YthLR%Es0bqI6L~b!jJLo_awQ3_JcH=+34$K)H{%ze3
zs@MKlkA{A}9xlQg&-?dqgVf<JBjfK`t#+jlwGou;f+EZ5+@fZAYaMC+Oj=du3kf}C
zvH60MvmSP;D&Jz&d_uOknku1rY4P<|Ku*yuW+~9flVJ+gmPZTt-~8chUM{N2*AoN^
z1pcP*Vg3f0T(Mz{%jhd+<6TCswdz$yp0(;#M*U{vT}B7ZXWhm>{X4zKOw@22zQDmA
z0R;}MdCHEq3c~ogve2@=+iX2p5)N$&hfMZRg)@qQK|EmmBCvb4rN>0x3WqphEp7^1
z-wK!P;-+1smVH;I%kH-I_-k`-e6Qs0-E{Y^P0;t~s|u@dkE-3L@R;{3>cN8t&tg~j
zRRsAua%p5e^VW`%P29A>&R)!wYgzn;r^JeztZ1@gg+bo$Fl*l^`}<6PU%92T9Ox?h
zJInr#a@$bF#Ra%~2;e@2wdu9#iWk{kv=W~1@zB<@vu0ayuz^)DW2qSc7&Rg6xa_**
zf_!%`Fy`!cznES4e0IT1X3e}_nk{b57C|(MPcr7wINY#UfkjRINxTQ&KFQp#BJhER
z7(3ccuC3hEQT83F*liBx9<muii!dT=!RK%V{Xf)bcHQdpg`u!88@4-c^BC~I?0)W9

delta 1381
zcmaJ>O>ERg6rQoyUhmqmy?@?K0>Sx##vw`qq*9@2Xp*KhN(oy+4_H;BkTFZD-E0`|
zCILbTi2kUm=+9IthgPaY6^e)|5JBo8r79e%9*Ud`vR2|y_0rz95{Ihd&>3%9DLM4b
zzRz#o_vXEsH>3TwcdcE$URM_dRK7kM&GjjtsF@4>5P;7C0tn`TUHd{C(m9vs3cSrj
z=Hv6Bf?x}Q4fCQca=;Pa3bRa1s4dkj09HcFG~-RGHGv?Ajr==zqMP`Gz?P9<D~{sI
zG_Vedn`m$ea^Mg^;mZI?%N$KiwI;Ue$X{vXS3PwhvUyEVl1I@Md9&ZvX$1i(sLs)m
zinJAdS)duIZe)y6UUd`{TZwO}6-OzQybSFGg5$t4{$LBnELcgsQqJYbYkUX!lDFVI
z`H63N(e$o)x4paGs`uUcCGYO~#q|$6&Gq-Feb4lMppGR<u2KD-j$7*=nTJQlj~yL&
zbz&epGBG-QoE%XMc#HhbuVz@+FuO2|uY_DDe-hU`WHs~#Ox%eHKXU^aN}rX-iY9j4
z+2V}rSdk#AB4Our*>Ov5B{W-fOF^EBjwh#GXX0cLPq?MYQrX2#q*1C*)v-zm28|51
zG1yK<q)w=kIcYCUlN-`wRj2MTi^DNS$H`}rZc}4s2Zb8|Ga9Q5*mDRVPJWBz#DI`U
zHu`~)I#nvo9_a4Q7pErkr|7;8?BCndLq3-Gghj`lDC0c&MeZUul}@M-Uuojj;I+RL
z*s>@thLD<rNce&u2d$u_&=MQc2Jq<lFnOx9YFdeHXhRmm<f_`5&~pH7^*>a1q{DBA
z$LZ@nPrvhU9xU+3!928LcS71W2zQZY-JpkmPCp2T$%4KQ>SRSPqz7qk{1OG1y+qTj
zo%aywiS2_g6GTxbS7Q$wYAvE67Qdyh<a+$HB;hueXeTifRxfK`B{IA)<;<3*i!->3
zEt4t&y*_*fKhFZ&gAi1lDi@rYl8av;R}yKXk9ncvvUZyrf>rV;VMfQPtDmZCf0}F%
zV!|AJOm-x1C;i5jZ(jF?&Uk}oJ?$L(N#{s?>f#;|{0Ij^gCe*uMu!^t`}zPFGWZ8N
zq>~4RK+%GDKh0<*D_qgV&E#pS5%MHsY=<Fo%-AjNrX+47?-<P|g8Sb|=ar~aIN=~P
zJ(F|sC<{2ufb9-<Gx+b#{4Mnrim04-`th5TV1FR)B*iKZA@o6HDHuM`mV!avP46Q2
z(!E_GNIbiw`C+39Ak1=pH2Ho<6#(N1_a^{ymU|K)^$1z6Dv?B{8}^W)Ovf`?v!|NA
q+~R8uzMAosl&^PGWl^K+C_G`XBv<3)QGHXpnt*e#3jPm+ll%i{Jv?&&

diff --git a/Backend/config.json b/Backend/config.json
new file mode 100644
index 0000000..54e135e
--- /dev/null
+++ b/Backend/config.json
@@ -0,0 +1,3 @@
+{
+  "useCloud" : false
+}
\ No newline at end of file
diff --git a/Backend/indexCloud.py b/Backend/indexCloud.py
new file mode 100644
index 0000000..bb897d9
--- /dev/null
+++ b/Backend/indexCloud.py
@@ -0,0 +1,41 @@
+from elasticsearch import Elasticsearch
+from langchain_huggingface import HuggingFaceEmbeddings
+import json
+import sys
+
+es = Elasticsearch(
+    cloud_id="tt:dXMtZWFzdC0yLmF3cy5lbGFzdGljLWNsb3VkLmNvbTo0NDMkOGM3ODQ0ZWVhZTEyNGY3NmFjNjQyNDFhNjI4NmVhYzMkZTI3YjlkNTQ0ODdhNGViNmEyMTcxMjMxNmJhMWI0ZGU=",
+    basic_auth=("elastic", "sSz2BEGv56JRNjGFwoQ191RJ")
+)
+
+embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
+
+
+def load_drug_data(json_path):
+    with open(json_path, 'r', encoding='utf-8') as f:
+        data = json.load(f)
+    return data
+
+
+def index_documents(data):
+    total_documents = len(data)
+    for i, item in enumerate(data, start=1):
+        doc_text = f"{item['link']} {item.get('pribalovy_letak', '')} {item.get('spc', '')}"
+
+        vector = embeddings.embed_query(doc_text)
+
+        es.index(index='drug_docs', id=i, body={
+            'text': doc_text,
+            'vector': vector,
+            'full_data': item
+        })
+
+        sys.stdout.write(f"\rПроиндексировано {i} из {total_documents} документов")
+        sys.stdout.flush()
+
+    print("\nИндексирование завершено.")
+
+
+data_path = "../../data_adc_databaza/cleaned_general_info_additional.json"
+drug_data = load_drug_data(data_path)
+index_documents(drug_data)
diff --git a/Backend/index_JSON.py b/Backend/index_JSON.py
index e353fcb..b5d7e1f 100644
--- a/Backend/index_JSON.py
+++ b/Backend/index_JSON.py
@@ -2,10 +2,8 @@ import json
 from elasticsearch import Elasticsearch
 from langchain_huggingface import HuggingFaceEmbeddings
 
-
 es = Elasticsearch([{'host': 'localhost', 'port': 9200, 'scheme': 'http'}])
 
-
 embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
 
 
diff --git a/Backend/model.py b/Backend/model.py
index cfecdb3..1477e41 100644
--- a/Backend/model.py
+++ b/Backend/model.py
@@ -1,7 +1,11 @@
-import os
+from elasticsearch import Elasticsearch
+import json
 import requests
+from langchain.chains import SequentialChain
+from langchain.chains import LLMChain, SequentialChain
 from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_elasticsearch import ElasticsearchStore
+from langchain.text_splitter import RecursiveCharacterTextSplitter
 import logging
 
 
@@ -9,10 +13,11 @@ logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
+
+
 mistral_api_key = "hXDC4RBJk1qy5pOlrgr01GtOlmyCBaNs"
 if not mistral_api_key:
-    raise ValueError("API ключ не найден. Убедитесь, что переменная MISTRAL_API_KEY установлена.")
-
+    raise ValueError("API ключ не найден.")
 
 
 class CustomMistralLLM:
@@ -38,30 +43,47 @@ class CustomMistralLLM:
         return result.get("choices", [{}])[0].get("message", {}).get("content", "No response")
 
 
-
 logger.info("Загрузка модели HuggingFaceEmbeddings...")
 embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2")
 
 
-vectorstore = ElasticsearchStore(
-    es_url="http://localhost:9200",
-    index_name='drug_docs',
-    embedding=embeddings,
-    es_user='elastic',
-    es_password='sSz2BEGv56JRNjGFwoQ191RJ'
-)
+config_file_path = "config.json"
 
 
+with open(config_file_path, 'r') as config_file:
+    config = json.load(config_file)
+
+#  Cloud ID
+if config.get("useCloud", False):
+    logger.info("CLOUD ELASTIC")
+    cloud_id = "tt:dXMtZWFzdC0yLmF3cy5lbGFzdGljLWNsb3VkLmNvbTo0NDMkOGM3ODQ0ZWVhZTEyNGY3NmFjNjQyNDFhNjI4NmVhYzMkZTI3YjlkNTQ0ODdhNGViNmEyMTcxMjMxNmJhMWI0ZGU="  # Замените на ваш Cloud ID
+    vectorstore = ElasticsearchStore(
+        es_cloud_id=cloud_id,
+        index_name='drug_docs',
+        embedding=embeddings,
+        es_user = "elastic",
+        es_password = "sSz2BEGv56JRNjGFwoQ191RJ",
+    )
+else:
+    logger.info("LOCAL ELASTIC")
+    vectorstore = ElasticsearchStore(
+        es_url="http://localhost:9200",
+        index_name='drug_docs',
+        embedding=embeddings,
+    )
+
+logger.info(f"Подключение установлено к {'облачному' if config.get('useCloud', False) else 'локальному'} Elasticsearch")
+
+# LLM
 llm = CustomMistralLLM(
     api_key=mistral_api_key,
     endpoint_url="https://api.mistral.ai/v1/chat/completions"
 )
 
-
-
 def process_query_with_mistral(query, k=10):
     logger.info("Обработка запроса началась.")
     try:
+        # Elasticsearch LangChain
         response = vectorstore.similarity_search(query, k=k)
         if not response:
             return {"summary": "Ничего не найдено", "links": [], "status_log": ["Ничего не найдено."]}
@@ -75,8 +97,12 @@ def process_query_with_mistral(query, k=10):
         )
 
         summary = llm.generate_text(prompt=structured_prompt, max_tokens=512, temperature=0.7)
-        return {"summary": summary, "links": links, "status_log": ["Ответ получен от модели Mistral."]}
+
+        #TextSplitter
+        splitter = RecursiveCharacterTextSplitter(chunk_size=200, chunk_overlap=20)
+        split_summary = splitter.split_text(summary)
+
+        return {"summary": split_summary, "links": links, "status_log": ["Ответ получен от модели Mistral."]}
     except Exception as e:
         logger.info(f"Ошибка: {str(e)}")
         return {"summary": "Произошла ошибка", "links": [], "status_log": [f"Ошибка: {str(e)}"]}
-