diff --git a/config/settings/common.py b/config/settings/common.py index 82bbe70fe..96cbd6a24 100644 --- a/config/settings/common.py +++ b/config/settings/common.py @@ -514,3 +514,11 @@ OPENAI_API_ENGINE_4 = env.str("OPENAI_API_ENGINE_4", "") OPENAI_API_ENGINE_4_MAX_TOKENS = env.int("OPENAI_API_ENGINE_4_MAX_TOKENS", 8000) OPENAI_API_TEMPERATURE = env.float("OPENAI_API_TEMPERATURE", 0.0) +OPENAI_API_ENGINE_EMBEDDINGS = env.str("OPENAI_API_ENGINE_EMBEDDINGS", "") +OPENAI_API_ENGINE_EMBEDDINGS_MAX_TOKENS = env.int( + "OPENAI_API_ENGINE_EMBEDDINGS_MAX_TOKENS", 8000 +) +LLM_VECTOR_DATA_STORE_ENV = env( + "LLM_VECTOR_DATA_STORE", default="llm_vector_data_store" +) +LLM_VECTOR_DATA_STORE = os.path.join(MEDIA_ROOT, LLM_VECTOR_DATA_STORE_ENV) diff --git a/config/settings/local.py b/config/settings/local.py index 8d43e8495..f7c8c02f9 100644 --- a/config/settings/local.py +++ b/config/settings/local.py @@ -68,3 +68,4 @@ MEDIA_ROOT_ENV = env("MEDIA_ROOT_ENV", default="media_dev") MEDIA_ROOT = str(APPS_DIR(MEDIA_ROOT_ENV)) SENDFILE_ROOT = MEDIA_ROOT +LLM_VECTOR_DATA_STORE = os.path.join(MEDIA_ROOT, LLM_VECTOR_DATA_STORE_ENV) diff --git a/docker-compose.yml b/docker-compose.yml index 84e80d423..44134c8f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,11 +65,13 @@ services: EMAILLABS_SECRET_KEY: ${EMAILLABS_SECRET_KEY} OPENAI_API_KEY: ${OPENAI_API_KEY} AZURE_ENDPOINT: ${OPENAI_API_BASE} + OPENAI_API_ENGINE_EMBEDDINGS: "text-embedding-ada-002" OPENAI_API_ENGINE_35: "gpt-35-turbo-16k" OPENAI_API_ENGINE_4: "gpt-4" OPENAI_API_TYPE: "azure" OPENAI_API_ENGINE_35_MAX_TOKENS: 16000 OPENAI_API_ENGINE_4_MAX_TOKENS: 8000 + OPENAI_API_ENGINE_EMBEDDINGS_MAX_TOKENS: 8000 OPENAI_API_TEMPERATURE: 0.0 OPENAI_API_VERSION: "2023-07-01-preview" ports: @@ -112,7 +114,7 @@ services: # SENTRY_ENVIRONMENT: "feder-dev" # COMPRESS_EML: "True" # env_file: - # - secrets_imap-to-webhook-url.env:optional + # - secrets_imap-to-webhook-url.env volumes: mysql-data: diff --git a/feder/llm_evaluation/llm_tools.py b/feder/llm_evaluation/llm_tools.py index afe2c4759..b3a4d4ddb 100644 --- a/feder/llm_evaluation/llm_tools.py +++ b/feder/llm_evaluation/llm_tools.py @@ -1,12 +1,14 @@ import json import logging +import os import time import tiktoken from django.conf import settings from langchain.schema.output_parser import StrOutputParser from langchain_community.callbacks import get_openai_callback -from langchain_openai import AzureChatOpenAI +from langchain_community.vectorstores.chroma import Chroma +from langchain_openai import AzureChatOpenAI, AzureOpenAIEmbeddings logger = logging.getLogger(__name__) @@ -24,7 +26,7 @@ def get_llm_response(prompt, prompt_kwargs_dict): openai_api_key=settings.OPENAI_API_KEY, openai_api_version=settings.OPENAI_API_VERSION, azure_endpoint=settings.AZURE_ENDPOINT, - deployment_name=settings.OPENAI_API_DEPLOYMENT_NAME, + deployment_name=settings.OPENAI_API_ENGINE_35, temperature=settings.OPENAI_API_TEMPERATURE, ) chain = prompt | model | StrOutputParser() @@ -46,3 +48,30 @@ def serializable_dict(obj): def get_serializable_dict(obj): return {k: v for k, v in vars(obj).items() if serializable_dict(v)} + + +def create_vectordb_data_for_monitoring_chat(monitoring): + if not os.path.exists(settings.LLM_VECTOR_DATA_STORE): + os.makedirs(settings.LLM_VECTOR_DATA_STORE) + texts = monitoring.responses_chat_context["chat_context_texts"] + embeddings = AzureOpenAIEmbeddings( + azure_endpoint=settings.AZURE_ENDPOINT, + deployment=settings.OPENAI_API_ENGINE_EMBEDDINGS, + openai_api_type=settings.OPENAI_API_TYPE, + openai_api_key=settings.OPENAI_API_KEY, + openai_api_version=settings.OPENAI_API_VERSION, + ) + vectordb = Chroma( + persist_directory=settings.LLM_VECTOR_DATA_STORE, + collection_name=monitoring.slug, + ) + embedding_ids = vectordb.get().get("ids") + if embedding_ids: + vectordb.delete(embedding_ids) + vectordb.from_texts( + texts, + embedding=embeddings, + collection_name=monitoring.slug, + persist_directory=settings.LLM_VECTOR_DATA_STORE, + ) + vectordb.persist() diff --git a/feder/llm_evaluation/models.py b/feder/llm_evaluation/models.py index 770229ede..efa9d6fc9 100644 --- a/feder/llm_evaluation/models.py +++ b/feder/llm_evaluation/models.py @@ -8,9 +8,11 @@ from django.utils import timezone from django.utils.translation import gettext_lazy as _ from jsonfield import JSONField +from langchain.chains import load_summarize_chain from langchain.schema.output_parser import StrOutputParser from langchain.text_splitter import TokenTextSplitter from langchain_community.callbacks import get_openai_callback +from langchain_core.documents import Document from langchain_openai import AzureChatOpenAI from model_utils import Choices from model_utils.models import TimeStampedModel @@ -23,6 +25,8 @@ letter_categorization, letter_evaluation_intro, letter_response_normalization, + monitoring_chat_prompt_template, + monitoring_chat_refine_template, monitoring_response_normalized_template, ) diff --git a/feder/llm_evaluation/prompts.py b/feder/llm_evaluation/prompts.py index 944f3afc5..0e11ac95a 100644 --- a/feder/llm_evaluation/prompts.py +++ b/feder/llm_evaluation/prompts.py @@ -132,24 +132,65 @@ template_format="f-string", ) -answer_categorization = PromptTemplate.from_template( +monitoring_chat_prompt_template = PromptTemplate.from_template( """ - Oceń odpowiedź z {institution} udzieloną do pytanie, przypisując ją do jednej z - kategorii z listy poniżej wybierając odpowiednią literę wraz z opisem, nie dodając - żadnych innych komentarzy. Przy ocenie weź pod uwagę tylko zrozumiały tekst w - języku Polskim. - Pytanie zadane {institution}: + Skorzystaj z poniższych fragmentów kontekstu, aby odpowiedzieć na pytanie na końcu. + Jeśli nie znasz odpowiedzi, po prostu powiedz, że nie wiesz, nie próbuj wymyślać + odpowiedzi. Kontekst zawiera zebrane odpowiedzi w formacie JSON na ankietę + wysłaną do instytucji. Struktura JSONa z odpowiedziami jest następująca: ``` - {question} - ```. - Odpowiedź udzielona na powyższe pytanie przez {institution}: + {{ + {{ Nazwa_instytucji_1: {{ + "1": {{"Pytanie":"treść pytania 1", "Odpowiedź":"treść odpowiedzi 1"}}, + "2": {{"Pytanie":"treść pytania 2", "Odpowiedź":"treść odpowiedzi 2"}} + }}, + {{ Nazwa_instytucji_2: {{ + "1": {{"Pytanie":"treść pytania 1", "Odpowiedź":"treść odpowiedzi 1"}}, + "2": {{"Pytanie":"treść pytania 2", "Odpowiedź":"treść odpowiedzi 2"}} + }} + }} ``` - {answer} - ```. - Lista kategorii odpowiedzi: + Jeśli instytucja nie udzieliła odpowiedzi to JSON dla tej instytucji będzie + zawierał pusty obiekt: {{}}. Udziel odpowiedzi po polsku. + + Kontekst: + --- + {text} + --- + Pytanie: {question} + """, + template_format="f-string", +) + +monitoring_chat_refine_template = PromptTemplate.from_template( + """ + Uzupełnij odpowiedź na pytanie z poprzedniego zadania, korzystając z poniższych + fragmentów kontekstu, aby otrzymać wyczerpującą odpowiedzieć na pytanie na końcu. + Jeśli nie znasz odpowiedzi, po prostu powiedz, że nie wiesz, nie próbuj wymyślać + odpowiedzi. Kontekst zawiera zebrane odpowiedzi w formacie JSON na ankietę + wysłaną do instytucji. Struktura JSONa z odpowiedziami jest następująca: ``` - {answer_categories} + {{ + {{ Nazwa_instytucji_1: {{ + "1": {{"Pytanie":"treść pytania 1", "Odpowiedź":"treść odpowiedzi 1"}}, + "2": {{"Pytanie":"treść pytania 2", "Odpowiedź":"treść odpowiedzi 2"}} + }}, + {{ Nazwa_instytucji_2: {{ + "1": {{"Pytanie":"treść pytania 1", "Odpowiedź":"treść odpowiedzi 1"}}, + "2": {{"Pytanie":"treść pytania 2", "Odpowiedź":"treść odpowiedzi 2"}} + }} + }} ``` + Jeśli instytucja nie udzieliła odpowiedzi to JSON dla tej instytucji będzie + zawierał pusty obiekt: {{}}. Udziel odpowiedzi po polsku. + + Odpowiedź z poprzedniego zadania: {existing_answer} + Kontekst: + --- + {text} + --- + Pytanie: {question} + """, template_format="f-string", ) diff --git a/feder/monitorings/locale/pl/LC_MESSAGES/django.po b/feder/monitorings/locale/pl/LC_MESSAGES/django.po index 1d3c1858c..4dcddb13a 100644 --- a/feder/monitorings/locale/pl/LC_MESSAGES/django.po +++ b/feder/monitorings/locale/pl/LC_MESSAGES/django.po @@ -459,6 +459,10 @@ msgid "Drafts" msgstr "Projekty" #: feder/monitorings/templates/monitorings/_tabs.html:36 +#: feder/monitorings/templates/monitorings/monitoring_chat.html:19 +msgid "Chat" +msgstr "" + #: feder/monitorings/templates/monitorings/monitoring_answers_categories.html:60 msgid "Answers categories" msgstr "Kategorie odpowiedzi" @@ -601,6 +605,18 @@ msgstr "Kwarantanna? (kol. Kwar.?)" msgid "Filter area and tags" msgstr "Filtruj obszar i tagi" +#: feder/monitorings/templates/monitorings/monitoring_chat.html:12 +#, fuzzy +#| msgid "Monitoring Cases" +msgid "Monitoring Chat" +msgstr "Sprawy w Monitoringu" + +#: feder/monitorings/templates/monitorings/monitoring_chat.html:14 +#, fuzzy +#| msgid "Monitoring Cases" +msgid "Monitoring Chat subject" +msgstr "Sprawy w Monitoringu" + #: feder/monitorings/templates/monitorings/monitoring_confirm_delete.html:9 msgid "Confirm delete" msgstr "Potwierdz usuwanie" diff --git a/feder/monitorings/models.py b/feder/monitorings/models.py index 84fa1e88c..56954976d 100644 --- a/feder/monitorings/models.py +++ b/feder/monitorings/models.py @@ -1,6 +1,8 @@ import json +import logging from itertools import groupby +import pytz import reversion from autoslug.fields import AutoSlugField from django.conf import settings @@ -14,7 +16,10 @@ from model_utils.models import TimeStampedModel from feder.domains.models import Domain -from feder.llm_evaluation.prompts import EMAIL_IS_ANSWER, answer_categorization +from feder.llm_evaluation.llm_tools import ( + create_vectordb_data_for_monitoring_chat, + num_tokens_from_string, +) from feder.main.utils import ( FormattedDatetimeMixin, RenderBooleanFieldMixin, @@ -24,6 +29,8 @@ from .validators import validate_nested_lists, validate_template_syntax +logger = logging.getLogger(__name__) + _("Monitorings index") _("Can add Monitoring") _("Can change Monitoring") diff --git a/feder/monitorings/templates/monitorings/_tabs.html b/feder/monitorings/templates/monitorings/_tabs.html index 471eafde3..ab4abffff 100644 --- a/feder/monitorings/templates/monitorings/_tabs.html +++ b/feder/monitorings/templates/monitorings/_tabs.html @@ -12,11 +12,11 @@ {% endif %} >{% trans 'Letters' %} {% if 'reply' in monitoring_perms or 'add_draft' in monitoring_perms %} - {% trans 'Drafts' %} + {% trans 'Drafts' %} {% endif %} {% trans 'Results' %} - {% if user.is_superuser and object.use_llm and object.normalized_response_template %} - {% trans 'Answers categories' %} + >{% trans 'Chat' %} {% endif %} diff --git a/feder/monitorings/templates/monitorings/monitoring_chat.html b/feder/monitorings/templates/monitorings/monitoring_chat.html new file mode 100644 index 000000000..ef4dc9c4c --- /dev/null +++ b/feder/monitorings/templates/monitorings/monitoring_chat.html @@ -0,0 +1,212 @@ +{% extends 'monitorings/base_monitoring_detail.html' %} +{% load i18n humanize bootstrap_pagination %} +{% block content_object %} + {% include 'monitorings/_tabs.html' with tab='chat' %} + +

{% trans "Monitoring Chat" %}

+
+
    + {% for chat in chats %} +
  • +
    +
    + {% trans "Your question regarding monitoring responses:" %} +
    +
    + {{chat.message}} +
    +
    +
    • +
  • +
    +
    + {% trans "LLM response based on monitoring responses content (prepared in " %} + {{chat.resp_time}}s): +
    +
    + {{chat.response|linebreaksbr}} +
    +
    +
    • + {% endfor %} +
+ +
+ {%csrf_token%} +
+ +
+ +
+
+
+
+ + + +{% endblock %} diff --git a/feder/monitorings/urls.py b/feder/monitorings/urls.py index 3b24e8dcc..62fb844d5 100644 --- a/feder/monitorings/urls.py +++ b/feder/monitorings/urls.py @@ -83,14 +83,9 @@ name="results", ), re_path( - r"^(?P[\w-]+)/answers-categories", - views.MonitoringAnswersCategoriesView.as_view(), - name="answers-categories", - ), - re_path( - r"^(?P[\w-]+)/answer-categories-prompt", - views.MonitoringAnswerCategoriesPromptView.as_view(), - name="answer-categories-prompt", + r"^(?P[\w-]+)/chat", + views.MonitoringChatView.as_view(), + name="chat", ), re_path( r"^(?P[\w-]+)/responses-report", diff --git a/feder/monitorings/views.py b/feder/monitorings/views.py index 4e25796b8..5870a4e75 100644 --- a/feder/monitorings/views.py +++ b/feder/monitorings/views.py @@ -709,6 +709,50 @@ def get_excel_workbook(self, monitoring_responses_data): return wb +class MonitoringChatView(DetailView): + model = Monitoring + template_name_suffix = "_chat" + select_related = ["user"] + + def get_queryset(self): + qs = super().get_queryset() + qs = qs.for_user(self.request.user) + return qs + + def get_context_data(self, **kwargs): + kwargs["url_extra_kwargs"] = {"slug": self.object.slug} + context = super().get_context_data(**kwargs) + context["voivodeship_table"] = mark_safe( + self.object.generate_voivodeship_table() + ) + context["chats"] = [ + { + "message": mark_safe(llm_monitoring_request.request_prompt), + "response": llm_monitoring_request.response_text, + "resp_time": llm_monitoring_request.completion_time_str, + } + for llm_monitoring_request in LlmMonitoringRequest.objects.filter( + evaluated_monitoring=self.object, chat_request=True + ).order_by("created") + ] + context["chat_post_url"] = reverse( + "monitorings:chat", + kwargs={"slug": self.kwargs.get("slug")}, + ) + return context + + def post(self, request, *args, **kwargs): + self.object = self.get_object() + if request.POST.get("message"): + chat_question = request.POST.get("message") + response_data = { + "response": LlmMonitoringRequest.get_monitoring_chat_response( + monitoring=self.object, chat_question=chat_question + ), + } + return JsonResponse(response_data) + + class MonitoringCreateView( LoginRequiredMixin, PermissionRequiredMixin, diff --git a/package-lock.json b/package-lock.json index 90965c625..758e075a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2168,6 +2168,15 @@ } } }, + "node_modules/gulp-autoprefixer/node_modules/through2": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/through2/-/through2-4.0.2.tgz", + "integrity": "sha512-iOqSav00cVxEEICeD7TjLB1sueEL+81Wpzp2bY17uZjZN0pWZPuo4suZ/61VujxmqSGFfgOcNuTZ85QJwNZQpw==", + "dev": true, + "dependencies": { + "readable-stream": "3" + } + }, "node_modules/gulp-clean-css": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/gulp-clean-css/-/gulp-clean-css-4.3.0.tgz", diff --git a/requirements/base.txt b/requirements/base.txt index b908e4b4b..3d7c676f8 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -96,3 +96,4 @@ langchain==0.1.19 openai==1.28.0 langchain-openai==0.1.6 tiktoken==0.6.0 +chromadb==0.4.22