<?xml version="1.0" encoding="utf-8"?>
<AlvaoApplication xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:xsd="http://www.w3.org/2001/XMLSchema" ModelVersion="1">
  <Applications>
    <Application id="1056">
      <Name>Knowledge Base AI Assistant</Name>
      <Description>Create Knowledge base article. 
Version: 2.0

Change log:
* 2.0
- use AI client in Alvao.API
- changed to certified application


* 1.3
- optimize returned solution text

* 1.2
- send more tickets to get solution from ticket cluster
- do not add person names to the article
- ...

* 1.1
- better exception handling

* 1.0 
- initial version</Description>
      <UniqueId>a5764038-7ef7-40b6-ad85-73e4c20d0504</UniqueId>
      <Version>2</Version>
      <AdvancedSettings>
        <Setting>
          <Name>KnowledgeBaseAIAssistant.MinTicketSimilarity</Name>
          <Value>90</Value>
        </Setting>
        <Setting>
          <Name>KnowledgeBaseAIAssistant.MinSimilarTicketCount</Name>
          <Value>3</Value>
        </Setting>
        <Setting>
          <Name>KnowledgeBaseAIAssistant.AnalyseTicketCount</Name>
          <Value>5</Value>
        </Setting>
        <Setting>
          <Name>KnowledgeBaseAIAssistant.ShowRelatedTickets</Name>
          <Value>false</Value>
        </Setting>
        <Setting>
          <Name>KnowledgeBaseAIAssistant.NotificationSchedule</Name>
          <Value>7,1,8</Value>
        </Setting>
      </AdvancedSettings>
      <Scripts>
        <Script id="1153">
          <Name>CreateArticles</Name>
          <Code>using System;
using Microsoft.Data.SqlClient;
using Alvao.Apps.API;
using Alvao.API.AI;
using Alvao.API.Common;
using Dapper;
using Alvao.Context;
using System.Linq;
using System.Collections.Generic;
using Alvao.Context.DB;

public class CreateArticles : IPeriodicAction
{
    public string Name
    {
        get =&gt; "CreateArticles";
        set { }
    }

    public void OnPeriod(SqlConnection con)
    {
        try
        {
            if (!Activation.IsModuleActivated(Alvao.Global.ModuleInfo.ModuleId.KnowledgeBaseAIAssistant))
            {
                throw new Exception("KnowledgeBaseAIAssistant module is not activated.");
            }
            var activeHours = GetActiveHours();
            int currentHour = DateTime.Now.Hour;
            if (!activeHours.Any(h =&gt; h == currentHour)) return;

            var articleGenerator = new KnowledgeBaseArticleGenerator();
            articleGenerator.CreateArticles();

            var notificationDays = GetNotificationDays();
            int dayOfWeek = (int)DateTime.Now.DayOfWeek;
            if (currentHour == 8 &amp;&amp; notificationDays.Any(d =&gt; d == dayOfWeek))
            {
                articleGenerator.SendNotification();
            }
        }
        catch (Exception ex)
        {
            Log.Error("OnPeriod", ex, "Failed");
        }
    }

    private static IEnumerable&lt;int&gt; GetActiveHours()
    {
        try
        {
            return DbProperty.ServiceDeskAIAssistant_CalculateVectorsHours.Split(',').Select(h =&gt; Convert.ToInt32(h.Trim()));
        }
        catch
        {
            return new int[0];
        }
    }

    private static IEnumerable&lt;int&gt; GetNotificationDays()
    {
        try
        {
            return DbProperty.KnowledgeBaseAIAssistant_NotificationSchedule.Split(',').Select(h =&gt; Convert.ToInt32(h.Trim()));
        }
        catch
        {
            return new int[0];
        }
    }
}</Code>
          <IsLibCode>false</IsLibCode>
          <Codesign>R0BBWdUykJQIdiLQnhagSJpBPfYcxWEIMspOwgRLpwfgOrXMaNP1LGjJNrh69+oeQloWSD+r1WFng/2V7xbbHkUPF5116V9tXVhZhzmjY4uveCy3qF1MhVRkGPdqlyE8z7GiAo3K+Th9LYtsGyHWnH37hvaHEsCkXZdrR6kPBa+8FsKpl43zGH4CRBMazXFsn3A/EYUQoULSUwXf4Z6hlmmElWyy1ZBogjY2AnX8xEImviMA9sFUrvq4r9fmFz2XjLcKvV9h/l0ZfGAiMVLhMV3hjAqhYzE7PrDBM+16U4esgd6yGRq+7RTX8IIv+8mHCXgiFnekWhjo0/JvE3Kz5g==</Codesign>
        </Script>
        <Script id="1154">
          <Name>CommonLib</Name>
          <Code>using System;
using System.Globalization;
using System.Collections.Generic;
using System.Text;
using System.Text.RegularExpressions;
using System.Linq;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Web;
using Microsoft.Data.SqlClient;
using Alvao.Apps.API;
using Alvao.API;
using Alvao.API.SD;
using Alvao.API.AI;
using Alvao.API.AI.Model;
using Alvao.API.AI.Utils;
using Alvao.API.Common;
using Alvao.API.Common.Model.Database;
using Alvao.API.Utils;
using Alvao.Global;
using Alvao.Context;
using Alvao.Context.DB;
using Dapper;
using Dapper.Contrib.Extensions;
using Newtonsoft.Json;

public class Settings
{
    private static AssistantSettings settings = Assistant.GetSettings();

    public static int MinSimilarTicketCount =&gt; settings.NewKnowledge.MinSimilarTicketCount;
    public static int AnalyseTicketCount =&gt; settings.NewKnowledge.AnalyseTicketCount;
    public static double MinTicketSimilarity = settings.NewKnowledge.MinTicketSimilarity;
    public static double SimilarityTreshold = MinTicketSimilarity * 0.01;
    public static bool ShowRelatedTickets = settings.NewKnowledge.ShowRelatedTickets;
    
    public static string ChatGptModelName = settings.GptModel;
    public const int MaxResponseTokenCount = 4000;

    // set the LogDirectoryPath to empty string to log to database table TenantDiagnosticsLog
    // otherwise text logs are written to specified directory
    public const string LogDirectoryPath = @""; // C:\tmp\Logs
    public const string LogNamePrefix = @"KnowledgeBaseAIAssistant";

    private static CultureInfo dbCultureInfo;

    public static CultureInfo GetDbCultureInfo()
    {
        if (dbCultureInfo != null) return dbCultureInfo;

        using var scope = AlvaoContext.GetConnectionScope();
        int cultureId = scope.Connection.ExecuteScalar&lt;int&gt;(
            @"SELECT COALESCE((SELECT iPropertyValue FROM tProperty WHERE sProperty = 'DefaultLanguageId'), iDbLocaleId) FROM tDb", 
            transaction: scope.Transaction);
        dbCultureInfo = CultureInfo.GetCultureInfo(cultureId);
        return dbCultureInfo;
    }
}

public class KnowledgeBaseArticleGenerator
{
    private static readonly Random random = new Random();
    private static readonly Regex AlvaoMessageSeparator = new("&lt;hr id=\"origmsg\"&gt;", RegexOptions.IgnoreCase);
    private static readonly Regex OutlookOnlineMessageSeparator = new("&lt;hr[^&gt;]+tabindex=\"-1\"", RegexOptions.IgnoreCase);
    private static readonly Regex OutlookDesktopMessageSeparator = new("&lt;div\\s+style=\"[^\"]*border-top:[^;]*solid", RegexOptions.IgnoreCase);
    private static readonly Regex MessageReferenceEn = new("This message refers to request[^\\.]*\\.");
    private static readonly Regex MessageReferenceCs = new("Tato zpráva se týká požadavku[^\\.]*\\.");

    private static readonly AIClient client = Assistant.GetAIClient();

    public void CreateArticles()
    {
        Log.Info("CreateArticles", "Started");
        var start = DateTime.Now;

        var modifiedClusters = GetModifiedClusters();
        if (modifiedClusters.Count == 0)
        {
            Log.Info("CreateArticles", "Finished {duration} - found no new clusters", DateTime.Now - start);
            return;
        }

        var newClusters = modifiedClusters.Where(c =&gt; c.IsNew).ToList();
        Log.Info("CreateArticles", "Modified {modifiedClustersCount}, New {newClustersCount}", modifiedClusters.Count, newClusters.Count);

        foreach (var cluster in newClusters)
        {
            ProcessNewCluster(cluster);
        }

        UpdateArticleTicketClusters(modifiedClusters.Where(c =&gt; !c.IsNew));
        Log.Info("CreateArticles", "Finished {duration} - new clusters found: {count}", DateTime.Now - start, newClusters.Count);
    }

    private List&lt;TicketCluster&gt; GetModifiedClusters()
    {
        var clusters = GetArticleTicketClusters().ToList();
        var clusterTicketIds = clusters.SelectMany(c =&gt; c.Tickets.Select(t =&gt; t.Id)).ToHashSet();
        var tickets = GetTickets().Where(t =&gt; !clusterTicketIds.Contains(t.Id)).ToList();
        var vectors = tickets.Select(t =&gt; t.CommunicationVector.ToVector()).ToList();

        Log.Info("GetModifiedClusters", "Ticket count {ticketCount}", tickets.Count);

        int compareSimilarityCount = 0;

        while (tickets.Count &gt; 1)
        {
            // get random index of ticket
            var firstIndex = tickets.Count &gt; 1 ? random.Next(tickets.Count) : 0;
            var ticket1 = tickets[firstIndex];
            var vector1 = vectors[firstIndex];
            tickets.RemoveAt(firstIndex);
            vectors.RemoveAt(firstIndex);

            if (string.IsNullOrEmpty(ticket1.Summary)) continue;

            var cluster = GetSimilarCluster(clusters, vector1);
            compareSimilarityCount += clusters.Count;
            if (cluster != null)
            {
                cluster.Add(ticket1, vector1);
                if (cluster.ArticleId &gt; 0)
                {
                    Log.Debug("GetModifiedClusters", "Ticket {ticketId} added to article {articleId}", ticket1.Id, cluster.ArticleId);
                }
                continue;
            }

            cluster = new TicketCluster(ticket1, vector1);
            clusters.Add(cluster);
        }

        Log.Info("GetModifiedClusters", "Compare Similarity Count {compareSimilarityCount}", compareSimilarityCount);

        var modifiedClusters = clusters.Where(c =&gt; c.IsModified &amp;&amp; c.Tickets.Count &gt;= Settings.MinSimilarTicketCount).ToList();
        return modifiedClusters;
    }

    private void ProcessNewCluster(TicketCluster cluster)
    {
        var conversationTexts = new List&lt;string&gt;();
        var clusterTickets = new List&lt;EntityInfo&gt;(cluster.Tickets);
        var selectedTicketIds = new List&lt;int&gt;();
        while (clusterTickets.Count &gt; 0 &amp;&amp; selectedTicketIds.Count &lt; Settings.AnalyseTicketCount)
        {
            int index = random.Next(clusterTickets.Count);
            int ticketId = clusterTickets[index].Id;
            selectedTicketIds.Add(ticketId);
            clusterTickets.RemoveAt(index);
            var ticket = Ticket.GetById(ticketId);
            string conversationText = $"Subject: {ticket.sHdTicket}\n{GetTicketCommunication(ticket)}";
            conversationTexts.Add(conversationText);
        }
        
        var culture = cluster.GetTicketsCulture();
        string responseLanguage = culture.EnglishName;
        string json = ExtractStructuredDataFromCommunication(conversationTexts, responseLanguage);

        if (string.IsNullOrEmpty(json))
        {
            Log.Warn("ProcessNewCluster", "No data returned for {ticketIds}, {lastErrorResponse}", selectedTicketIds, client.LastErrorResponse);
            return;
        }

        try
        {
            var data = JsonConvert.DeserializeObject&lt;ArticleData&gt;(json);
            if (data == null)
            {
                Log.Warn("ProcessNewCluster", "No data found for {ticketIds}", selectedTicketIds);
                return;
            }

            if (!string.IsNullOrEmpty(data.Error) || string.IsNullOrEmpty(data.Title))
            {
                Log.Warn("ProcessNewCluster", "Could not retrieve data for {ticketIds}. AI returned {data}", selectedTicketIds, data);
                return;
            }

            cluster.Title = data.Title ?? string.Empty;
            cluster.Annotation = data.Annotation ?? string.Empty;
            cluster.Description = data.Description ?? string.Empty;
            cluster.Solution = data.Solution ?? string.Empty;
            cluster.ProposedSolution = data.ProposedSolution ?? string.Empty;
            cluster.References = data.References ?? new List&lt;WebReference&gt;();
            cluster.Error = data.Error ?? string.Empty;
            cluster.Save(culture);
        }
        catch (Exception ex)
        {
            Log.Error("ProcessNewCluster", ex, "Failed to parse response for {ticketIds}, {json}", selectedTicketIds, json);
        }
    }

    public void SendNotification()
    {
        string propertyName = "CustomApp.KnowledgeBaseAIAssistant.HistoryData";
        var history = JsonProperty.Load&lt;HistoryData&gt;(propertyName);

        var recipients = GetNotificationEmails(history.LastArticleId);
        if (!recipients.Any()) return;

        var texts = Texts.En;
        using var scope = AlvaoContext.GetConnectionScope();
        var articles = scope.Connection.Query&lt;tArticle&gt;(@"
SELECT DISTINCT a.*
FROM tArticle AS a
INNER JOIN tHdTicketCust AS tc ON tc.GeneratedKnowledgeId = a.iArticleId
WHERE a.dArticleCreated &gt; @lastArticleId
	AND a.Removed IS NULL
        ", new { lastArticleId = history.LastArticleId }, transaction: scope.Transaction);

        int newArticleCount = articles.Count();
        Log.Info("SendNotification", "New articles {newArticleCount}", newArticleCount);

        if (newArticleCount == 0) return;

        var manageKBUrl = DbProperty.WebAppUrl.TrimEnd('/') + "/KnowledgeBase/Manage";
        var lastNotificationDate = history.LastNotificationTime &gt; DateTime.MinValue ? history.LastNotificationTime.ToShortDateString() : "last time";
        var mail = new Rebex.Mail.MailMessage();
        mail.DefaultCharset = System.Text.Encoding.UTF8;
        mail.Subject = texts.NotificationSubject;
        mail.BodyHtml = string.Format(texts.NotificationBody, lastNotificationDate, manageKBUrl);

        mail.From.Add(DbProperty.AlvaoStandardSenderAddress);
        foreach (var recipient in recipients)
        {
            mail.To.Add(recipient);
        }
        Email.Queue(mail);
        Log.Info("SendNotification", "Email notification sent {recipientCount}", recipients.Count());

        history.LastArticleId = articles.Select(a =&gt; a.iArticleId).Max();
        history.LastNotificationTime = DateTime.UtcNow;
        JsonProperty.Save(propertyName, history);
    }

    private IEnumerable&lt;string&gt; GetNotificationEmails(int lastArticleId)
    {
        // get main solvers and managers of services with new created articles from last time
        using var scope = AlvaoContext.GetConnectionScope();
        string sql = @"
SELECT DISTINCT p.sPersonEmail
FROM
(
	SELECT DISTINCT s.SectionId
	FROM tArticle AS a
	INNER JOIN tHdTicketCust AS tc ON tc.GeneratedKnowledgeId = a.iArticleId
	INNER JOIN ArticleHdSection AS s ON s.ArticleId = a.iArticleId
	WHERE a.iArticleId &gt; @lastArticleId AND a.Removed IS NULL
) AS s
INNER JOIN (
	SELECT HdSectionId, PersonId FROM vHdSectionOperator
	UNION
	SELECT HdSectionId, PersonId FROM HdSectionManager
) AS r ON r.HdSectionId = s.SectionId
INNER JOIN tPerson AS p ON p.iPersonId = r.PersonId AND p.dPersonRemoved IS NULL AND p.sPersonEmail IS NOT NULL
        ";
        return scope.Connection.Query&lt;string&gt;(sql, new { lastArticleId }, transaction: scope.Transaction);
    }

    private static TicketCluster GetSimilarCluster(List&lt;TicketCluster&gt; clusters, List&lt;double&gt; vector)
    {
        TicketCluster mostSimilarCluster = null;
        double maxSimilarity = double.MinValue;
        foreach (var cluster in clusters)
        {
            double similarity = cluster.GetSimilarity(vector);
            if (similarity &gt; Settings.SimilarityTreshold &amp;&amp; similarity &gt; maxSimilarity)
            {
                mostSimilarCluster = cluster;
                maxSimilarity = similarity;
            }
        }

        return mostSimilarCluster;
    }

    private IEnumerable&lt;TicketCluster&gt; GetArticleTicketClusters()
    {
        using var scope = AlvaoContext.GetConnectionScope();
        var articles = scope.Connection.Query&lt;tArticle&gt;(@"
SELECT DISTINCT a.*
FROM tArticle AS a
INNER JOIN tHdTicketCust AS tc ON tc.GeneratedKnowledgeId = a.iArticleId
", transaction: scope.Transaction);

        var relations = scope.Connection.Query&lt;(int ArticleId, int TicketId)&gt;(@"
SELECT tc.GeneratedKnowledgeId, tc.liHdTicketId AS TicketId
FROM tHdTicketCust AS tc
WHERE tc.GeneratedKnowledgeId IS NOT NULL
ORDER BY tc.GeneratedKnowledgeId, tc.liHdTicketId
", transaction: scope.Transaction);

        var clusters = new List&lt;TicketCluster&gt;();
        foreach (var article in articles)
        {
            var ticketIds = relations.Where(r =&gt; r.ArticleId == article.iArticleId).Select(r =&gt; r.TicketId);
            var tickets = GetTickets(ticketIds).ToList();
            var cluster = new TicketCluster(article, tickets);
            clusters.Add(cluster);
        }
        return clusters;
    }

    public IEnumerable&lt;EntityInfo&gt; GetTickets(IEnumerable&lt;int&gt; ticketIds = null)
    {
        // read the data in smaller chunks (e.g. 100 records max) to prevent SQL timeouts
        int minTicketId = -1;
        var tickets = new List&lt;EntityInfo&gt;();
        if (ticketIds != null)
        {
            foreach (var ticketIdsPart in ticketIds.Chunk(100))
            {
                tickets.AddRange(GetTicketsInternal(ticketIdsPart, minTicketId));
            }

            return tickets;
        }

        bool found = false;
        do
        {
            var ticketsPart = GetTicketsInternal(null, minTicketId);
            found = ticketsPart.Any();
            if (found)
            {
                tickets.AddRange(ticketsPart);
                minTicketId = ticketsPart.Select(t =&gt; t.Id).Max();
            }
        } while (found);

        return tickets;
    }

    private IEnumerable&lt;EntityInfo&gt; GetTicketsInternal(IEnumerable&lt;int&gt; ticketIds, int minTicketId)
    {
        // read the data in smaller chunks (e.g. 100 records max) to prevent SQL timeouts
        using var scope = AlvaoContext.GetConnectionScope();
        return scope.Connection.Query&lt;EntityInfo&gt;($@"
select top(100)
    t.iHdTicketId as Id,
    t.sHdTicket as [Subject],
    tc.Summary,
    a.ActHtml as [Message],
    tc.SolutionProposal,
    t.liHdTicketHdSectionId as SectionId,
    isnull(sc.AssistantDetectMajorIncident, 0) as DetectMajorIncident,
    isnull(sc.AssistantFindSimilarTickets, 0) as FindSimilarTickets,
    isnull(sc.AssistantFindTicketsSolution, 0) as FindTicketsSolution,
    isnull(sc.AssistantRecommendServicesForTickets, 0) as RecommendServicesForTickets,
    isnull(sc.AssistantShowSummary, 0) as ShowSummary,
    isnull(sc.AssistantShowSolutionProposal, 0) as ShowSolutionProposal,
    tc.CommunicationVector,
    tc.FieldsVector,
    tc.ObjectsVector
from tHdTicket t
    inner join tHdSectionCust as sc on sc.liHdSectionId = t.liHdTicketHdSectionId and sc.AssistantAutoCreateKnowledgeFromTickets = 1
    inner join tHdTicketCust as tc on tc.liHdTicketId = t.iHdTicketId and tc.CommunicationVector &lt;&gt; '[]'
    left join tAct as a on a.iActId = t.liHdTicketStartingActId
    where t.iHdTicketId &gt; @minTicketId and t.dHdTicketResolved is not null
{ (ticketIds != null ? "and t.iHdTicketId in (select id from dbo.ftCommaListToTableIds(@ticketIds)) " : "") }
order by t.iHdTicketId", new { minTicketId, ticketIds = string.Join(",", ticketIds ?? new List&lt;int&gt;()) }, scope.Transaction);
    }

    private string GetTicketCommunication(tHdTicket ticket)
    {
        int ticketId = ticket.iHdTicketId;
        var acts = GetTicketCommunicationActs(ticketId);
        var conversation = new StringBuilder();
        bool first = true;
        foreach (var act in acts)
        {
            bool removeParameterTable = ticket.liHdTicketStartingActId == act.iActId;
            string actText = StripHtml(act.ActHtml, removeParameterTable);
            if (string.IsNullOrWhiteSpace(actText))
            {
                continue;
            }

            if (!first)
            {
                conversation.AppendLine();
            }
            conversation.Append("=== ");
            conversation.Append(act.sActFrom);
            if (act.ActMarkId == 2) // marked as solution
            {
                conversation.Append(" === Marked as solution");
            }
            conversation.AppendLine(" ===");
            conversation.AppendLine(actText);
            first = false;
        }

        if (first) return null;

        return conversation.ToString();
    }

    private IEnumerable&lt;tAct&gt; GetTicketCommunicationActs(int ticketId)
    {
        var systemPerson = Alvao.API.Common.Person.GetSystem();
        int systemPersonId = systemPerson.iPersonId;

        using var scope = AlvaoContext.GetConnectionScope();
        return scope.Connection.Query&lt;tAct&gt;(@"
SELECT a.*
FROM tAct AS a
INNER JOIN tHdTicket AS t ON t.iHdTicketId = a.liActHdTicketId
WHERE a.liActHdTicketId = @ticketId
    AND a.dActRemoved IS NULL
    AND (a.ActOperationId IS NULL OR a.ActOperationId = 3) -- not operation log or Resolved
    AND a.liActFromPersonId &lt;&gt; @systemPersonId -- no system messages
    AND 
    (
        a.ActMarkId in (1, 2) -- marked important or solution
        OR -- communication with requester
        (
            a.UserRead = 1
            OR (a.liActFromPersonId = t.liHdTicketUserPersonId)
            OR (a.liActToPersonId = t.liHdTicketUserPersonId)
            OR (a.liActFromPersonId IS NULL AND a.sActFromEmail = t.sHdTicketUserEmail)
            OR (a.liActFromPersonId IS NULL AND a.sActFrom = t.sHdTicketUser)
            OR (a.sActTo LIKE '%&lt;' + t.sHdTicketUserEmail + '&gt;%')
            OR (a.sActCc LIKE '%&lt;' + t.sHdTicketUserEmail + '&gt;%')
            OR (a.sActToEmail LIKE t.sHdTicketUserEmail)
        )
    )
ORDER BY a.iActId
        ", new
        {
            ticketId,
            systemPersonId
        }, transaction: scope.Transaction);
    }

    private void UpdateArticleTicketClusters(IEnumerable&lt;TicketCluster&gt; clusters)
    {
        foreach (var cluster in clusters)
        {
            cluster.SaveTicketRelations();
        }
    }

    private string ExtractStructuredDataFromCommunication(IEnumerable&lt;string&gt; texts, string responseLanguage)
    {
        string systemMessage = @$"Extract problem description and solution from the conversation between users for an article in the knowledge base.
Retrieve the solution text from the conversation as exact as possible without making modifications or loosing details.
Prioritize finding solutions from messages that are marked as solution.
Do not include any names, email signatures or people who participated in the conversation.
Return the solution text only if you find it at least in two conversations.
Set error text if data cannot be retrieved or if the conversations contain text about different issues.
Do not add advice to contact support into proposed solution.
Article data should be returned in JSON format.
Links in the values that should be formatted as HTML should use the &lt;a&gt; tag with href attribute.

Each message in the conversation begins with the username enclosed in ""==="".
Do not mention or add user names in the answer.
All text values in the JSON response must be written in the {responseLanguage} language.

In case that data cannot be extracted then set the property error in the returned JSON object with the reason.
";
        var conversations = new StringBuilder();
        int conversationIndex = 0;
        foreach (string text in texts)
        {
            conversationIndex++;
            conversations.AppendLine($"--- Conversation {conversationIndex} start ---");
            conversations.AppendLine(text);
            conversations.AppendLine($"--- Conversation {conversationIndex} end ---");
        }
        
        var data = new
        {
            max_tokens = Settings.MaxResponseTokenCount,
            messages = new List&lt;object&gt;()
            {
                new
                {
                    role = "system",
                    content = systemMessage
                },
                new
                {
                    role = "user",
                    content = $"Extract data from conversations.\n\n{conversations}\n"
                }
            },
            response_format = new
            {
                type = "json_schema",
                json_schema = new
                {
                    name = "ArticleDataResponse",
                    strict = true,
                    schema = new
                    {
                        type = "object",
                        description = "Values for article in knowledge base",
                        properties = new
                        {
                            title = new { type = "string", description = "Simple text with title of the article" },
                            annotation = new { type = "string", description = "Short text that contains keywords and describes main content of the article." },
                            description = new { type = "string", description = "Description of the problem. Text must be formatted in HTML." },
                            solution = new { type = "string", description = "Exact solution of the problem found in texts in conversation between users. Text must be formatted in HTML." },
                            proposedSolution = new { type = "string", description = "General solution for the problem proposed by assistant. Text must be formatted in HTML." },
                            references = new
                            {
                                type = "array",
                                description = "Web links used in proposed solution.",
                                items = new
                                {
                                    type = "object",
                                    description = "Web link with title and URL",
                                    properties = new
                                    {
                                        title = new { type = "string", description = "Title text of web link" },
                                        url = new { type = "string", description = "URL address of the web link" },
                                    },
                                    required = new[] { "title", "url" },
                                    additionalProperties = false,
                                },
                            },
                            error = new { type = "string", description = "Contains text of error if the data cannot be retrieved." },
                        },
                        required = new[] { "title", "annotation", "description", "solution", "proposedSolution", "references", "error" },
                        additionalProperties = false,
                    }
                }
            }
        };

        var response = client.GetAzureResponse($"{Settings.ChatGptModelName}/chat/completions", data);
        var result = client.GetResult&lt;ChatCompletionCreateResponse&gt;(response);
        string json = result != null ? AIClient.GetResponseText(result) : null;
        return json;
    }

    // Remove original message and sentense "This message refers to request xxx."
    // Put everything in one line
    private static string StripHtml(string html, bool removeParameterTable)
    {
        if (string.IsNullOrEmpty(html)) return html;

        if (removeParameterTable)
        {
            var tableEnd = "&lt;/table&gt;";
            var tableEndPosition = html.IndexOf(tableEnd);
            var descriptionStartPosition = html.IndexOf("&lt;p&gt;");
            if (descriptionStartPosition &lt; 0 &amp;&amp; tableEndPosition &gt;= 0) descriptionStartPosition = tableEndPosition + tableEnd.Length;
            if (tableEndPosition &gt; 0 &amp;&amp; tableEndPosition &lt;= descriptionStartPosition)
            {
                html = "&lt;html&gt;&lt;body&gt;&lt;div&gt;" + html.Substring(descriptionStartPosition);
            }
        }

        var separatorPosition = GetSeparatorPosition(html);
        if (separatorPosition &gt;= 0)
        {
            html = html.Substring(0, separatorPosition) + "&lt;/div&gt;&lt;/body&gt;&lt;/html&gt;";
        }

        var text = StringLib.StripHtml(html);
        text = MessageReferenceEn.Replace(text, string.Empty);
        text = MessageReferenceCs.Replace(text, string.Empty);
        text = text.ReplaceLineEndings(" ");
        return text;
    }

    private static int GetSeparatorPosition(string html)
    {
        var match = AlvaoMessageSeparator.Match(html);
        if (match.Success)
        {
            return match.Index;
        }

        match = OutlookOnlineMessageSeparator.Match(html);
        if (match.Success)
        {
            return match.Index;
        }

        match = OutlookDesktopMessageSeparator.Match(html);
        if (match.Success)
        {
            return match.Index;
        }

        return -1;
    }
}

public class TicketCluster
{
    private static tPerson SystemPerson = Alvao.API.Common.Person.GetSystem();

    public List&lt;EntityInfo&gt; Tickets { get; } = new List&lt;EntityInfo&gt;();
    public List&lt;List&lt;double&gt;&gt; Vectors { get; } = new List&lt;List&lt;double&gt;&gt;();
    public List&lt;double&gt; Vector { get; set; }

    public int ArticleId { get; set; }
    public string Title { get; set; } = string.Empty;
    public string Annotation { get; set; } = string.Empty;
    public string Description { get; set; } = string.Empty;
    public string Solution { get; set; } = string.Empty;
    public string ProposedSolution { get; set; } = string.Empty;
    public List&lt;WebReference&gt; References { get; set; } = new List&lt;WebReference&gt;();
    public string Error { get; set; } = string.Empty;
    public bool IsNew { get; private set; }
    public bool IsModified { get; private set; }

    public TicketCluster(EntityInfo ticket, List&lt;double&gt; vector)
    {
        Vector = vector;
        Vectors.Add(vector);
        Tickets.Add(ticket);
        IsNew = true;
        IsModified = true;
    }

    public TicketCluster(tArticle article, IEnumerable&lt;EntityInfo&gt; tickets)
    {
        ArticleId = article.iArticleId;
        Annotation = article.mArticleAnnotation;
        Title = article.sArticle;
        Description = article.mArticle;
        Solution = string.Empty;
        ProposedSolution = string.Empty;
        foreach (var ticket in tickets)
        {
            Tickets.Add(ticket);
            Vectors.Add(ticket.CommunicationVector.ToVector());
        }
        Vector = Similarity.GetMeanVector(Vectors);
    }

    public double GetSimilarity(List&lt;double&gt; vector)
    {
        return Similarity.GetCosineSimilarity(Vector, vector);
    }

    public void Add(EntityInfo ticket, List&lt;double&gt; vector)
    {
        Tickets.Add(ticket);
        Vectors.Add(vector);
        // Comment next line to skip mean vector calculation - try to add only tickets similar to the first one
        Vector = Similarity.GetMeanVector(Vectors);
        IsModified = true;
    }

    public bool AddIfSimilar(EntityInfo ticket, List&lt;double&gt; vector)
    {
        var similarity = GetSimilarity(vector);
        bool isSimilar = similarity &gt; Settings.SimilarityTreshold;
        if (isSimilar)
        {
            Add(ticket, vector);
        }
        
        return isSimilar;
    }

    public string GetHtml(CultureInfo culture)
    {
        var texts = Texts.Get(culture);
        string webLinks = string.Empty;
        if (References != null &amp;&amp; References.Count &gt; 0)
        {
            string links = string.Join("\r\n", References.Select(
                r =&gt; $@"&lt;li&gt;&lt;a target=""_blank"" rel=""noopener noreferrer"" href=""{r.Url}""&gt;{HttpUtility.HtmlEncode(r.Title)}&lt;/a&gt;&lt;/li&gt;"));
            webLinks = $@"
&lt;h3&gt;{texts.References}&lt;/h3&gt;
  &lt;ul&gt;
    {links}
  &lt;/ul&gt;
";
        }
        string relatedTicketsHtml = string.Empty;
        if (Settings.ShowRelatedTickets)
        {
            string webAppUrl = DbProperty.WebAppUrl.Trim('/');
            string links = string.Join("\r\n", Tickets.Select(
                t =&gt; $@"&lt;li&gt;&lt;a target=""_blank"" rel=""noopener noreferrer"" href=""{webAppUrl}/Ticket?id={t.Id}""&gt;{HttpUtility.HtmlEncode(t.Subject)}&lt;/a&gt;&lt;/li&gt;"));
            relatedTicketsHtml = $@"
&lt;h2&gt;{texts.RelatedTickets}&lt;/h2&gt;
  &lt;ul&gt;
    {links}
  &lt;/ul&gt;
";
        }

        string errorHtml = string.Empty;
        if (!string.IsNullOrEmpty(Error))
        {
            errorHtml = $"&lt;h2&gt;Error&lt;/h2&gt;&lt;pre&gt;{HttpUtility.HtmlEncode(Error)}&lt;/pre&gt;";
        }
        
        return $@"
  &lt;h2&gt;{texts.Description}&lt;/h2&gt;
  &lt;p&gt;{Description.StripJavaScript(true)}&lt;/p&gt;
  &lt;h2&gt;{texts.Solution}&lt;/h2&gt;
  &lt;p&gt;{Solution.StripJavaScript(true)}&lt;/p&gt;
  &lt;h2&gt;{texts.ProposedSolution}&lt;/h2&gt;
  &lt;p&gt;{ProposedSolution.StripJavaScript(true)}&lt;/p&gt;
  {webLinks}
  &lt;hr /&gt;
  &lt;div style=""opacity: 0.6""&gt;{texts.AINotice}&lt;/div&gt;
  {errorHtml}
  {relatedTicketsHtml}
";
    }

    public string GetTicketsLanguage()
    {
        var culture = GetTicketsCulture();
        return culture.EnglishName;
    }

    public CultureInfo GetTicketsCulture()
    {
        int sectionId = GetTicketsSectionId();
        var section = Section.GetById(sectionId);
        if (section?.DefaultLanguageId == null) return CultureInfo.GetCultureInfo("en-US");
        return Settings.GetDbCultureInfo();
    }

    private int GetTicketsSectionId()
    {
        if (Tickets.Count == 0) return 0;
        
        return Tickets.GroupBy(t =&gt; t.SectionId)
            .Select(g =&gt; new { SectionId = g.Key, Count = g.Count() })
            .OrderByDescending(x =&gt; x.Count).First().SectionId;
    }

    public void Save(CultureInfo culture)
    {
        if (ArticleId == 0)
        {
            string title = "[AI] " + Title;
            string html = GetHtml(culture);

            var article = new tArticle
            {
                sArticle = title,
                mArticleAnnotation = Annotation,
                HtmlArticle = html,
                liArticleAuthorPersonId = SystemPerson.iPersonId,
                ModifierPersonId = SystemPerson.iPersonId,
                dArticleCreated = System.DateTime.UtcNow,
                dArticleModified = System.DateTime.UtcNow,
                LocaleId = culture.LCID,
                Published = false,
            };
    
            using var scope = AlvaoContext.GetConnectionScope();
            ArticleId = (int)scope.Connection.Insert(article, transaction: scope.Transaction);

            var sectionIds = Tickets.Select(t =&gt; t.SectionId).Distinct();
            foreach (int sectionId in sectionIds)
            {
                var model = new ArticleHdSection()
                {
                    ArticleId = ArticleId,
                    SectionId = sectionId
                };
                scope.Connection.Insert(model, transaction: scope.Transaction);
            }

            // calculate vector
            Assistant.UpdateArticles([ArticleId], DateTime.Now.AddHours(1));
        }

        SaveTicketRelations();
    }

    public void SaveTicketRelations()
    {
        using var scope = AlvaoContext.GetConnectionScope();
        foreach (var ticket in Tickets)
        {
            scope.Connection.Execute(
                @"UPDATE tHdTicketCust SET GeneratedKnowledgeId = @articleId WHERE liHdTicketId = @ticketId",
                new { articleId = ArticleId, ticketId = ticket.Id },
                scope.Transaction);
        }
    }
}

public class ArticleData
{
    [JsonProperty("title")]
    public string Title { get; set; }

    [JsonProperty("annotation")]
    public string Annotation { get; set; }

    [JsonProperty("description")]
    public string Description { get; set; }

    [JsonProperty("solution")]
    public string Solution { get; set; }

    [JsonProperty("proposedSolution")]
    public string ProposedSolution { get; set; }

    [JsonProperty("references")]
    public List&lt;WebReference&gt; References { get; set; }

    [JsonProperty("error")]
    public string Error { get; set; }
}

public class WebReference
{
    [JsonProperty("title")]
    public string Title { get; set; }

    [JsonProperty("url")]
    public string Url { get; set; }
}


public static class Log
{
    public enum LogLevel
    {
        Trace = 0,
        Debug,
        Info,
        Warn,
        Error,
        Fatal,
        Off
    }

    private static readonly Regex argumentRegex = new Regex(@"\{[^\}]*\}");

    public static void Debug(string source, string message, params object[] args)
    {
        Write(LogLevel.Debug, source, null, message, args);
    }

    public static void Info(string source, string message, params object[] args)
    {
        Write(LogLevel.Info, source, null, message, args);
    }

    public static void Warn(string source, string message, params object[] args)
    {
        Write(LogLevel.Warn, source, null, message, args);
    }

    public static void Warn(string source, Exception exception, string message, params object[] args)
    {
        Write(LogLevel.Warn, source, exception, message, args);
    }

    public static void Error(string source, string message, params object[] args)
    {
        Write(LogLevel.Error, source, null, message, args);
    }

    public static void Error(string source, Exception exception, string message, params object[] args)
    {
        Write(LogLevel.Error, source, exception, message, args);
    }

    private static void Write(LogLevel logLevel, string source, Exception exception, string message, params object[] args)
    {
        if (string.IsNullOrEmpty(Settings.LogDirectoryPath))
            WriteDbLog(logLevel, source, exception, message, args);
        else
            WriteJsonLog(logLevel, source, exception, message, args);
        // WriteTextLog(logLevel, source, exception, message, args);
    }

    private static void WriteDbLog(LogLevel logLevel, string source, Exception exception, string message, params object[] args)
    {
        try
        {
            using var scope = AlvaoContext.GetConnectionScope();
            TenantDiagnosticsLog model = new()
            {
                LogLevelId = (int)logLevel,
                LoggedDate = DateTime.UtcNow,
                Application = "CustomApp." + Settings.LogNamePrefix,
                Callsite = source,
                Message = message,
                Exception = GetExceptionDetails(exception),
                Parameters = JsonConvert.SerializeObject(GetArgValues(message, args)),
            };
            scope.Connection.Insert(model, transaction: scope.Transaction);
        }
        catch {}
    }    

    private static void WriteTextLog(LogLevel logLevel, string source, Exception exception, string message, params object[] args)
    {
        try
        {
            var logMessage = GetMessage(message, args);
            string exceptionDetails = GetExceptionDetails(exception);
            if (!Directory.Exists(Settings.LogDirectoryPath))
            {
                Directory.CreateDirectory(Settings.LogDirectoryPath);
            }
            string path = Path.Combine(Settings.LogDirectoryPath, $"{Settings.LogNamePrefix}-{DateTime.Now:yyyy-MM-dd}.log");
            using (var writer = new StreamWriter(path, true))
            {
                writer.WriteLine($"{DateTime.Now:s}|{logLevel}|{source}|{logMessage}{(!string.IsNullOrEmpty(exceptionDetails) ? $"|{exceptionDetails}" : "")}");
            }
        }
        catch {}
    }

    private class LogRecord
    {
        public string time { get; set; }
        public string level  { get; set; }
        public string source  { get; set; }
        public string message  { get; set; }
        public Dictionary&lt;string, object&gt; args { get; set; }
        public string exception  { get; set; }

        public LogRecord()
        {
            time = DateTime.Now.ToString("o");
        }
    }

    private static void WriteJsonLog(LogLevel level, string source, Exception exception, string message, params object[] args)
    {
        try
        {
            string exceptionDetails = GetExceptionDetails(exception);
            var record = new LogRecord
            {
                level = level.ToString(),
                source = source,
                message = message,
                exception = GetExceptionDetails(exception),
                args = GetArgValues(message, args)
            };

            if (!Directory.Exists(Settings.LogDirectoryPath))
            {
                Directory.CreateDirectory(Settings.LogDirectoryPath);
            }

            string path = Path.Combine(Settings.LogDirectoryPath, $"{Settings.LogNamePrefix}-{DateTime.Now:yyyy-MM-dd}.json.log");
            using (var writer = new StreamWriter(path, true))
            {
                string line = JsonConvert.SerializeObject(record);
                writer.WriteLine(line);
            }
        }
        catch {}
    }

    private static string GetExceptionDetails(Exception exception)
    {
        if (exception == null) return null;

        var exceptionDetails = new StringBuilder();
        var ex = exception;
        int index = 0;
        while (ex != null)
        {
            if (index &gt; 0) exceptionDetails.AppendLine();
            index++;
            exceptionDetails.Append(index);
            exceptionDetails.Append(" - ");
            exceptionDetails.Append(ex.GetType().Name);
            exceptionDetails.Append(": ");
            exceptionDetails.AppendLine(ex.Message);
            exceptionDetails.Append(ex.StackTrace);
            ex = ex.InnerException;
        }

        return exceptionDetails.ToString();
    }

    private static string GetMessage(string message, params object[] args)
    {
        if (args == null || args.Length == 0)
        {
            return message;
        }

        var formatString = message;
        var matches = argumentRegex.Matches(formatString);
        for (int i = 0; i &lt; matches.Count &amp;&amp; i &lt; args.Length; i++)
        {
            string replaceString = matches[i].Value;
            var paramName = replaceString.Substring(1, replaceString.Length - 2);
            string quote = args[i] is string ? "'" : string.Empty;
            formatString = formatString.Replace(replaceString, paramName + "=" + quote + "{" + i + "}" + quote);
        }

        return string.Format(formatString, args);
    }

    private static Dictionary&lt;string, object&gt; GetArgValues(string message, params object[] args)
    {
        if (args == null || args.Length == 0)
        {
            return null;
        }

        var values = new Dictionary&lt;string, object&gt;();
        var matches = argumentRegex.Matches(message);
        for (int i = 0; i &lt; matches.Count &amp;&amp; i &lt; args.Length; i++)
        {
            string replaceString = matches[i].Value;
            var paramName = replaceString.Substring(1, replaceString.Length - 2);
            values[paramName] = args[i];
        }

        return values;
    }
}

public class TokenUsage
{
    [JsonProperty("promptTokens")]
    public long PromptTokens { get; set; }

    [JsonProperty("completionTokens")]
    public long CompletionTokens { get; set; }

    [JsonProperty("totalTokens")]
    public long TotalTokens { get; set; }
}

public static class JsonProperty
{
    public static void Save(string propertyName, object data)
    {
        string json = JsonConvert.SerializeObject(data);
        using var scope = AlvaoContext.GetConnectionScope();
        scope.Connection.Execute(@"spUpdateInsertProperty", new
            {
                PropertyName = propertyName,
                sPropertyValue = json,
                bPropertyValue = (bool?)null,
                iPropertyValue = (int?)null,
                dPropertyValue = (DateTime?)null
            },
            transaction: scope.Transaction,
            commandType: System.Data.CommandType.StoredProcedure);
    }

    public static T Load&lt;T&gt;(string propertyName) where T : new()
    {
        var sql = "select sPropertyValue from tProperty where sProperty = @propertyName";
        using var scope = AlvaoContext.GetConnectionScope();
        string json = scope.Connection.QueryFirstOrDefault&lt;string&gt;(sql, new { propertyName }, transaction: scope.Transaction);
        if (string.IsNullOrEmpty(json)) return new T();

        try
        {
            return JsonConvert.DeserializeObject&lt;T&gt;(json);
        }
        catch
        {
            return new T();
        }
    }
}

public class HistoryData
{
    [JsonProperty("lastNotificationTime")]
    public DateTime LastNotificationTime { get; set; }
    [JsonProperty("lastArticleId")]
    public int LastArticleId { get; set; }
}

public class Texts
{
    public static Texts En = new Texts
    {
        NotificationSubject = "New articles awaiting your review",
        NotificationBody = @"&lt;html&gt;&lt;body&gt;
&lt;p&gt;Hey team,&lt;/p&gt;
&lt;p&gt;
Great news! Since {0}, some fresh articles for the knowledge base were generated based on recurring tickets.
&lt;/p&gt;
&lt;p&gt;
Since they're AI-generated, they might need a few tweaks.
Review them in the &lt;a href=""{1}""&gt;knowledge base&lt;/a&gt; and publish them to the users so they can help themselves through the self-service portal.
That way, you'll see fewer repeated tickets.
&lt;p&gt;
&lt;p&gt;
Thanks for keeping your knowledge base sharp.
&lt;/p&gt;
&lt;p&gt;ALVAO&lt;p&gt;
&lt;/body&gt;&lt;/html&gt;",
        Description = "Description",
        Solution = "Solution",
        ProposedSolution = "AI Proposed solution",
        RelatedTickets = "Related tickets",
        AINotice = "This article was generated by AI and may be incorrect.",
        References = "References",
    };

    public static Texts Cs = new Texts
    {
        NotificationSubject = "Nové články čekají na vaši recenzi",
        NotificationBody = @"&lt;html&gt;&lt;body&gt;
&lt;p&gt;Ahoj týme,&lt;/p&gt;
&lt;p&gt;
Skvělé zprávy! Od {0} byly na základě opakujících se tiketů vygenerovány nové články pro znalostní bázi.
&lt;/p&gt;
&lt;p&gt;
Protože jsou generovány umělou inteligencí, mohou potřebovat několik úprav.
Prohlédněte si je ve &lt;a href=""{1}""&gt;znalostní bázi&lt;/a&gt; a publikujte je uživatelům, aby si mohli pomoci sami prostřednictvím samoobslužného portálu.
Tímto způsobem uvidíte méně opakujících se tiketů.
&lt;p&gt;
&lt;p&gt;
Díky, že udržujete svou znalostní bázi aktuální.
&lt;/p&gt;
&lt;p&gt;ALVAO&lt;p&gt;
&lt;/body&gt;&lt;/html&gt;
",
        Description = "Popis",
        Solution = "Řešení",
        ProposedSolution = "Navrhované řešení s využitím AI",
        RelatedTickets = "Related tickets",
        AINotice = "Tento článek byl vygenerován umělou inteligencí a může být nesprávný.",
        References = "Odkazy",
    };

    public string NotificationSubject { get; set; }
    public string NotificationBody { get; set; }
    public string Description { get; set; }
    public string Solution { get; set; }
    public string ProposedSolution { get; set; }
    public string RelatedTickets { get; set; }
    public string AINotice { get; set; }
    public string References { get; set; }

    public static Texts Get(CultureInfo culture)
    {
        return culture.TwoLetterISOLanguageName == "cs" ? Cs : En;
    }
}</Code>
          <IsLibCode>true</IsLibCode>
          <Codesign>Tqg63tlMRJtGyjLmxVqaBZvE6xVWMhoTEXvZLatJKLJx9G2h5blDB+JXVVcHubiyuaM9qrWS+yjrnE4kkNBF1cemz2YHvgixGAF+r2OVBQeOBq7BOCOakl942SLqVWPhYRY6Lo+IchgnARuRC/pnUvR9EH7WGRns4bPFqmnJ2HESEUbC/B46Shakr+TcbQxAmVXHYVQEdlP0jmAi9IhDgItYs7JOWvtY70jovswBy52LGAwxe5HLjrzzZ0IiqJOV0KLderrpXDyrJj5i2SrCT/oBlAyIucxbIymdMa3FxyKbcYD+Cj0GAoPu2Hc1lvkP82TDhFHaOZ8DFEi015nOlQ==</Codesign>
        </Script>
      </Scripts>
    </Application>
  </Applications>
</AlvaoApplication>