From 9218545a43db68f86d71ff9a09cca4232ea9147c Mon Sep 17 00:00:00 2001
From: Artemy <artegoser@gmail.com>
Date: Wed, 22 May 2024 21:25:38 +0300
Subject: [PATCH] feat: llm parser

---
 packages/plugins/src/engines/ai.ts    | 65 +++++++++++++++++++++++++++
 packages/plugins/src/engines/index.ts |  3 +-
 2 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 packages/plugins/src/engines/ai.ts

diff --git a/packages/plugins/src/engines/ai.ts b/packages/plugins/src/engines/ai.ts
new file mode 100644
index 0000000..e760c87
--- /dev/null
+++ b/packages/plugins/src/engines/ai.ts
@@ -0,0 +1,65 @@
+import { Engine, EngineParseError, Route } from '@txtdot/sdk';
+
+const AI = new Engine('Ai', 'Engine for parsing content with LLM', []);
+
+AI.route('*path', async (input, ro: Route<{ path: string }>) => {
+  let conversation = [
+    {
+      role: 'system',
+      content:
+        'You are a super intelligence that turns a junk HTML page into a beautiful output without style classes, etc. You return only the necessary information in HTML form. You are not allowed to use any styles, scripts, etc. Your output should contain only pure HTML tags.',
+    },
+    {
+      role: 'user',
+      content: input.data,
+    },
+  ];
+
+  const content = await fetch(process.env.COMPLETIONS_URL || '', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      Authorization: `Bearer ${process.env.KEY}`,
+    },
+    body: JSON.stringify({
+      model: process.env.MODEL,
+      messages: conversation,
+    }),
+  })
+    .then((r) => r.json())
+    .then((r) => {
+      conversation.push(r.choices[0].message);
+      conversation.push({
+        role: 'system',
+        content: 'Now you need to generate title. Write it without explanation',
+      });
+      return r.choices[0].message.content;
+    })
+    .catch((e) => {
+      throw new EngineParseError(e);
+    });
+
+  const title = await fetch(process.env.COMPLETIONS_URL || '', {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      Authorization: `Bearer ${process.env.KEY}`,
+    },
+    body: JSON.stringify({
+      model: process.env.MODEL,
+      messages: conversation,
+    }),
+  })
+    .then((r) => r.json())
+    .then((r) => r.choices[0].message.content)
+    .catch((e) => {
+      throw new EngineParseError(e);
+    });
+
+  return {
+    content,
+    title,
+  };
+});
+
+export default AI;
diff --git a/packages/plugins/src/engines/index.ts b/packages/plugins/src/engines/index.ts
index b5f8527..a99c2ed 100644
--- a/packages/plugins/src/engines/index.ts
+++ b/packages/plugins/src/engines/index.ts
@@ -1,5 +1,6 @@
 import StackOverflow from './stackoverflow';
 import Readability from './readability';
 import SearX from './searx';
+import AI from './ai';
 
-export { StackOverflow, Readability, SearX };
+export { StackOverflow, Readability, SearX, AI };