mirror of
https://github.com/txtdot/txtdot
synced 2024-10-18 06:32:32 +03:00
feat: llm parser
This commit is contained in:
parent
13bd8fbd24
commit
9218545a43
65
packages/plugins/src/engines/ai.ts
Normal file
65
packages/plugins/src/engines/ai.ts
Normal file
@ -0,0 +1,65 @@
|
||||
import { Engine, EngineParseError, Route } from '@txtdot/sdk';
|
||||
|
||||
const AI = new Engine('Ai', 'Engine for parsing content with LLM', []);
|
||||
|
||||
AI.route('*path', async (input, ro: Route<{ path: string }>) => {
|
||||
let conversation = [
|
||||
{
|
||||
role: 'system',
|
||||
content:
|
||||
'You are a super intelligence that turns a junk HTML page into a beautiful output without style classes, etc. You return only the necessary information in HTML form. You are not allowed to use any styles, scripts, etc. Your output should contain only pure HTML tags.',
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: input.data,
|
||||
},
|
||||
];
|
||||
|
||||
const content = await fetch(process.env.COMPLETIONS_URL || '', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
Authorization: `Bearer ${process.env.KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: process.env.MODEL,
|
||||
messages: conversation,
|
||||
}),
|
||||
})
|
||||
.then((r) => r.json())
|
||||
.then((r) => {
|
||||
conversation.push(r.choices[0].message);
|
||||
conversation.push({
|
||||
role: 'system',
|
||||
content: 'Now you need to generate title. Write it without explanation',
|
||||
});
|
||||
return r.choices[0].message.content;
|
||||
})
|
||||
.catch((e) => {
|
||||
throw new EngineParseError(e);
|
||||
});
|
||||
|
||||
const title = await fetch(process.env.COMPLETIONS_URL || '', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
Authorization: `Bearer ${process.env.KEY}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: process.env.MODEL,
|
||||
messages: conversation,
|
||||
}),
|
||||
})
|
||||
.then((r) => r.json())
|
||||
.then((r) => r.choices[0].message.content)
|
||||
.catch((e) => {
|
||||
throw new EngineParseError(e);
|
||||
});
|
||||
|
||||
return {
|
||||
content,
|
||||
title,
|
||||
};
|
||||
});
|
||||
|
||||
export default AI;
|
@ -1,5 +1,6 @@
|
||||
import StackOverflow from './stackoverflow';
|
||||
import Readability from './readability';
|
||||
import SearX from './searx';
|
||||
import AI from './ai';
|
||||
|
||||
export { StackOverflow, Readability, SearX };
|
||||
export { StackOverflow, Readability, SearX, AI };
|
||||
|
Loading…
Reference in New Issue
Block a user