mirror of
https://github.com/txtdot/txtdot
synced 2024-10-18 06:32:32 +03:00
feat: middlewares
feat: highlighter
This commit is contained in:
parent
752939d099
commit
d477de027a
@ -1,7 +1,5 @@
|
||||
import { Readability as OReadability } from '@mozilla/readability';
|
||||
|
||||
import { Engine, EngineParseError } from '@txtdot/sdk';
|
||||
import { parseHTML } from 'linkedom';
|
||||
import { Engine, EngineParseError, Route } from '@txtdot/sdk';
|
||||
|
||||
const Readability = new Engine(
|
||||
'Readability',
|
||||
@ -9,7 +7,7 @@ const Readability = new Engine(
|
||||
['*']
|
||||
);
|
||||
|
||||
Readability.route('*path', async (input, ro) => {
|
||||
Readability.route('*path', async (input, ro: Route<{ path: string }>) => {
|
||||
const reader = new OReadability(input.document);
|
||||
const parsed = reader.parse();
|
||||
|
||||
|
@ -1,6 +1,5 @@
|
||||
import { Engine, JSX } from '@txtdot/sdk';
|
||||
import { HandlerInput, Route } from '@txtdot/sdk';
|
||||
import { parseHTML } from 'linkedom';
|
||||
import { PageFooter, ResultItem } from '../components/searchers';
|
||||
|
||||
const SearX = new Engine('SearX', "Engine for searching with 'SearXNG'", [
|
||||
|
@ -1,15 +1,16 @@
|
||||
import * as engines from './engines';
|
||||
|
||||
export { engines };
|
||||
|
||||
export const engineList = [
|
||||
engines.StackOverflow,
|
||||
engines.SearX,
|
||||
engines.Readability,
|
||||
];
|
||||
|
||||
import { compile } from 'html-to-text';
|
||||
import * as middlewares from './middlewares';
|
||||
export { middlewares };
|
||||
export const middlewareList = [middlewares.Highlight];
|
||||
|
||||
import { compile } from 'html-to-text';
|
||||
export const html2text = compile({
|
||||
longWordSplit: {
|
||||
forceWrapOnLimit: true,
|
||||
|
39
packages/plugins/src/middlewares/highlight.tsx
Normal file
39
packages/plugins/src/middlewares/highlight.tsx
Normal file
@ -0,0 +1,39 @@
|
||||
import { Middleware, JSX } from '@txtdot/sdk';
|
||||
|
||||
const Highlight = new Middleware(
|
||||
'highlight',
|
||||
'Highlights code with highlight.js',
|
||||
['*']
|
||||
);
|
||||
|
||||
Highlight.use(async (input, ro, out) => {
|
||||
if (out.content.indexOf('<code') !== -1)
|
||||
return {
|
||||
...out,
|
||||
content: <Highlighter content={out.content} />,
|
||||
};
|
||||
|
||||
return out;
|
||||
});
|
||||
|
||||
function Highlighter({ content }: { content: string }) {
|
||||
return (
|
||||
<>
|
||||
<style>
|
||||
@import
|
||||
"https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/styles/atom-one-light.min.css";
|
||||
@import
|
||||
"https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/styles/atom-one-dark.min.css"
|
||||
screen and (prefers-color-scheme: dark);
|
||||
</style>
|
||||
<script
|
||||
src="https://cdn.jsdelivr.net/gh/highlightjs/cdn-release@11.9.0/build/highlight.min.js"
|
||||
type="text/javascript"
|
||||
/>
|
||||
<script>hljs.highlightAll();</script>
|
||||
{content}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default Highlight;
|
3
packages/plugins/src/middlewares/index.ts
Normal file
3
packages/plugins/src/middlewares/index.ts
Normal file
@ -0,0 +1,3 @@
|
||||
import Highlight from './highlight';
|
||||
|
||||
export { Highlight };
|
@ -34,7 +34,7 @@ export class Engine {
|
||||
}
|
||||
|
||||
async handle(input: HandlerInput): Promise<EngineOutput> {
|
||||
const url = new URL(input.getUrl());
|
||||
const url = new URL(input.url);
|
||||
const path = url.pathname + url.search + url.hash;
|
||||
for (const route of this.routes) {
|
||||
const match = route.route.match(path);
|
||||
|
@ -24,9 +24,7 @@ export function createElement(
|
||||
})
|
||||
.join(' ');
|
||||
|
||||
return inner.length === 0
|
||||
? `<${name} ${propsstr}/>`
|
||||
: `<${name} ${propsstr}>${content}</${name}>`;
|
||||
return `<${name} ${propsstr}>${content}</${name}>`;
|
||||
} else if (typeof name === 'function') {
|
||||
return name(props, content);
|
||||
} else {
|
||||
|
@ -1,4 +1,5 @@
|
||||
import { Engine } from './engine';
|
||||
import { Middleware } from './middleware';
|
||||
|
||||
import {
|
||||
EngineParseError,
|
||||
@ -16,17 +17,22 @@ import {
|
||||
HandlerOutput,
|
||||
Route,
|
||||
handlerSchema,
|
||||
EngineOutput,
|
||||
MiddleFunction,
|
||||
} from './types/handler';
|
||||
|
||||
import * as JSX from './jsx';
|
||||
|
||||
export {
|
||||
Engine,
|
||||
Middleware,
|
||||
EngineParseError,
|
||||
NoHandlerFoundError,
|
||||
TxtDotError,
|
||||
EngineFunction,
|
||||
MiddleFunction,
|
||||
EngineMatch,
|
||||
EngineOutput,
|
||||
Engines,
|
||||
RouteValues,
|
||||
EnginesMatch,
|
||||
|
61
packages/sdk/src/middleware.ts
Normal file
61
packages/sdk/src/middleware.ts
Normal file
@ -0,0 +1,61 @@
|
||||
import Route from 'route-parser';
|
||||
|
||||
import {
|
||||
HandlerInput,
|
||||
RouteValues,
|
||||
EngineOutput,
|
||||
MiddleFunction,
|
||||
} from './types/handler';
|
||||
|
||||
interface IMiddle<TParams extends RouteValues> {
|
||||
route: Route;
|
||||
handler: MiddleFunction<TParams>;
|
||||
}
|
||||
|
||||
export class Middleware {
|
||||
name: string;
|
||||
description: string;
|
||||
domains: string[];
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
middles: IMiddle<any>[] = [];
|
||||
constructor(name: string, description: string, domains: string[] = []) {
|
||||
this.domains = domains;
|
||||
this.name = name;
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
route<TParams extends RouteValues>(
|
||||
path: string,
|
||||
handler: MiddleFunction<TParams>
|
||||
) {
|
||||
this.middles.push({ route: new Route<TParams>(path), handler });
|
||||
}
|
||||
|
||||
use<TParams extends RouteValues>(handler: MiddleFunction<TParams>) {
|
||||
this.middles.push({ route: new Route<{ path: string }>('*path'), handler });
|
||||
}
|
||||
|
||||
async handle(input: HandlerInput, out: EngineOutput): Promise<EngineOutput> {
|
||||
const url = new URL(input.url);
|
||||
const path = url.pathname + url.search + url.hash;
|
||||
|
||||
let processed_out = out;
|
||||
|
||||
for (const middle of this.middles) {
|
||||
const match = middle.route.match(path);
|
||||
|
||||
if (match) {
|
||||
processed_out = await middle.handler(
|
||||
input,
|
||||
{
|
||||
q: match,
|
||||
reverse: (req) => middle.route.reverse(req),
|
||||
},
|
||||
out
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return processed_out;
|
||||
}
|
||||
}
|
@ -2,26 +2,30 @@ import { parseHTML } from 'linkedom';
|
||||
import { Engine } from '../engine';
|
||||
|
||||
export class HandlerInput {
|
||||
private data: string;
|
||||
private url: string;
|
||||
private window?: Window;
|
||||
private _data: string;
|
||||
private _url: string;
|
||||
private _window?: Window;
|
||||
|
||||
constructor(data: string, url: string) {
|
||||
this.data = data;
|
||||
this.url = url;
|
||||
this._data = data;
|
||||
this._url = url;
|
||||
}
|
||||
|
||||
getUrl(): string {
|
||||
return this.url;
|
||||
get url(): string {
|
||||
return this._url;
|
||||
}
|
||||
|
||||
get data(): string {
|
||||
return this._data;
|
||||
}
|
||||
|
||||
get document(): Document {
|
||||
if (this.window) {
|
||||
return this.window.document;
|
||||
if (this._window) {
|
||||
return this._window.document;
|
||||
}
|
||||
|
||||
this.window = parseHTML(this.data);
|
||||
return this.window.document;
|
||||
this._window = parseHTML(this._data);
|
||||
return this._window.document;
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,6 +79,12 @@ export type EngineFunction<TParams extends RouteValues> = (
|
||||
ro: Route<TParams>
|
||||
) => Promise<EngineOutput>;
|
||||
|
||||
export type MiddleFunction<TParams extends RouteValues> = (
|
||||
input: HandlerInput,
|
||||
ro: Route<TParams>,
|
||||
out: EngineOutput
|
||||
) => Promise<EngineOutput>;
|
||||
|
||||
export type EnginesMatch<TParams extends RouteValues> = EngineMatch<TParams>[];
|
||||
|
||||
export interface Route<TParams extends RouteValues> {
|
||||
|
@ -1,5 +1,5 @@
|
||||
import { IAppConfig } from '../types/pluginConfig';
|
||||
import { engineList, html2text } from '@txtdot/plugins';
|
||||
import { engineList, middlewareList, html2text } from '@txtdot/plugins';
|
||||
|
||||
/**
|
||||
* Configuration of plugins
|
||||
@ -7,6 +7,7 @@ import { engineList, html2text } from '@txtdot/plugins';
|
||||
*/
|
||||
const plugin_config: IAppConfig = {
|
||||
engines: [...engineList],
|
||||
middlewares: [...middlewareList],
|
||||
html2text,
|
||||
};
|
||||
|
||||
|
@ -6,7 +6,7 @@ import { NotHtmlMimetypeError } from './errors/main';
|
||||
import { decodeStream, parseEncodingName } from './utils/http';
|
||||
import replaceHref from './utils/replace-href';
|
||||
|
||||
import { Engine } from '@txtdot/sdk';
|
||||
import { Engine, EngineOutput, Middleware } from '@txtdot/sdk';
|
||||
import { HandlerInput, HandlerOutput } from '@txtdot/sdk';
|
||||
import config from './config';
|
||||
import { parseHTML } from 'linkedom';
|
||||
@ -18,14 +18,25 @@ interface IEngineId {
|
||||
|
||||
export class Distributor {
|
||||
engines_id: IEngineId = {};
|
||||
fallback: Engine[] = [];
|
||||
list: string[] = [];
|
||||
engines_fallback: Engine[] = [];
|
||||
engines_list: string[] = [];
|
||||
|
||||
middles_id: IEngineId = {};
|
||||
middles_fallback: Middleware[] = [];
|
||||
middles_list: string[] = [];
|
||||
|
||||
constructor() {}
|
||||
|
||||
engine(engine: Engine) {
|
||||
this.engines_id[engine.name] = this.list.length;
|
||||
this.fallback.push(engine);
|
||||
this.list.push(engine.name);
|
||||
this.engines_id[engine.name] = this.engines_list.length;
|
||||
this.engines_fallback.push(engine);
|
||||
this.engines_list.push(engine.name);
|
||||
}
|
||||
|
||||
middleware(middleware: Middleware) {
|
||||
this.middles_id[middleware.name] = this.middles_list.length;
|
||||
this.middles_fallback.push(middleware);
|
||||
this.middles_list.push(middleware.name);
|
||||
}
|
||||
|
||||
async handlePage(
|
||||
@ -54,13 +65,13 @@ export class Distributor {
|
||||
|
||||
const engine = this.getFallbackEngine(urlObj.hostname, engineName);
|
||||
|
||||
const output = await engine.handle(
|
||||
new HandlerInput(
|
||||
await decodeStream(data, parseEncodingName(mime)),
|
||||
remoteUrl
|
||||
)
|
||||
const input = new HandlerInput(
|
||||
await decodeStream(data, parseEncodingName(mime)),
|
||||
remoteUrl
|
||||
);
|
||||
|
||||
const output = await engine.handle(input);
|
||||
|
||||
const dom = parseHTML(output.content);
|
||||
|
||||
// Get text content before link replacement, because in text format we need original links
|
||||
@ -77,15 +88,27 @@ export class Distributor {
|
||||
);
|
||||
|
||||
const purify = DOMPurify(dom);
|
||||
const content = purify.sanitize(dom.document.toString());
|
||||
const title = output.title || dom.document.title;
|
||||
const lang = output.lang || dom.document.documentElement.lang;
|
||||
const purified_content = purify.sanitize(dom.document.toString());
|
||||
|
||||
const purified = {
|
||||
...output,
|
||||
content: purified_content,
|
||||
};
|
||||
|
||||
const processed = await this.processMiddlewares(
|
||||
urlObj.hostname,
|
||||
input,
|
||||
purified
|
||||
);
|
||||
|
||||
const title = processed.title || dom.document.title;
|
||||
const lang = processed.lang || dom.document.documentElement.lang;
|
||||
const textContent =
|
||||
html2text(stdTextContent, output, title) ||
|
||||
html2text(stdTextContent, processed, title) ||
|
||||
'Text output cannot be generated.';
|
||||
|
||||
return {
|
||||
content,
|
||||
content: processed.content,
|
||||
textContent,
|
||||
title,
|
||||
lang,
|
||||
@ -94,15 +117,31 @@ export class Distributor {
|
||||
|
||||
getFallbackEngine(host: string, specified?: string): Engine {
|
||||
if (specified) {
|
||||
return this.fallback[this.engines_id[specified]];
|
||||
return this.engines_fallback[this.engines_id[specified]];
|
||||
}
|
||||
|
||||
for (const engine of this.fallback) {
|
||||
for (const engine of this.engines_fallback) {
|
||||
if (micromatch.isMatch(host, engine.domains)) {
|
||||
return engine;
|
||||
}
|
||||
}
|
||||
|
||||
return this.fallback[0];
|
||||
return this.engines_fallback[0];
|
||||
}
|
||||
|
||||
async processMiddlewares(
|
||||
host: string,
|
||||
input: HandlerInput,
|
||||
output: EngineOutput
|
||||
): Promise<EngineOutput> {
|
||||
let processed_output = output;
|
||||
|
||||
for (const middle of this.middles_fallback) {
|
||||
if (micromatch.isMatch(host, middle.domains)) {
|
||||
processed_output = await middle.handle(input, processed_output);
|
||||
}
|
||||
}
|
||||
|
||||
return processed_output;
|
||||
}
|
||||
}
|
||||
|
@ -7,5 +7,9 @@ for (const engine of plugin_config.engines) {
|
||||
distributor.engine(engine);
|
||||
}
|
||||
|
||||
export const engineList = distributor.list;
|
||||
for (const middleware of plugin_config.middlewares || []) {
|
||||
distributor.middleware(middleware);
|
||||
}
|
||||
|
||||
export const engineList = distributor.engines_list;
|
||||
export { distributor };
|
||||
|
@ -8,7 +8,7 @@ import config from '../../config';
|
||||
export default async function configurationRoute(fastify: FastifyInstance) {
|
||||
fastify.get('/configuration', { schema: indexSchema }, async (_, reply) => {
|
||||
return reply.view('/templates/configuration.ejs', {
|
||||
engines: distributor.fallback,
|
||||
engines: distributor.engines_fallback,
|
||||
config,
|
||||
});
|
||||
});
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { Engine } from '@txtdot/sdk';
|
||||
import { Engine, Middleware } from '@txtdot/sdk';
|
||||
|
||||
type Html2TextConverter = (html: string) => string;
|
||||
|
||||
@ -7,6 +7,10 @@ export interface IAppConfig {
|
||||
* List of engines, ordered
|
||||
*/
|
||||
engines: Engine[];
|
||||
/**
|
||||
* List of middlewares, ordered
|
||||
*/
|
||||
middlewares?: Middleware[];
|
||||
/**
|
||||
* HTML to text converter, if engine doesn't support text
|
||||
*/
|
||||
|
Loading…
Reference in New Issue
Block a user