Plugins config (#147)

* refactor: rename configs

* refactor: create plugin config

* refactor: universal config

* fix: engine plugins default list

package.json

@@ -9,7 +9,7 @@
     "@fastify/swagger": "^8.14.0",
     "@fastify/swagger-ui": "^3.0.0",
     "@fastify/view": "^9.0.0",
-    "@txtdot/plugins": "^1.0.0",
+    "@txtdot/plugins": "^1.1.1",
     "@txtdot/sdk": "^1.1.1",
     "axios": "^1.6.8",
     "dompurify": "^3.1.0",

pnpm-lock.yaml

@@ -21,8 +21,8 @@ importers:
         specifier: ^9.0.0
         version: 9.0.0
       '@txtdot/plugins':
-        specifier: ^1.0.0
+        specifier: ^1.1.1
-        version: 1.0.0
+        version: 1.1.1
       '@txtdot/sdk':
         specifier: ^1.1.1
         version: 1.1.1
@@ -310,8 +310,8 @@ packages:
     resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==}
     engines: {node: '>=14'}
 
-  '@txtdot/plugins@1.0.0':
+  '@txtdot/plugins@1.1.1':
-    resolution: {integrity: sha512-RJLVNHFBxiEUo8my6eSgkfrSpxPDIUYdGsI0hfw8lyI5aZ30/OjhFfzfK0Jsf1b7xxw5acZn1My0yrF9RpdHBA==}
+    resolution: {integrity: sha512-rCRCzi18xdFK/JpdB+dQF1PaN+w6tNhVJum5YJu05PY0vsz5Rgx9ct7HmPi8W01Ek+iwec8GUeP1IKkUDHF6BQ==}
 
   '@txtdot/sdk@1.1.1':
     resolution: {integrity: sha512-H1YYXdDX3sjZdBfb+osDlDVBVlW8lNeyAIl0j7DhwbOmW3bNCFI9pdMiWGDIyotH6UqOncFP0u+jBugW/5N4Wg==}
@@ -1666,7 +1666,7 @@ snapshots:
   '@pkgjs/parseargs@0.11.0':
     optional: true
 
-  '@txtdot/plugins@1.0.0':
+  '@txtdot/plugins@1.1.1':
     dependencies:
       '@mozilla/readability': 0.5.0
       '@txtdot/sdk': 1.1.1

src/app.ts

@@ -13,22 +13,19 @@ import proxyRoute from './routes/browser/proxy';
 import parseRoute from './routes/api/parse';
 import rawHtml from './routes/api/raw-html';
 
-import packageJSON from './package';
 import errorHandler from './errors/handler';
-import getConfig from './config/main';
 import redirectRoute from './routes/browser/redirect';
 
-import dynConfig from './config/dynamic.config';
 import configurationRoute from './routes/browser/configuration';
 
+import config from './config';
+
 class App {
   async init() {
-    const config = getConfig();
-
     const fastify = Fastify({
       logger: true,
-      trustProxy: config.reverse_proxy,
+      trustProxy: config.env.reverse_proxy,
-      connectionTimeout: config.timeout,
+      connectionTimeout: config.env.timeout,
     });
 
     fastify.setErrorHandler(errorHandler);
@@ -44,14 +41,14 @@ class App {
       },
     });
 
-    if (config.swagger) {
+    if (config.env.swagger) {
-      dynConfig.addRoute('/doc');
+      config.dyn.addRoute('/doc');
       await fastify.register(fastifySwagger, {
         swagger: {
           info: {
             title: 'TXTDot API',
-            description: packageJSON.description,
+            description: config.package.description,
-            version: packageJSON.version,
+            version: config.package.version,
           },
         },
       });
@@ -59,20 +56,20 @@ class App {
     }
 
     fastify.addHook('onRoute', (route) => {
-      dynConfig.addRoute(route.url);
+      config.dyn.addRoute(route.url);
     });
 
     fastify.register(indexRoute);
     fastify.register(getRoute);
     fastify.register(configurationRoute);
 
-    config.third_party.searx_url && fastify.register(redirectRoute);
+    config.env.third_party.searx_url && fastify.register(redirectRoute);
-    config.proxy.enabled && fastify.register(proxyRoute);
+    config.env.proxy.enabled && fastify.register(proxyRoute);
 
     fastify.register(parseRoute);
     fastify.register(rawHtml);
 
-    fastify.listen({ host: config.host, port: config.port }, (err) => {
+    fastify.listen({ host: config.env.host, port: config.env.port }, (err) => {
       err && console.log(err);
     });
   }

src/config/dynConfig.ts

@@ -1,4 +1,4 @@
-class DynConfigService {
+class DynConfig {
   public routes: Set<string> = new Set();
   constructor() {}
   addRoute(route: string) {
@@ -6,5 +6,5 @@ class DynConfigService {
   }
 }
 
-const config = new DynConfigService();
+const dyn_config = new DynConfig();
-export default config;
+export default dyn_config;

src/config/envConfig.ts

@@ -1,6 +1,6 @@
-import { config } from 'dotenv';
+import { config as dconfig } from 'dotenv';
 
-export class ConfigService {
+class EnvConfig {
   public readonly host: string;
   public readonly port: number;
   public readonly timeout: number;
@@ -10,7 +10,7 @@ export class ConfigService {
   public readonly third_party: ThirdPartyConfig;
 
   constructor() {
-    config();
+    dconfig();
 
     this.host = process.env.HOST || '0.0.0.0';
     this.port = Number(process.env.PORT) || 8080;
@@ -37,6 +37,8 @@ export class ConfigService {
     return value === 'true' || value === '1';
   }
 }
+const env_config = new EnvConfig();
+export default env_config;
 
 interface ProxyConfig {
   enabled: boolean;

src/config/index.ts

@@ -0,0 +1,13 @@
+import dyn_config from './dynConfig';
+import env_config from './envConfig';
+import package_config from './packageConfig';
+import plugin_config from './pluginConfig';
+
+const config = {
+  dyn: dyn_config,
+  env: env_config,
+  plugin: plugin_config,
+  package: package_config,
+};
+
+export default config;

src/config/main.ts

@@ -1,12 +0,0 @@
-import { ConfigService } from './config.service';
-
-let configSvc: ConfigService | undefined;
-
-export default function getConfig(): ConfigService {
-  if (configSvc) {
-    return configSvc;
-  }
-
-  configSvc = new ConfigService();
-  return configSvc;
-}

src/config/packageConfig.ts

@@ -0,0 +1,3 @@
+import * as package_config from '../../package.json';
+
+export default package_config;

src/config/pluginConfig.ts

@@ -0,0 +1,12 @@
+import { IAppConfig } from '../types/appConfig';
+import { engineList } from '@txtdot/plugins';
+
+/**
+ * Configuration of plugins
+ * Here you can add your own plugins
+ */
+const plugin_config: IAppConfig = {
+  engines: [...engineList],
+};
+
+export default plugin_config;

src/distributor.ts

@@ -1,15 +1,15 @@
-import axios, { oaxios } from '../types/axios';
+import axios, { oaxios } from './types/axios';
 import micromatch from 'micromatch';
 import DOMPurify from 'dompurify';
 import { Readable } from 'stream';
-import { NotHtmlMimetypeError } from '../errors/main';
+import { NotHtmlMimetypeError } from './errors/main';
-import { decodeStream, parseEncodingName } from '../utils/http';
+import { decodeStream, parseEncodingName } from './utils/http';
-import replaceHref from '../utils/replace-href';
+import replaceHref from './utils/replace-href';
 import { parseHTML } from 'linkedom';
 
-import getConfig from '../config/main';
 import { Engine } from '@txtdot/sdk';
 import { HandlerInput, IHandlerOutput } from '@txtdot/sdk/dist/types/handler';
+import config from './config';
 
 interface IEngineId {
   [key: string]: number;
@@ -35,7 +35,7 @@ export class Distributor {
   ): Promise<IHandlerOutput> {
     const urlObj = new URL(remoteUrl);
 
-    const webder_url = getConfig().third_party.webder_url;
+    const webder_url = config.env.third_party.webder_url;
 
     const response = webder_url
       ? await oaxios.get(

src/errors/handler.ts

@@ -5,7 +5,7 @@ import { getFastifyError } from './validation';
 import { TxtDotError } from '@txtdot/sdk/dist/types/errors';
 
 import { IGetSchema } from '../types/requests/browser';
-import getConfig from '../config/main';
+import config from '../config';
 
 export default function errorHandler(
   error: Error,
@@ -58,7 +58,7 @@ function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) {
       code: error.code,
       description: error.description,
       proxyBtn:
-        error instanceof NotHtmlMimetypeError && getConfig().proxy.enabled,
+        error instanceof NotHtmlMimetypeError && config.env.proxy.enabled,
     });
   }
 

src/errors/main.ts

@@ -1,4 +1,4 @@
-import getConfig from '../config/main';
+import config from '../config';
 import { TxtDotError } from '@txtdot/sdk/dist/types/errors';
 
 export class LocalResourceError extends TxtDotError {
@@ -23,7 +23,7 @@ export class NotHtmlMimetypeError extends TxtDotError {
       421,
       'NotHtmlMimetypeError',
       'Received non-HTML content, ' +
-        (getConfig().proxy.enabled
+        (config.env.proxy.enabled
           ? 'use proxy instead of parser.'
           : 'proxying is disabled by the instance admin.')
     );

src/handlers/main.ts

@@ -1,11 +0,0 @@
-import { Distributor } from './distributor';
-import { engines } from '@txtdot/plugins';
-
-const distributor = new Distributor();
-
-distributor.engine(engines.StackOverflow);
-distributor.engine(engines.SearX);
-distributor.engine(engines.Readability);
-
-export const engineList = distributor.list;
-export default distributor;

src/package.ts

@@ -1,3 +0,0 @@
-import * as config from '../package.json';
-
-export default config;

src/plugin_manager.ts

@@ -0,0 +1,11 @@
+import { Distributor } from './distributor';
+import plugin_config from './config/pluginConfig';
+
+const distributor = new Distributor();
+
+for (const engine of plugin_config.engines) {
+  distributor.engine(engine);
+}
+
+export const engineList = distributor.list;
+export { distributor };

src/routes/api/parse.ts

@@ -6,7 +6,7 @@ import {
   parseSchema,
 } from '../../types/requests/api';
 
-import distributor from '../../handlers/main';
+import { distributor } from '../../plugin_manager';
 import { generateRequestUrl } from '../../utils/generate';
 
 export default async function parseRoute(fastify: FastifyInstance) {

src/routes/api/raw-html.ts

@@ -2,7 +2,7 @@ import { FastifyInstance } from 'fastify';
 
 import { IParseSchema, rawHtmlSchema } from '../../types/requests/api';
 
-import distributor from '../../handlers/main';
+import { distributor } from '../../plugin_manager';
 import { generateRequestUrl } from '../../utils/generate';
 
 export default async function rawHtml(fastify: FastifyInstance) {

src/routes/browser/configuration.ts

@@ -1,19 +1,15 @@
 import { FastifyInstance } from 'fastify';
 
-import packageJSON from '../../package';
+import { distributor } from '../../plugin_manager';
-import distributor from '../../handlers/main';
 import { indexSchema } from '../../types/requests/browser';
 
-import getConfig from '../../config/main';
+import config from '../../config';
-import dynConfig from '../../config/dynamic.config';
 
 export default async function configurationRoute(fastify: FastifyInstance) {
   fastify.get('/configuration', { schema: indexSchema }, async (_, reply) => {
     return reply.view('/templates/configuration.ejs', {
-      packageJSON,
       engines: distributor.fallback,
-      dynConfig,
+      config,
-      config: getConfig(),
     });
   });
 }

src/routes/browser/get.ts

@@ -1,10 +1,9 @@
 import { FastifyInstance } from 'fastify';
 
 import { GetSchema, IGetSchema } from '../../types/requests/browser';
-import distributor from '../../handlers/main';
+import { distributor } from '../../plugin_manager';
 import { generateRequestUrl } from '../../utils/generate';
-
+import config from '../../config';
-import getConfig from '../../config/main';
 
 export default async function getRoute(fastify: FastifyInstance) {
   fastify.get<IGetSchema>(
@@ -32,7 +31,7 @@ export default async function getRoute(fastify: FastifyInstance) {
         return reply.view('/templates/get.ejs', {
           parsed,
           remoteUrl,
-          config: getConfig(),
+          config,
         });
       }
     }

src/routes/browser/index.ts

@@ -1,17 +1,14 @@
 import { FastifyInstance } from 'fastify';
 
-import packageJSON from '../../package';
+import { engineList } from '../../plugin_manager';
-import { engineList } from '../../handlers/main';
 import { indexSchema } from '../../types/requests/browser';
-
+import config from '../../config';
-import getConfig from '../../config/main';
 
 export default async function indexRoute(fastify: FastifyInstance) {
   fastify.get('/', { schema: indexSchema }, async (_, reply) => {
     return reply.view('/templates/index.ejs', {
-      packageJSON,
       engineList,
-      config: getConfig(),
+      config,
     });
   });
 }

src/routes/browser/proxy.ts

@@ -2,8 +2,8 @@ import { FastifyInstance } from 'fastify';
 import { IProxySchema, ProxySchema } from '../../types/requests/browser';
 import axios from '../../types/axios';
 import sharp from 'sharp';
-import getConfig from '../../config/main';
 import { UnsupportedMimetypeError } from '../../errors/main';
+import config from '../../config';
 
 export default async function proxyRoute(fastify: FastifyInstance) {
   fastify.get<IProxySchema>(
@@ -21,7 +21,7 @@ export default async function proxyRoute(fastify: FastifyInstance) {
     }
   );
 
-  if (getConfig().proxy.img_compress)
+  if (config.env.proxy.img_compress)
     fastify.get<IProxySchema>(
       '/proxy/img',
       { schema: ProxySchema },

src/types/appConfig.ts

@@ -0,0 +1,5 @@
+import { Engine } from '@txtdot/sdk';
+
+export interface IAppConfig {
+  engines: Engine[];
+}

src/types/requests/api.ts

@@ -1,6 +1,6 @@
 import { FastifySchema, FastifyRequest } from 'fastify';
 import { IApiError, errorResponseSchema } from '../../errors/api';
-import { engineList } from '../../handlers/main';
+import { engineList } from '../../plugin_manager';
 import { FromSchema } from 'json-schema-to-ts';
 import { handlerSchema } from '@txtdot/sdk/dist/types/handler';
 

src/types/requests/browser.ts

@@ -1,5 +1,5 @@
 import { FastifySchema } from 'fastify';
-import { engineList } from '../../handlers/main';
+import { engineList } from '../../plugin_manager';
 import { FromSchema } from 'json-schema-to-ts';
 
 export interface IGetSchema {

src/utils/replace-href.ts

@@ -1,5 +1,5 @@
+import config from '../config';
 import { generateParserUrl, generateProxyUrl } from './generate';
-import getConfig from '../config/main';
 
 export default function replaceHref(
   dom: Window,
@@ -21,9 +21,7 @@ export default function replaceHref(
   modifyLinks(doc.querySelectorAll('a[href]'), 'href', parserUrl);
   modifyLinks(doc.querySelectorAll('frame,iframe'), 'src', parserUrl);
 
-  const config = getConfig();
+  if (config.env.proxy.enabled) {
-
-  if (config.proxy.enabled) {
     modifyLinks(
       doc.querySelectorAll('video,audio,embed,track,source'),
       'src',
@@ -33,7 +31,7 @@ export default function replaceHref(
     modifyLinks(
       doc.querySelectorAll('img,image'),
       'src',
-      config.proxy.img_compress ? imgProxyUrl : proxyUrl
+      config.env.proxy.img_compress ? imgProxyUrl : proxyUrl
     );
 
     modifyLinks(doc.getElementsByTagName('object'), 'data', proxyUrl);

templates/components/form-main.ejs

@@ -1,4 +1,4 @@
-<% search = config.third_party.searx_url %>
+<% search = config.env.third_party.searx_url %>
 
 <% 
 
@@ -19,7 +19,7 @@ if (search) {
       <div class="input">
         <input type="submit" id="submit" class="button" value="Go">
       </div>
-      <input type="hidden" name="url" value="<%= config.third_party.searx_url %>/search"/>
+      <input type="hidden" name="url" value="<%= config.env.third_party.searx_url %>/search"/>
     </form>
   <% 
 }

templates/components/menu.ejs

@@ -2,12 +2,12 @@
   <a class="button secondary" href="/">Home</a>
   <a class="button secondary" href="<%= remoteUrl %>">Original page</a>
   <% 
-    if (config.third_party.searx_url) {
+    if (config.env.third_party.searx_url) {
       %>
       <form class="form-search" action="/redirect" method="get">
         <input type="text" name="q" id="search" placeholder="Search">
         <input class="button" type="submit" value="Go"/>
-        <input type="hidden" name="url" value="<%= config.third_party.searx_url %>/search"/>
+        <input type="hidden" name="url" value="<%= config.env.third_party.searx_url %>/search"/>
       </form>
       <%
     }

templates/components/search-styles.ejs

@@ -1,5 +1,5 @@
 <% 
-  if (config.third_party.searx_url) {
+  if (config.env.third_party.searx_url) {
     %><link rel="stylesheet" href="/static/search.css">
     <link rel="stylesheet" href="/static/form-inputs.css"><%
   }

templates/configuration.ejs

@@ -17,7 +17,7 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="description" content="<%= packageJSON.description %>">
+    <meta name="description" content="<%= config.package.description %>">
     <title>txt. configuration</title>
     <link rel="stylesheet" href="/static/common.css">
     <link rel="stylesheet" href="/static/configuration.css">
@@ -29,11 +29,11 @@
         <div class="menu">
           <a class="button secondary" href="/">Home</a>
         </div>
-        <p><%= packageJSON.description %></p>
+        <p><%= config.package.description %></p>
       </header>
       <div class="configuration">
         <h2>Configuration</h2>
-        <pre>  version: <%= packageJSON.version %><%= to_pretty(config) %></pre>
+        <pre>  version: <%= config.package.version %><%= to_pretty(config.env) %></pre>
         <h2>Available engines</h2>
         <ol>
           <%
@@ -44,7 +44,7 @@
         </ol>
         <h2>Available routes</h2>
         <% 
-          for (const route of dynConfig.routes) {
+          for (const route of config.dyn.routes) {
             %><a class="button secondary" href="<%= route %>"><%= route %></a><%
           }
         %>

templates/index.ejs

@@ -4,7 +4,7 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, initial-scale=1">
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
-    <meta name="description" content="<%= packageJSON.description %>">
+    <meta name="description" content="<%= config.package.description %>">
     <title>txt. main page</title>
     <link rel="stylesheet" href="/static/common.css">
     <link rel="stylesheet" href="/static/index.css">
@@ -16,11 +16,11 @@
       <header>
         <h1>txt<span class="dot">.</span></h1>
         <div class="menu">
-          <a href="https://github.com/TxtDot/txtdot/releases/tag/v<%= packageJSON.version %>" class="button secondary">v<%= packageJSON.version %></a>
+          <a href="https://github.com/TxtDot/txtdot/releases/tag/v<%= config.package.version %>" class="button secondary">v<%= config.package.version %></a>
           <a href="https://github.com/txtdot/txtdot" class="button secondary">GitHub</a>
           <a href="https://txtdot.github.io/documentation" class="button secondary">Docs</a>
         </div>
-        <p><%= packageJSON.description %></p>
+        <p><%= config.package.description %></p>
       </header>
       <%- include('./components/form-main.ejs') %>
     </main>