Webder support (#145)

* Webder support (#144)

* refactor: remove search configuration.

Create third_party

* feat: webder support

.env.example

@@ -15,6 +15,6 @@ IMG_COMPRESS=true  # enable image compressing; proxy_res is required
 ## Documentation
 SWAGGER=false  # whether to add API docs route
 
-## Search
-SEARCH_ENABLED=false  # searx_url is required when enabled
-SEARX_URL=""  # SearXNG base URL, e.g. https://searx.dc09.ru
+## Third-party
+SEARX_URL="https://searx.dc09.ru"  # SearXNG base URL, enables search.
+WEBDER_URL="http://webder.example.com" # WebDer base URL, enables browser rendering.

README.md

@@ -104,3 +104,5 @@ between original page and proxied one.
 - [MicroMatch](https://github.com/micromatch/micromatch)
 - [RouteParser](https://github.com/rcs/route-parser)
 - [IconvLite](https://github.com/ashtuchkin/iconv-lite)
+
+<a href="https://www.producthunt.com/posts/txtdot?utm_source=badge-featured&utm_medium=badge&utm_souce=badge-txtdot" target="_blank"><img src="https://api.producthunt.com/widgets/embed-image/v1/featured.svg?post_id=443317&theme=neutral" alt="txtdot - HTTP&#0032;proxy&#0032;that&#0032;saves&#0032;bandwidth&#0044;&#0032;removing&#0032;ads&#0032;and&#0032;scripts&#0046; | Product Hunt" style="width: 250px; height: 54px;" width="250" height="54" /></a>

package.json

@@ -5,39 +5,39 @@
   "description": "txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts",
   "main": "dist/app.js",
   "dependencies": {
-    "@fastify/static": "^6.12.0",
+    "@fastify/static": "^7.0.3",
     "@fastify/swagger": "^8.14.0",
-    "@fastify/swagger-ui": "^2.1.0",
+    "@fastify/swagger-ui": "^3.0.0",
     "@fastify/view": "^9.0.0",
     "@mozilla/readability": "^0.5.0",
-    "axios": "^1.6.5",
-    "dompurify": "^3.0.8",
+    "axios": "^1.6.8",
+    "dompurify": "^3.1.0",
     "dotenv": "^16.3.1",
-    "ejs": "^3.1.9",
-    "fastify": "^4.25.2",
+    "ejs": "^3.1.10",
+    "fastify": "^4.26.2",
     "iconv-lite": "^0.6.3",
     "ip-range-check": "^0.2.0",
-    "json-schema-to-ts": "^3.0.0",
-    "linkedom": "^0.16.8",
+    "json-schema-to-ts": "^3.0.1",
+    "linkedom": "^0.16.11",
     "micromatch": "^4.0.5",
     "route-parser": "^0.0.5",
-    "sharp": "^0.33.2"
+    "sharp": "^0.33.3"
   },
   "devDependencies": {
     "@types/dompurify": "^3.0.5",
     "@types/ejs": "^3.1.5",
     "@types/jsdom": "^21.1.6",
-    "@types/micromatch": "^4.0.6",
-    "@types/node": "^20.11.24",
+    "@types/micromatch": "^4.0.7",
+    "@types/node": "^20.12.7",
     "@types/route-parser": "^0.1.7",
-    "@typescript-eslint/eslint-plugin": "^7.1.0",
-    "@typescript-eslint/parser": "^7.1.0",
+    "@typescript-eslint/eslint-plugin": "^7.7.0",
+    "@typescript-eslint/parser": "^7.7.0",
     "clean-css-cli": "^5.6.3",
     "copyfiles": "^2.4.1",
     "eslint": "^8.56.0",
     "prettier": "^3.1.1",
-    "tsc-watch": "^6.0.4",
-    "typescript": "^5.3.3"
+    "tsc-watch": "^6.2.0",
+    "typescript": "^5.4.5"
   },
   "scripts": {
     "test": "echo \"Error: no test specified\" && exit 1",

src/app.ts

@@ -66,7 +66,7 @@ class App {
     fastify.register(getRoute);
     fastify.register(configurationRoute);
 
-    config.search.enabled && fastify.register(redirectRoute);
+    config.third_party.searx_url && fastify.register(redirectRoute);
     config.proxy.enabled && fastify.register(proxyRoute);
 
     fastify.register(parseRoute);

src/config/config.service.ts

@@ -7,7 +7,7 @@ export class ConfigService {
   public readonly reverse_proxy: boolean;
   public readonly proxy: ProxyConfig;
   public readonly swagger: boolean;
-  public readonly search: SearchConfig;
+  public readonly third_party: ThirdPartyConfig;
 
   constructor() {
     config();
@@ -26,9 +26,9 @@ export class ConfigService {
 
     this.swagger = this.parseBool(process.env.SWAGGER, false);
 
-    this.search = {
-      enabled: this.parseBool(process.env.SEARCH_ENABLED, false),
+    this.third_party = {
       searx_url: process.env.SEARX_URL,
+      webder_url: process.env.WEBDER_URL,
     };
   }
 
@@ -43,7 +43,7 @@ interface ProxyConfig {
   img_compress: boolean;
 }
 
-interface SearchConfig {
-  enabled: boolean;
+interface ThirdPartyConfig {
   searx_url?: string;
+  webder_url?: string;
 }

src/handlers/distributor.ts

@@ -1,5 +1,5 @@
 import { IHandlerOutput } from './handler.interface';
-import axios from '../types/axios';
+import axios, { oaxios } from '../types/axios';
 
 import micromatch from 'micromatch';
 
@@ -13,6 +13,7 @@ import { decodeStream, parseEncodingName } from '../utils/http';
 import replaceHref from '../utils/replace-href';
 import { parseHTML } from 'linkedom';
 import { Engine } from './engine';
+import getConfig from '../config/main';
 
 interface IEngineId {
   [key: string]: number;
@@ -38,7 +39,14 @@ export class Distributor {
   ): Promise<IHandlerOutput> {
     const urlObj = new URL(remoteUrl);
 
-    const response = await axios.get(remoteUrl);
+    const webder_url = getConfig().third_party.webder_url;
+
+    const response = webder_url
+      ? await oaxios.get(
+          `${webder_url}/render?url=${encodeURIComponent(remoteUrl)}`
+        )
+      : await axios.get(remoteUrl);
+
     const data: Readable = response.data;
     const mime: string | undefined =
       response.headers['content-type']?.toString();

src/types/axios.ts

@@ -1,14 +1,17 @@
-import origAxios from 'axios';
+import origAxios, { CreateAxiosDefaults } from 'axios';
 import { isLocalResource, isLocalResourceURL } from '../utils/islocal';
 import { LocalResourceError } from '../errors/main';
 
-const axios = origAxios.create({
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const config: CreateAxiosDefaults<any> = {
   headers: {
     'User-Agent':
       'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0',
   },
   responseType: 'stream',
-});
+};
+
+const axios = origAxios.create(config);
 
 axios.interceptors.response.use(
   (response) => {
@@ -27,4 +30,12 @@ axios.interceptors.response.use(
   }
 );
 
+/**
+ * Modified axios for blocking local resources
+ */
 export default axios;
+
+/**
+ *  Original axios
+ */
+export const oaxios = origAxios.create(config);

templates/components/form-main.ejs

@@ -1,4 +1,4 @@
-<% search = config.search.enabled %>
+<% search = config.third_party.searx_url %>
 
 <% 
 
@@ -19,7 +19,7 @@ if (search) {
       <div class="input">
         <input type="submit" id="submit" class="button" value="Go">
       </div>
-      <input type="hidden" name="url" value="<%= config.search.searx_url %>/search"/>
+      <input type="hidden" name="url" value="<%= config.third_party.searx_url %>/search"/>
     </form>
   <% 
 }

templates/components/menu.ejs

@@ -2,12 +2,12 @@
   <a class="button secondary" href="/">Home</a>
   <a class="button secondary" href="<%= remoteUrl %>">Original page</a>
   <% 
-    if (config.search.enabled) {
+    if (config.third_party.searx_url) {
       %>
       <form class="form-search" action="/redirect" method="get">
         <input type="text" name="q" id="search" placeholder="Search">
         <input class="button" type="submit" value="Go"/>
-        <input type="hidden" name="url" value="<%= config.search.searx_url %>/search"/>
+        <input type="hidden" name="url" value="<%= config.third_party.searx_url %>/search"/>
       </form>
       <%
     }

templates/components/search-styles.ejs

@@ -1,5 +1,5 @@
 <% 
-  if (config.search.enabled) {
+  if (config.third_party.searx_url) {
     %><link rel="stylesheet" href="/static/search.css">
     <link rel="stylesheet" href="/static/form-inputs.css"><%
   }