From 1fc1d8e88b5948ee13737bf2ab35cb0481a758d6 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Thu, 21 Sep 2023 13:18:12 +0400 Subject: [PATCH 01/16] NotHtmlMimetype error handling --- src/errors/handler.ts | 9 +-------- src/errors/main.ts | 32 ++++++++++++++++++++++---------- src/handlers/main.ts | 2 +- templates/error.ejs | 8 ++++++++ 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/src/errors/handler.ts b/src/errors/handler.ts index 1b0e472..909757a 100644 --- a/src/errors/handler.ts +++ b/src/errors/handler.ts @@ -29,10 +29,6 @@ function apiErrorHandler(error: Error, reply: FastifyReply) { }); } - if (error instanceof NotHtmlMimetypeError) { - return generateResponse(501); - } - if (getFastifyError(error)?.statusCode === 400) { return generateResponse(400); } @@ -45,10 +41,6 @@ function apiErrorHandler(error: Error, reply: FastifyReply) { } function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) { - if (error instanceof NotHtmlMimetypeError) { - return reply.redirect(301, error.url); - } - if (getFastifyError(error)?.statusCode === 400) { return reply.code(400).view("/templates/error.ejs", { url, @@ -62,6 +54,7 @@ function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) { url, code: error.code, description: error.description, + proxyBtn: error instanceof NotHtmlMimetypeError, }); } diff --git a/src/errors/main.ts b/src/errors/main.ts index 29190c9..bee68f3 100644 --- a/src/errors/main.ts +++ b/src/errors/main.ts @@ -3,7 +3,11 @@ export abstract class TxtDotError extends Error { name: string; description: string; - constructor(code: number, name: string, description: string) { + constructor( + code: number, + name: string, + description: string, + ) { super(description); this.code = code; this.name = name; @@ -13,22 +17,30 @@ export abstract class TxtDotError extends Error { export class EngineParseError extends TxtDotError { constructor(message: string) { - super(422, "EngineParseError", `Parse error: ${message}`); + super( + 422, + "EngineParseError", + `Parse error: ${message}`, + ); } } export class LocalResourceError extends TxtDotError { constructor() { - super(403, "LocalResourceError", "Proxying local resources is forbidden."); + super( + 403, + "LocalResourceError", + "Proxying local resources is forbidden.", + ); } } -export class NotHtmlMimetypeError extends Error { - name: string = "NotHtmlMimetypeError"; - url: string; - - constructor(url: string) { - super(); - this.url = url; +export class NotHtmlMimetypeError extends TxtDotError { + constructor() { + super( + 421, + "NotHtmlMimetypeError", + "Received non-HTML content, use proxy", + ); } } diff --git a/src/handlers/main.ts b/src/handlers/main.ts index d3fa83a..27875a2 100644 --- a/src/handlers/main.ts +++ b/src/handlers/main.ts @@ -32,7 +32,7 @@ export default async function handlePage( const mime: string | undefined = response.headers["content-type"]?.toString(); if (mime && mime.indexOf("text/html") === -1) { - throw new NotHtmlMimetypeError(url); + throw new NotHtmlMimetypeError(); } return getFallbackEngine(urlObj.hostname, engine)( diff --git a/templates/error.ejs b/templates/error.ejs index 127c7da..2094b13 100644 --- a/templates/error.ejs +++ b/templates/error.ejs @@ -15,6 +15,14 @@

txt.

Home + <% if (proxyBtn) { %> + + Proxy + + <% } %> Original page

<%= description %>

From 47e4db1fc8a61b6e0fb047fb9e5318f73fc5d59e Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Thu, 21 Sep 2023 13:51:25 +0400 Subject: [PATCH 02/16] Proxying --- src/app.ts | 6 ++++-- src/routes/browser/proxy.ts | 18 ++++++++++++++++++ src/types/requests/browser.ts | 30 ++++++++++++++++++++++++++---- 3 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 src/routes/browser/proxy.ts diff --git a/src/app.ts b/src/app.ts index 65f5a5d..8c1f59f 100644 --- a/src/app.ts +++ b/src/app.ts @@ -9,9 +9,10 @@ import fastifySwagger from "@fastify/swagger"; import fastifySwaggerUi from "@fastify/swagger-ui"; import ejs from "ejs"; -import getRoute from "./routes/browser/get"; -import parseRoute from "./routes/api/parse"; import indexRoute from "./routes/browser/index"; +import getRoute from "./routes/browser/get"; +import proxyRoute from "./routes/browser/proxy"; +import parseRoute from "./routes/api/parse"; import rawHtml from "./routes/api/raw-html"; import publicConfig from "./publicConfig"; @@ -54,6 +55,7 @@ class App { fastify.register(indexRoute); fastify.register(getRoute); + fastify.register(proxyRoute); fastify.register(parseRoute); fastify.register(rawHtml); diff --git a/src/routes/browser/proxy.ts b/src/routes/browser/proxy.ts new file mode 100644 index 0000000..4947abf --- /dev/null +++ b/src/routes/browser/proxy.ts @@ -0,0 +1,18 @@ +import { FastifyInstance } from "fastify"; +import { IProxySchema, ProxySchema } from "../../types/requests/browser"; +import axios from "../../types/axios"; + +export default async function proxyRoute(fastify: FastifyInstance) { + fastify.get( + "/proxy", + { schema: ProxySchema }, + async (request, reply) => { + const response = await axios.get(request.query.url); + const mime: string | undefined = response.headers["content-type"]?.toString(); + const clen: string | undefined = response.headers["content-length"]?.toString(); + mime && reply.header("Content-Type", mime); + clen && reply.header("Content-Length", Number(clen)); + return reply.send(response.data); + } + ); +} diff --git a/src/types/requests/browser.ts b/src/types/requests/browser.ts index e9891ed..4507b11 100644 --- a/src/types/requests/browser.ts +++ b/src/types/requests/browser.ts @@ -6,10 +6,9 @@ export interface IGetSchema { Querystring: IGetQuerySchema; } -export const indexSchema = { - produces: ["text/html"], - hide: true -}; +export interface IProxySchema { + Querystring: IProxyQuerySchema; +} export const getQuerySchema = { type: "object", @@ -32,9 +31,32 @@ export const getQuerySchema = { } as const; export type IGetQuerySchema = FromSchema; +export const proxyQuerySchema = { + type: "object", + required: ["url"], + properties: { + url: { + type: "string", + description: "URL", + }, + } +} as const; +export type IProxyQuerySchema = FromSchema; + +export const indexSchema = { + hide: true, + produces: ["text/html"], +}; + export const GetSchema: FastifySchema = { description: "Get page", hide: true, querystring: getQuerySchema, produces: ["text/html", "text/plain"], }; + +export const ProxySchema: FastifySchema = { + description: "Proxy resource", + hide: true, + querystring: proxyQuerySchema, +} From 553efb1da68fa4af5b305e4212cd1d8ae3a6d179 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Thu, 21 Sep 2023 16:26:32 +0400 Subject: [PATCH 03/16] New config entries --- .env.example | 2 ++ src/app.ts | 5 ++++- src/config/config.service.ts | 3 +++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 22b7787..26f6507 100644 --- a/.env.example +++ b/.env.example @@ -2,3 +2,5 @@ HOST=127.0.0.1 # 0.0.0.0 if txtdot is not behind reverse proxy PORT=8080 REVERSE_PROXY=true # only for reverse proxy; see docs + +PROXY_RES=true diff --git a/src/app.ts b/src/app.ts index 8c1f59f..6843d3c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -55,7 +55,10 @@ class App { fastify.register(indexRoute); fastify.register(getRoute); - fastify.register(proxyRoute); + + if (this.config.proxy_res) + fastify.register(proxyRoute); + fastify.register(parseRoute); fastify.register(rawHtml); diff --git a/src/config/config.service.ts b/src/config/config.service.ts index fa870ab..13ddb8d 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -4,6 +4,7 @@ export class ConfigService { public readonly host: string; public readonly port: number; public readonly reverse_proxy: boolean; + public readonly proxy_res: boolean; constructor() { config(); @@ -12,5 +13,7 @@ export class ConfigService { this.port = Number(process.env.PORT) || 8080; this.reverse_proxy = Boolean(process.env.REVERSE_PROXY) || false; + + this.proxy_res = Boolean(process.env.PROXY_RES) || true; } } From f56b79c5633681cff1766ecfefbd140a834630d3 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Thu, 21 Sep 2023 16:28:55 +0400 Subject: [PATCH 04/16] getConfig() --- src/app.ts | 15 ++++++--------- src/config/main.ts | 12 ++++++++++++ 2 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 src/config/main.ts diff --git a/src/app.ts b/src/app.ts index 6843d3c..fff4239 100644 --- a/src/app.ts +++ b/src/app.ts @@ -17,18 +17,15 @@ import rawHtml from "./routes/api/raw-html"; import publicConfig from "./publicConfig"; import errorHandler from "./errors/handler"; +import getConfig from "./config/main"; class App { - config: ConfigService; - - constructor() { - this.config = new ConfigService(); - } - async init() { + const config = getConfig(); + const fastify = Fastify({ logger: true, - trustProxy: this.config.reverse_proxy, + trustProxy: config.reverse_proxy, }); fastify.register(fastifyStatic, { @@ -56,7 +53,7 @@ class App { fastify.register(indexRoute); fastify.register(getRoute); - if (this.config.proxy_res) + if (config.proxy_res) fastify.register(proxyRoute); fastify.register(parseRoute); @@ -65,7 +62,7 @@ class App { fastify.setErrorHandler(errorHandler); fastify.listen( - { host: this.config.host, port: this.config.port }, + { host: config.host, port: config.port }, (err) => { err && console.log(err); } diff --git a/src/config/main.ts b/src/config/main.ts new file mode 100644 index 0000000..37dcad3 --- /dev/null +++ b/src/config/main.ts @@ -0,0 +1,12 @@ +import { ConfigService } from "./config.service"; + +let configSvc: ConfigService | undefined; + +export default function getConfig(): ConfigService { + if (configSvc) { + return configSvc; + } + + configSvc = new ConfigService(); + return configSvc; +} From 86c033bdaaf3470da4b587dd62ba718dcc5cd300 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Thu, 21 Sep 2023 16:32:08 +0400 Subject: [PATCH 05/16] Use common getConfig() --- src/errors/handler.ts | 6 +++++- src/errors/main.ts | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/errors/handler.ts b/src/errors/handler.ts index 909757a..5d28722 100644 --- a/src/errors/handler.ts +++ b/src/errors/handler.ts @@ -3,6 +3,7 @@ import { NotHtmlMimetypeError, TxtDotError } from "./main"; import { getFastifyError } from "./validation"; import { IGetSchema } from "../types/requests/browser"; +import getConfig from "../config/main"; export default function errorHandler( error: Error, @@ -54,7 +55,10 @@ function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) { url, code: error.code, description: error.description, - proxyBtn: error instanceof NotHtmlMimetypeError, + proxyBtn: ( + error instanceof NotHtmlMimetypeError && + getConfig().proxy_res + ), }); } diff --git a/src/errors/main.ts b/src/errors/main.ts index bee68f3..81f983a 100644 --- a/src/errors/main.ts +++ b/src/errors/main.ts @@ -1,3 +1,5 @@ +import getConfig from "../config/main"; + export abstract class TxtDotError extends Error { code: number; name: string; @@ -40,7 +42,11 @@ export class NotHtmlMimetypeError extends TxtDotError { super( 421, "NotHtmlMimetypeError", - "Received non-HTML content, use proxy", + "Received non-HTML content, " + ( + getConfig().proxy_res ? + "use proxy instead of parser." : + "proxying is disabled by the instance admin." + ), ); } } From 645542795daf9e64116ddc0fe1f4febd74eae93a Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Thu, 21 Sep 2023 17:58:23 +0400 Subject: [PATCH 06/16] Template variable bugfix --- templates/error.ejs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/error.ejs b/templates/error.ejs index 2094b13..2bea449 100644 --- a/templates/error.ejs +++ b/templates/error.ejs @@ -15,7 +15,7 @@

txt.

Home - <% if (proxyBtn) { %> + <% if (locals.proxyBtn && proxyBtn) { %> Date: Thu, 21 Sep 2023 17:58:43 +0400 Subject: [PATCH 07/16] Replacing all hrefs/srcs --- src/handlers/handler-input.ts | 87 +++++++++++++++++++++++++++++------ src/utils/generate.ts | 10 +++- 2 files changed, 82 insertions(+), 15 deletions(-) diff --git a/src/handlers/handler-input.ts b/src/handlers/handler-input.ts index 7c7693e..7877c84 100644 --- a/src/handlers/handler-input.ts +++ b/src/handlers/handler-input.ts @@ -1,5 +1,6 @@ import { JSDOM } from "jsdom"; -import { generateProxyUrl } from "../utils/generate"; +import { generateParserUrl, generateProxyUrl } from "../utils/generate"; +import getConfig from "../config/main"; export class HandlerInput { private data: string; @@ -23,6 +24,10 @@ export class HandlerInput { this.redirectPath = redirectPath; } + getUrl(): string { + return this.url; + } + parseDom(): JSDOM { if (this.dom) { return this.dom; @@ -30,24 +35,78 @@ export class HandlerInput { this.dom = new JSDOM(this.data, { url: this.url }); - const links = this.dom.window.document.getElementsByTagName("a"); - for (const link of links) { - try { - link.href = generateProxyUrl( - this.requestUrl, - link.href, - this.engine, - this.redirectPath, - ); - } catch (_err) { - // ignore TypeError: Invalid URL + const bytag = + (dom: JSDOM, tag: string) => dom.window.document.getElementsByTagName(tag); + const bycss = + (dom: JSDOM, css: string) => dom.window.document.querySelectorAll(css); + + const parserUrl = (href: string) => generateParserUrl( + this.requestUrl, + href, + this.engine, + this.redirectPath, + ); + const proxyUrl = (href: string) => generateProxyUrl( + this.requestUrl, + href, + ); + + this.modifyLinks( + bytag(this.dom, "a"), + "href", + parserUrl, + ); + this.modifyLinks( + bycss(this.dom, "frame,iframe"), + "src", + parserUrl, + ); + + if (getConfig().proxy_res) { + this.modifyLinks( + bycss(this.dom, "img,image,video,audio,embed,track,source"), + "src", + proxyUrl, + ); + + this.modifyLinks( + bytag(this.dom, "object"), + "data", + proxyUrl, + ); + + const sources = bytag(this.dom, "source"); + for (const source of sources) { + // split srcset by comma + // @ts-ignore + source.srcset = source.srcset.split(",").map( + (src: string) => { + // split src by space + const parts = src.split(" "); + try { + // first part is URL + parts[0] = proxyUrl(parts[0]); + } catch (_err) { } + // join by space after splitting + return parts.join(" "); + } + ).join(","); // join by comma } } return this.dom; } - getUrl(): string { - return this.url; + private modifyLinks( + nodeList: NodeListOf | HTMLCollectionOf, + property: string, + generateLink: (value: string) => string, + ) { + for (const node of nodeList) { + try { + // @ts-ignore + node[property] = generateLink(node[property]); + } catch (_err) { } + } } } diff --git a/src/utils/generate.ts b/src/utils/generate.ts index 4d5a006..a275585 100644 --- a/src/utils/generate.ts +++ b/src/utils/generate.ts @@ -6,7 +6,7 @@ export function generateRequestUrl( return new URL(`${protocol}://${host}${originalUrl}`); } -export function generateProxyUrl( +export function generateParserUrl( requestUrl: URL, href: string, engine?: string, @@ -22,3 +22,11 @@ export function generateProxyUrl( return `${requestUrl.origin}/${redirect_url}${urlParam}${engineParam}${hash}`; } + +export function generateProxyUrl( + requestUrl: URL, + href: string, +): string { + const urlParam = `?url=${encodeURIComponent(href)}`; + return `${requestUrl.origin}/proxy${urlParam}`; +} From 4994bd5e5427875a19da0baa995300f2416c0540 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Fri, 22 Sep 2023 12:53:25 +0400 Subject: [PATCH 08/16] Refactor: move links replacement into utils, call it after Readability --- src/handlers/handler-input.ts | 84 ----------------------------------- src/handlers/main.ts | 17 +++++-- src/utils/replace-href.ts | 82 ++++++++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 88 deletions(-) create mode 100644 src/utils/replace-href.ts diff --git a/src/handlers/handler-input.ts b/src/handlers/handler-input.ts index 7877c84..ad510b9 100644 --- a/src/handlers/handler-input.ts +++ b/src/handlers/handler-input.ts @@ -1,27 +1,16 @@ import { JSDOM } from "jsdom"; -import { generateParserUrl, generateProxyUrl } from "../utils/generate"; -import getConfig from "../config/main"; export class HandlerInput { private data: string; private url: string; - private requestUrl: URL; - private engine?: string; - private redirectPath: string; private dom?: JSDOM; constructor( data: string, url: string, - requestUrl: URL, - engine?: string, - redirectPath: string = "get", ) { this.data = data; this.url = url; - this.requestUrl = requestUrl; - this.engine = engine; - this.redirectPath = redirectPath; } getUrl(): string { @@ -34,79 +23,6 @@ export class HandlerInput { } this.dom = new JSDOM(this.data, { url: this.url }); - - const bytag = - (dom: JSDOM, tag: string) => dom.window.document.getElementsByTagName(tag); - const bycss = - (dom: JSDOM, css: string) => dom.window.document.querySelectorAll(css); - - const parserUrl = (href: string) => generateParserUrl( - this.requestUrl, - href, - this.engine, - this.redirectPath, - ); - const proxyUrl = (href: string) => generateProxyUrl( - this.requestUrl, - href, - ); - - this.modifyLinks( - bytag(this.dom, "a"), - "href", - parserUrl, - ); - this.modifyLinks( - bycss(this.dom, "frame,iframe"), - "src", - parserUrl, - ); - - if (getConfig().proxy_res) { - this.modifyLinks( - bycss(this.dom, "img,image,video,audio,embed,track,source"), - "src", - proxyUrl, - ); - - this.modifyLinks( - bytag(this.dom, "object"), - "data", - proxyUrl, - ); - - const sources = bytag(this.dom, "source"); - for (const source of sources) { - // split srcset by comma - // @ts-ignore - source.srcset = source.srcset.split(",").map( - (src: string) => { - // split src by space - const parts = src.split(" "); - try { - // first part is URL - parts[0] = proxyUrl(parts[0]); - } catch (_err) { } - // join by space after splitting - return parts.join(" "); - } - ).join(","); // join by comma - } - } - return this.dom; } - - private modifyLinks( - nodeList: NodeListOf | HTMLCollectionOf, - property: string, - generateLink: (value: string) => string, - ) { - for (const node of nodeList) { - try { - // @ts-ignore - node[property] = generateLink(node[property]); - } catch (_err) { } - } - } } diff --git a/src/handlers/main.ts b/src/handlers/main.ts index 27875a2..45e5e2f 100644 --- a/src/handlers/main.ts +++ b/src/handlers/main.ts @@ -4,6 +4,8 @@ import axios from "../types/axios"; import micromatch from "micromatch"; +import { JSDOM } from "jsdom"; + import readability from "./readability"; import google, { GoogleDomains } from "./google"; import stackoverflow, { StackOverflowDomains } from "./stackoverflow/main"; @@ -14,6 +16,7 @@ import { LocalResourceError, NotHtmlMimetypeError } from "../errors/main"; import { HandlerInput } from "./handler-input"; import { Readable } from "stream"; import { decodeStream, parseEncodingName } from "../utils/http"; +import replaceHref from "../utils/replace-href"; export default async function handlePage( url: string, // remote URL @@ -35,15 +38,21 @@ export default async function handlePage( throw new NotHtmlMimetypeError(); } - return getFallbackEngine(urlObj.hostname, engine)( + const handler = getFallbackEngine(urlObj.hostname, engine); + const output = await handler( new HandlerInput( await decodeStream(data, parseEncodingName(mime)), url, - requestUrl, - engine, - redirectPath, ) ); + + // post-process + const dom = new JSDOM(output.content, { url }); + replaceHref(dom, requestUrl, engine, redirectPath); + output.content = dom.serialize(); + // TODO: DomPurify + + return output; } function getFallbackEngine(host: string, specified?: string): EngineFunction { diff --git a/src/utils/replace-href.ts b/src/utils/replace-href.ts new file mode 100644 index 0000000..bd16d33 --- /dev/null +++ b/src/utils/replace-href.ts @@ -0,0 +1,82 @@ +import { JSDOM } from "jsdom"; +import { generateParserUrl, generateProxyUrl } from "./generate"; +import getConfig from "../config/main"; + +export default function replaceHref( + dom: JSDOM, + requestUrl: URL, + engine?: string, + redirectPath: string = "get", +) { + const bytag = + (dom: JSDOM, tag: string) => dom.window.document.getElementsByTagName(tag); + const bycss = + (dom: JSDOM, css: string) => dom.window.document.querySelectorAll(css); + + const parserUrl = (href: string) => generateParserUrl( + requestUrl, + href, + engine, + redirectPath, + ); + const proxyUrl = (href: string) => generateProxyUrl( + requestUrl, + href, + ); + + modifyLinks( + bytag(dom, "a"), + "href", + parserUrl, + ); + modifyLinks( + bycss(dom, "frame,iframe"), + "src", + parserUrl, + ); + + if (getConfig().proxy_res) { + modifyLinks( + bycss(dom, "img,image,video,audio,embed,track,source"), + "src", + proxyUrl, + ); + + modifyLinks( + bytag(dom, "object"), + "data", + proxyUrl, + ); + + const sources = bytag(dom, "source"); + for (const source of sources) { + // split srcset by comma + // @ts-ignore + source.srcset = source.srcset.split(",").map( + (src: string) => { + // split src by space + const parts = src.split(" "); + try { + // first part is URL + parts[0] = proxyUrl(parts[0]); + } catch (_err) { } + // join by space after splitting + return parts.join(" "); + } + ).join(","); // join by comma + } + } +} + +function modifyLinks( + nodeList: NodeListOf | HTMLCollectionOf, + property: string, + generateLink: (value: string) => string, +) { + for (const node of nodeList) { + try { + // @ts-ignore + node[property] = generateLink(node[property]); + } catch (_err) { } + } +} From 40092d8227dd55c184a879e3b5bf8a460e4e7f51 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Fri, 22 Sep 2023 13:00:16 +0400 Subject: [PATCH 09/16] replace-href: code cleanup --- src/utils/replace-href.ts | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/utils/replace-href.ts b/src/utils/replace-href.ts index bd16d33..1ac4f3c 100644 --- a/src/utils/replace-href.ts +++ b/src/utils/replace-href.ts @@ -8,10 +8,7 @@ export default function replaceHref( engine?: string, redirectPath: string = "get", ) { - const bytag = - (dom: JSDOM, tag: string) => dom.window.document.getElementsByTagName(tag); - const bycss = - (dom: JSDOM, css: string) => dom.window.document.querySelectorAll(css); + const doc = dom.window.document; const parserUrl = (href: string) => generateParserUrl( requestUrl, @@ -25,30 +22,30 @@ export default function replaceHref( ); modifyLinks( - bytag(dom, "a"), + doc.getElementsByTagName("a"), "href", parserUrl, ); modifyLinks( - bycss(dom, "frame,iframe"), + doc.querySelectorAll("frame,iframe"), "src", parserUrl, ); if (getConfig().proxy_res) { modifyLinks( - bycss(dom, "img,image,video,audio,embed,track,source"), + doc.querySelectorAll("img,image,video,audio,embed,track,source"), "src", proxyUrl, ); modifyLinks( - bytag(dom, "object"), + doc.getElementsByTagName("object"), "data", proxyUrl, ); - const sources = bytag(dom, "source"); + const sources = doc.getElementsByTagName("source"); for (const source of sources) { // split srcset by comma // @ts-ignore From 1cec7ac3f0acd7a0f4f311d92b8148546a5b49c6 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Fri, 22 Sep 2023 13:00:44 +0400 Subject: [PATCH 10/16] app.ts: unused import --- src/app.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app.ts b/src/app.ts index fff4239..497418c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,5 +1,3 @@ -import { ConfigService } from "./config/config.service"; - import path from "path"; import Fastify from "fastify"; From d0d3dfd7b9e96ffd4de9ad4dcb4822684c14c28f Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Fri, 22 Sep 2023 13:23:02 +0400 Subject: [PATCH 11/16] Bugfix: trim spaces in srcset, replace srcset in imgs --- src/utils/replace-href.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/utils/replace-href.ts b/src/utils/replace-href.ts index 1ac4f3c..4a8a3c9 100644 --- a/src/utils/replace-href.ts +++ b/src/utils/replace-href.ts @@ -45,14 +45,17 @@ export default function replaceHref( proxyUrl, ); - const sources = doc.getElementsByTagName("source"); + const sources = doc.querySelectorAll("source,img"); for (const source of sources) { // split srcset by comma // @ts-ignore + if (!source.srcset) + continue; + // @ts-ignore source.srcset = source.srcset.split(",").map( (src: string) => { // split src by space - const parts = src.split(" "); + const parts = src.trim().split(" "); try { // first part is URL parts[0] = proxyUrl(parts[0]); From 3a8198ea14d7fc72260699f1f0d2626d4f7342ac Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Fri, 22 Sep 2023 13:37:17 +0400 Subject: [PATCH 12/16] Bugfix: do not convert base64 and non-http(s):// --- src/utils/replace-href.ts | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/utils/replace-href.ts b/src/utils/replace-href.ts index 4a8a3c9..b964870 100644 --- a/src/utils/replace-href.ts +++ b/src/utils/replace-href.ts @@ -10,16 +10,18 @@ export default function replaceHref( ) { const doc = dom.window.document; - const parserUrl = (href: string) => generateParserUrl( - requestUrl, - href, - engine, - redirectPath, - ); - const proxyUrl = (href: string) => generateProxyUrl( - requestUrl, - href, - ); + const parserUrl = (href: string) => + href.startsWith("http") ? generateParserUrl( + requestUrl, + href, + engine, + redirectPath, + ) : href; + const proxyUrl = (href: string) => + href.startsWith("http") ? generateProxyUrl( + requestUrl, + href, + ) : href; modifyLinks( doc.getElementsByTagName("a"), @@ -58,6 +60,7 @@ export default function replaceHref( const parts = src.trim().split(" "); try { // first part is URL + // (srcset="http 200w 1x,...") parts[0] = proxyUrl(parts[0]); } catch (_err) { } // join by space after splitting From a8d4b3a2d3f7cafa375f17e6edeb54f1e30a8a06 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Mon, 25 Sep 2023 10:48:34 +0400 Subject: [PATCH 13/16] SECURITY BUGFIX: "false" env var value was parsed as true --- src/config/config.service.ts | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/config/config.service.ts b/src/config/config.service.ts index 13ddb8d..a84e5b2 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -12,8 +12,19 @@ export class ConfigService { this.host = process.env.HOST || "0.0.0.0"; this.port = Number(process.env.PORT) || 8080; - this.reverse_proxy = Boolean(process.env.REVERSE_PROXY) || false; + this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false); - this.proxy_res = Boolean(process.env.PROXY_RES) || true; + this.proxy_res = this.parseBool(process.env.PROXY_RES, true); + } + + parseBool(value: string | undefined, def: boolean): boolean { + if (!value) return def; + switch (value) { + case "false": + case "0": + return false; + default: + return Boolean(value); + } } } From 29e923cce6cc93e930d8b15e447f840f72c52b39 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Mon, 25 Sep 2023 10:55:06 +0400 Subject: [PATCH 14/16] Simpified boolean check (see fix in prev commit) --- src/config/config.service.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/config/config.service.ts b/src/config/config.service.ts index a84e5b2..ec3e1a2 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -19,12 +19,6 @@ export class ConfigService { parseBool(value: string | undefined, def: boolean): boolean { if (!value) return def; - switch (value) { - case "false": - case "0": - return false; - default: - return Boolean(value); - } + return value === "true" || value === "1"; } } From 809a389b6a6d4a1f3268bd27eb6d51061b72d546 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Mon, 25 Sep 2023 11:14:19 +0400 Subject: [PATCH 15/16] New config entry: swagger --- .env.example | 1 + src/config/config.service.ts | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.env.example b/.env.example index 26f6507..5f3fb65 100644 --- a/.env.example +++ b/.env.example @@ -4,3 +4,4 @@ PORT=8080 REVERSE_PROXY=true # only for reverse proxy; see docs PROXY_RES=true +SWAGGER=false # whether to add API docs route or not diff --git a/src/config/config.service.ts b/src/config/config.service.ts index ec3e1a2..3ff93e3 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -5,6 +5,7 @@ export class ConfigService { public readonly port: number; public readonly reverse_proxy: boolean; public readonly proxy_res: boolean; + public readonly swagger: boolean; constructor() { config(); @@ -15,6 +16,7 @@ export class ConfigService { this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false); this.proxy_res = this.parseBool(process.env.PROXY_RES, true); + this.swagger = this.parseBool(process.env.SWAGGER, false); } parseBool(value: string | undefined, def: boolean): boolean { From 9ffba003e6edb5221b1242465305a165db3991d8 Mon Sep 17 00:00:00 2001 From: DarkCat09 Date: Mon, 25 Sep 2023 11:16:15 +0400 Subject: [PATCH 16/16] Use "swagger" config entry --- src/app.ts | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/app.ts b/src/app.ts index 497418c..70a935a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -37,16 +37,18 @@ class App { }, }); - await fastify.register(fastifySwagger, { - swagger: { - info: { - title: "TXTDot API", - description: publicConfig.description, - version: publicConfig.version, - }, - } - }); - await fastify.register(fastifySwaggerUi, { routePrefix: "/doc" }); + if (config.swagger) { + await fastify.register(fastifySwagger, { + swagger: { + info: { + title: "TXTDot API", + description: publicConfig.description, + version: publicConfig.version, + }, + } + }); + await fastify.register(fastifySwaggerUi, { routePrefix: "/doc" }); + } fastify.register(indexRoute); fastify.register(getRoute);