diff --git a/package-lock.json b/package-lock.json index cd76dc2..2380cc5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "txtdot", - "version": "1.5.3", + "version": "1.6.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "txtdot", - "version": "1.5.3", + "version": "1.6.0", "license": "MIT", "dependencies": { "@fastify/static": "^6.12.0", diff --git a/package.json b/package.json index 0dc8b46..e5cfec3 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,8 @@ { "name": "txtdot", - "version": "1.5.3", + "version": "1.6.0", "private": true, - "description": "", + "description": "txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts", "main": "dist/app.js", "dependencies": { "@fastify/static": "^6.12.0", diff --git a/src/app.ts b/src/app.ts index ab4685c..b35d4e2 100644 --- a/src/app.ts +++ b/src/app.ts @@ -13,10 +13,10 @@ import proxyRoute from './routes/browser/proxy'; import parseRoute from './routes/api/parse'; import rawHtml from './routes/api/raw-html'; -import publicConfig from './publicConfig'; +import packageJSON from './package'; import errorHandler from './errors/handler'; import getConfig from './config/main'; -import searchRoute from './routes/browser/search'; +import redirectRoute from './routes/browser/redirect'; class App { async init() { @@ -46,8 +46,8 @@ class App { swagger: { info: { title: 'TXTDot API', - description: publicConfig.description, - version: publicConfig.version, + description: packageJSON.description, + version: packageJSON.version, }, }, }); @@ -58,7 +58,7 @@ class App { fastify.register(getRoute); if (config.search.enabled) { - fastify.register(searchRoute); + fastify.register(redirectRoute); } if (config.proxy_res) fastify.register(proxyRoute); diff --git a/src/errors/main.ts b/src/errors/main.ts index 2513a9d..62c54bf 100644 --- a/src/errors/main.ts +++ b/src/errors/main.ts @@ -19,6 +19,12 @@ export class EngineParseError extends TxtDotError { } } +export class NoHandlerFoundError extends TxtDotError { + constructor(message: string) { + super(404, 'NoHandlerFoundError', `No handler found for: ${message}`); + } +} + export class LocalResourceError extends TxtDotError { constructor() { super(403, 'LocalResourceError', 'Proxying local resources is forbidden.'); diff --git a/src/handlers/engine.ts b/src/handlers/engine.ts index 3cc54aa..22aea15 100644 --- a/src/handlers/engine.ts +++ b/src/handlers/engine.ts @@ -1,25 +1,29 @@ import Route from 'route-parser'; import { HandlerInput } from './handler-input'; import { IHandlerOutput } from './handler.interface'; -import { EngineParseError } from '../errors/main'; -import { EngineFunction } from '../types/handlers'; +import { NoHandlerFoundError } from '../errors/main'; +import { EngineFunction, RouteValues } from '../types/handlers'; -interface IRoute { +interface IRoute { route: Route; - handler: EngineFunction; + handler: EngineFunction; } export class Engine { name: string; domains: string[]; - routes: IRoute[] = []; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + routes: IRoute[] = []; constructor(name: string, domains: string[] = []) { this.domains = domains; this.name = name; } - route(path: string, handler: EngineFunction) { - this.routes.push({ route: new Route(path), handler: handler }); + route( + path: string, + handler: EngineFunction + ) { + this.routes.push({ route: new Route(path), handler }); } async handle(input: HandlerInput): Promise { @@ -29,10 +33,13 @@ export class Engine { const match = route.route.match(path); if (match) { - return await route.handler(input, match); + return await route.handler(input, { + q: match, + reverse: (req) => route.route.reverse(req), + }); } } - throw new EngineParseError(`No handler for ${path}. [${this.name}]`); + throw new NoHandlerFoundError(`${path}. [${this.name}]`); } } diff --git a/src/handlers/engines/readability.ts b/src/handlers/engines/readability.ts index c8a7e79..c52b10b 100644 --- a/src/handlers/engines/readability.ts +++ b/src/handlers/engines/readability.ts @@ -5,14 +5,12 @@ import { Engine } from '../engine'; const ReadabilityEngine = new Engine('Readability'); -ReadabilityEngine.route('*path', async (input, req) => { +ReadabilityEngine.route('*path', async (input, ro) => { const reader = new Readability(input.parseDom().window.document); const parsed = reader.parse(); if (!parsed) { - throw new EngineParseError( - `Parse error (${req.path}). [${ReadabilityEngine.name}]` - ); + throw new EngineParseError(`(${ro.q.path}). [${ReadabilityEngine.name}]`); } return { diff --git a/src/handlers/engines/searx.ts b/src/handlers/engines/searx.ts index 9f9ba3a..cf2b7e1 100644 --- a/src/handlers/engines/searx.ts +++ b/src/handlers/engines/searx.ts @@ -1,22 +1,22 @@ +import { Route } from '../../types/handlers'; import { Engine } from '../engine'; +import { HandlerInput } from '../handler-input'; const SearXEngine = new Engine('SearX', ['searx.*']); -SearXEngine.route('/search?q=:search', async (input, req) => { +async function search( + input: HandlerInput, + ro: Route<{ search: string; pageno?: string }> +) { const document = input.parseDom().window.document; - const search = req.search; - const url = new URL(input.getUrl()); - const page = parseInt(url.searchParams.get('pageno') || '1'); + const search = ro.q.search; + const page = parseInt(ro.q.pageno || '1'); const page_footer = `${ page !== 1 - ? `Previous |` + ? `Previous |` : '' - } Next`; + } Next`; const articles = Array.from(document.querySelectorAll('.result')); const articles_parsed = articles.map((a) => { @@ -49,6 +49,9 @@ SearXEngine.route('/search?q=:search', async (input, req) => { title: `${search} - Searx - Page ${page}`, lang: document.documentElement.lang, }; -}); +} + +SearXEngine.route('/search?q=:search&pageno=:pageno', search); +SearXEngine.route('/search?q=:search', search); export default SearXEngine; diff --git a/src/handlers/engines/stackoverflow.ts b/src/handlers/engines/stackoverflow.ts deleted file mode 100644 index e8fe6af..0000000 --- a/src/handlers/engines/stackoverflow.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { Engine } from '../engine'; - -const SOE = new Engine('StackOverflow', [ - 'stackoverflow.com', - '*.stackoverflow.com', - '*.stackexchange.com', - 'askubuntu.com', - 'stackapps.com', - 'mathoverflow.net', - 'superuser.com', - 'serverfault.com', -]); - -SOE.route('/questions/:id/:slug', async (input, req) => { - const document = input.parseDom().window.document; - - const questionEl = document.getElementById('question'); - const question = postParser(questionEl); - - const title = document.querySelector('.question-hyperlink')?.innerHTML || ''; - - const allAnswers = [...document.querySelectorAll('.answer')]; - const answers = allAnswers.map((a) => postParser(a)); - - return { - content: `${question}
${answers.length} answers
${answers.join( - '
' - )}`, - textContent: `${req.id}/${req.slug}\n`, - title, - lang: 'en', - }; -}); - -function postParser(el: Element | null): string { - if (!el) { - return ''; - } - const body = el.querySelector('.js-post-body')?.innerHTML || ''; - const voteCount = el.querySelector('.js-vote-count')?.textContent || ''; - - return `

${voteCount} votes

${body}`; -} - -export default SOE; diff --git a/src/handlers/engines/stackoverflow/main.ts b/src/handlers/engines/stackoverflow/main.ts new file mode 100644 index 0000000..43608bf --- /dev/null +++ b/src/handlers/engines/stackoverflow/main.ts @@ -0,0 +1,18 @@ +import { Engine } from '../../engine'; +import questions from './questions'; +import users from './users'; +const soEngine = new Engine('StackOverflow', [ + 'stackoverflow.com', + '*.stackoverflow.com', + '*.stackexchange.com', + 'askubuntu.com', + 'stackapps.com', + 'mathoverflow.net', + 'superuser.com', + 'serverfault.com', +]); + +soEngine.route('/questions/:id/*slug', questions); +soEngine.route('/users/:id/*slug', users); + +export default soEngine; diff --git a/src/handlers/engines/stackoverflow/questions.ts b/src/handlers/engines/stackoverflow/questions.ts new file mode 100644 index 0000000..38ac121 --- /dev/null +++ b/src/handlers/engines/stackoverflow/questions.ts @@ -0,0 +1,49 @@ +import { Route } from '../../../types/handlers'; +import { HandlerInput } from '../../handler-input'; + +async function questions( + input: HandlerInput, + ro: Route<{ id: string; slug: string }> +) { + const document = input.parseDom().window.document; + + const questionEl = document.getElementById('question'); + const question = postParser(questionEl); + + const title = document.querySelector('.question-hyperlink')?.innerHTML || ''; + + const allAnswers = [...document.querySelectorAll('.answer')]; + const answers = allAnswers.map((a) => postParser(a)); + + return { + content: `${question}
${answers.length} answers
${answers.join( + '
' + )}`, + textContent: `${ro.q.id}/${ro.q.slug}\n`, // TODO + title, + lang: document.documentElement.lang, + }; +} + +function postParser(el: Element | null): string { + if (!el) { + return ''; + } + const body = el.querySelector('.js-post-body')?.innerHTML || ''; + const voteCount = el.querySelector('.js-vote-count')?.textContent || ''; + + const footer = [...el.querySelectorAll('.post-signature')].map((el) => { + const userName = el.querySelector('.user-details a')?.textContent || ''; + const userUrl = + (el.querySelector('.user-details a') as HTMLAnchorElement)?.href || ''; + const userTitle = el.querySelector('.user-action-time')?.textContent || ''; + + return `

${userTitle}${ + userUrl ? ` by ${userName}` : '' + }

`; + }); + + return `

${voteCount} votes

${body}${footer.join('')}`; +} + +export default questions; diff --git a/src/handlers/engines/stackoverflow/users.ts b/src/handlers/engines/stackoverflow/users.ts new file mode 100644 index 0000000..d250b35 --- /dev/null +++ b/src/handlers/engines/stackoverflow/users.ts @@ -0,0 +1,37 @@ +import { Route } from '../../../types/handlers'; +import { HandlerInput } from '../../handler-input'; + +async function users( + input: HandlerInput, + ro: Route<{ id: string; slug: string }> +) { + const document = input.parseDom().window.document; + + const userInfo = + document.querySelector('.md\\:ai-start > div:nth-child(2)')?.textContent || + ''; + + const topPosts = [ + ...(document.querySelector('#js-top-posts > div:nth-child(2)')?.children || + []), + ] + .map((el) => { + const title = el.querySelector('a')?.textContent || ''; + const url = el.querySelector('a')?.href || ''; + const votes = el.querySelector('.s-badge__votes')?.textContent || ''; + const type = + el.querySelector('.iconAnswer, .iconQuestion')?.textContent || ''; + + return `${type} (${votes}) ${title}`; + }) + .join('
'); + + return { + content: `${userInfo}

Top Posts

${topPosts}`, + textContent: `${ro.q.id}/${ro.q.slug}\n`, // TODO + title: document.querySelector('title')?.textContent || '', + lang: document.documentElement.lang, + }; +} + +export default users; diff --git a/src/handlers/main.ts b/src/handlers/main.ts index 59c819f..377d613 100644 --- a/src/handlers/main.ts +++ b/src/handlers/main.ts @@ -1,7 +1,7 @@ import { Distributor } from './distributor'; import Readability from './engines/readability'; import SearX from './engines/searx'; -import StackOverflow from './engines/stackoverflow'; +import StackOverflow from './engines/stackoverflow/main'; const distributor = new Distributor(); diff --git a/src/package.ts b/src/package.ts new file mode 100644 index 0000000..01ff519 --- /dev/null +++ b/src/package.ts @@ -0,0 +1,3 @@ +import * as config from '../package.json'; + +export default config; diff --git a/src/publicConfig.ts b/src/publicConfig.ts deleted file mode 100644 index e92b96e..0000000 --- a/src/publicConfig.ts +++ /dev/null @@ -1,5 +0,0 @@ -export default { - version: '1.5.3', - description: - 'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts', -}; diff --git a/src/routes/browser/index.ts b/src/routes/browser/index.ts index 413de53..d7ceb2a 100644 --- a/src/routes/browser/index.ts +++ b/src/routes/browser/index.ts @@ -1,6 +1,6 @@ import { FastifyInstance } from 'fastify'; -import publicConfig from '../../publicConfig'; +import packageJSON from '../../package'; import { engineList } from '../../handlers/main'; import { indexSchema } from '../../types/requests/browser'; @@ -9,7 +9,7 @@ import getConfig from '../../config/main'; export default async function indexRoute(fastify: FastifyInstance) { fastify.get('/', { schema: indexSchema }, async (_, reply) => { return reply.view('/templates/index.ejs', { - publicConfig, + packageJSON, engineList, config: getConfig(), }); diff --git a/src/routes/browser/redirect.ts b/src/routes/browser/redirect.ts new file mode 100644 index 0000000..64d0e4e --- /dev/null +++ b/src/routes/browser/redirect.ts @@ -0,0 +1,20 @@ +import { FastifyInstance } from 'fastify'; + +import { redirectSchema, IRedirectSchema } from '../../types/requests/browser'; + +export default async function redirectRoute(fastify: FastifyInstance) { + fastify.get( + '/redirect', + { schema: redirectSchema }, + async (request, reply) => { + const params = new URLSearchParams(request.query); + params.delete('url'); + + reply.redirect( + `/get?url=${encodeURIComponent( + request.query.url + '?' + params.toString() + )}` + ); + } + ); +} diff --git a/src/routes/browser/search.ts b/src/routes/browser/search.ts deleted file mode 100644 index 77f00c5..0000000 --- a/src/routes/browser/search.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { FastifyInstance } from 'fastify'; - -import { searchSchema, ISearchSchema } from '../../types/requests/browser'; - -import getConfig from '../../config/main'; - -export default async function searchRoute(fastify: FastifyInstance) { - fastify.get( - '/search', - { schema: searchSchema }, - async (request, reply) => { - const query = request.query.q; - - const config = getConfig(); - - if (config.search.enabled) { - const searchUrl = `${config.search.searx_url}/search?q=${query}`; - reply.redirect(`/get?url=${encodeURI(searchUrl)}`); - } else { - throw new Error('Search is not enabled'); - } - } - ); -} diff --git a/src/types/handlers.ts b/src/types/handlers.ts index 4ddb92d..6bcc3f1 100644 --- a/src/types/handlers.ts +++ b/src/types/handlers.ts @@ -1,3 +1,4 @@ +// import Route from 'route-parser'; import { Engine } from '../handlers/engine'; import { HandlerInput } from '../handlers/handler-input'; import { IHandlerOutput } from '../handlers/handler.interface'; @@ -6,17 +7,25 @@ export interface Engines { [key: string]: Engine; } -export type EngineMatch = { +export type EngineMatch = { pattern: string | string[]; - engine: EngineFunction; + engine: EngineFunction; }; export interface RouteValues { [key: string]: string; } -export type EngineFunction = ( +export type EngineFunction = ( input: HandlerInput, - req: RouteValues + ro: Route ) => Promise; -export type EnginesMatch = EngineMatch[]; + +export type EnginesMatch = EngineMatch[]; + +export interface Route { + q: TParams; + reverse: (req: { [K in keyof TParams]: string | number | boolean }) => + | string + | false; +} diff --git a/src/types/requests/browser.ts b/src/types/requests/browser.ts index 45d1716..350848f 100644 --- a/src/types/requests/browser.ts +++ b/src/types/requests/browser.ts @@ -10,21 +10,27 @@ export interface IProxySchema { Querystring: IProxyQuerySchema; } -export interface ISearchSchema { - Querystring: ISearchQuerySchema; +export interface IRedirectSchema { + Querystring: IRedirectQuerySchema; } -export const searchQuerySchema = { +export const redirectQuerySchema = { type: 'object', - required: ['q'], + required: ['url'], properties: { - q: { + url: { type: 'string', - description: 'Search query', + description: 'URL to redirect without querystring', }, }, + patternProperties: { + '^(?!url).*$': { type: 'string' }, + }, } as const; -export type ISearchQuerySchema = FromSchema; +export type IRedirectQuerySchema = { + url: string; + [key: string]: string; +}; export const getQuerySchema = { type: 'object', @@ -64,10 +70,10 @@ export const indexSchema = { produces: ['text/html'], }; -export const searchSchema: FastifySchema = { - description: 'Search redirection page', +export const redirectSchema: FastifySchema = { + description: 'Universal redirection page', hide: true, - querystring: searchQuerySchema, + querystring: redirectQuerySchema, }; export const GetSchema: FastifySchema = { diff --git a/templates/components/form-main.ejs b/templates/components/form-main.ejs index f4768fc..2d225ec 100644 --- a/templates/components/form-main.ejs +++ b/templates/components/form-main.ejs @@ -10,13 +10,14 @@ Search -
+
+
<% } %> diff --git a/templates/components/menu.ejs b/templates/components/menu.ejs index 9073a6f..5fe5521 100644 --- a/templates/components/menu.ejs +++ b/templates/components/menu.ejs @@ -4,9 +4,10 @@ <% if (config.search.enabled) { %> -
+ +
<% } diff --git a/templates/index.ejs b/templates/index.ejs index a2a216e..f19ffbd 100644 --- a/templates/index.ejs +++ b/templates/index.ejs @@ -4,7 +4,7 @@ - + txt. main page @@ -16,11 +16,11 @@

txt.

- v<%= publicConfig.version %> + v<%= packageJSON.version %> GitHub Docs
-

<%= publicConfig.description %>

+

<%= packageJSON.description %>

<%- include('./components/form-main.ejs') %> diff --git a/tsconfig.json b/tsconfig.json index 96b6021..b7d87b0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -39,7 +39,7 @@ // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ - // "resolveJsonModule": true, /* Enable importing .json files. */ + "resolveJsonModule": true /* Enable importing .json files. */, // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ @@ -55,7 +55,7 @@ // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./dist/src/" /* Specify an output folder for all emitted files. */, + "outDir": "./dist/" /* Specify an output folder for all emitted files. */, // "removeComments": true, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */