From ec7d100867aa8204c819c7db7fe3632c2428800e Mon Sep 17 00:00:00 2001 From: Artemy Date: Mon, 14 Aug 2023 14:17:53 +0300 Subject: [PATCH 1/4] feat: custom axios instance --- src/handlers/readability.ts | 2 +- src/types/axios.ts | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 src/types/axios.ts diff --git a/src/handlers/readability.ts b/src/handlers/readability.ts index cafc43a..dc66edb 100644 --- a/src/handlers/readability.ts +++ b/src/handlers/readability.ts @@ -1,5 +1,5 @@ import { Readability } from "@mozilla/readability"; -import axios from "axios"; +import axios from "../types/axios"; import { JSDOM } from "jsdom"; import { IHandlerOutput } from "./handler.interface"; diff --git a/src/types/axios.ts b/src/types/axios.ts new file mode 100644 index 0000000..fbd3bd6 --- /dev/null +++ b/src/types/axios.ts @@ -0,0 +1,7 @@ +import axios from "axios"; + +export default axios.create({ + headers: { + "User-Agent": "txtdot", + }, +}); From fb33669c6793bba45573d978284b1852b9555324 Mon Sep 17 00:00:00 2001 From: Artemy Date: Mon, 14 Aug 2023 14:37:30 +0300 Subject: [PATCH 2/4] doc: update information --- README.md | 22 ++++++++++++++++++++++ package-lock.json | 2 +- package.json | 8 ++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index bd0fb54..3c3ec73 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,29 @@ # txt. + HTTP proxy that parses only text, links and pictures from pages reducing internet traffic, removing ads and heavy scripts. +## Installation + +```bash +npm install +``` + +## Running + +### Dev + +```bash +npm run dev +``` + +### Prod + +```bash +npm run build +npm run start +``` + Uses [Mozilla's readability.js](https://github.com/mozilla/readability), [JSDOM](https://github.com/jsdom/jsdom), [Fastify web framework](https://github.com/fastify/fastify). diff --git a/package-lock.json b/package-lock.json index a5787c8..efab8ef 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,7 +7,7 @@ "": { "name": "txtdot", "version": "1.0.0", - "license": "ISC", + "license": "MIT", "dependencies": { "@fastify/middie": "^8.3.0", "@mozilla/readability": "^0.4.4", diff --git a/package.json b/package.json index e4722ea..b6cf40c 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,10 @@ "start": "node dist/app.js" }, "keywords": [], - "author": "", - "license": "ISC" + "authors": [ + "Artemy Egorov https://github.com/artegoser", + "DarkCat09 https://dc09.ru/", + "megahomyak https://github.com/megahomyak" + ], + "license": "MIT" } From 45b426f6717bdbbbc6c201e516ea7395ac772498 Mon Sep 17 00:00:00 2001 From: Artemy Date: Mon, 14 Aug 2023 14:39:36 +0300 Subject: [PATCH 3/4] Create LICENSE --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c82b042 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 TxtDot + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 53259410785ce36d3243c0cbffc5421a3b2ab399 Mon Sep 17 00:00:00 2001 From: Artemy Date: Mon, 14 Aug 2023 15:02:52 +0300 Subject: [PATCH 4/4] feat: engine choose --- src/handlers/main.ts | 9 ++++++++- src/routes/parseRoute.ts | 6 ++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/handlers/main.ts b/src/handlers/main.ts index d0923ee..b67922c 100644 --- a/src/handlers/main.ts +++ b/src/handlers/main.ts @@ -1,7 +1,14 @@ import { IHandlerOutput } from "./handler.interface"; import { readability } from "./readability"; -export default function handlePage(url: string): Promise { +export default function handlePage( + url: string, + engine?: string +): Promise { + if (engine) { + return engines[engine](url); + } + const host = new URL(url).hostname; return fallback[host]?.(url) || fallback["*"](url); } diff --git a/src/routes/parseRoute.ts b/src/routes/parseRoute.ts index e23d7a1..7270a54 100644 --- a/src/routes/parseRoute.ts +++ b/src/routes/parseRoute.ts @@ -1,14 +1,12 @@ import NodeCache from "node-cache"; import { EngineRequest } from "../types/requests"; import { FastifyInstance } from "fastify"; -import { engines } from "../handlers/main"; +import handlePage from "../handlers/main"; export default function parseRoute(cache: NodeCache) { return async (fastify: FastifyInstance) => { fastify.get("/parse", async (req: EngineRequest) => { - const url = req.query.url; - const engine = req.query.engine || "readability"; - const parsed = await engines[engine](url); + const parsed = await handlePage(req.query.url, req.query.engine); cache.set(req.originalUrl || req.url, { content: parsed,