mirror of
https://github.com/txtdot/txtdot
synced 2024-10-18 14:40:19 +03:00
Merge branch 'main' into new-routing
This commit is contained in:
commit
aec8960d32
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2023 TxtDot
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
22
README.md
22
README.md
@ -1,7 +1,29 @@
|
|||||||
# txt.
|
# txt.
|
||||||
|
|
||||||
HTTP proxy that parses only text, links and pictures from pages
|
HTTP proxy that parses only text, links and pictures from pages
|
||||||
reducing internet traffic, removing ads and heavy scripts.
|
reducing internet traffic, removing ads and heavy scripts.
|
||||||
|
|
||||||
|
## Installation
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install
|
||||||
|
```
|
||||||
|
|
||||||
|
## Running
|
||||||
|
|
||||||
|
### Dev
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run dev
|
||||||
|
```
|
||||||
|
|
||||||
|
### Prod
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm run build
|
||||||
|
npm run start
|
||||||
|
```
|
||||||
|
|
||||||
Uses [Mozilla's readability.js](https://github.com/mozilla/readability),
|
Uses [Mozilla's readability.js](https://github.com/mozilla/readability),
|
||||||
[JSDOM](https://github.com/jsdom/jsdom),
|
[JSDOM](https://github.com/jsdom/jsdom),
|
||||||
[Fastify web framework](https://github.com/fastify/fastify).
|
[Fastify web framework](https://github.com/fastify/fastify).
|
||||||
|
2
package-lock.json
generated
2
package-lock.json
generated
@ -7,7 +7,7 @@
|
|||||||
"": {
|
"": {
|
||||||
"name": "txtdot",
|
"name": "txtdot",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"license": "ISC",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@fastify/middie": "^8.3.0",
|
"@fastify/middie": "^8.3.0",
|
||||||
"@fastify/view": "^8.0.0",
|
"@fastify/view": "^8.0.0",
|
||||||
|
@ -33,6 +33,10 @@
|
|||||||
"start": "node dist/app.js"
|
"start": "node dist/app.js"
|
||||||
},
|
},
|
||||||
"keywords": [],
|
"keywords": [],
|
||||||
"author": "",
|
"authors": [
|
||||||
"license": "ISC"
|
"Artemy Egorov <me@artegoser.ru> https://github.com/artegoser",
|
||||||
|
"DarkCat09 <darkcat09@vivaldi.net> https://dc09.ru/",
|
||||||
|
"megahomyak https://github.com/megahomyak"
|
||||||
|
],
|
||||||
|
"license": "MIT"
|
||||||
}
|
}
|
||||||
|
@ -1,28 +1,27 @@
|
|||||||
import { IHandlerOutput } from "./handler.interface";
|
import { IHandlerOutput } from "./handler.interface";
|
||||||
import { readability } from "./readability";
|
import { readability } from "./readability";
|
||||||
|
|
||||||
export default function handlePage(url: string, engine: string): Promise<IHandlerOutput> {
|
type EngineFunction = (url: string) => Promise<IHandlerOutput>
|
||||||
const func = engines[engine];
|
|
||||||
if (!func) {
|
export default function handlePage(url: string, engine?: string): Promise<IHandlerOutput> {
|
||||||
throw new Error('No such engine')
|
let func: EngineFunction | undefined = engines[engine || ""];
|
||||||
|
|
||||||
|
if (func === undefined) {
|
||||||
|
const host = new URL(url).hostname;
|
||||||
|
func = fallback[host] || fallback["*"];
|
||||||
}
|
}
|
||||||
|
|
||||||
return func(url)
|
return func(url);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Engines {
|
||||||
|
[key: string]: EngineFunction;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const engines: Engines = {
|
export const engines: Engines = {
|
||||||
"readability": readability,
|
"readability": readability,
|
||||||
};
|
};
|
||||||
|
|
||||||
interface Engines {
|
|
||||||
[name: string]: (url: string) => Promise<IHandlerOutput>;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
|
||||||
const fallback: Engines = {
|
const fallback: Engines = {
|
||||||
"*": engines.readability,
|
"*": engines.readability,
|
||||||
};
|
};
|
||||||
interface Engines {
|
|
||||||
[host: string]: (url: string) => Promise<IHandlerOutput>;
|
|
||||||
}
|
|
||||||
*/
|
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
import { Readability } from "@mozilla/readability";
|
import { Readability } from "@mozilla/readability";
|
||||||
import axios from "axios";
|
import axios from "../types/axios";
|
||||||
import { JSDOM } from "jsdom";
|
import { JSDOM } from "jsdom";
|
||||||
import { IHandlerOutput } from "./handler.interface";
|
import { IHandlerOutput } from "./handler.interface";
|
||||||
|
|
||||||
|
@ -1,14 +1,12 @@
|
|||||||
import NodeCache from "node-cache";
|
import NodeCache from "node-cache";
|
||||||
import { EngineRequest } from "../types/requests";
|
import { EngineRequest } from "../types/requests";
|
||||||
import { FastifyInstance } from "fastify";
|
import { FastifyInstance } from "fastify";
|
||||||
import { engines } from "../handlers/main";
|
import handlePage from "../handlers/main";
|
||||||
|
|
||||||
export default function parseRoute(cache: NodeCache) {
|
export default function parseRoute(cache: NodeCache) {
|
||||||
return async (fastify: FastifyInstance) => {
|
return async (fastify: FastifyInstance) => {
|
||||||
fastify.get("/parse", async (req: EngineRequest) => {
|
fastify.get("/parse", async (req: EngineRequest) => {
|
||||||
const url = req.query.url;
|
const parsed = await handlePage(req.query.url, req.query.engine);
|
||||||
const engine = req.query.engine || "readability";
|
|
||||||
const parsed = await engines[engine](url);
|
|
||||||
|
|
||||||
cache.set(req.originalUrl || req.url, {
|
cache.set(req.originalUrl || req.url, {
|
||||||
content: parsed,
|
content: parsed,
|
||||||
|
7
src/types/axios.ts
Normal file
7
src/types/axios.ts
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
import axios from "axios";
|
||||||
|
|
||||||
|
export default axios.create({
|
||||||
|
headers: {
|
||||||
|
"User-Agent": "txtdot",
|
||||||
|
},
|
||||||
|
});
|
Loading…
Reference in New Issue
Block a user