mirror of
https://github.com/txtdot/txtdot
synced 2024-10-18 14:40:19 +03:00
Merge branch 'main' into new-routing
This commit is contained in:
commit
aec8960d32
21
LICENSE
Normal file
21
LICENSE
Normal file
@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023 TxtDot
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
22
README.md
22
README.md
@ -1,7 +1,29 @@
|
||||
# txt.
|
||||
|
||||
HTTP proxy that parses only text, links and pictures from pages
|
||||
reducing internet traffic, removing ads and heavy scripts.
|
||||
|
||||
## Installation
|
||||
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
## Running
|
||||
|
||||
### Dev
|
||||
|
||||
```bash
|
||||
npm run dev
|
||||
```
|
||||
|
||||
### Prod
|
||||
|
||||
```bash
|
||||
npm run build
|
||||
npm run start
|
||||
```
|
||||
|
||||
Uses [Mozilla's readability.js](https://github.com/mozilla/readability),
|
||||
[JSDOM](https://github.com/jsdom/jsdom),
|
||||
[Fastify web framework](https://github.com/fastify/fastify).
|
||||
|
2
package-lock.json
generated
2
package-lock.json
generated
@ -7,7 +7,7 @@
|
||||
"": {
|
||||
"name": "txtdot",
|
||||
"version": "1.0.0",
|
||||
"license": "ISC",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@fastify/middie": "^8.3.0",
|
||||
"@fastify/view": "^8.0.0",
|
||||
|
@ -33,6 +33,10 @@
|
||||
"start": "node dist/app.js"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC"
|
||||
"authors": [
|
||||
"Artemy Egorov <me@artegoser.ru> https://github.com/artegoser",
|
||||
"DarkCat09 <darkcat09@vivaldi.net> https://dc09.ru/",
|
||||
"megahomyak https://github.com/megahomyak"
|
||||
],
|
||||
"license": "MIT"
|
||||
}
|
||||
|
@ -1,28 +1,27 @@
|
||||
import { IHandlerOutput } from "./handler.interface";
|
||||
import { readability } from "./readability";
|
||||
|
||||
export default function handlePage(url: string, engine: string): Promise<IHandlerOutput> {
|
||||
const func = engines[engine];
|
||||
if (!func) {
|
||||
throw new Error('No such engine')
|
||||
type EngineFunction = (url: string) => Promise<IHandlerOutput>
|
||||
|
||||
export default function handlePage(url: string, engine?: string): Promise<IHandlerOutput> {
|
||||
let func: EngineFunction | undefined = engines[engine || ""];
|
||||
|
||||
if (func === undefined) {
|
||||
const host = new URL(url).hostname;
|
||||
func = fallback[host] || fallback["*"];
|
||||
}
|
||||
|
||||
return func(url)
|
||||
return func(url);
|
||||
}
|
||||
|
||||
interface Engines {
|
||||
[key: string]: EngineFunction;
|
||||
}
|
||||
|
||||
export const engines: Engines = {
|
||||
"readability": readability,
|
||||
};
|
||||
|
||||
interface Engines {
|
||||
[name: string]: (url: string) => Promise<IHandlerOutput>;
|
||||
}
|
||||
|
||||
/*
|
||||
const fallback: Engines = {
|
||||
"*": engines.readability,
|
||||
};
|
||||
interface Engines {
|
||||
[host: string]: (url: string) => Promise<IHandlerOutput>;
|
||||
}
|
||||
*/
|
||||
|
@ -1,5 +1,5 @@
|
||||
import { Readability } from "@mozilla/readability";
|
||||
import axios from "axios";
|
||||
import axios from "../types/axios";
|
||||
import { JSDOM } from "jsdom";
|
||||
import { IHandlerOutput } from "./handler.interface";
|
||||
|
||||
|
@ -1,14 +1,12 @@
|
||||
import NodeCache from "node-cache";
|
||||
import { EngineRequest } from "../types/requests";
|
||||
import { FastifyInstance } from "fastify";
|
||||
import { engines } from "../handlers/main";
|
||||
import handlePage from "../handlers/main";
|
||||
|
||||
export default function parseRoute(cache: NodeCache) {
|
||||
return async (fastify: FastifyInstance) => {
|
||||
fastify.get("/parse", async (req: EngineRequest) => {
|
||||
const url = req.query.url;
|
||||
const engine = req.query.engine || "readability";
|
||||
const parsed = await engines[engine](url);
|
||||
const parsed = await handlePage(req.query.url, req.query.engine);
|
||||
|
||||
cache.set(req.originalUrl || req.url, {
|
||||
content: parsed,
|
||||
|
7
src/types/axios.ts
Normal file
7
src/types/axios.ts
Normal file
@ -0,0 +1,7 @@
|
||||
import axios from "axios";
|
||||
|
||||
export default axios.create({
|
||||
headers: {
|
||||
"User-Agent": "txtdot",
|
||||
},
|
||||
});
|
Loading…
Reference in New Issue
Block a user