mirror of
https://github.com/txtdot/txtdot
synced 2024-10-18 14:40:19 +03:00
refactor: remove node-cache
Let nginx or something similar take care of the cache
This commit is contained in:
Artemy
parent
974ef6cd6a
commit
b62a566aba
158
package-lock.json
generated
158
package-lock.json
generated
@ -9,15 +9,13 @@
|
||||
"version": "1.0.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@fastify/middie": "^8.3.0",
|
||||
"@fastify/view": "^8.0.0",
|
||||
"@mozilla/readability": "^0.4.4",
|
||||
"axios": "^1.4.0",
|
||||
"dotenv": "^16.3.1",
|
||||
"ejs": "^3.1.9",
|
||||
"fastify": "^4.21.0",
|
||||
"jsdom": "^22.1.0",
|
||||
"node-cache": "^5.1.2"
|
||||
"jsdom": "^22.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/ejs": "^3.1.2",
|
||||
@ -28,6 +26,7 @@
|
||||
"@typescript-eslint/parser": "^6.3.0",
|
||||
"eslint": "^8.47.0",
|
||||
"ts-node": "^10.9.1",
|
||||
"tsc-watch": "^6.0.4",
|
||||
"typescript": "^5.1.6"
|
||||
}
|
||||
},
|
||||
@ -150,17 +149,6 @@
|
||||
"fast-json-stringify": "^5.7.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@fastify/middie": {
|
||||
"version": "8.3.0",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/middie/-/middie-8.3.0.tgz",
|
||||
"integrity": "sha512-h+zBxCzMlkEkh4fM7pZaSGzqS7P9M0Z6rXnWPdUEPfe7x1BCj++wEk/pQ5jpyYY4pF8AknFqb77n7uwh8HdxEA==",
|
||||
"dependencies": {
|
||||
"@fastify/error": "^3.2.0",
|
||||
"fastify-plugin": "^4.0.0",
|
||||
"path-to-regexp": "^6.1.0",
|
||||
"reusify": "^1.0.4"
|
||||
}
|
||||
},
|
||||
"node_modules/@fastify/view": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/@fastify/view/-/view-8.0.0.tgz",
|
||||
@ -880,14 +868,6 @@
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/clone": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/clone/-/clone-2.1.2.tgz",
|
||||
"integrity": "sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/color-convert": {
|
||||
"version": "2.0.1",
|
||||
"license": "MIT",
|
||||
@ -1052,6 +1032,12 @@
|
||||
"url": "https://github.com/motdotla/dotenv?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/duplexer": {
|
||||
"version": "0.1.2",
|
||||
"resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz",
|
||||
"integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/ejs": {
|
||||
"version": "3.1.9",
|
||||
"resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.9.tgz",
|
||||
@ -1222,6 +1208,21 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/event-stream": {
|
||||
"version": "3.3.4",
|
||||
"resolved": "https://registry.npmjs.org/event-stream/-/event-stream-3.3.4.tgz",
|
||||
"integrity": "sha512-QHpkERcGsR0T7Qm3HNJSyXKEEj8AHNxkY3PK8TS2KJvQ7NiSHe3DDpwVKKtoYprL/AreyzFBeIkBIWChAqn60g==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"duplexer": "~0.1.1",
|
||||
"from": "~0",
|
||||
"map-stream": "~0.1.0",
|
||||
"pause-stream": "0.0.11",
|
||||
"split": "0.3",
|
||||
"stream-combiner": "~0.0.4",
|
||||
"through": "~2.3.1"
|
||||
}
|
||||
},
|
||||
"node_modules/event-target-shim": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
|
||||
@ -1511,6 +1512,12 @@
|
||||
"node": ">= 0.6"
|
||||
}
|
||||
},
|
||||
"node_modules/from": {
|
||||
"version": "0.1.7",
|
||||
"resolved": "https://registry.npmjs.org/from/-/from-0.1.7.tgz",
|
||||
"integrity": "sha512-twe20eF1OxVxp/ML/kq2p1uc6KvFK/+vs8WjEbeKmV2He22MKm7YF2ANIt+EOqhJ5L3K/SuuPhk0hWQDjOM23g==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/fs.realpath": {
|
||||
"version": "1.0.0",
|
||||
"dev": true,
|
||||
@ -1884,6 +1891,12 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/map-stream": {
|
||||
"version": "0.1.0",
|
||||
"resolved": "https://registry.npmjs.org/map-stream/-/map-stream-0.1.0.tgz",
|
||||
"integrity": "sha512-CkYQrPYZfWnu/DAmVCpTSX/xHpKZ80eKh2lAkyA6AJTef6bW+6JpbQZN5rofum7da+SyN1bi5ctTm+lTfcCW3g==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/merge2": {
|
||||
"version": "1.4.1",
|
||||
"dev": true,
|
||||
@ -1947,16 +1960,11 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-cache": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/node-cache/-/node-cache-5.1.2.tgz",
|
||||
"integrity": "sha512-t1QzWwnk4sjLWaQAS8CHgOJ+RAfmHpxFWmc36IWTiWHQfs0w5JDMBS1b1ZxQteo0vVVuWJvIUKHDkkeK7vIGCg==",
|
||||
"dependencies": {
|
||||
"clone": "2.x"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 8.0.0"
|
||||
}
|
||||
"node_modules/node-cleanup": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/node-cleanup/-/node-cleanup-2.1.2.tgz",
|
||||
"integrity": "sha512-qN8v/s2PAJwGUtr1/hYTpNKlD6Y9rc4p8KSmJXyGdYGZsDGKXrGThikLFP9OCHFeLeEpQzPwiAtdIvBLqm//Hw==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/nwsapi": {
|
||||
"version": "2.2.7",
|
||||
@ -2066,11 +2074,6 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/path-to-regexp": {
|
||||
"version": "6.2.1",
|
||||
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-6.2.1.tgz",
|
||||
"integrity": "sha512-JLyh7xT1kizaEvcaXOQwOc2/Yhw6KZOvPf1S8401UyLk86CU79LN3vl7ztXGm/pZ+YjoyAJ4rxmHwbkBXJX+yw=="
|
||||
},
|
||||
"node_modules/path-type": {
|
||||
"version": "4.0.0",
|
||||
"dev": true,
|
||||
@ -2079,6 +2082,15 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/pause-stream": {
|
||||
"version": "0.0.11",
|
||||
"resolved": "https://registry.npmjs.org/pause-stream/-/pause-stream-0.0.11.tgz",
|
||||
"integrity": "sha512-e3FBlXLmN/D1S+zHzanP4E/4Z60oFAa3O051qt1pxa7DEJWKAyil6upYVXCWadEnuoqa4Pkc9oUx9zsxYeRv8A==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"through": "~2.3"
|
||||
}
|
||||
},
|
||||
"node_modules/picomatch": {
|
||||
"version": "2.3.1",
|
||||
"dev": true,
|
||||
@ -2163,6 +2175,21 @@
|
||||
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
|
||||
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
|
||||
},
|
||||
"node_modules/ps-tree": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/ps-tree/-/ps-tree-1.2.0.tgz",
|
||||
"integrity": "sha512-0VnamPPYHl4uaU/nSFeZZpR21QAWRz+sRv4iW9+v/GS/J5U5iZB5BNN6J0RMoOvdx2gWM2+ZFMIm58q24e4UYA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"event-stream": "=3.3.4"
|
||||
},
|
||||
"bin": {
|
||||
"ps-tree": "bin/ps-tree.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/psl": {
|
||||
"version": "1.9.0",
|
||||
"resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
|
||||
@ -2419,6 +2446,18 @@
|
||||
"atomic-sleep": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/split": {
|
||||
"version": "0.3.3",
|
||||
"resolved": "https://registry.npmjs.org/split/-/split-0.3.3.tgz",
|
||||
"integrity": "sha512-wD2AeVmxXRBoX44wAycgjVpMhvbwdI2aZjCkvfNcH1YqHQvJVa1duWc73OyVGJUc05fhFaTZeQ/PYsrmyH0JVA==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"through": "2"
|
||||
},
|
||||
"engines": {
|
||||
"node": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/split2": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
|
||||
@ -2427,6 +2466,15 @@
|
||||
"node": ">= 10.x"
|
||||
}
|
||||
},
|
||||
"node_modules/stream-combiner": {
|
||||
"version": "0.0.4",
|
||||
"resolved": "https://registry.npmjs.org/stream-combiner/-/stream-combiner-0.0.4.tgz",
|
||||
"integrity": "sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"duplexer": "~0.1.1"
|
||||
}
|
||||
},
|
||||
"node_modules/string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
@ -2435,6 +2483,15 @@
|
||||
"safe-buffer": "~5.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/string-argv": {
|
||||
"version": "0.3.2",
|
||||
"resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.2.tgz",
|
||||
"integrity": "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==",
|
||||
"dev": true,
|
||||
"engines": {
|
||||
"node": ">=0.6.19"
|
||||
}
|
||||
},
|
||||
"node_modules/strip-ansi": {
|
||||
"version": "6.0.1",
|
||||
"dev": true,
|
||||
@ -2485,6 +2542,12 @@
|
||||
"real-require": "^0.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/through": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
"integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/tiny-lru": {
|
||||
"version": "11.0.1",
|
||||
"resolved": "https://registry.npmjs.org/tiny-lru/-/tiny-lru-11.0.1.tgz",
|
||||
@ -2583,6 +2646,27 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/tsc-watch": {
|
||||
"version": "6.0.4",
|
||||
"resolved": "https://registry.npmjs.org/tsc-watch/-/tsc-watch-6.0.4.tgz",
|
||||
"integrity": "sha512-cHvbvhjO86w2aGlaHgSCeQRl+Aqw6X6XN4sQMPZKF88GoP30O+oTuh5lRIJr5pgFWrRpF1AgXnJJ2DoFEIPHyg==",
|
||||
"dev": true,
|
||||
"dependencies": {
|
||||
"cross-spawn": "^7.0.3",
|
||||
"node-cleanup": "^2.1.2",
|
||||
"ps-tree": "^1.2.0",
|
||||
"string-argv": "^0.3.1"
|
||||
},
|
||||
"bin": {
|
||||
"tsc-watch": "dist/lib/tsc-watch.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.12.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "*"
|
||||
}
|
||||
},
|
||||
"node_modules/type-check": {
|
||||
"version": "0.4.0",
|
||||
"dev": true,
|
||||
|
||||
@ -5,15 +5,13 @@
|
||||
"description": "",
|
||||
"main": "dist/app.js",
|
||||
"dependencies": {
|
||||
"@fastify/middie": "^8.3.0",
|
||||
"@fastify/view": "^8.0.0",
|
||||
"@mozilla/readability": "^0.4.4",
|
||||
"axios": "^1.4.0",
|
||||
"dotenv": "^16.3.1",
|
||||
"ejs": "^3.1.9",
|
||||
"fastify": "^4.21.0",
|
||||
"jsdom": "^22.1.0",
|
||||
"node-cache": "^5.1.2"
|
||||
"jsdom": "^22.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/ejs": "^3.1.2",
|
||||
@ -24,13 +22,15 @@
|
||||
"@typescript-eslint/parser": "^6.3.0",
|
||||
"eslint": "^8.47.0",
|
||||
"ts-node": "^10.9.1",
|
||||
"tsc-watch": "^6.0.4",
|
||||
"typescript": "^5.1.6"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1",
|
||||
"build": "tsc",
|
||||
"dev": "ts-node src/app.ts",
|
||||
"start": "node dist/app.js"
|
||||
"start": "node dist/app.js",
|
||||
"watch": "tsc-watch --onSuccess \"node dist/app.js\""
|
||||
},
|
||||
"keywords": [],
|
||||
"authors": [
|
||||
|
||||
@ -1,10 +1,7 @@
|
||||
import { IConfigService } from "./config/config.interface";
|
||||
import { ConfigService } from "./config/config.service";
|
||||
|
||||
import NodeCache from "node-cache";
|
||||
|
||||
import Fastify from "fastify";
|
||||
import middie from "@fastify/middie";
|
||||
import fastifyView from "@fastify/view";
|
||||
import ejs from "ejs";
|
||||
|
||||
@ -13,11 +10,9 @@ import parseRoute from "./routes/parseRoute";
|
||||
|
||||
class App {
|
||||
config: IConfigService;
|
||||
cache: NodeCache;
|
||||
|
||||
constructor() {
|
||||
this.config = new ConfigService();
|
||||
this.cache = new NodeCache({ stdTTL: 100, checkperiod: 120 });
|
||||
}
|
||||
|
||||
async init() {
|
||||
@ -25,11 +20,10 @@ class App {
|
||||
logger: true,
|
||||
});
|
||||
|
||||
await fastify.register(middie);
|
||||
await fastify.register(fastifyView, {
|
||||
engine: {
|
||||
ejs: ejs,
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
fastify.register(mainRoute);
|
||||
|
||||
@ -1,47 +1,29 @@
|
||||
import { FastifyInstance } from "fastify";
|
||||
import NodeCache from "node-cache";
|
||||
|
||||
import { GetRequest } from "../types/requests";
|
||||
import { IHandlerOutput } from "../handlers/handler.interface";
|
||||
import handlePage from "../handlers/main";
|
||||
|
||||
export default function mainRoute(cache: NodeCache) {
|
||||
return async (fastify: FastifyInstance) => {
|
||||
fastify.get("/", async (req: GetRequest, res) => {
|
||||
const remoteUrl = req.query.url;
|
||||
const engine = req.query.engine || "readability";
|
||||
export default async function mainRoute(fastify: FastifyInstance) {
|
||||
fastify.get("/", async (request: GetRequest, reply) => {
|
||||
const remoteUrl = request.query.url;
|
||||
const engine = request.query.engine || "readability";
|
||||
|
||||
let format: string;
|
||||
let format: string;
|
||||
|
||||
if (req.query.format === "text") {
|
||||
res.type("text/plain; charset=utf-8");
|
||||
format = "text";
|
||||
}
|
||||
else {
|
||||
res.type("text/html; charset=utf-8");
|
||||
format = "html";
|
||||
}
|
||||
if (request.query.format === "text") {
|
||||
reply.type("text/plain; charset=utf-8");
|
||||
format = "text";
|
||||
} else {
|
||||
reply.type("text/html; charset=utf-8");
|
||||
format = "html";
|
||||
}
|
||||
|
||||
const cacheKey = req.originalUrl || req.url;
|
||||
const cached: string | undefined = cache.get(cacheKey);
|
||||
const parsed = await handlePage(remoteUrl, engine);
|
||||
|
||||
if (cached) {
|
||||
res.send(cached);
|
||||
return;
|
||||
}
|
||||
|
||||
const parsed = await handlePage(remoteUrl, engine);
|
||||
|
||||
if (format === "text") {
|
||||
const content = parsed.textContent;
|
||||
cache.set(cacheKey, content);
|
||||
res.send(content);
|
||||
}
|
||||
else {
|
||||
const content = parsed.content;
|
||||
cache.set(cacheKey, content);
|
||||
res.view("/template/index.ejs", { parsed: parsed });
|
||||
}
|
||||
});
|
||||
};
|
||||
if (format === "text") {
|
||||
return parsed.textContent;
|
||||
} else {
|
||||
return reply.view("/templates/index.ejs", { parsed: parsed });
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@ -1,18 +1,11 @@
|
||||
import NodeCache from "node-cache";
|
||||
import { EngineRequest } from "../types/requests";
|
||||
import { FastifyInstance } from "fastify";
|
||||
import handlePage from "../handlers/main";
|
||||
|
||||
export default function parseRoute(cache: NodeCache) {
|
||||
return async (fastify: FastifyInstance) => {
|
||||
fastify.get("/parse", async (req: EngineRequest) => {
|
||||
const parsed = await handlePage(req.query.url, req.query.engine);
|
||||
export default async function parseRoute(fastify: FastifyInstance) {
|
||||
fastify.get("/parse", async (req: EngineRequest) => {
|
||||
const parsed = await handlePage(req.query.url, req.query.engine);
|
||||
|
||||
cache.set(req.originalUrl || req.url, {
|
||||
content: parsed,
|
||||
contentType: "text/json",
|
||||
});
|
||||
return parsed;
|
||||
});
|
||||
};
|
||||
return parsed;
|
||||
});
|
||||
}
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
import { FastifyInstance, FastifyRequest } from "fastify";
|
||||
import NodeCache from "node-cache";
|
||||
import { FastifyRequest } from "fastify";
|
||||
|
||||
export type GetRequest = FastifyRequest<{
|
||||
Querystring: {
|
||||
@ -15,7 +14,3 @@ export type EngineRequest = FastifyRequest<{
|
||||
engine?: string;
|
||||
};
|
||||
}>;
|
||||
|
||||
export interface IFastifyInstance extends FastifyInstance {
|
||||
cache: NodeCache;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user