diff --git a/.eslintrc.json b/.eslintrc.json index 6751428..380b417 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,20 +1,14 @@ { - "env": { - "browser": true, - "es2021": true - }, - "extends": [ - "eslint:recommended", - "plugin:@typescript-eslint/recommended" - ], - "parser": "@typescript-eslint/parser", - "parserOptions": { - "ecmaVersion": "latest", - "sourceType": "module" - }, - "plugins": [ - "@typescript-eslint" - ], - "rules": { - } + "env": { + "browser": true, + "es2021": true + }, + "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"], + "parser": "@typescript-eslint/parser", + "parserOptions": { + "ecmaVersion": "latest", + "sourceType": "module" + }, + "plugins": ["@typescript-eslint"], + "rules": {} } diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5d5040e..3cba37f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,8 +5,8 @@ version: 2 updates: - - package-ecosystem: "npm" # See documentation for possible values - directory: "/" # Location of package manifests - target-branch: "dependabot" + - package-ecosystem: 'npm' # See documentation for possible values + directory: '/' # Location of package manifests + target-branch: 'dependabot' schedule: - interval: "weekly" + interval: 'weekly' diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..65261d6 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,5 @@ +{ + "trailingComma": "es5", + "tabWidth": 2, + "singleQuote": true +} diff --git a/docker-compose.yml b/docker-compose.yml index 533ca5f..273f5d5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ -version: "3" +version: '3' services: txtdot: image: ghcr.io/txtdot/txtdot:latest ports: - - "8080:8080" + - '8080:8080' restart: unless-stopped volumes: - - ".env:/app/.env" + - '.env:/app/.env' diff --git a/package-lock.json b/package-lock.json index 8fa04d1..adb2a72 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,8 +21,8 @@ "fastify": "^4.21.0", "iconv-lite": "^0.6.3", "ip-range-check": "^0.2.0", - "jsdom": "^22.1.0", "json-schema-to-ts": "^2.9.2", + "linkedom": "^0.16.4", "micromatch": "^4.0.5" }, "devDependencies": { @@ -314,14 +314,6 @@ "node": ">= 8" } }, - "node_modules/@tootallnate/once": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "engines": { - "node": ">= 10" - } - }, "node_modules/@types/braces": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.4.tgz", @@ -590,11 +582,6 @@ "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", "dev": true }, - "node_modules/abab": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", - "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==" - }, "node_modules/abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -632,17 +619,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/agent-base": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", - "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", - "dependencies": { - "debug": "4" - }, - "engines": { - "node": ">= 6.0.0" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -822,6 +798,11 @@ "node": ">=8" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -1096,30 +1077,37 @@ "node": ">= 8" } }, - "node_modules/cssstyle": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz", - "integrity": "sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==", + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", "dependencies": { - "rrweb-cssom": "^0.6.0" + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" }, - "engines": { - "node": ">=14" + "funding": { + "url": "https://github.com/sponsors/fb55" } }, - "node_modules/data-urls": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", - "integrity": "sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==", - "dependencies": { - "abab": "^2.0.6", - "whatwg-mimetype": "^3.0.0", - "whatwg-url": "^12.0.0" - }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", "engines": { - "node": ">=14" + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" } }, + "node_modules/cssom": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" + }, "node_modules/debug": { "version": "4.3.4", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", @@ -1136,11 +1124,6 @@ } } }, - "node_modules/decimal.js": { - "version": "10.4.3", - "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz", - "integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA==" - }, "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", @@ -1187,15 +1170,42 @@ "node": ">=6.0.0" } }, - "node_modules/domexception": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", - "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", "dependencies": { - "webidl-conversions": "^7.0.0" + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ] + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "dependencies": { + "domelementtype": "^2.3.0" }, "engines": { - "node": ">=12" + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" } }, "node_modules/dompurify": { @@ -1203,6 +1213,19 @@ "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.6.tgz", "integrity": "sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==" }, + "node_modules/domutils": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz", + "integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dotenv": { "version": "16.3.1", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", @@ -1872,15 +1895,27 @@ "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz", "integrity": "sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A==" }, - "node_modules/html-encoding-sniffer": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", - "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", + "node_modules/html-escaper": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", + "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==" + }, + "node_modules/htmlparser2": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.0.0.tgz", + "integrity": "sha512-uxbSI98wmFT/G4P2zXx4OVx04qWUmyFPrD2/CNepa2Zo3GPNaCaaxElDgwUrwYWkK1nr9fft0Ya8dws8coDLLQ==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], "dependencies": { - "whatwg-encoding": "^2.0.0" - }, - "engines": { - "node": ">=12" + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.1.0", + "entities": "^4.5.0" } }, "node_modules/http-errors": { @@ -1898,31 +1933,6 @@ "node": ">= 0.8" } }, - "node_modules/http-proxy-agent": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", - "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "dependencies": { - "@tootallnate/once": "2", - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/https-proxy-agent": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", - "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==", - "dependencies": { - "agent-base": "6", - "debug": "4" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -2076,11 +2086,6 @@ "node": ">=8" } }, - "node_modules/is-potential-custom-element-name": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", - "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" - }, "node_modules/isarray": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", @@ -2122,47 +2127,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/jsdom": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-22.1.0.tgz", - "integrity": "sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==", - "dependencies": { - "abab": "^2.0.6", - "cssstyle": "^3.0.0", - "data-urls": "^4.0.0", - "decimal.js": "^10.4.3", - "domexception": "^4.0.0", - "form-data": "^4.0.0", - "html-encoding-sniffer": "^3.0.0", - "http-proxy-agent": "^5.0.0", - "https-proxy-agent": "^5.0.1", - "is-potential-custom-element-name": "^1.0.1", - "nwsapi": "^2.2.4", - "parse5": "^7.1.2", - "rrweb-cssom": "^0.6.0", - "saxes": "^6.0.0", - "symbol-tree": "^3.2.4", - "tough-cookie": "^4.1.2", - "w3c-xmlserializer": "^4.0.0", - "webidl-conversions": "^7.0.0", - "whatwg-encoding": "^2.0.0", - "whatwg-mimetype": "^3.0.0", - "whatwg-url": "^12.0.1", - "ws": "^8.13.0", - "xml-name-validator": "^4.0.0" - }, - "engines": { - "node": ">=16" - }, - "peerDependencies": { - "canvas": "^2.5.0" - }, - "peerDependenciesMeta": { - "canvas": { - "optional": true - } - } - }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -2250,6 +2214,18 @@ "set-cookie-parser": "^2.4.1" } }, + "node_modules/linkedom": { + "version": "0.16.4", + "resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.16.4.tgz", + "integrity": "sha512-SykvDVh/jAnaO+WiPqH5vX3QpZrIRImuppzYhIHons3RXPhDwqN2dOyfopOVaHleqWtoS+3vWCqen+m8M3HToQ==", + "dependencies": { + "css-select": "^5.1.0", + "cssom": "^0.5.0", + "html-escaper": "^3.0.3", + "htmlparser2": "^9.0.0", + "uhyphen": "^0.2.0" + } + }, "node_modules/locate-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", @@ -2398,10 +2374,16 @@ "node": ">=0.10.0" } }, - "node_modules/nwsapi": { - "version": "2.2.7", - "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz", - "integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==" + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } }, "node_modules/on-exit-leak-free": { "version": "2.1.2", @@ -2486,6 +2468,7 @@ "version": "7.1.2", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dev": true, "dependencies": { "entities": "^4.4.0" }, @@ -2667,11 +2650,6 @@ "node": ">= 0.10" } }, - "node_modules/psl": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz", - "integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==" - }, "node_modules/punycode": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", @@ -2680,11 +2658,6 @@ "node": ">=6" } }, - "node_modules/querystringify": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", - "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" - }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -2764,11 +2737,6 @@ "node": ">=0.10.0" } }, - "node_modules/requires-port": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", - "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" - }, "node_modules/resolve-from": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", @@ -2835,11 +2803,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/rrweb-cssom": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz", - "integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw==" - }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -2903,17 +2866,6 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "node_modules/saxes": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", - "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", - "dependencies": { - "xmlchars": "^2.2.0" - }, - "engines": { - "node": ">=v12.22.7" - } - }, "node_modules/secure-json-parse": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", @@ -3091,11 +3043,6 @@ "node": ">=8" } }, - "node_modules/symbol-tree": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" - }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -3189,31 +3136,6 @@ "node": ">=0.6" } }, - "node_modules/tough-cookie": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz", - "integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==", - "dependencies": { - "psl": "^1.1.33", - "punycode": "^2.1.1", - "universalify": "^0.2.0", - "url-parse": "^1.5.3" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/tr46": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz", - "integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==", - "dependencies": { - "punycode": "^2.3.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/ts-algebra": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-1.2.2.tgz", @@ -3289,20 +3211,17 @@ "node": ">=14.17" } }, + "node_modules/uhyphen": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz", + "integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA==" + }, "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "dev": true }, - "node_modules/universalify": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", - "integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==", - "engines": { - "node": ">= 4.0.0" - } - }, "node_modules/untildify": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", @@ -3320,71 +3239,12 @@ "punycode": "^2.1.0" } }, - "node_modules/url-parse": { - "version": "1.5.10", - "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", - "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", - "dependencies": { - "querystringify": "^2.1.1", - "requires-port": "^1.0.0" - } - }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, - "node_modules/w3c-xmlserializer": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz", - "integrity": "sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==", - "dependencies": { - "xml-name-validator": "^4.0.0" - }, - "engines": { - "node": ">=14" - } - }, - "node_modules/webidl-conversions": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", - "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", - "engines": { - "node": ">=12" - } - }, - "node_modules/whatwg-encoding": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", - "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", - "dependencies": { - "iconv-lite": "0.6.3" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/whatwg-mimetype": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", - "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", - "engines": { - "node": ">=12" - } - }, - "node_modules/whatwg-url": { - "version": "12.0.1", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz", - "integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==", - "dependencies": { - "tr46": "^4.1.1", - "webidl-conversions": "^7.0.0" - }, - "engines": { - "node": ">=14" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -3422,39 +3282,6 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" }, - "node_modules/ws": { - "version": "8.14.2", - "resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz", - "integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==", - "engines": { - "node": ">=10.0.0" - }, - "peerDependencies": { - "bufferutil": "^4.0.1", - "utf-8-validate": ">=5.0.2" - }, - "peerDependenciesMeta": { - "bufferutil": { - "optional": true - }, - "utf-8-validate": { - "optional": true - } - } - }, - "node_modules/xml-name-validator": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", - "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==", - "engines": { - "node": ">=12" - } - }, - "node_modules/xmlchars": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", - "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" - }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/package.json b/package.json index 5a1d16a..15f19ba 100644 --- a/package.json +++ b/package.json @@ -17,8 +17,8 @@ "fastify": "^4.21.0", "iconv-lite": "^0.6.3", "ip-range-check": "^0.2.0", - "jsdom": "^22.1.0", "json-schema-to-ts": "^2.9.2", + "linkedom": "^0.16.4", "micromatch": "^4.0.5" }, "devDependencies": { @@ -41,7 +41,9 @@ "start": "cd ./dist && node ./src/app.js", "start:docker": "node ./src/app.js", "clean-css": "cleancss --batch static/*.css -o dist/static --batch-suffix \"\"", - "dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"" + "dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"", + "format": "prettier --write .", + "format:check": "prettier --check ." }, "keywords": [], "authors": [ diff --git a/src/app.ts b/src/app.ts index 70a935a..c5869ad 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,21 +1,21 @@ -import path from "path"; +import path from 'path'; -import Fastify from "fastify"; -import fastifyStatic from "@fastify/static"; -import fastifyView from "@fastify/view"; -import fastifySwagger from "@fastify/swagger"; -import fastifySwaggerUi from "@fastify/swagger-ui"; -import ejs from "ejs"; +import Fastify from 'fastify'; +import fastifyStatic from '@fastify/static'; +import fastifyView from '@fastify/view'; +import fastifySwagger from '@fastify/swagger'; +import fastifySwaggerUi from '@fastify/swagger-ui'; +import ejs from 'ejs'; -import indexRoute from "./routes/browser/index"; -import getRoute from "./routes/browser/get"; -import proxyRoute from "./routes/browser/proxy"; -import parseRoute from "./routes/api/parse"; -import rawHtml from "./routes/api/raw-html"; +import indexRoute from './routes/browser/index'; +import getRoute from './routes/browser/get'; +import proxyRoute from './routes/browser/proxy'; +import parseRoute from './routes/api/parse'; +import rawHtml from './routes/api/raw-html'; -import publicConfig from "./publicConfig"; -import errorHandler from "./errors/handler"; -import getConfig from "./config/main"; +import publicConfig from './publicConfig'; +import errorHandler from './errors/handler'; +import getConfig from './config/main'; class App { async init() { @@ -24,11 +24,12 @@ class App { const fastify = Fastify({ logger: true, trustProxy: config.reverse_proxy, + connectionTimeout: config.timeout, }); fastify.register(fastifyStatic, { - root: path.join(process.cwd(), "static"), - prefix: "/static/", + root: path.join(process.cwd(), 'static'), + prefix: '/static/', }); fastify.register(fastifyView, { @@ -41,32 +42,28 @@ class App { await fastify.register(fastifySwagger, { swagger: { info: { - title: "TXTDot API", + title: 'TXTDot API', description: publicConfig.description, version: publicConfig.version, }, - } + }, }); - await fastify.register(fastifySwaggerUi, { routePrefix: "/doc" }); + await fastify.register(fastifySwaggerUi, { routePrefix: '/doc' }); } fastify.register(indexRoute); fastify.register(getRoute); - if (config.proxy_res) - fastify.register(proxyRoute); + if (config.proxy_res) fastify.register(proxyRoute); fastify.register(parseRoute); fastify.register(rawHtml); fastify.setErrorHandler(errorHandler); - fastify.listen( - { host: config.host, port: config.port }, - (err) => { - err && console.log(err); - } - ); + fastify.listen({ host: config.host, port: config.port }, (err) => { + err && console.log(err); + }); } } diff --git a/src/config/config.service.ts b/src/config/config.service.ts index 3ff93e3..415e246 100644 --- a/src/config/config.service.ts +++ b/src/config/config.service.ts @@ -1,8 +1,9 @@ -import { config } from "dotenv"; +import { config } from 'dotenv'; export class ConfigService { public readonly host: string; public readonly port: number; + public readonly timeout: number; public readonly reverse_proxy: boolean; public readonly proxy_res: boolean; public readonly swagger: boolean; @@ -10,9 +11,11 @@ export class ConfigService { constructor() { config(); - this.host = process.env.HOST || "0.0.0.0"; + this.host = process.env.HOST || '0.0.0.0'; this.port = Number(process.env.PORT) || 8080; + this.timeout = 1000; + this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false); this.proxy_res = this.parseBool(process.env.PROXY_RES, true); @@ -21,6 +24,6 @@ export class ConfigService { parseBool(value: string | undefined, def: boolean): boolean { if (!value) return def; - return value === "true" || value === "1"; + return value === 'true' || value === '1'; } } diff --git a/src/config/main.ts b/src/config/main.ts index 37dcad3..ea4e219 100644 --- a/src/config/main.ts +++ b/src/config/main.ts @@ -1,4 +1,4 @@ -import { ConfigService } from "./config.service"; +import { ConfigService } from './config.service'; let configSvc: ConfigService | undefined; diff --git a/src/errors/api.ts b/src/errors/api.ts index 4d0728c..8515501 100644 --- a/src/errors/api.ts +++ b/src/errors/api.ts @@ -5,28 +5,28 @@ export interface IApiError { } export const errorSchema = { - type: "object", + type: 'object', properties: { code: { - type: "number", - description: "HTTP error code", + type: 'number', + description: 'HTTP error code', }, name: { - type: "string", - description: "Exception class name", + type: 'string', + description: 'Exception class name', }, message: { - type: "string", - description: "Exception message", + type: 'string', + description: 'Exception message', }, }, }; export const errorResponseSchema = { - type: "object", + type: 'object', properties: { data: { - type: "object", + type: 'object', nullable: true, }, error: errorSchema, diff --git a/src/errors/handler.ts b/src/errors/handler.ts index 5d28722..f5cf4c3 100644 --- a/src/errors/handler.ts +++ b/src/errors/handler.ts @@ -1,16 +1,16 @@ -import { FastifyReply, FastifyRequest } from "fastify"; -import { NotHtmlMimetypeError, TxtDotError } from "./main"; -import { getFastifyError } from "./validation"; +import { FastifyReply, FastifyRequest } from 'fastify'; +import { NotHtmlMimetypeError, TxtDotError } from './main'; +import { getFastifyError } from './validation'; -import { IGetSchema } from "../types/requests/browser"; -import getConfig from "../config/main"; +import { IGetSchema } from '../types/requests/browser'; +import getConfig from '../config/main'; export default function errorHandler( error: Error, req: FastifyRequest, reply: FastifyReply ) { - if (req.originalUrl.startsWith("/api/")) { + if (req.originalUrl.startsWith('/api/')) { return apiErrorHandler(error, reply); } @@ -43,26 +43,23 @@ function apiErrorHandler(error: Error, reply: FastifyReply) { function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) { if (getFastifyError(error)?.statusCode === 400) { - return reply.code(400).view("/templates/error.ejs", { + return reply.code(400).view('/templates/error.ejs', { url, code: 400, description: `Invalid parameter specified: ${error.message}`, - }) - } - - if (error instanceof TxtDotError) { - return reply.code(error.code).view("/templates/error.ejs", { - url, - code: error.code, - description: error.description, - proxyBtn: ( - error instanceof NotHtmlMimetypeError && - getConfig().proxy_res - ), }); } - return reply.code(500).view("/templates/error.ejs", { + if (error instanceof TxtDotError) { + return reply.code(error.code).view('/templates/error.ejs', { + url, + code: error.code, + description: error.description, + proxyBtn: error instanceof NotHtmlMimetypeError && getConfig().proxy_res, + }); + } + + return reply.code(500).view('/templates/error.ejs', { url, code: 500, description: `${error.name}: ${error.message}`, diff --git a/src/errors/main.ts b/src/errors/main.ts index 81f983a..2513a9d 100644 --- a/src/errors/main.ts +++ b/src/errors/main.ts @@ -1,15 +1,11 @@ -import getConfig from "../config/main"; +import getConfig from '../config/main'; export abstract class TxtDotError extends Error { code: number; name: string; description: string; - constructor( - code: number, - name: string, - description: string, - ) { + constructor(code: number, name: string, description: string) { super(description); this.code = code; this.name = name; @@ -19,21 +15,13 @@ export abstract class TxtDotError extends Error { export class EngineParseError extends TxtDotError { constructor(message: string) { - super( - 422, - "EngineParseError", - `Parse error: ${message}`, - ); + super(422, 'EngineParseError', `Parse error: ${message}`); } } export class LocalResourceError extends TxtDotError { constructor() { - super( - 403, - "LocalResourceError", - "Proxying local resources is forbidden.", - ); + super(403, 'LocalResourceError', 'Proxying local resources is forbidden.'); } } @@ -41,12 +29,11 @@ export class NotHtmlMimetypeError extends TxtDotError { constructor() { super( 421, - "NotHtmlMimetypeError", - "Received non-HTML content, " + ( - getConfig().proxy_res ? - "use proxy instead of parser." : - "proxying is disabled by the instance admin." - ), + 'NotHtmlMimetypeError', + 'Received non-HTML content, ' + + (getConfig().proxy_res + ? 'use proxy instead of parser.' + : 'proxying is disabled by the instance admin.') ); } } diff --git a/src/handlers/google.ts b/src/handlers/google.ts index 6783d8a..392edd5 100644 --- a/src/handlers/google.ts +++ b/src/handlers/google.ts @@ -1,6 +1,6 @@ -import { HandlerInput } from "./handler-input"; -import { IHandlerOutput } from "./handler.interface"; -import { EngineParseError } from "../errors/main"; +import { HandlerInput } from './handler-input'; +import { IHandlerOutput } from './handler.interface'; +import { EngineParseError } from '../errors/main'; export default async function google( input: HandlerInput @@ -8,18 +8,18 @@ export default async function google( const window = input.parseDom().window; const googleAnchors = [ - ...window.document.querySelectorAll("a[jsname=UWckNb]"), + ...window.document.querySelectorAll('a[jsname=UWckNb]'), ] as HTMLAnchorElement[]; if (!googleAnchors) { throw new EngineParseError( - "Failed to find anchors in search result [google]" + 'Failed to find anchors in search result [google]' ); } const results = googleAnchors .map((a: HTMLAnchorElement): GoogleProps => { - const parsedHref = new URL(new URL(a.href).searchParams.get("url")!); + const parsedHref = new URL(new URL(a.href).searchParams.get('url')!); return { href: a.href!, siteName: parsedHref.hostname, @@ -43,7 +43,7 @@ export default async function google( }); const search = window.document.getElementById( - "APjFqb" + 'APjFqb' ) as HTMLTextAreaElement; const searchForm = ` @@ -54,18 +54,18 @@ export default async function google( `; return { - content: `${searchForm}${content.join("")}`, - textContent: textContent.join("\n"), + content: `${searchForm}${content.join('')}`, + textContent: textContent.join('\n'), }; } export const GoogleDomains = [ - "google.*", - "google.co.*", - "google.com.*", - "www.google.*", - "www.google.co.*", - "www.google.com.*", + 'google.*', + 'google.co.*', + 'google.com.*', + 'www.google.*', + 'www.google.co.*', + 'www.google.com.*', ]; interface GoogleProps { diff --git a/src/handlers/handler-input.ts b/src/handlers/handler-input.ts index ad510b9..4be0a60 100644 --- a/src/handlers/handler-input.ts +++ b/src/handlers/handler-input.ts @@ -1,14 +1,11 @@ -import { JSDOM } from "jsdom"; +import { parseHTML } from 'linkedom'; export class HandlerInput { private data: string; private url: string; - private dom?: JSDOM; + private dom?: Window; - constructor( - data: string, - url: string, - ) { + constructor(data: string, url: string) { this.data = data; this.url = url; } @@ -17,12 +14,12 @@ export class HandlerInput { return this.url; } - parseDom(): JSDOM { + parseDom(): Window { if (this.dom) { return this.dom; } - this.dom = new JSDOM(this.data, { url: this.url }); + this.dom = parseHTML(this.data); return this.dom; } } diff --git a/src/handlers/handler.interface.ts b/src/handlers/handler.interface.ts index 1bcd7c2..2b5bd8f 100644 --- a/src/handlers/handler.interface.ts +++ b/src/handlers/handler.interface.ts @@ -6,19 +6,19 @@ export interface IHandlerOutput { } export const handlerSchema = { - type: "object", + type: 'object', properties: { content: { - type: "string", + type: 'string', }, textContent: { - type: "string", + type: 'string', }, title: { - type: "string", + type: 'string', }, lang: { - type: "string", + type: 'string', }, }, }; diff --git a/src/handlers/main.ts b/src/handlers/main.ts index 01c79f7..9dba929 100644 --- a/src/handlers/main.ts +++ b/src/handlers/main.ts @@ -1,42 +1,42 @@ -import { IHandlerOutput } from "./handler.interface"; -import { Engines, EngineFunction, EnginesMatch } from "../types/handlers"; -import axios from "../types/axios"; +import { IHandlerOutput } from './handler.interface'; +import { Engines, EngineFunction, EnginesMatch } from '../types/handlers'; +import axios from '../types/axios'; -import micromatch from "micromatch"; +import micromatch from 'micromatch'; -import { JSDOM } from "jsdom"; -import DOMPurify from "dompurify"; +import DOMPurify from 'dompurify'; -import { Readable } from "stream"; +import { Readable } from 'stream'; -import readability from "./readability"; -import google, { GoogleDomains } from "./google"; -import stackoverflow, { StackOverflowDomains } from "./stackoverflow/main"; +import readability from './readability'; +import google, { GoogleDomains } from './google'; +import stackoverflow, { StackOverflowDomains } from './stackoverflow/main'; -import isLocalResource from "../utils/islocal"; +import isLocalResource from '../utils/islocal'; -import { LocalResourceError, NotHtmlMimetypeError } from "../errors/main"; -import { HandlerInput } from "./handler-input"; -import { decodeStream, parseEncodingName } from "../utils/http"; -import replaceHref from "../utils/replace-href"; +import { LocalResourceError, NotHtmlMimetypeError } from '../errors/main'; +import { HandlerInput } from './handler-input'; +import { decodeStream, parseEncodingName } from '../utils/http'; +import replaceHref from '../utils/replace-href'; +import { parseHTML } from 'linkedom'; export default async function handlePage( - url: string, // remote URL + remoteUrl: string, // remote URL requestUrl: URL, // proxy URL engine?: string, - redirectPath: string = "get", + redirectPath: string = 'get' ): Promise { - const urlObj = new URL(url); + const urlObj = new URL(remoteUrl); if (await isLocalResource(urlObj)) { throw new LocalResourceError(); } - const response = await axios.get(url); + const response = await axios.get(remoteUrl); const data: Readable = response.data; - const mime: string | undefined = response.headers["content-type"]?.toString(); + const mime: string | undefined = response.headers['content-type']?.toString(); - if (mime && mime.indexOf("text/html") === -1) { + if (mime && mime.indexOf('text/html') === -1) { throw new NotHtmlMimetypeError(); } @@ -44,17 +44,17 @@ export default async function handlePage( const output = await handler( new HandlerInput( await decodeStream(data, parseEncodingName(mime)), - url, + remoteUrl ) ); // post-process - const dom = new JSDOM(output.content, { url }); - replaceHref(dom, requestUrl, engine, redirectPath); + const dom = parseHTML(output.content); + replaceHref(dom, requestUrl, new URL(remoteUrl), engine, redirectPath); const purify = DOMPurify(dom.window); - output.content = purify.sanitize(dom.serialize()); + output.content = purify.sanitize(dom.document.toString()); return output; } diff --git a/src/handlers/readability.ts b/src/handlers/readability.ts index a22cc66..f8e2fcb 100644 --- a/src/handlers/readability.ts +++ b/src/handlers/readability.ts @@ -1,16 +1,16 @@ -import { Readability } from "@mozilla/readability"; -import { HandlerInput } from "./handler-input"; -import { IHandlerOutput } from "./handler.interface"; -import { EngineParseError } from "../errors/main"; +import { Readability } from '@mozilla/readability'; +import { HandlerInput } from './handler-input'; +import { IHandlerOutput } from './handler.interface'; +import { EngineParseError } from '../errors/main'; export default async function readability( - input: HandlerInput, + input: HandlerInput ): Promise { const reader = new Readability(input.parseDom().window.document); const parsed = reader.parse(); if (!parsed) { - throw new EngineParseError("Failed to parse [readability]"); + throw new EngineParseError('Failed to parse [readability]'); } return { diff --git a/src/handlers/stackoverflow/main.ts b/src/handlers/stackoverflow/main.ts index f1b9c6c..f69344d 100644 --- a/src/handlers/stackoverflow/main.ts +++ b/src/handlers/stackoverflow/main.ts @@ -1,30 +1,30 @@ -import { HandlerInput } from "../handler-input"; -import { IHandlerOutput } from "../handler.interface"; -import { EngineParseError } from "../../errors/main"; -import qPostsHandler from "./questions-posts"; +import { HandlerInput } from '../handler-input'; +import { IHandlerOutput } from '../handler.interface'; +import { EngineParseError } from '../../errors/main'; +import qPostsHandler from './questions-posts'; export default async function stackoverflow( - input: HandlerInput, + input: HandlerInput ): Promise { const window = input.parseDom().window; const url = new URL(window.location.href); - const path = url.pathname.split("/").filter((p) => p !== ""); + const path = url.pathname.split('/').filter((p) => p !== ''); let result: IHandlerOutput = { - content: "", - textContent: "", - title: "", - lang: "", + content: '', + textContent: '', + title: '', + lang: '', }; - if (path[0] === "questions") { + if (path[0] === 'questions') { if (path.length === 3) { result = await qPostsHandler(window); } else if (path.length === 1) { - result.content = "questions"; + result.content = 'questions'; } else { - throw new EngineParseError("Invalid URL [stackoverflow]"); + throw new EngineParseError('Invalid URL [stackoverflow]'); } } @@ -32,12 +32,12 @@ export default async function stackoverflow( } export const StackOverflowDomains = [ - "stackoverflow.com", - "*.stackoverflow.com", - "*.stackexchange.com", - "askubuntu.com", - "stackapps.com", - "mathoverflow.net", - "superuser.com", - "serverfault.com", + 'stackoverflow.com', + '*.stackoverflow.com', + '*.stackexchange.com', + 'askubuntu.com', + 'stackapps.com', + 'mathoverflow.net', + 'superuser.com', + 'serverfault.com', ]; diff --git a/src/handlers/stackoverflow/post-parser.ts b/src/handlers/stackoverflow/post-parser.ts index e5e6999..7b06a3e 100644 --- a/src/handlers/stackoverflow/post-parser.ts +++ b/src/handlers/stackoverflow/post-parser.ts @@ -1,9 +1,9 @@ export default function postParser(el: Element | null): string { if (!el) { - return ""; + return ''; } - const body = el.querySelector(".js-post-body")?.innerHTML || ""; - const voteCount = el.querySelector(".js-vote-count")?.textContent || ""; + const body = el.querySelector('.js-post-body')?.innerHTML || ''; + const voteCount = el.querySelector('.js-vote-count')?.textContent || ''; return `

${voteCount} votes

${body}`; } diff --git a/src/handlers/stackoverflow/questions-posts.ts b/src/handlers/stackoverflow/questions-posts.ts index 3d011f8..c1eb08c 100644 --- a/src/handlers/stackoverflow/questions-posts.ts +++ b/src/handlers/stackoverflow/questions-posts.ts @@ -1,26 +1,25 @@ -import { DOMWindow } from "jsdom"; -import { IHandlerOutput } from "../handler.interface"; -import postParser from "./post-parser"; +import { IHandlerOutput } from '../handler.interface'; +import postParser from './post-parser'; export default async function qPostsHandler( - window: DOMWindow + window: Window ): Promise { - const questionEl = window.document.getElementById("question"); + const questionEl = window.document.getElementById('question'); const question = postParser(questionEl); const title = - window.document.querySelector(".question-hyperlink")?.innerHTML || ""; + window.document.querySelector('.question-hyperlink')?.innerHTML || ''; - const allAnswers = [...window.document.querySelectorAll(".answer")]; + const allAnswers = [...window.document.querySelectorAll('.answer')]; const answers = allAnswers.map((a) => postParser(a)); return { content: `${question}
${answers.length} answers
${answers.join( - "
" + '
' )}`, - textContent: "question", + textContent: 'question', title, - lang: "en", + lang: 'en', }; } diff --git a/src/publicConfig.ts b/src/publicConfig.ts index 355f63c..04daf30 100644 --- a/src/publicConfig.ts +++ b/src/publicConfig.ts @@ -1,5 +1,5 @@ export default { - version: "1.4.0", + version: '1.4.0', description: - "txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts", + 'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts', }; diff --git a/src/routes/api/parse.ts b/src/routes/api/parse.ts index 5d51014..bff8af4 100644 --- a/src/routes/api/parse.ts +++ b/src/routes/api/parse.ts @@ -1,13 +1,17 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import { EngineRequest, IParseSchema, parseSchema } from "../../types/requests/api"; +import { + EngineRequest, + IParseSchema, + parseSchema, +} from '../../types/requests/api'; -import handlePage from "../../handlers/main"; -import { generateRequestUrl } from "../../utils/generate"; +import handlePage from '../../handlers/main'; +import { generateRequestUrl } from '../../utils/generate'; export default async function parseRoute(fastify: FastifyInstance) { fastify.get( - "/api/parse", + '/api/parse', { schema: parseSchema }, async (request: EngineRequest) => { return { diff --git a/src/routes/api/raw-html.ts b/src/routes/api/raw-html.ts index b9d9de5..4deb197 100644 --- a/src/routes/api/raw-html.ts +++ b/src/routes/api/raw-html.ts @@ -1,16 +1,16 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import { IParseSchema, rawHtmlSchema } from "../../types/requests/api"; +import { IParseSchema, rawHtmlSchema } from '../../types/requests/api'; -import handlePage from "../../handlers/main"; -import { generateRequestUrl } from "../../utils/generate"; +import handlePage from '../../handlers/main'; +import { generateRequestUrl } from '../../utils/generate'; export default async function rawHtml(fastify: FastifyInstance) { fastify.get( - "/api/raw-html", + '/api/raw-html', { schema: rawHtmlSchema }, async (request, reply) => { - reply.type("text/html; charset=utf-8"); + reply.type('text/html; charset=utf-8'); return ( await handlePage( request.query.url, @@ -20,7 +20,7 @@ export default async function rawHtml(fastify: FastifyInstance) { request.originalUrl ), request.query.engine, - "api/raw-html" + 'api/raw-html' ) ).content; } diff --git a/src/routes/browser/get.ts b/src/routes/browser/get.ts index ef94428..4ee62f2 100644 --- a/src/routes/browser/get.ts +++ b/src/routes/browser/get.ts @@ -1,12 +1,12 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import { GetSchema, IGetSchema } from "../../types/requests/browser"; -import handlePage from "../../handlers/main"; -import { generateRequestUrl } from "../../utils/generate"; +import { GetSchema, IGetSchema } from '../../types/requests/browser'; +import handlePage from '../../handlers/main'; +import { generateRequestUrl } from '../../utils/generate'; export default async function getRoute(fastify: FastifyInstance) { fastify.get( - "/get", + '/get', { schema: GetSchema }, async (request, reply) => { const remoteUrl = request.query.url; @@ -22,12 +22,12 @@ export default async function getRoute(fastify: FastifyInstance) { engine ); - if (request.query.format === "text") { - reply.type("text/plain; charset=utf-8"); + if (request.query.format === 'text') { + reply.type('text/plain; charset=utf-8'); return parsed.textContent; } else { - reply.type("text/html; charset=utf-8"); - return reply.view("/templates/get.ejs", { parsed, remoteUrl }); + reply.type('text/html; charset=utf-8'); + return reply.view('/templates/get.ejs', { parsed, remoteUrl }); } } ); diff --git a/src/routes/browser/index.ts b/src/routes/browser/index.ts index 1d69770..83ee81b 100644 --- a/src/routes/browser/index.ts +++ b/src/routes/browser/index.ts @@ -1,11 +1,11 @@ -import { FastifyInstance } from "fastify"; +import { FastifyInstance } from 'fastify'; -import publicConfig from "../../publicConfig"; -import { engineList } from "../../handlers/main"; -import { indexSchema } from "../../types/requests/browser"; +import publicConfig from '../../publicConfig'; +import { engineList } from '../../handlers/main'; +import { indexSchema } from '../../types/requests/browser'; export default async function indexRoute(fastify: FastifyInstance) { - fastify.get("/", { schema: indexSchema }, async (_, reply) => { - return reply.view("/templates/index.ejs", { publicConfig, engineList }); + fastify.get('/', { schema: indexSchema }, async (_, reply) => { + return reply.view('/templates/index.ejs', { publicConfig, engineList }); }); } diff --git a/src/routes/browser/proxy.ts b/src/routes/browser/proxy.ts index 4947abf..20aa0c4 100644 --- a/src/routes/browser/proxy.ts +++ b/src/routes/browser/proxy.ts @@ -1,17 +1,19 @@ -import { FastifyInstance } from "fastify"; -import { IProxySchema, ProxySchema } from "../../types/requests/browser"; -import axios from "../../types/axios"; +import { FastifyInstance } from 'fastify'; +import { IProxySchema, ProxySchema } from '../../types/requests/browser'; +import axios from '../../types/axios'; export default async function proxyRoute(fastify: FastifyInstance) { fastify.get( - "/proxy", + '/proxy', { schema: ProxySchema }, async (request, reply) => { const response = await axios.get(request.query.url); - const mime: string | undefined = response.headers["content-type"]?.toString(); - const clen: string | undefined = response.headers["content-length"]?.toString(); - mime && reply.header("Content-Type", mime); - clen && reply.header("Content-Length", Number(clen)); + const mime: string | undefined = + response.headers['content-type']?.toString(); + const clen: string | undefined = + response.headers['content-length']?.toString(); + mime && reply.header('Content-Type', mime); + clen && reply.header('Content-Length', Number(clen)); return reply.send(response.data); } ); diff --git a/src/types/axios.ts b/src/types/axios.ts index 947ccfa..6e4628b 100644 --- a/src/types/axios.ts +++ b/src/types/axios.ts @@ -1,9 +1,9 @@ -import axios from "axios"; +import axios from 'axios'; export default axios.create({ headers: { - "User-Agent": - "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0", + 'User-Agent': + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0', }, - responseType: "stream", + responseType: 'stream', }); diff --git a/src/types/handlers.ts b/src/types/handlers.ts index 846eeda..58138c7 100644 --- a/src/types/handlers.ts +++ b/src/types/handlers.ts @@ -1,5 +1,5 @@ -import { HandlerInput } from "../handlers/handler-input"; -import { IHandlerOutput } from "../handlers/handler.interface"; +import { HandlerInput } from '../handlers/handler-input'; +import { IHandlerOutput } from '../handlers/handler.interface'; export interface Engines { [key: string]: EngineFunction; diff --git a/src/types/requests/api.ts b/src/types/requests/api.ts index 0472505..db8ee91 100644 --- a/src/types/requests/api.ts +++ b/src/types/requests/api.ts @@ -1,8 +1,8 @@ -import { FastifySchema, FastifyRequest } from "fastify"; -import { IApiError, errorResponseSchema } from "../../errors/api"; -import { handlerSchema } from "../../handlers/handler.interface"; -import { engineList } from "../../handlers/main"; -import { FromSchema } from "json-schema-to-ts"; +import { FastifySchema, FastifyRequest } from 'fastify'; +import { IApiError, errorResponseSchema } from '../../errors/api'; +import { handlerSchema } from '../../handlers/handler.interface'; +import { engineList } from '../../handlers/main'; +import { FromSchema } from 'json-schema-to-ts'; export interface IApiResponse { data?: T; @@ -10,38 +10,38 @@ export interface IApiResponse { } export const parseQuerySchema = { - type: "object", - required: ["url"], + type: 'object', + required: ['url'], properties: { url: { - type: "string", - description: "URL", + type: 'string', + description: 'URL', }, engine: { - type: "string", - enum: [...engineList, ""], + type: 'string', + enum: [...engineList, ''], }, }, } as const; export const parseSchema: FastifySchema = { - description: "Parse the page and get all data from the engine", + description: 'Parse the page and get all data from the engine', querystring: parseQuerySchema, response: { - "2xx": { - type: "object", + '2xx': { + type: 'object', properties: { data: handlerSchema, error: { - type: "object", + type: 'object', nullable: true, }, }, }, - "4xx": errorResponseSchema, - "5xx": errorResponseSchema, + '4xx': errorResponseSchema, + '5xx': errorResponseSchema, }, - produces: ["text/json"], + produces: ['text/json'], }; export interface IParseSchema { @@ -49,9 +49,9 @@ export interface IParseSchema { } export const rawHtmlSchema: FastifySchema = { - description: "Parse the page and get raw HTML from the engine", + description: 'Parse the page and get raw HTML from the engine', querystring: parseQuerySchema, - produces: ["text/html"], + produces: ['text/html'], }; export type EngineRequest = FastifyRequest<{ diff --git a/src/types/requests/browser.ts b/src/types/requests/browser.ts index 4507b11..4504b0c 100644 --- a/src/types/requests/browser.ts +++ b/src/types/requests/browser.ts @@ -1,6 +1,6 @@ -import { FastifySchema } from "fastify"; -import { engineList } from "../../handlers/main"; -import { FromSchema } from "json-schema-to-ts"; +import { FastifySchema } from 'fastify'; +import { engineList } from '../../handlers/main'; +import { FromSchema } from 'json-schema-to-ts'; export interface IGetSchema { Querystring: IGetQuerySchema; @@ -11,52 +11,52 @@ export interface IProxySchema { } export const getQuerySchema = { - type: "object", - required: ["url"], + type: 'object', + required: ['url'], properties: { url: { - type: "string", - description: "URL", + type: 'string', + description: 'URL', }, format: { - type: "string", - enum: ["text", "html", ""], - default: "html", + type: 'string', + enum: ['text', 'html', ''], + default: 'html', }, engine: { - type: "string", - enum: [...engineList, ""], + type: 'string', + enum: [...engineList, ''], }, }, } as const; export type IGetQuerySchema = FromSchema; export const proxyQuerySchema = { - type: "object", - required: ["url"], + type: 'object', + required: ['url'], properties: { url: { - type: "string", - description: "URL", + type: 'string', + description: 'URL', }, - } + }, } as const; export type IProxyQuerySchema = FromSchema; export const indexSchema = { hide: true, - produces: ["text/html"], + produces: ['text/html'], }; export const GetSchema: FastifySchema = { - description: "Get page", + description: 'Get page', hide: true, querystring: getQuerySchema, - produces: ["text/html", "text/plain"], + produces: ['text/html', 'text/plain'], }; export const ProxySchema: FastifySchema = { - description: "Proxy resource", + description: 'Proxy resource', hide: true, querystring: proxyQuerySchema, -} +}; diff --git a/src/utils/generate.ts b/src/utils/generate.ts index a275585..f5a3295 100644 --- a/src/utils/generate.ts +++ b/src/utils/generate.ts @@ -8,25 +8,35 @@ export function generateRequestUrl( export function generateParserUrl( requestUrl: URL, + remoteUrl: URL, href: string, engine?: string, - redirect_url: string = "get" + redirect_url: string = 'get' ): string { - const parsedHref = new URL(href); + const realURL = getRealURL(href, remoteUrl); - const hash = parsedHref.hash; // save #hash - parsedHref.hash = ""; // remove + const hash = realURL.hash; // save #hash + realURL.hash = ''; // remove - const urlParam = `?url=${encodeURIComponent(parsedHref.toString())}`; - const engineParam = engine ? `&engine=${engine}` : ""; + const urlParam = `?url=${encodeURIComponent(realURL.toString())}`; + const engineParam = engine ? `&engine=${engine}` : ''; return `${requestUrl.origin}/${redirect_url}${urlParam}${engineParam}${hash}`; } export function generateProxyUrl( requestUrl: URL, - href: string, + remoteUrl: URL, + href: string ): string { - const urlParam = `?url=${encodeURIComponent(href)}`; + const realHref = getRealURL(href, remoteUrl); + + const urlParam = `?url=${encodeURIComponent(realHref.href)}`; return `${requestUrl.origin}/proxy${urlParam}`; } + +function getRealURL(href: string, remoteUrl: URL) { + return href.startsWith('http') + ? new URL(href) + : new URL(href, remoteUrl.href); +} diff --git a/src/utils/http.ts b/src/utils/http.ts index 86e436a..b5eecea 100644 --- a/src/utils/http.ts +++ b/src/utils/http.ts @@ -1,12 +1,12 @@ -import { Readable } from "stream"; -import iconv from "iconv-lite"; +import { Readable } from 'stream'; +import iconv from 'iconv-lite'; export async function decodeStream( data: Readable, - charset: string = "utf-8", + charset: string = 'utf-8' ): Promise { const strm = data.pipe(iconv.decodeStream(charset)) as IconvStream; - return await new Promise(resolve => { + return await new Promise((resolve) => { strm.collect((_err: Error, body: string) => { resolve(body); }); @@ -16,7 +16,7 @@ export async function decodeStream( export function parseEncodingName(ctype?: string): string { const match = ctype?.match(/charset=([A-Za-z0-9-]+)$/); if (!match) { - return "utf-8"; + return 'utf-8'; } return match[1]; } diff --git a/src/utils/islocal.ts b/src/utils/islocal.ts index c997c01..752cb7f 100644 --- a/src/utils/islocal.ts +++ b/src/utils/islocal.ts @@ -1,45 +1,43 @@ -import dns from "dns"; -import ipRangeCheck from "ip-range-check"; +import dns from 'dns'; +import ipRangeCheck from 'ip-range-check'; const subnets = [ - "0.0.0.0/8", - "127.0.0.0/8", - "10.0.0.0/8", - "100.64.0.0/10", - "169.254.0.0/16", - "172.16.0.0/12", - "192.0.0.0/24", - "192.0.2.0/24", - "192.88.99.0/24", - "192.168.0.0/16", - "198.18.0.0/15", - "198.51.100.0/24", - "203.0.113.0/24", - "224.0.0.0/4", - "233.252.0.0/24", - "240.0.0.0/4", - "255.255.255.255/32", - "::/128", - "::1/128", - "::ffff:0:0/96", - "::ffff:0:0:0/96", - "64:ff9b::/96", - "64:ff9b:1::/48", - "100::/64", - "2001:0000::/32", - "2001:20::/28", - "2001:db8::/32", - "2002::/16", - "fc00::/7", - "fe80::/64", - "ff00::/8", + '0.0.0.0/8', + '127.0.0.0/8', + '10.0.0.0/8', + '100.64.0.0/10', + '169.254.0.0/16', + '172.16.0.0/12', + '192.0.0.0/24', + '192.0.2.0/24', + '192.88.99.0/24', + '192.168.0.0/16', + '198.18.0.0/15', + '198.51.100.0/24', + '203.0.113.0/24', + '224.0.0.0/4', + '233.252.0.0/24', + '240.0.0.0/4', + '255.255.255.255/32', + '::/128', + '::1/128', + '::ffff:0:0/96', + '::ffff:0:0:0/96', + '64:ff9b::/96', + '64:ff9b:1::/48', + '100::/64', + '2001:0000::/32', + '2001:20::/28', + '2001:db8::/32', + '2002::/16', + 'fc00::/7', + 'fe80::/64', + 'ff00::/8', ]; export default async function isLocalResource(url: URL): Promise { // Resolve domain name - const addr = ( - await dns.promises.lookup(url.hostname) - ).address; + const addr = (await dns.promises.lookup(url.hostname)).address; // Check if IP is in local network return ipRangeCheck(addr, subnets); diff --git a/src/utils/replace-href.ts b/src/utils/replace-href.ts index b964870..522159f 100644 --- a/src/utils/replace-href.ts +++ b/src/utils/replace-href.ts @@ -1,72 +1,53 @@ -import { JSDOM } from "jsdom"; -import { generateParserUrl, generateProxyUrl } from "./generate"; -import getConfig from "../config/main"; +import { generateParserUrl, generateProxyUrl } from './generate'; +import getConfig from '../config/main'; export default function replaceHref( - dom: JSDOM, + dom: Window, requestUrl: URL, + remoteUrl: URL, engine?: string, - redirectPath: string = "get", + redirectPath: string = 'get' ) { - const doc = dom.window.document; - + const doc: Document = dom.window.document; const parserUrl = (href: string) => - href.startsWith("http") ? generateParserUrl( - requestUrl, - href, - engine, - redirectPath, - ) : href; - const proxyUrl = (href: string) => - href.startsWith("http") ? generateProxyUrl( - requestUrl, - href, - ) : href; + generateParserUrl(requestUrl, remoteUrl, href, engine, redirectPath); - modifyLinks( - doc.getElementsByTagName("a"), - "href", - parserUrl, - ); - modifyLinks( - doc.querySelectorAll("frame,iframe"), - "src", - parserUrl, - ); + const proxyUrl = (href: string) => + generateProxyUrl(requestUrl, remoteUrl, href); + + modifyLinks(doc.querySelectorAll('a[href]'), 'href', parserUrl); + modifyLinks(doc.querySelectorAll('frame,iframe'), 'src', parserUrl); if (getConfig().proxy_res) { modifyLinks( - doc.querySelectorAll("img,image,video,audio,embed,track,source"), - "src", - proxyUrl, + doc.querySelectorAll('img,image,video,audio,embed,track,source'), + 'src', + proxyUrl ); - modifyLinks( - doc.getElementsByTagName("object"), - "data", - proxyUrl, - ); - - const sources = doc.querySelectorAll("source,img"); + modifyLinks(doc.getElementsByTagName('object'), 'data', proxyUrl); + const sources = doc.querySelectorAll('source,img'); for (const source of sources) { // split srcset by comma - // @ts-ignore - if (!source.srcset) - continue; - // @ts-ignore - source.srcset = source.srcset.split(",").map( - (src: string) => { + // @ts-expect-error because I don't know what to do about it. + if (!source.srcset) continue; + // @ts-expect-error because I don't know what to do about it. + source.srcset = source.srcset + .split(',') + .map((src: string) => { // split src by space - const parts = src.trim().split(" "); + const parts = src.trim().split(' '); try { // first part is URL // (srcset="http 200w 1x,...") parts[0] = proxyUrl(parts[0]); - } catch (_err) { } + } catch (_err) { + /* empty */ + } // join by space after splitting - return parts.join(" "); - } - ).join(","); // join by comma + return parts.join(' '); + }) + .join(','); // join by comma } } } @@ -74,12 +55,14 @@ export default function replaceHref( function modifyLinks( nodeList: NodeListOf | HTMLCollectionOf, property: string, - generateLink: (value: string) => string, + generateLink: (value: string) => string ) { for (const node of nodeList) { try { - // @ts-ignore + // @ts-expect-error because I don't know what to do about it. node[property] = generateLink(node[property]); - } catch (_err) { } + } catch (_err) { + /* empty */ + } } } diff --git a/static/form.css b/static/form.css index 8dd12af..b5bed02 100644 --- a/static/form.css +++ b/static/form.css @@ -31,7 +31,7 @@ label { #url { width: 100%; - height: 100%; /* shrink to #submit height */ + height: 100%; /* shrink to #submit height */ outline: none; border: 0;