mirrors/txtdot
3
0
Fork 0
mirror of https://github.com/txtdot/txtdot synced 2024-10-18 14:40:19 +03:00
Code Issues Packages Projects Releases Wiki Activity

Timeout (#70)

Browse Source 
* fix: long response time due to many <a>...

... without hrefs. It's a temporary measure until it's clear how to deal with such performance issues.

* perf: remove jsdom install linkedom

* feat: timeout

But still this timeout works only for the time of transfer of the page itself, not its processing by the server

* fix: links

* format
This commit is contained in:
Artemy Egorov 2023-12-13 20:08:24 +03:00 committed by GitHub
parent b805f19f78
commit 8f707c800e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 492 additions and 687 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
.eslintrc.json
View File

@ -1,20 +1,14 @@
{ {
"env": { "env": {
"browser": true, "browser": true,
"es2021": true "es2021": true
}, },
"extends": [ "extends": ["eslint:recommended", "plugin:@typescript-eslint/recommended"],
"eslint:recommended", "parser": "@typescript-eslint/parser",
"plugin:@typescript-eslint/recommended" "parserOptions": {
], "ecmaVersion": "latest",
"parser": "@typescript-eslint/parser", "sourceType": "module"
"parserOptions": { },
"ecmaVersion": "latest", "plugins": ["@typescript-eslint"],
"sourceType": "module" "rules": {}
},
"plugins": [
"@typescript-eslint"
],
"rules": {
}
} }

8
.github/dependabot.yml vendored
View File

@ -5,8 +5,8 @@
version: 2 version: 2
updates: updates:
- package-ecosystem: "npm" # See documentation for possible values - package-ecosystem: 'npm' # See documentation for possible values
directory: "/" # Location of package manifests directory: '/' # Location of package manifests
target-branch: "dependabot" target-branch: 'dependabot'
schedule: schedule:
interval: "weekly" interval: 'weekly'

5
.prettierrc Normal file
View File

@ -0,0 +1,5 @@
{
"trailingComma": "es5",
"tabWidth": 2,
"singleQuote": true
}

6
docker-compose.yml
View File

@ -1,10 +1,10 @@
version: "3" version: '3'
services: services:
txtdot: txtdot:
image: ghcr.io/txtdot/txtdot:latest image: ghcr.io/txtdot/txtdot:latest
ports: ports:
- "8080:8080" - '8080:8080'
restart: unless-stopped restart: unless-stopped
volumes: volumes:
- ".env:/app/.env" - '.env:/app/.env'

421
package-lock.json generated
View File

@ -21,8 +21,8 @@
"fastify": "^4.21.0", "fastify": "^4.21.0",
"iconv-lite": "^0.6.3", "iconv-lite": "^0.6.3",
"ip-range-check": "^0.2.0", "ip-range-check": "^0.2.0",
"jsdom": "^22.1.0",
"json-schema-to-ts": "^2.9.2", "json-schema-to-ts": "^2.9.2",
"linkedom": "^0.16.4",
"micromatch": "^4.0.5" "micromatch": "^4.0.5"
}, },
"devDependencies": { "devDependencies": {
@ -314,14 +314,6 @@
"node": ">= 8" "node": ">= 8"
} }
}, },
"node_modules/@tootallnate/once": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz",
"integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==",
"engines": {
"node": ">= 10"
}
},
"node_modules/@types/braces": { "node_modules/@types/braces": {
"version": "3.0.4", "version": "3.0.4",
"resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.4.tgz", "resolved": "https://registry.npmjs.org/@types/braces/-/braces-3.0.4.tgz",
@ -590,11 +582,6 @@
"integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==", "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==",
"dev": true "dev": true
}, },
"node_modules/abab": {
"version": "2.0.6",
"resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz",
"integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA=="
},
"node_modules/abort-controller": { "node_modules/abort-controller": {
"version": "3.0.0", "version": "3.0.0",
"resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
@ -632,17 +619,6 @@
"acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0"
} }
}, },
"node_modules/agent-base": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
"integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
"dependencies": {
"debug": "4"
},
"engines": {
"node": ">= 6.0.0"
}
},
"node_modules/ajv": { "node_modules/ajv": {
"version": "6.12.6", "version": "6.12.6",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@ -822,6 +798,11 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/boolbase": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww=="
},
"node_modules/brace-expansion": { "node_modules/brace-expansion": {
"version": "1.1.11", "version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@ -1096,30 +1077,37 @@
"node": ">= 8" "node": ">= 8"
} }
}, },
"node_modules/cssstyle": { "node_modules/css-select": {
"version": "3.0.0", "version": "5.1.0",
"resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-3.0.0.tgz", "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz",
"integrity": "sha512-N4u2ABATi3Qplzf0hWbVCdjenim8F3ojEXpBDF5hBpjzW182MjNGLqfmQ0SkSPeQ+V86ZXgeH8aXj6kayd4jgg==", "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==",
"dependencies": { "dependencies": {
"rrweb-cssom": "^0.6.0" "boolbase": "^1.0.0",
"css-what": "^6.1.0",
"domhandler": "^5.0.2",
"domutils": "^3.0.1",
"nth-check": "^2.0.1"
}, },
"engines": { "funding": {
"node": ">=14" "url": "https://github.com/sponsors/fb55"
} }
}, },
"node_modules/data-urls": { "node_modules/css-what": {
"version": "4.0.0", "version": "6.1.0",
"resolved": "https://registry.npmjs.org/data-urls/-/data-urls-4.0.0.tgz", "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz",
"integrity": "sha512-/mMTei/JXPqvFqQtfyTowxmJVwr2PVAeCcDxyFf6LhoOu/09TX2OX3kb2wzi4DMXcfj4OItwDOnhl5oziPnT6g==", "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==",
"dependencies": {
"abab": "^2.0.6",
"whatwg-mimetype": "^3.0.0",
"whatwg-url": "^12.0.0"
},
"engines": { "engines": {
"node": ">=14" "node": ">= 6"
},
"funding": {
"url": "https://github.com/sponsors/fb55"
} }
}, },
"node_modules/cssom": {
"version": "0.5.0",
"resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz",
"integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw=="
},
"node_modules/debug": { "node_modules/debug": {
"version": "4.3.4", "version": "4.3.4",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz",
@ -1136,11 +1124,6 @@
} }
} }
}, },
"node_modules/decimal.js": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
"integrity": "sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA=="
},
"node_modules/deep-is": { "node_modules/deep-is": {
"version": "0.1.4", "version": "0.1.4",
"resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz",
@ -1187,15 +1170,42 @@
"node": ">=6.0.0" "node": ">=6.0.0"
} }
}, },
"node_modules/domexception": { "node_modules/dom-serializer": {
"version": "4.0.0", "version": "2.0.0",
"resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
"integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
"dependencies": { "dependencies": {
"webidl-conversions": "^7.0.0" "domelementtype": "^2.3.0",
"domhandler": "^5.0.2",
"entities": "^4.2.0"
},
"funding": {
"url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
}
},
"node_modules/domelementtype": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
"integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
]
},
"node_modules/domhandler": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
"integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
"dependencies": {
"domelementtype": "^2.3.0"
}, },
"engines": { "engines": {
"node": ">=12" "node": ">= 4"
},
"funding": {
"url": "https://github.com/fb55/domhandler?sponsor=1"
} }
}, },
"node_modules/dompurify": { "node_modules/dompurify": {
@ -1203,6 +1213,19 @@
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.6.tgz", "resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.6.tgz",
"integrity": "sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w==" "integrity": "sha512-ilkD8YEnnGh1zJ240uJsW7AzE+2qpbOUYjacomn3AvJ6J4JhKGSZ2nh4wUIXPZrEPppaCLx5jFe8T89Rk8tQ7w=="
}, },
"node_modules/domutils": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-3.1.0.tgz",
"integrity": "sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==",
"dependencies": {
"dom-serializer": "^2.0.0",
"domelementtype": "^2.3.0",
"domhandler": "^5.0.3"
},
"funding": {
"url": "https://github.com/fb55/domutils?sponsor=1"
}
},
"node_modules/dotenv": { "node_modules/dotenv": {
"version": "16.3.1", "version": "16.3.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz",
@ -1872,15 +1895,27 @@
"resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz", "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz",
"integrity": "sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A==" "integrity": "sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A=="
}, },
"node_modules/html-encoding-sniffer": { "node_modules/html-escaper": {
"version": "3.0.0", "version": "3.0.3",
"resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz",
"integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ=="
},
"node_modules/htmlparser2": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.0.0.tgz",
"integrity": "sha512-uxbSI98wmFT/G4P2zXx4OVx04qWUmyFPrD2/CNepa2Zo3GPNaCaaxElDgwUrwYWkK1nr9fft0Ya8dws8coDLLQ==",
"funding": [
"https://github.com/fb55/htmlparser2?sponsor=1",
{
"type": "github",
"url": "https://github.com/sponsors/fb55"
}
],
"dependencies": { "dependencies": {
"whatwg-encoding": "^2.0.0" "domelementtype": "^2.3.0",
}, "domhandler": "^5.0.3",
"engines": { "domutils": "^3.1.0",
"node": ">=12" "entities": "^4.5.0"
} }
}, },
"node_modules/http-errors": { "node_modules/http-errors": {
@ -1898,31 +1933,6 @@
"node": ">= 0.8" "node": ">= 0.8"
} }
}, },
"node_modules/http-proxy-agent": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz",
"integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==",
"dependencies": {
"@tootallnate/once": "2",
"agent-base": "6",
"debug": "4"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/https-proxy-agent": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
"integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
"dependencies": {
"agent-base": "6",
"debug": "4"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/iconv-lite": { "node_modules/iconv-lite": {
"version": "0.6.3", "version": "0.6.3",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
@ -2076,11 +2086,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/is-potential-custom-element-name": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz",
"integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ=="
},
"node_modules/isarray": { "node_modules/isarray": {
"version": "0.0.1", "version": "0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz",
@ -2122,47 +2127,6 @@
"js-yaml": "bin/js-yaml.js" "js-yaml": "bin/js-yaml.js"
} }
}, },
"node_modules/jsdom": {
"version": "22.1.0",
"resolved": "https://registry.npmjs.org/jsdom/-/jsdom-22.1.0.tgz",
"integrity": "sha512-/9AVW7xNbsBv6GfWho4TTNjEo9fe6Zhf9O7s0Fhhr3u+awPwAJMKwAMXnkk5vBxflqLW9hTHX/0cs+P3gW+cQw==",
"dependencies": {
"abab": "^2.0.6",
"cssstyle": "^3.0.0",
"data-urls": "^4.0.0",
"decimal.js": "^10.4.3",
"domexception": "^4.0.0",
"form-data": "^4.0.0",
"html-encoding-sniffer": "^3.0.0",
"http-proxy-agent": "^5.0.0",
"https-proxy-agent": "^5.0.1",
"is-potential-custom-element-name": "^1.0.1",
"nwsapi": "^2.2.4",
"parse5": "^7.1.2",
"rrweb-cssom": "^0.6.0",
"saxes": "^6.0.0",
"symbol-tree": "^3.2.4",
"tough-cookie": "^4.1.2",
"w3c-xmlserializer": "^4.0.0",
"webidl-conversions": "^7.0.0",
"whatwg-encoding": "^2.0.0",
"whatwg-mimetype": "^3.0.0",
"whatwg-url": "^12.0.1",
"ws": "^8.13.0",
"xml-name-validator": "^4.0.0"
},
"engines": {
"node": ">=16"
},
"peerDependencies": {
"canvas": "^2.5.0"
},
"peerDependenciesMeta": {
"canvas": {
"optional": true
}
}
},
"node_modules/json-buffer": { "node_modules/json-buffer": {
"version": "3.0.1", "version": "3.0.1",
"resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
@ -2250,6 +2214,18 @@
"set-cookie-parser": "^2.4.1" "set-cookie-parser": "^2.4.1"
} }
}, },
"node_modules/linkedom": {
"version": "0.16.4",
"resolved": "https://registry.npmjs.org/linkedom/-/linkedom-0.16.4.tgz",
"integrity": "sha512-SykvDVh/jAnaO+WiPqH5vX3QpZrIRImuppzYhIHons3RXPhDwqN2dOyfopOVaHleqWtoS+3vWCqen+m8M3HToQ==",
"dependencies": {
"css-select": "^5.1.0",
"cssom": "^0.5.0",
"html-escaper": "^3.0.3",
"htmlparser2": "^9.0.0",
"uhyphen": "^0.2.0"
}
},
"node_modules/locate-path": { "node_modules/locate-path": {
"version": "6.0.0", "version": "6.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
@ -2398,10 +2374,16 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/nwsapi": { "node_modules/nth-check": {
"version": "2.2.7", "version": "2.1.1",
"resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.7.tgz", "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
"integrity": "sha512-ub5E4+FBPKwAZx0UwIQOjYWGHTEq5sPqHQNRN8Z9e4A7u3Tj1weLJsL59yH9vmvqEtBHaOmT6cYQKIZOxp35FQ==" "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
"dependencies": {
"boolbase": "^1.0.0"
},
"funding": {
"url": "https://github.com/fb55/nth-check?sponsor=1"
}
}, },
"node_modules/on-exit-leak-free": { "node_modules/on-exit-leak-free": {
"version": "2.1.2", "version": "2.1.2",
@ -2486,6 +2468,7 @@
"version": "7.1.2", "version": "7.1.2",
"resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz",
"integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==",
"dev": true,
"dependencies": { "dependencies": {
"entities": "^4.4.0" "entities": "^4.4.0"
}, },
@ -2667,11 +2650,6 @@
"node": ">= 0.10" "node": ">= 0.10"
} }
}, },
"node_modules/psl": {
"version": "1.9.0",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.9.0.tgz",
"integrity": "sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag=="
},
"node_modules/punycode": { "node_modules/punycode": {
"version": "2.3.1", "version": "2.3.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.3.1.tgz",
@ -2680,11 +2658,6 @@
"node": ">=6" "node": ">=6"
} }
}, },
"node_modules/querystringify": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz",
"integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ=="
},
"node_modules/queue-microtask": { "node_modules/queue-microtask": {
"version": "1.2.3", "version": "1.2.3",
"resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz",
@ -2764,11 +2737,6 @@
"node": ">=0.10.0" "node": ">=0.10.0"
} }
}, },
"node_modules/requires-port": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz",
"integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ=="
},
"node_modules/resolve-from": { "node_modules/resolve-from": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
@ -2835,11 +2803,6 @@
"url": "https://github.com/sponsors/isaacs" "url": "https://github.com/sponsors/isaacs"
} }
}, },
"node_modules/rrweb-cssom": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/rrweb-cssom/-/rrweb-cssom-0.6.0.tgz",
"integrity": "sha512-APM0Gt1KoXBz0iIkkdB/kfvGOwC4UuJFeG/c+yV7wSc7q96cG/kJ0HiYCnzivD9SB53cLV1MlHFNfOuPaadYSw=="
},
"node_modules/run-parallel": { "node_modules/run-parallel": {
"version": "1.2.0", "version": "1.2.0",
"resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz",
@ -2903,17 +2866,6 @@
"resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
}, },
"node_modules/saxes": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz",
"integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==",
"dependencies": {
"xmlchars": "^2.2.0"
},
"engines": {
"node": ">=v12.22.7"
}
},
"node_modules/secure-json-parse": { "node_modules/secure-json-parse": {
"version": "2.7.0", "version": "2.7.0",
"resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz",
@ -3091,11 +3043,6 @@
"node": ">=8" "node": ">=8"
} }
}, },
"node_modules/symbol-tree": {
"version": "3.2.4",
"resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz",
"integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw=="
},
"node_modules/text-table": { "node_modules/text-table": {
"version": "0.2.0", "version": "0.2.0",
"resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz",
@ -3189,31 +3136,6 @@
"node": ">=0.6" "node": ">=0.6"
} }
}, },
"node_modules/tough-cookie": {
"version": "4.1.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.1.3.tgz",
"integrity": "sha512-aX/y5pVRkfRnfmuX+OdbSdXvPe6ieKX/G2s7e98f4poJHnqH3281gDPm/metm6E/WRamfx7WC4HUqkWHfQHprw==",
"dependencies": {
"psl": "^1.1.33",
"punycode": "^2.1.1",
"universalify": "^0.2.0",
"url-parse": "^1.5.3"
},
"engines": {
"node": ">=6"
}
},
"node_modules/tr46": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/tr46/-/tr46-4.1.1.tgz",
"integrity": "sha512-2lv/66T7e5yNyhAAC4NaKe5nVavzuGJQVVtRYLyQ2OI8tsJ61PMLlelehb0wi2Hx6+hT/OJUWZcw8MjlSRnxvw==",
"dependencies": {
"punycode": "^2.3.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/ts-algebra": { "node_modules/ts-algebra": {
"version": "1.2.2", "version": "1.2.2",
"resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-1.2.2.tgz", "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-1.2.2.tgz",
@ -3289,20 +3211,17 @@
"node": ">=14.17" "node": ">=14.17"
} }
}, },
"node_modules/uhyphen": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/uhyphen/-/uhyphen-0.2.0.tgz",
"integrity": "sha512-qz3o9CHXmJJPGBdqzab7qAYuW8kQGKNEuoHFYrBwV6hWIMcpAmxDLXojcHfFr9US1Pe6zUswEIJIbLI610fuqA=="
},
"node_modules/undici-types": { "node_modules/undici-types": {
"version": "5.26.5", "version": "5.26.5",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
"integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==",
"dev": true "dev": true
}, },
"node_modules/universalify": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz",
"integrity": "sha512-CJ1QgKmNg3CwvAv/kOFmtnEN05f0D/cn9QntgNOQlQF9dgvVTHj3t+8JPdjqawCHk7V/KA+fbUqzZ9XWhcqPUg==",
"engines": {
"node": ">= 4.0.0"
}
},
"node_modules/untildify": { "node_modules/untildify": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz", "resolved": "https://registry.npmjs.org/untildify/-/untildify-4.0.0.tgz",
@ -3320,71 +3239,12 @@
"punycode": "^2.1.0" "punycode": "^2.1.0"
} }
}, },
"node_modules/url-parse": {
"version": "1.5.10",
"resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz",
"integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==",
"dependencies": {
"querystringify": "^2.1.1",
"requires-port": "^1.0.0"
}
},
"node_modules/util-deprecate": { "node_modules/util-deprecate": {
"version": "1.0.2", "version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
"dev": true "dev": true
}, },
"node_modules/w3c-xmlserializer": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-4.0.0.tgz",
"integrity": "sha512-d+BFHzbiCx6zGfz0HyQ6Rg69w9k19nviJspaj4yNscGjrHu94sVP+aRm75yEbCh+r2/yR+7q6hux9LVtbuTGBw==",
"dependencies": {
"xml-name-validator": "^4.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/webidl-conversions": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz",
"integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==",
"engines": {
"node": ">=12"
}
},
"node_modules/whatwg-encoding": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz",
"integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==",
"dependencies": {
"iconv-lite": "0.6.3"
},
"engines": {
"node": ">=12"
}
},
"node_modules/whatwg-mimetype": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz",
"integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==",
"engines": {
"node": ">=12"
}
},
"node_modules/whatwg-url": {
"version": "12.0.1",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-12.0.1.tgz",
"integrity": "sha512-Ed/LrqB8EPlGxjS+TrsXcpUond1mhccS3pchLhzSgPCnTimUCKj3IZE75pAs5m6heB2U2TMerKFUXheyHY+VDQ==",
"dependencies": {
"tr46": "^4.1.1",
"webidl-conversions": "^7.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/which": { "node_modules/which": {
"version": "2.0.2", "version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
@ -3422,39 +3282,6 @@
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="
}, },
"node_modules/ws": {
"version": "8.14.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.14.2.tgz",
"integrity": "sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g==",
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
},
"node_modules/xml-name-validator": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz",
"integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==",
"engines": {
"node": ">=12"
}
},
"node_modules/xmlchars": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz",
"integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw=="
},
"node_modules/xtend": { "node_modules/xtend": {
"version": "4.0.2", "version": "4.0.2",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz",

6
package.json
View File

@ -17,8 +17,8 @@
"fastify": "^4.21.0", "fastify": "^4.21.0",
"iconv-lite": "^0.6.3", "iconv-lite": "^0.6.3",
"ip-range-check": "^0.2.0", "ip-range-check": "^0.2.0",
"jsdom": "^22.1.0",
"json-schema-to-ts": "^2.9.2", "json-schema-to-ts": "^2.9.2",
"linkedom": "^0.16.4",
"micromatch": "^4.0.5" "micromatch": "^4.0.5"
}, },
"devDependencies": { "devDependencies": {
@ -41,7 +41,9 @@
"start": "cd ./dist && node ./src/app.js", "start": "cd ./dist && node ./src/app.js",
"start:docker": "node ./src/app.js", "start:docker": "node ./src/app.js",
"clean-css": "cleancss --batch static/*.css -o dist/static --batch-suffix \"\"", "clean-css": "cleancss --batch static/*.css -o dist/static --batch-suffix \"\"",
"dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"" "dev": "tsc-watch --onSuccess \"node ./dist/src/app.js\"",
"format": "prettier --write .",
"format:check": "prettier --check ."
}, },
"keywords": [], "keywords": [],
"authors": [ "authors": [

53
src/app.ts
View File

@ -1,21 +1,21 @@
import path from "path"; import path from 'path';
import Fastify from "fastify"; import Fastify from 'fastify';
import fastifyStatic from "@fastify/static"; import fastifyStatic from '@fastify/static';
import fastifyView from "@fastify/view"; import fastifyView from '@fastify/view';
import fastifySwagger from "@fastify/swagger"; import fastifySwagger from '@fastify/swagger';
import fastifySwaggerUi from "@fastify/swagger-ui"; import fastifySwaggerUi from '@fastify/swagger-ui';
import ejs from "ejs"; import ejs from 'ejs';
import indexRoute from "./routes/browser/index"; import indexRoute from './routes/browser/index';
import getRoute from "./routes/browser/get"; import getRoute from './routes/browser/get';
import proxyRoute from "./routes/browser/proxy"; import proxyRoute from './routes/browser/proxy';
import parseRoute from "./routes/api/parse"; import parseRoute from './routes/api/parse';
import rawHtml from "./routes/api/raw-html"; import rawHtml from './routes/api/raw-html';
import publicConfig from "./publicConfig"; import publicConfig from './publicConfig';
import errorHandler from "./errors/handler"; import errorHandler from './errors/handler';
import getConfig from "./config/main"; import getConfig from './config/main';
class App { class App {
async init() { async init() {
@ -24,11 +24,12 @@ class App {
const fastify = Fastify({ const fastify = Fastify({
logger: true, logger: true,
trustProxy: config.reverse_proxy, trustProxy: config.reverse_proxy,
connectionTimeout: config.timeout,
}); });
fastify.register(fastifyStatic, { fastify.register(fastifyStatic, {
root: path.join(process.cwd(), "static"), root: path.join(process.cwd(), 'static'),
prefix: "/static/", prefix: '/static/',
}); });
fastify.register(fastifyView, { fastify.register(fastifyView, {
@ -41,32 +42,28 @@ class App {
await fastify.register(fastifySwagger, { await fastify.register(fastifySwagger, {
swagger: { swagger: {
info: { info: {
title: "TXTDot API", title: 'TXTDot API',
description: publicConfig.description, description: publicConfig.description,
version: publicConfig.version, version: publicConfig.version,
}, },
} },
}); });
await fastify.register(fastifySwaggerUi, { routePrefix: "/doc" }); await fastify.register(fastifySwaggerUi, { routePrefix: '/doc' });
} }
fastify.register(indexRoute); fastify.register(indexRoute);
fastify.register(getRoute); fastify.register(getRoute);
if (config.proxy_res) if (config.proxy_res) fastify.register(proxyRoute);
fastify.register(proxyRoute);
fastify.register(parseRoute); fastify.register(parseRoute);
fastify.register(rawHtml); fastify.register(rawHtml);
fastify.setErrorHandler(errorHandler); fastify.setErrorHandler(errorHandler);
fastify.listen( fastify.listen({ host: config.host, port: config.port }, (err) => {
{ host: config.host, port: config.port }, err && console.log(err);
(err) => { });
err && console.log(err);
}
);
} }
} }

9
src/config/config.service.ts
View File

@ -1,8 +1,9 @@
import { config } from "dotenv"; import { config } from 'dotenv';
export class ConfigService { export class ConfigService {
public readonly host: string; public readonly host: string;
public readonly port: number; public readonly port: number;
public readonly timeout: number;
public readonly reverse_proxy: boolean; public readonly reverse_proxy: boolean;
public readonly proxy_res: boolean; public readonly proxy_res: boolean;
public readonly swagger: boolean; public readonly swagger: boolean;
@ -10,9 +11,11 @@ export class ConfigService {
constructor() { constructor() {
config(); config();
this.host = process.env.HOST || "0.0.0.0"; this.host = process.env.HOST || '0.0.0.0';
this.port = Number(process.env.PORT) || 8080; this.port = Number(process.env.PORT) || 8080;
this.timeout = 1000;
this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false); this.reverse_proxy = this.parseBool(process.env.REVERSE_PROXY, false);
this.proxy_res = this.parseBool(process.env.PROXY_RES, true); this.proxy_res = this.parseBool(process.env.PROXY_RES, true);
@ -21,6 +24,6 @@ export class ConfigService {
parseBool(value: string | undefined, def: boolean): boolean { parseBool(value: string | undefined, def: boolean): boolean {
if (!value) return def; if (!value) return def;
return value === "true" || value === "1"; return value === 'true' || value === '1';
} }
} }

2
src/config/main.ts
View File

@ -1,4 +1,4 @@
import { ConfigService } from "./config.service"; import { ConfigService } from './config.service';
let configSvc: ConfigService | undefined; let configSvc: ConfigService | undefined;

18
src/errors/api.ts
View File

@ -5,28 +5,28 @@ export interface IApiError {
} }
export const errorSchema = { export const errorSchema = {
type: "object", type: 'object',
properties: { properties: {
code: { code: {
type: "number", type: 'number',
description: "HTTP error code", description: 'HTTP error code',
}, },
name: { name: {
type: "string", type: 'string',
description: "Exception class name", description: 'Exception class name',
}, },
message: { message: {
type: "string", type: 'string',
description: "Exception message", description: 'Exception message',
}, },
}, },
}; };
export const errorResponseSchema = { export const errorResponseSchema = {
type: "object", type: 'object',
properties: { properties: {
data: { data: {
type: "object", type: 'object',
nullable: true, nullable: true,
}, },
error: errorSchema, error: errorSchema,

37
src/errors/handler.ts
View File

@ -1,16 +1,16 @@
import { FastifyReply, FastifyRequest } from "fastify"; import { FastifyReply, FastifyRequest } from 'fastify';
import { NotHtmlMimetypeError, TxtDotError } from "./main"; import { NotHtmlMimetypeError, TxtDotError } from './main';
import { getFastifyError } from "./validation"; import { getFastifyError } from './validation';
import { IGetSchema } from "../types/requests/browser"; import { IGetSchema } from '../types/requests/browser';
import getConfig from "../config/main"; import getConfig from '../config/main';
export default function errorHandler( export default function errorHandler(
error: Error, error: Error,
req: FastifyRequest, req: FastifyRequest,
reply: FastifyReply reply: FastifyReply
) { ) {
if (req.originalUrl.startsWith("/api/")) { if (req.originalUrl.startsWith('/api/')) {
return apiErrorHandler(error, reply); return apiErrorHandler(error, reply);
} }
@ -43,26 +43,23 @@ function apiErrorHandler(error: Error, reply: FastifyReply) {
function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) { function htmlErrorHandler(error: Error, reply: FastifyReply, url: string) {
if (getFastifyError(error)?.statusCode === 400) { if (getFastifyError(error)?.statusCode === 400) {
return reply.code(400).view("/templates/error.ejs", { return reply.code(400).view('/templates/error.ejs', {
url, url,
code: 400, code: 400,
description: `Invalid parameter specified: ${error.message}`, description: `Invalid parameter specified: ${error.message}`,
})
}
if (error instanceof TxtDotError) {
return reply.code(error.code).view("/templates/error.ejs", {
url,
code: error.code,
description: error.description,
proxyBtn: (
error instanceof NotHtmlMimetypeError &&
getConfig().proxy_res
),
}); });
} }
return reply.code(500).view("/templates/error.ejs", { if (error instanceof TxtDotError) {
return reply.code(error.code).view('/templates/error.ejs', {
url,
code: error.code,
description: error.description,
proxyBtn: error instanceof NotHtmlMimetypeError && getConfig().proxy_res,
});
}
return reply.code(500).view('/templates/error.ejs', {
url, url,
code: 500, code: 500,
description: `${error.name}: ${error.message}`, description: `${error.name}: ${error.message}`,

31
src/errors/main.ts
View File

@ -1,15 +1,11 @@
import getConfig from "../config/main"; import getConfig from '../config/main';
export abstract class TxtDotError extends Error { export abstract class TxtDotError extends Error {
code: number; code: number;
name: string; name: string;
description: string; description: string;
constructor( constructor(code: number, name: string, description: string) {
code: number,
name: string,
description: string,
) {
super(description); super(description);
this.code = code; this.code = code;
this.name = name; this.name = name;
@ -19,21 +15,13 @@ export abstract class TxtDotError extends Error {
export class EngineParseError extends TxtDotError { export class EngineParseError extends TxtDotError {
constructor(message: string) { constructor(message: string) {
super( super(422, 'EngineParseError', `Parse error: ${message}`);
422,
"EngineParseError",
`Parse error: ${message}`,
);
} }
} }
export class LocalResourceError extends TxtDotError { export class LocalResourceError extends TxtDotError {
constructor() { constructor() {
super( super(403, 'LocalResourceError', 'Proxying local resources is forbidden.');
403,
"LocalResourceError",
"Proxying local resources is forbidden.",
);
} }
} }
@ -41,12 +29,11 @@ export class NotHtmlMimetypeError extends TxtDotError {
constructor() { constructor() {
super( super(
421, 421,
"NotHtmlMimetypeError", 'NotHtmlMimetypeError',
"Received non-HTML content, " + ( 'Received non-HTML content, ' +
getConfig().proxy_res ? (getConfig().proxy_res
"use proxy instead of parser." : ? 'use proxy instead of parser.'
"proxying is disabled by the instance admin." : 'proxying is disabled by the instance admin.')
),
); );
} }
} }

30
src/handlers/google.ts
View File

@ -1,6 +1,6 @@
import { HandlerInput } from "./handler-input"; import { HandlerInput } from './handler-input';
import { IHandlerOutput } from "./handler.interface"; import { IHandlerOutput } from './handler.interface';
import { EngineParseError } from "../errors/main"; import { EngineParseError } from '../errors/main';
export default async function google( export default async function google(
input: HandlerInput input: HandlerInput
@ -8,18 +8,18 @@ export default async function google(
const window = input.parseDom().window; const window = input.parseDom().window;
const googleAnchors = [ const googleAnchors = [
...window.document.querySelectorAll("a[jsname=UWckNb]"), ...window.document.querySelectorAll('a[jsname=UWckNb]'),
] as HTMLAnchorElement[]; ] as HTMLAnchorElement[];
if (!googleAnchors) { if (!googleAnchors) {
throw new EngineParseError( throw new EngineParseError(
"Failed to find anchors in search result [google]" 'Failed to find anchors in search result [google]'
); );
} }
const results = googleAnchors const results = googleAnchors
.map((a: HTMLAnchorElement): GoogleProps => { .map((a: HTMLAnchorElement): GoogleProps => {
const parsedHref = new URL(new URL(a.href).searchParams.get("url")!); const parsedHref = new URL(new URL(a.href).searchParams.get('url')!);
return { return {
href: a.href!, href: a.href!,
siteName: parsedHref.hostname, siteName: parsedHref.hostname,
@ -43,7 +43,7 @@ export default async function google(
}); });
const search = window.document.getElementById( const search = window.document.getElementById(
"APjFqb" 'APjFqb'
) as HTMLTextAreaElement; ) as HTMLTextAreaElement;
const searchForm = ` const searchForm = `
@ -54,18 +54,18 @@ export default async function google(
`; `;
return { return {
content: `${searchForm}${content.join("")}`, content: `${searchForm}${content.join('')}`,
textContent: textContent.join("\n"), textContent: textContent.join('\n'),
}; };
} }
export const GoogleDomains = [ export const GoogleDomains = [
"google.*", 'google.*',
"google.co.*", 'google.co.*',
"google.com.*", 'google.com.*',
"www.google.*", 'www.google.*',
"www.google.co.*", 'www.google.co.*',
"www.google.com.*", 'www.google.com.*',
]; ];
interface GoogleProps { interface GoogleProps {

13
src/handlers/handler-input.ts
View File

@ -1,14 +1,11 @@
import { JSDOM } from "jsdom"; import { parseHTML } from 'linkedom';
export class HandlerInput { export class HandlerInput {
private data: string; private data: string;
private url: string; private url: string;
private dom?: JSDOM; private dom?: Window;
constructor( constructor(data: string, url: string) {
data: string,
url: string,
) {
this.data = data; this.data = data;
this.url = url; this.url = url;
} }
@ -17,12 +14,12 @@ export class HandlerInput {
return this.url; return this.url;
} }
parseDom(): JSDOM { parseDom(): Window {
if (this.dom) { if (this.dom) {
return this.dom; return this.dom;
} }
this.dom = new JSDOM(this.data, { url: this.url }); this.dom = parseHTML(this.data);
return this.dom; return this.dom;
} }
} }

10
src/handlers/handler.interface.ts
View File

@ -6,19 +6,19 @@ export interface IHandlerOutput {
} }
export const handlerSchema = { export const handlerSchema = {
type: "object", type: 'object',
properties: { properties: {
content: { content: {
type: "string", type: 'string',
}, },
textContent: { textContent: {
type: "string", type: 'string',
}, },
title: { title: {
type: "string", type: 'string',
}, },
lang: { lang: {
type: "string", type: 'string',
}, },
}, },
}; };

50
src/handlers/main.ts
View File

@ -1,42 +1,42 @@
import { IHandlerOutput } from "./handler.interface"; import { IHandlerOutput } from './handler.interface';
import { Engines, EngineFunction, EnginesMatch } from "../types/handlers"; import { Engines, EngineFunction, EnginesMatch } from '../types/handlers';
import axios from "../types/axios"; import axios from '../types/axios';
import micromatch from "micromatch"; import micromatch from 'micromatch';
import { JSDOM } from "jsdom"; import DOMPurify from 'dompurify';
import DOMPurify from "dompurify";
import { Readable } from "stream"; import { Readable } from 'stream';
import readability from "./readability"; import readability from './readability';
import google, { GoogleDomains } from "./google"; import google, { GoogleDomains } from './google';
import stackoverflow, { StackOverflowDomains } from "./stackoverflow/main"; import stackoverflow, { StackOverflowDomains } from './stackoverflow/main';
import isLocalResource from "../utils/islocal"; import isLocalResource from '../utils/islocal';
import { LocalResourceError, NotHtmlMimetypeError } from "../errors/main"; import { LocalResourceError, NotHtmlMimetypeError } from '../errors/main';
import { HandlerInput } from "./handler-input"; import { HandlerInput } from './handler-input';
import { decodeStream, parseEncodingName } from "../utils/http"; import { decodeStream, parseEncodingName } from '../utils/http';
import replaceHref from "../utils/replace-href"; import replaceHref from '../utils/replace-href';
import { parseHTML } from 'linkedom';
export default async function handlePage( export default async function handlePage(
url: string, // remote URL remoteUrl: string, // remote URL
requestUrl: URL, // proxy URL requestUrl: URL, // proxy URL
engine?: string, engine?: string,
redirectPath: string = "get", redirectPath: string = 'get'
): Promise<IHandlerOutput> { ): Promise<IHandlerOutput> {
const urlObj = new URL(url); const urlObj = new URL(remoteUrl);
if (await isLocalResource(urlObj)) { if (await isLocalResource(urlObj)) {
throw new LocalResourceError(); throw new LocalResourceError();
} }
const response = await axios.get(url); const response = await axios.get(remoteUrl);
const data: Readable = response.data; const data: Readable = response.data;
const mime: string | undefined = response.headers["content-type"]?.toString(); const mime: string | undefined = response.headers['content-type']?.toString();
if (mime && mime.indexOf("text/html") === -1) { if (mime && mime.indexOf('text/html') === -1) {
throw new NotHtmlMimetypeError(); throw new NotHtmlMimetypeError();
} }
@ -44,17 +44,17 @@ export default async function handlePage(
const output = await handler( const output = await handler(
new HandlerInput( new HandlerInput(
await decodeStream(data, parseEncodingName(mime)), await decodeStream(data, parseEncodingName(mime)),
url, remoteUrl
) )
); );
// post-process // post-process
const dom = new JSDOM(output.content, { url }); const dom = parseHTML(output.content);
replaceHref(dom, requestUrl, engine, redirectPath); replaceHref(dom, requestUrl, new URL(remoteUrl), engine, redirectPath);
const purify = DOMPurify(dom.window); const purify = DOMPurify(dom.window);
output.content = purify.sanitize(dom.serialize()); output.content = purify.sanitize(dom.document.toString());
return output; return output;
} }

12
src/handlers/readability.ts
View File

@ -1,16 +1,16 @@
import { Readability } from "@mozilla/readability"; import { Readability } from '@mozilla/readability';
import { HandlerInput } from "./handler-input"; import { HandlerInput } from './handler-input';
import { IHandlerOutput } from "./handler.interface"; import { IHandlerOutput } from './handler.interface';
import { EngineParseError } from "../errors/main"; import { EngineParseError } from '../errors/main';
export default async function readability( export default async function readability(
input: HandlerInput, input: HandlerInput
): Promise<IHandlerOutput> { ): Promise<IHandlerOutput> {
const reader = new Readability(input.parseDom().window.document); const reader = new Readability(input.parseDom().window.document);
const parsed = reader.parse(); const parsed = reader.parse();
if (!parsed) { if (!parsed) {
throw new EngineParseError("Failed to parse [readability]"); throw new EngineParseError('Failed to parse [readability]');
} }
return { return {

42
src/handlers/stackoverflow/main.ts
View File

@ -1,30 +1,30 @@
import { HandlerInput } from "../handler-input"; import { HandlerInput } from '../handler-input';
import { IHandlerOutput } from "../handler.interface"; import { IHandlerOutput } from '../handler.interface';
import { EngineParseError } from "../../errors/main"; import { EngineParseError } from '../../errors/main';
import qPostsHandler from "./questions-posts"; import qPostsHandler from './questions-posts';
export default async function stackoverflow( export default async function stackoverflow(
input: HandlerInput, input: HandlerInput
): Promise<IHandlerOutput> { ): Promise<IHandlerOutput> {
const window = input.parseDom().window; const window = input.parseDom().window;
const url = new URL(window.location.href); const url = new URL(window.location.href);
const path = url.pathname.split("/").filter((p) => p !== ""); const path = url.pathname.split('/').filter((p) => p !== '');
let result: IHandlerOutput = { let result: IHandlerOutput = {
content: "", content: '',
textContent: "", textContent: '',
title: "", title: '',
lang: "", lang: '',
}; };
if (path[0] === "questions") { if (path[0] === 'questions') {
if (path.length === 3) { if (path.length === 3) {
result = await qPostsHandler(window); result = await qPostsHandler(window);
} else if (path.length === 1) { } else if (path.length === 1) {
result.content = "questions"; result.content = 'questions';
} else { } else {
throw new EngineParseError("Invalid URL [stackoverflow]"); throw new EngineParseError('Invalid URL [stackoverflow]');
} }
} }
@ -32,12 +32,12 @@ export default async function stackoverflow(
} }
export const StackOverflowDomains = [ export const StackOverflowDomains = [
"stackoverflow.com", 'stackoverflow.com',
"*.stackoverflow.com", '*.stackoverflow.com',
"*.stackexchange.com", '*.stackexchange.com',
"askubuntu.com", 'askubuntu.com',
"stackapps.com", 'stackapps.com',
"mathoverflow.net", 'mathoverflow.net',
"superuser.com", 'superuser.com',
"serverfault.com", 'serverfault.com',
]; ];

6
src/handlers/stackoverflow/post-parser.ts
View File

@ -1,9 +1,9 @@
export default function postParser(el: Element | null): string { export default function postParser(el: Element | null): string {
if (!el) { if (!el) {
return ""; return '';
} }
const body = el.querySelector(".js-post-body")?.innerHTML || ""; const body = el.querySelector('.js-post-body')?.innerHTML || '';
const voteCount = el.querySelector(".js-vote-count")?.textContent || ""; const voteCount = el.querySelector('.js-vote-count')?.textContent || '';
return `<h3>${voteCount} votes</h3>${body}`; return `<h3>${voteCount} votes</h3>${body}`;
} }

19
src/handlers/stackoverflow/questions-posts.ts
View File

@ -1,26 +1,25 @@
import { DOMWindow } from "jsdom"; import { IHandlerOutput } from '../handler.interface';
import { IHandlerOutput } from "../handler.interface"; import postParser from './post-parser';
import postParser from "./post-parser";
export default async function qPostsHandler( export default async function qPostsHandler(
window: DOMWindow window: Window
): Promise<IHandlerOutput> { ): Promise<IHandlerOutput> {
const questionEl = window.document.getElementById("question"); const questionEl = window.document.getElementById('question');
const question = postParser(questionEl); const question = postParser(questionEl);
const title = const title =
window.document.querySelector(".question-hyperlink")?.innerHTML || ""; window.document.querySelector('.question-hyperlink')?.innerHTML || '';
const allAnswers = [...window.document.querySelectorAll(".answer")]; const allAnswers = [...window.document.querySelectorAll('.answer')];
const answers = allAnswers.map((a) => postParser(a)); const answers = allAnswers.map((a) => postParser(a));
return { return {
content: `${question}<hr>${answers.length} answers <hr>${answers.join( content: `${question}<hr>${answers.length} answers <hr>${answers.join(
"<hr>" '<hr>'
)}`, )}`,
textContent: "question", textContent: 'question',
title, title,
lang: "en", lang: 'en',
}; };
} }

4
src/publicConfig.ts
View File

@ -1,5 +1,5 @@
export default { export default {
version: "1.4.0", version: '1.4.0',
description: description:
"txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts", 'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts',
}; };

14
src/routes/api/parse.ts
View File

@ -1,13 +1,17 @@
import { FastifyInstance } from "fastify"; import { FastifyInstance } from 'fastify';
import { EngineRequest, IParseSchema, parseSchema } from "../../types/requests/api"; import {
EngineRequest,
IParseSchema,
parseSchema,
} from '../../types/requests/api';
import handlePage from "../../handlers/main"; import handlePage from '../../handlers/main';
import { generateRequestUrl } from "../../utils/generate"; import { generateRequestUrl } from '../../utils/generate';
export default async function parseRoute(fastify: FastifyInstance) { export default async function parseRoute(fastify: FastifyInstance) {
fastify.get<IParseSchema>( fastify.get<IParseSchema>(
"/api/parse", '/api/parse',
{ schema: parseSchema }, { schema: parseSchema },
async (request: EngineRequest) => { async (request: EngineRequest) => {
return { return {

14
src/routes/api/raw-html.ts
View File

@ -1,16 +1,16 @@
import { FastifyInstance } from "fastify"; import { FastifyInstance } from 'fastify';
import { IParseSchema, rawHtmlSchema } from "../../types/requests/api"; import { IParseSchema, rawHtmlSchema } from '../../types/requests/api';
import handlePage from "../../handlers/main"; import handlePage from '../../handlers/main';
import { generateRequestUrl } from "../../utils/generate"; import { generateRequestUrl } from '../../utils/generate';
export default async function rawHtml(fastify: FastifyInstance) { export default async function rawHtml(fastify: FastifyInstance) {
fastify.get<IParseSchema>( fastify.get<IParseSchema>(
"/api/raw-html", '/api/raw-html',
{ schema: rawHtmlSchema }, { schema: rawHtmlSchema },
async (request, reply) => { async (request, reply) => {
reply.type("text/html; charset=utf-8"); reply.type('text/html; charset=utf-8');
return ( return (
await handlePage( await handlePage(
request.query.url, request.query.url,
@ -20,7 +20,7 @@ export default async function rawHtml(fastify: FastifyInstance) {
request.originalUrl request.originalUrl
), ),
request.query.engine, request.query.engine,
"api/raw-html" 'api/raw-html'
) )
).content; ).content;
} }

18
src/routes/browser/get.ts
View File

@ -1,12 +1,12 @@
import { FastifyInstance } from "fastify"; import { FastifyInstance } from 'fastify';
import { GetSchema, IGetSchema } from "../../types/requests/browser"; import { GetSchema, IGetSchema } from '../../types/requests/browser';
import handlePage from "../../handlers/main"; import handlePage from '../../handlers/main';
import { generateRequestUrl } from "../../utils/generate"; import { generateRequestUrl } from '../../utils/generate';
export default async function getRoute(fastify: FastifyInstance) { export default async function getRoute(fastify: FastifyInstance) {
fastify.get<IGetSchema>( fastify.get<IGetSchema>(
"/get", '/get',
{ schema: GetSchema }, { schema: GetSchema },
async (request, reply) => { async (request, reply) => {
const remoteUrl = request.query.url; const remoteUrl = request.query.url;
@ -22,12 +22,12 @@ export default async function getRoute(fastify: FastifyInstance) {
engine engine
); );
if (request.query.format === "text") { if (request.query.format === 'text') {
reply.type("text/plain; charset=utf-8"); reply.type('text/plain; charset=utf-8');
return parsed.textContent; return parsed.textContent;
} else { } else {
reply.type("text/html; charset=utf-8"); reply.type('text/html; charset=utf-8');
return reply.view("/templates/get.ejs", { parsed, remoteUrl }); return reply.view('/templates/get.ejs', { parsed, remoteUrl });
} }
} }
); );

12
src/routes/browser/index.ts
View File

@ -1,11 +1,11 @@
import { FastifyInstance } from "fastify"; import { FastifyInstance } from 'fastify';
import publicConfig from "../../publicConfig"; import publicConfig from '../../publicConfig';
import { engineList } from "../../handlers/main"; import { engineList } from '../../handlers/main';
import { indexSchema } from "../../types/requests/browser"; import { indexSchema } from '../../types/requests/browser';
export default async function indexRoute(fastify: FastifyInstance) { export default async function indexRoute(fastify: FastifyInstance) {
fastify.get("/", { schema: indexSchema }, async (_, reply) => { fastify.get('/', { schema: indexSchema }, async (_, reply) => {
return reply.view("/templates/index.ejs", { publicConfig, engineList }); return reply.view('/templates/index.ejs', { publicConfig, engineList });
}); });
} }

18
src/routes/browser/proxy.ts
View File

@ -1,17 +1,19 @@
import { FastifyInstance } from "fastify"; import { FastifyInstance } from 'fastify';
import { IProxySchema, ProxySchema } from "../../types/requests/browser"; import { IProxySchema, ProxySchema } from '../../types/requests/browser';
import axios from "../../types/axios"; import axios from '../../types/axios';
export default async function proxyRoute(fastify: FastifyInstance) { export default async function proxyRoute(fastify: FastifyInstance) {
fastify.get<IProxySchema>( fastify.get<IProxySchema>(
"/proxy", '/proxy',
{ schema: ProxySchema }, { schema: ProxySchema },
async (request, reply) => { async (request, reply) => {
const response = await axios.get(request.query.url); const response = await axios.get(request.query.url);
const mime: string | undefined = response.headers["content-type"]?.toString(); const mime: string | undefined =
const clen: string | undefined = response.headers["content-length"]?.toString(); response.headers['content-type']?.toString();
mime && reply.header("Content-Type", mime); const clen: string | undefined =
clen && reply.header("Content-Length", Number(clen)); response.headers['content-length']?.toString();
mime && reply.header('Content-Type', mime);
clen && reply.header('Content-Length', Number(clen));
return reply.send(response.data); return reply.send(response.data);
} }
); );

8
src/types/axios.ts
View File

@ -1,9 +1,9 @@
import axios from "axios"; import axios from 'axios';
export default axios.create({ export default axios.create({
headers: { headers: {
"User-Agent": 'User-Agent':
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0", 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0',
}, },
responseType: "stream", responseType: 'stream',
}); });

4
src/types/handlers.ts
View File

@ -1,5 +1,5 @@
import { HandlerInput } from "../handlers/handler-input"; import { HandlerInput } from '../handlers/handler-input';
import { IHandlerOutput } from "../handlers/handler.interface"; import { IHandlerOutput } from '../handlers/handler.interface';
export interface Engines { export interface Engines {
[key: string]: EngineFunction; [key: string]: EngineFunction;

40
src/types/requests/api.ts
View File

@ -1,8 +1,8 @@
import { FastifySchema, FastifyRequest } from "fastify"; import { FastifySchema, FastifyRequest } from 'fastify';
import { IApiError, errorResponseSchema } from "../../errors/api"; import { IApiError, errorResponseSchema } from '../../errors/api';
import { handlerSchema } from "../../handlers/handler.interface"; import { handlerSchema } from '../../handlers/handler.interface';
import { engineList } from "../../handlers/main"; import { engineList } from '../../handlers/main';
import { FromSchema } from "json-schema-to-ts"; import { FromSchema } from 'json-schema-to-ts';
export interface IApiResponse<T> { export interface IApiResponse<T> {
data?: T; data?: T;
@ -10,38 +10,38 @@ export interface IApiResponse<T> {
} }
export const parseQuerySchema = { export const parseQuerySchema = {
type: "object", type: 'object',
required: ["url"], required: ['url'],
properties: { properties: {
url: { url: {
type: "string", type: 'string',
description: "URL", description: 'URL',
}, },
engine: { engine: {
type: "string", type: 'string',
enum: [...engineList, ""], enum: [...engineList, ''],
}, },
}, },
} as const; } as const;
export const parseSchema: FastifySchema = { export const parseSchema: FastifySchema = {
description: "Parse the page and get all data from the engine", description: 'Parse the page and get all data from the engine',
querystring: parseQuerySchema, querystring: parseQuerySchema,
response: { response: {
"2xx": { '2xx': {
type: "object", type: 'object',
properties: { properties: {
data: handlerSchema, data: handlerSchema,
error: { error: {
type: "object", type: 'object',
nullable: true, nullable: true,
}, },
}, },
}, },
"4xx": errorResponseSchema, '4xx': errorResponseSchema,
"5xx": errorResponseSchema, '5xx': errorResponseSchema,
}, },
produces: ["text/json"], produces: ['text/json'],
}; };
export interface IParseSchema { export interface IParseSchema {
@ -49,9 +49,9 @@ export interface IParseSchema {
} }
export const rawHtmlSchema: FastifySchema = { export const rawHtmlSchema: FastifySchema = {
description: "Parse the page and get raw HTML from the engine", description: 'Parse the page and get raw HTML from the engine',
querystring: parseQuerySchema, querystring: parseQuerySchema,
produces: ["text/html"], produces: ['text/html'],
}; };
export type EngineRequest = FastifyRequest<{ export type EngineRequest = FastifyRequest<{

44
src/types/requests/browser.ts
View File

@ -1,6 +1,6 @@
import { FastifySchema } from "fastify"; import { FastifySchema } from 'fastify';
import { engineList } from "../../handlers/main"; import { engineList } from '../../handlers/main';
import { FromSchema } from "json-schema-to-ts"; import { FromSchema } from 'json-schema-to-ts';
export interface IGetSchema { export interface IGetSchema {
Querystring: IGetQuerySchema; Querystring: IGetQuerySchema;
@ -11,52 +11,52 @@ export interface IProxySchema {
} }
export const getQuerySchema = { export const getQuerySchema = {
type: "object", type: 'object',
required: ["url"], required: ['url'],
properties: { properties: {
url: { url: {
type: "string", type: 'string',
description: "URL", description: 'URL',
}, },
format: { format: {
type: "string", type: 'string',
enum: ["text", "html", ""], enum: ['text', 'html', ''],
default: "html", default: 'html',
}, },
engine: { engine: {
type: "string", type: 'string',
enum: [...engineList, ""], enum: [...engineList, ''],
}, },
}, },
} as const; } as const;
export type IGetQuerySchema = FromSchema<typeof getQuerySchema>; export type IGetQuerySchema = FromSchema<typeof getQuerySchema>;
export const proxyQuerySchema = { export const proxyQuerySchema = {
type: "object", type: 'object',
required: ["url"], required: ['url'],
properties: { properties: {
url: { url: {
type: "string", type: 'string',
description: "URL", description: 'URL',
}, },
} },
} as const; } as const;
export type IProxyQuerySchema = FromSchema<typeof proxyQuerySchema>; export type IProxyQuerySchema = FromSchema<typeof proxyQuerySchema>;
export const indexSchema = { export const indexSchema = {
hide: true, hide: true,
produces: ["text/html"], produces: ['text/html'],
}; };
export const GetSchema: FastifySchema = { export const GetSchema: FastifySchema = {
description: "Get page", description: 'Get page',
hide: true, hide: true,
querystring: getQuerySchema, querystring: getQuerySchema,
produces: ["text/html", "text/plain"], produces: ['text/html', 'text/plain'],
}; };
export const ProxySchema: FastifySchema = { export const ProxySchema: FastifySchema = {
description: "Proxy resource", description: 'Proxy resource',
hide: true, hide: true,
querystring: proxyQuerySchema, querystring: proxyQuerySchema,
} };

26
src/utils/generate.ts
View File

@ -8,25 +8,35 @@ export function generateRequestUrl(
export function generateParserUrl( export function generateParserUrl(
requestUrl: URL, requestUrl: URL,
remoteUrl: URL,
href: string, href: string,
engine?: string, engine?: string,
redirect_url: string = "get" redirect_url: string = 'get'
): string { ): string {
const parsedHref = new URL(href); const realURL = getRealURL(href, remoteUrl);
const hash = parsedHref.hash; // save #hash const hash = realURL.hash; // save #hash
parsedHref.hash = ""; // remove realURL.hash = ''; // remove
const urlParam = `?url=${encodeURIComponent(parsedHref.toString())}`; const urlParam = `?url=${encodeURIComponent(realURL.toString())}`;
const engineParam = engine ? `&engine=${engine}` : ""; const engineParam = engine ? `&engine=${engine}` : '';
return `${requestUrl.origin}/${redirect_url}${urlParam}${engineParam}${hash}`; return `${requestUrl.origin}/${redirect_url}${urlParam}${engineParam}${hash}`;
} }
export function generateProxyUrl( export function generateProxyUrl(
requestUrl: URL, requestUrl: URL,
href: string, remoteUrl: URL,
href: string
): string { ): string {
const urlParam = `?url=${encodeURIComponent(href)}`; const realHref = getRealURL(href, remoteUrl);
const urlParam = `?url=${encodeURIComponent(realHref.href)}`;
return `${requestUrl.origin}/proxy${urlParam}`; return `${requestUrl.origin}/proxy${urlParam}`;
} }
function getRealURL(href: string, remoteUrl: URL) {
return href.startsWith('http')
? new URL(href)
: new URL(href, remoteUrl.href);
}

10
src/utils/http.ts
View File

@ -1,12 +1,12 @@
import { Readable } from "stream"; import { Readable } from 'stream';
import iconv from "iconv-lite"; import iconv from 'iconv-lite';
export async function decodeStream( export async function decodeStream(
data: Readable, data: Readable,
charset: string = "utf-8", charset: string = 'utf-8'
): Promise<string> { ): Promise<string> {
const strm = data.pipe(iconv.decodeStream(charset)) as IconvStream; const strm = data.pipe(iconv.decodeStream(charset)) as IconvStream;
return await new Promise(resolve => { return await new Promise((resolve) => {
strm.collect((_err: Error, body: string) => { strm.collect((_err: Error, body: string) => {
resolve(body); resolve(body);
}); });
@ -16,7 +16,7 @@ export async function decodeStream(
export function parseEncodingName(ctype?: string): string { export function parseEncodingName(ctype?: string): string {
const match = ctype?.match(/charset=([A-Za-z0-9-]+)$/); const match = ctype?.match(/charset=([A-Za-z0-9-]+)$/);
if (!match) { if (!match) {
return "utf-8"; return 'utf-8';
} }
return match[1]; return match[1];
} }

70
src/utils/islocal.ts
View File

@ -1,45 +1,43 @@
import dns from "dns"; import dns from 'dns';
import ipRangeCheck from "ip-range-check"; import ipRangeCheck from 'ip-range-check';
const subnets = [ const subnets = [
"0.0.0.0/8", '0.0.0.0/8',
"127.0.0.0/8", '127.0.0.0/8',
"10.0.0.0/8", '10.0.0.0/8',
"100.64.0.0/10", '100.64.0.0/10',
"169.254.0.0/16", '169.254.0.0/16',
"172.16.0.0/12", '172.16.0.0/12',
"192.0.0.0/24", '192.0.0.0/24',
"192.0.2.0/24", '192.0.2.0/24',
"192.88.99.0/24", '192.88.99.0/24',
"192.168.0.0/16", '192.168.0.0/16',
"198.18.0.0/15", '198.18.0.0/15',
"198.51.100.0/24", '198.51.100.0/24',
"203.0.113.0/24", '203.0.113.0/24',
"224.0.0.0/4", '224.0.0.0/4',
"233.252.0.0/24", '233.252.0.0/24',
"240.0.0.0/4", '240.0.0.0/4',
"255.255.255.255/32", '255.255.255.255/32',
"::/128", '::/128',
"::1/128", '::1/128',
"::ffff:0:0/96", '::ffff:0:0/96',
"::ffff:0:0:0/96", '::ffff:0:0:0/96',
"64:ff9b::/96", '64:ff9b::/96',
"64:ff9b:1::/48", '64:ff9b:1::/48',
"100::/64", '100::/64',
"2001:0000::/32", '2001:0000::/32',
"2001:20::/28", '2001:20::/28',
"2001:db8::/32", '2001:db8::/32',
"2002::/16", '2002::/16',
"fc00::/7", 'fc00::/7',
"fe80::/64", 'fe80::/64',
"ff00::/8", 'ff00::/8',
]; ];
export default async function isLocalResource(url: URL): Promise<boolean> { export default async function isLocalResource(url: URL): Promise<boolean> {
// Resolve domain name // Resolve domain name
const addr = ( const addr = (await dns.promises.lookup(url.hostname)).address;
await dns.promises.lookup(url.hostname)
).address;
// Check if IP is in local network // Check if IP is in local network
return ipRangeCheck(addr, subnets); return ipRangeCheck(addr, subnets);

87
src/utils/replace-href.ts
View File

@ -1,72 +1,53 @@
import { JSDOM } from "jsdom"; import { generateParserUrl, generateProxyUrl } from './generate';
import { generateParserUrl, generateProxyUrl } from "./generate"; import getConfig from '../config/main';
import getConfig from "../config/main";
export default function replaceHref( export default function replaceHref(
dom: JSDOM, dom: Window,
requestUrl: URL, requestUrl: URL,
remoteUrl: URL,
engine?: string, engine?: string,
redirectPath: string = "get", redirectPath: string = 'get'
) { ) {
const doc = dom.window.document; const doc: Document = dom.window.document;
const parserUrl = (href: string) => const parserUrl = (href: string) =>
href.startsWith("http") ? generateParserUrl( generateParserUrl(requestUrl, remoteUrl, href, engine, redirectPath);
requestUrl,
href,
engine,
redirectPath,
) : href;
const proxyUrl = (href: string) =>
href.startsWith("http") ? generateProxyUrl(
requestUrl,
href,
) : href;
modifyLinks( const proxyUrl = (href: string) =>
doc.getElementsByTagName("a"), generateProxyUrl(requestUrl, remoteUrl, href);
"href",
parserUrl, modifyLinks(doc.querySelectorAll('a[href]'), 'href', parserUrl);
); modifyLinks(doc.querySelectorAll('frame,iframe'), 'src', parserUrl);
modifyLinks(
doc.querySelectorAll("frame,iframe"),
"src",
parserUrl,
);
if (getConfig().proxy_res) { if (getConfig().proxy_res) {
modifyLinks( modifyLinks(
doc.querySelectorAll("img,image,video,audio,embed,track,source"), doc.querySelectorAll('img,image,video,audio,embed,track,source'),
"src", 'src',
proxyUrl, proxyUrl
); );
modifyLinks( modifyLinks(doc.getElementsByTagName('object'), 'data', proxyUrl);
doc.getElementsByTagName("object"), const sources = doc.querySelectorAll('source,img');
"data",
proxyUrl,
);
const sources = doc.querySelectorAll("source,img");
for (const source of sources) { for (const source of sources) {
// split srcset by comma // split srcset by comma
// @ts-ignore // @ts-expect-error because I don't know what to do about it.
if (!source.srcset) if (!source.srcset) continue;
continue; // @ts-expect-error because I don't know what to do about it.
// @ts-ignore source.srcset = source.srcset
source.srcset = source.srcset.split(",").map( .split(',')
(src: string) => { .map((src: string) => {
// split src by space // split src by space
const parts = src.trim().split(" "); const parts = src.trim().split(' ');
try { try {
// first part is URL // first part is URL
// (srcset="http 200w 1x,...") // (srcset="http 200w 1x,...")
parts[0] = proxyUrl(parts[0]); parts[0] = proxyUrl(parts[0]);
} catch (_err) { } } catch (_err) {
/* empty */
}
// join by space after splitting // join by space after splitting
return parts.join(" "); return parts.join(' ');
} })
).join(","); // join by comma .join(','); // join by comma
} }
} }
} }
@ -74,12 +55,14 @@ export default function replaceHref(
function modifyLinks( function modifyLinks(
nodeList: NodeListOf<Element> | HTMLCollectionOf<Element>, nodeList: NodeListOf<Element> | HTMLCollectionOf<Element>,
property: string, property: string,
generateLink: (value: string) => string, generateLink: (value: string) => string
) { ) {
for (const node of nodeList) { for (const node of nodeList) {
try { try {
// @ts-ignore // @ts-expect-error because I don't know what to do about it.
node[property] = generateLink(node[property]); node[property] = generateLink(node[property]);
} catch (_err) { } } catch (_err) {
/* empty */
}
} }
} }

2
static/form.css
View File

@ -31,7 +31,7 @@ label {
#url { #url {
width: 100%; width: 100%;
height: 100%; /* shrink to #submit height */ height: 100%; /* shrink to #submit height */
outline: none; outline: none;
border: 0; border: 0;