mirror of
https://github.com/txtdot/txtdot
synced 2024-10-18 14:40:19 +03:00
Search by default (#71)
* feat: searx parser * feat: search form in get page * feat: search in main page * fix: button naming * chore: update version
This commit is contained in:
parent
8524289f55
commit
fa6c9cc2a0
4
package-lock.json
generated
4
package-lock.json
generated
@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "txtdot",
|
"name": "txtdot",
|
||||||
"version": "1.4.0",
|
"version": "1.5.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "txtdot",
|
"name": "txtdot",
|
||||||
"version": "1.4.0",
|
"version": "1.5.0",
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@fastify/static": "^6.10.2",
|
"@fastify/static": "^6.10.2",
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
{
|
{
|
||||||
"name": "txtdot",
|
"name": "txtdot",
|
||||||
"version": "1.4.0",
|
"version": "1.5.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"description": "",
|
"description": "",
|
||||||
"main": "dist/app.js",
|
"main": "dist/app.js",
|
||||||
|
@ -16,6 +16,7 @@ import rawHtml from './routes/api/raw-html';
|
|||||||
import publicConfig from './publicConfig';
|
import publicConfig from './publicConfig';
|
||||||
import errorHandler from './errors/handler';
|
import errorHandler from './errors/handler';
|
||||||
import getConfig from './config/main';
|
import getConfig from './config/main';
|
||||||
|
import searchRoute from './routes/browser/search';
|
||||||
|
|
||||||
class App {
|
class App {
|
||||||
async init() {
|
async init() {
|
||||||
@ -54,6 +55,10 @@ class App {
|
|||||||
fastify.register(indexRoute);
|
fastify.register(indexRoute);
|
||||||
fastify.register(getRoute);
|
fastify.register(getRoute);
|
||||||
|
|
||||||
|
if (config.search.enabled) {
|
||||||
|
fastify.register(searchRoute);
|
||||||
|
}
|
||||||
|
|
||||||
if (config.proxy_res) fastify.register(proxyRoute);
|
if (config.proxy_res) fastify.register(proxyRoute);
|
||||||
|
|
||||||
fastify.register(parseRoute);
|
fastify.register(parseRoute);
|
||||||
|
@ -7,6 +7,7 @@ export class ConfigService {
|
|||||||
public readonly reverse_proxy: boolean;
|
public readonly reverse_proxy: boolean;
|
||||||
public readonly proxy_res: boolean;
|
public readonly proxy_res: boolean;
|
||||||
public readonly swagger: boolean;
|
public readonly swagger: boolean;
|
||||||
|
public readonly search: SearchConfig;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
config();
|
config();
|
||||||
@ -20,6 +21,11 @@ export class ConfigService {
|
|||||||
|
|
||||||
this.proxy_res = this.parseBool(process.env.PROXY_RES, true);
|
this.proxy_res = this.parseBool(process.env.PROXY_RES, true);
|
||||||
this.swagger = this.parseBool(process.env.SWAGGER, false);
|
this.swagger = this.parseBool(process.env.SWAGGER, false);
|
||||||
|
|
||||||
|
this.search = {
|
||||||
|
enabled: this.parseBool(process.env.SEARCH_ENABLED, false),
|
||||||
|
searx_url: process.env.SEARX_URL,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
parseBool(value: string | undefined, def: boolean): boolean {
|
parseBool(value: string | undefined, def: boolean): boolean {
|
||||||
@ -27,3 +33,8 @@ export class ConfigService {
|
|||||||
return value === 'true' || value === '1';
|
return value === 'true' || value === '1';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface SearchConfig {
|
||||||
|
enabled: boolean;
|
||||||
|
searx_url?: string;
|
||||||
|
}
|
||||||
|
@ -11,6 +11,7 @@ import { Readable } from 'stream';
|
|||||||
import readability from './readability';
|
import readability from './readability';
|
||||||
import google, { GoogleDomains } from './google';
|
import google, { GoogleDomains } from './google';
|
||||||
import stackoverflow, { StackOverflowDomains } from './stackoverflow/main';
|
import stackoverflow, { StackOverflowDomains } from './stackoverflow/main';
|
||||||
|
import searx, { SearxDomains } from './searx';
|
||||||
|
|
||||||
import isLocalResource from '../utils/islocal';
|
import isLocalResource from '../utils/islocal';
|
||||||
|
|
||||||
@ -75,6 +76,7 @@ export const engines: Engines = {
|
|||||||
readability,
|
readability,
|
||||||
google,
|
google,
|
||||||
stackoverflow,
|
stackoverflow,
|
||||||
|
searx,
|
||||||
};
|
};
|
||||||
|
|
||||||
export const engineList: string[] = Object.keys(engines);
|
export const engineList: string[] = Object.keys(engines);
|
||||||
@ -88,4 +90,8 @@ export const fallback: EnginesMatch = [
|
|||||||
pattern: StackOverflowDomains,
|
pattern: StackOverflowDomains,
|
||||||
engine: engines.stackoverflow,
|
engine: engines.stackoverflow,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
pattern: SearxDomains,
|
||||||
|
engine: engines.searx,
|
||||||
|
},
|
||||||
];
|
];
|
||||||
|
59
src/handlers/searx.ts
Normal file
59
src/handlers/searx.ts
Normal file
@ -0,0 +1,59 @@
|
|||||||
|
import { HandlerInput } from './handler-input';
|
||||||
|
import { IHandlerOutput } from './handler.interface';
|
||||||
|
|
||||||
|
export default async function searx(
|
||||||
|
input: HandlerInput
|
||||||
|
): Promise<IHandlerOutput> {
|
||||||
|
const document = input.parseDom().window.document;
|
||||||
|
|
||||||
|
const search = document.getElementById('q') as HTMLTextAreaElement;
|
||||||
|
|
||||||
|
const url = new URL(input.getUrl());
|
||||||
|
|
||||||
|
const page = parseInt(url.searchParams.get('pageno') || '1');
|
||||||
|
|
||||||
|
const page_footer = `${
|
||||||
|
page !== 1
|
||||||
|
? `<a href="${url.origin}${url.pathname}?q=${search.value}&pageno=${
|
||||||
|
page - 1
|
||||||
|
}">Previous </a>|`
|
||||||
|
: ''
|
||||||
|
}<a href="${url.origin}${url.pathname}?q=${search.value}&pageno=${
|
||||||
|
page + 1
|
||||||
|
}"> Next</a>`;
|
||||||
|
|
||||||
|
const articles = Array.from(document.querySelectorAll('.result'));
|
||||||
|
|
||||||
|
const articles_parsed = articles.map((a) => {
|
||||||
|
const parsed = {
|
||||||
|
url:
|
||||||
|
(a.getElementsByClassName('url_wrapper')[0] as HTMLAnchorElement)
|
||||||
|
.href || '',
|
||||||
|
title:
|
||||||
|
(a.getElementsByTagName('h3')[0] as HTMLHeadingElement).textContent ||
|
||||||
|
'',
|
||||||
|
content:
|
||||||
|
(a.getElementsByClassName('content')[0] as HTMLDivElement)
|
||||||
|
.textContent || '',
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
html: `<a href="${parsed.url}">${parsed.title}</a><p>${parsed.content}</p><hr>`,
|
||||||
|
text: `${parsed.title} (${parsed.url})\n${parsed.content}\n---\n\n`,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
const content = `${articles_parsed
|
||||||
|
.map((a) => a.html)
|
||||||
|
.join('')}${page_footer}`;
|
||||||
|
const textContent = articles_parsed.map((a) => a.text).join('');
|
||||||
|
|
||||||
|
return {
|
||||||
|
content,
|
||||||
|
textContent,
|
||||||
|
title: `${search.value} - Searx - Page ${page}`,
|
||||||
|
lang: document.documentElement.lang,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export const SearxDomains = ['searx.*'];
|
@ -1,5 +1,5 @@
|
|||||||
export default {
|
export default {
|
||||||
version: '1.4.0',
|
version: '1.5.0',
|
||||||
description:
|
description:
|
||||||
'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts',
|
'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts',
|
||||||
};
|
};
|
||||||
|
@ -4,6 +4,8 @@ import { GetSchema, IGetSchema } from '../../types/requests/browser';
|
|||||||
import handlePage from '../../handlers/main';
|
import handlePage from '../../handlers/main';
|
||||||
import { generateRequestUrl } from '../../utils/generate';
|
import { generateRequestUrl } from '../../utils/generate';
|
||||||
|
|
||||||
|
import getConfig from '../../config/main';
|
||||||
|
|
||||||
export default async function getRoute(fastify: FastifyInstance) {
|
export default async function getRoute(fastify: FastifyInstance) {
|
||||||
fastify.get<IGetSchema>(
|
fastify.get<IGetSchema>(
|
||||||
'/get',
|
'/get',
|
||||||
@ -27,7 +29,11 @@ export default async function getRoute(fastify: FastifyInstance) {
|
|||||||
return parsed.textContent;
|
return parsed.textContent;
|
||||||
} else {
|
} else {
|
||||||
reply.type('text/html; charset=utf-8');
|
reply.type('text/html; charset=utf-8');
|
||||||
return reply.view('/templates/get.ejs', { parsed, remoteUrl });
|
return reply.view('/templates/get.ejs', {
|
||||||
|
parsed,
|
||||||
|
remoteUrl,
|
||||||
|
config: getConfig(),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
@ -4,8 +4,14 @@ import publicConfig from '../../publicConfig';
|
|||||||
import { engineList } from '../../handlers/main';
|
import { engineList } from '../../handlers/main';
|
||||||
import { indexSchema } from '../../types/requests/browser';
|
import { indexSchema } from '../../types/requests/browser';
|
||||||
|
|
||||||
|
import getConfig from '../../config/main';
|
||||||
|
|
||||||
export default async function indexRoute(fastify: FastifyInstance) {
|
export default async function indexRoute(fastify: FastifyInstance) {
|
||||||
fastify.get('/', { schema: indexSchema }, async (_, reply) => {
|
fastify.get('/', { schema: indexSchema }, async (_, reply) => {
|
||||||
return reply.view('/templates/index.ejs', { publicConfig, engineList });
|
return reply.view('/templates/index.ejs', {
|
||||||
|
publicConfig,
|
||||||
|
engineList,
|
||||||
|
config: getConfig(),
|
||||||
|
});
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
24
src/routes/browser/search.ts
Normal file
24
src/routes/browser/search.ts
Normal file
@ -0,0 +1,24 @@
|
|||||||
|
import { FastifyInstance } from 'fastify';
|
||||||
|
|
||||||
|
import { searchSchema, ISearchSchema } from '../../types/requests/browser';
|
||||||
|
|
||||||
|
import getConfig from '../../config/main';
|
||||||
|
|
||||||
|
export default async function searchRoute(fastify: FastifyInstance) {
|
||||||
|
fastify.get<ISearchSchema>(
|
||||||
|
'/search',
|
||||||
|
{ schema: searchSchema },
|
||||||
|
async (request, reply) => {
|
||||||
|
const query = request.query.q;
|
||||||
|
|
||||||
|
const config = getConfig();
|
||||||
|
|
||||||
|
if (config.search.enabled) {
|
||||||
|
const searchUrl = `${config.search.searx_url}/search?q=${query}`;
|
||||||
|
reply.redirect(`/get?url=${encodeURI(searchUrl)}`);
|
||||||
|
} else {
|
||||||
|
throw new Error('Search is not enabled');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
|
}
|
@ -10,6 +10,22 @@ export interface IProxySchema {
|
|||||||
Querystring: IProxyQuerySchema;
|
Querystring: IProxyQuerySchema;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ISearchSchema {
|
||||||
|
Querystring: ISearchQuerySchema;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const searchQuerySchema = {
|
||||||
|
type: 'object',
|
||||||
|
required: ['q'],
|
||||||
|
properties: {
|
||||||
|
q: {
|
||||||
|
type: 'string',
|
||||||
|
description: 'Search query',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as const;
|
||||||
|
export type ISearchQuerySchema = FromSchema<typeof searchQuerySchema>;
|
||||||
|
|
||||||
export const getQuerySchema = {
|
export const getQuerySchema = {
|
||||||
type: 'object',
|
type: 'object',
|
||||||
required: ['url'],
|
required: ['url'],
|
||||||
@ -48,6 +64,12 @@ export const indexSchema = {
|
|||||||
produces: ['text/html'],
|
produces: ['text/html'],
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export const searchSchema: FastifySchema = {
|
||||||
|
description: 'Search redirection page',
|
||||||
|
hide: true,
|
||||||
|
querystring: searchQuerySchema,
|
||||||
|
};
|
||||||
|
|
||||||
export const GetSchema: FastifySchema = {
|
export const GetSchema: FastifySchema = {
|
||||||
description: 'Get page',
|
description: 'Get page',
|
||||||
hide: true,
|
hide: true,
|
||||||
|
@ -29,7 +29,8 @@ label {
|
|||||||
font-size: 0.9rem;
|
font-size: 0.9rem;
|
||||||
}
|
}
|
||||||
|
|
||||||
#url {
|
#url,
|
||||||
|
#search {
|
||||||
width: 100%;
|
width: 100%;
|
||||||
height: 100%; /* shrink to #submit height */
|
height: 100%; /* shrink to #submit height */
|
||||||
|
|
||||||
|
25
static/search.css
Normal file
25
static/search.css
Normal file
@ -0,0 +1,25 @@
|
|||||||
|
.right {
|
||||||
|
display: flex;
|
||||||
|
margin-left: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
#search {
|
||||||
|
width: 100%;
|
||||||
|
height: 100%; /* shrink to #submit height */
|
||||||
|
|
||||||
|
outline: none;
|
||||||
|
border: 0;
|
||||||
|
border-bottom: 0.125rem solid var(--fg2);
|
||||||
|
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--fg);
|
||||||
|
font-size: 1rem;
|
||||||
|
|
||||||
|
margin-right: 0.5rem;
|
||||||
|
margin-left: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
#search::placeholder {
|
||||||
|
color: var(--fg2);
|
||||||
|
opacity: 1;
|
||||||
|
}
|
@ -8,12 +8,27 @@
|
|||||||
<title><%= parsed.title %></title>
|
<title><%= parsed.title %></title>
|
||||||
<link rel="stylesheet" href="/static/common.css">
|
<link rel="stylesheet" href="/static/common.css">
|
||||||
<link rel="stylesheet" href="/static/get.css">
|
<link rel="stylesheet" href="/static/get.css">
|
||||||
|
<%
|
||||||
|
if (config.search.enabled) {
|
||||||
|
%><link rel="stylesheet" href="/static/search.css"><%
|
||||||
|
}
|
||||||
|
%>
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<main>
|
<main>
|
||||||
<div class="menu">
|
<div class="menu">
|
||||||
<a class="button secondary" href="/">Home</a>
|
<a class="button secondary" href="/">Home</a>
|
||||||
<a class="button secondary" href="<%= remoteUrl %>">Original page</a>
|
<a class="button secondary" href="<%= remoteUrl %>">Original page</a>
|
||||||
|
<%
|
||||||
|
if (config.search.enabled) {
|
||||||
|
%>
|
||||||
|
<form class="right" action="/search" method="get">
|
||||||
|
<input type="text" name="q" id="search" placeholder="Search">
|
||||||
|
<input class="button" type="submit" value="Go"/>
|
||||||
|
</form>
|
||||||
|
<%
|
||||||
|
}
|
||||||
|
%>
|
||||||
</div>
|
</div>
|
||||||
<p class="title">
|
<p class="title">
|
||||||
<%= parsed.title %>
|
<%= parsed.title %>
|
||||||
|
@ -21,6 +21,23 @@
|
|||||||
</p>
|
</p>
|
||||||
<p><%= publicConfig.description %></p>
|
<p><%= publicConfig.description %></p>
|
||||||
</header>
|
</header>
|
||||||
|
<%
|
||||||
|
if (config.search.enabled) {
|
||||||
|
|
||||||
|
%>
|
||||||
|
<form action="/search" method="get" class="input-grid">
|
||||||
|
<div class="input">
|
||||||
|
<input type="text" name="q" id="search" placeholder="Search">
|
||||||
|
</div>
|
||||||
|
<div class="input">
|
||||||
|
<input type="submit" id="submit" class="button" value="Go">
|
||||||
|
</div>
|
||||||
|
</form>
|
||||||
|
<%
|
||||||
|
|
||||||
|
%><details style="margin-top: 1rem;"><summary>Advanced</summary><%
|
||||||
|
}
|
||||||
|
%>
|
||||||
<form action="/get" method="get" class="input-grid">
|
<form action="/get" method="get" class="input-grid">
|
||||||
<div class="input">
|
<div class="input">
|
||||||
<input type="text" name="url" id="url" placeholder="URL">
|
<input type="text" name="url" id="url" placeholder="URL">
|
||||||
@ -52,6 +69,11 @@
|
|||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</form>
|
</form>
|
||||||
|
<%
|
||||||
|
if (config.search.enabled) {
|
||||||
|
%></details><%
|
||||||
|
}
|
||||||
|
%>
|
||||||
</main>
|
</main>
|
||||||
</body>
|
</body>
|
||||||
</html>
|
</html>
|
||||||
|
Loading…
Reference in New Issue
Block a user