mirror of
https://github.com/txtdot/txtdot
synced 2024-10-18 14:40:19 +03:00
Search by default (#71)
* feat: searx parser * feat: search form in get page * feat: search in main page * fix: button naming * chore: update version
This commit is contained in:
parent
8524289f55
commit
fa6c9cc2a0
4
package-lock.json
generated
4
package-lock.json
generated
@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "txtdot",
|
||||
"version": "1.4.0",
|
||||
"version": "1.5.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "txtdot",
|
||||
"version": "1.4.0",
|
||||
"version": "1.5.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@fastify/static": "^6.10.2",
|
||||
|
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "txtdot",
|
||||
"version": "1.4.0",
|
||||
"version": "1.5.0",
|
||||
"private": true,
|
||||
"description": "",
|
||||
"main": "dist/app.js",
|
||||
|
@ -16,6 +16,7 @@ import rawHtml from './routes/api/raw-html';
|
||||
import publicConfig from './publicConfig';
|
||||
import errorHandler from './errors/handler';
|
||||
import getConfig from './config/main';
|
||||
import searchRoute from './routes/browser/search';
|
||||
|
||||
class App {
|
||||
async init() {
|
||||
@ -54,6 +55,10 @@ class App {
|
||||
fastify.register(indexRoute);
|
||||
fastify.register(getRoute);
|
||||
|
||||
if (config.search.enabled) {
|
||||
fastify.register(searchRoute);
|
||||
}
|
||||
|
||||
if (config.proxy_res) fastify.register(proxyRoute);
|
||||
|
||||
fastify.register(parseRoute);
|
||||
|
@ -7,6 +7,7 @@ export class ConfigService {
|
||||
public readonly reverse_proxy: boolean;
|
||||
public readonly proxy_res: boolean;
|
||||
public readonly swagger: boolean;
|
||||
public readonly search: SearchConfig;
|
||||
|
||||
constructor() {
|
||||
config();
|
||||
@ -20,6 +21,11 @@ export class ConfigService {
|
||||
|
||||
this.proxy_res = this.parseBool(process.env.PROXY_RES, true);
|
||||
this.swagger = this.parseBool(process.env.SWAGGER, false);
|
||||
|
||||
this.search = {
|
||||
enabled: this.parseBool(process.env.SEARCH_ENABLED, false),
|
||||
searx_url: process.env.SEARX_URL,
|
||||
};
|
||||
}
|
||||
|
||||
parseBool(value: string | undefined, def: boolean): boolean {
|
||||
@ -27,3 +33,8 @@ export class ConfigService {
|
||||
return value === 'true' || value === '1';
|
||||
}
|
||||
}
|
||||
|
||||
interface SearchConfig {
|
||||
enabled: boolean;
|
||||
searx_url?: string;
|
||||
}
|
||||
|
@ -11,6 +11,7 @@ import { Readable } from 'stream';
|
||||
import readability from './readability';
|
||||
import google, { GoogleDomains } from './google';
|
||||
import stackoverflow, { StackOverflowDomains } from './stackoverflow/main';
|
||||
import searx, { SearxDomains } from './searx';
|
||||
|
||||
import isLocalResource from '../utils/islocal';
|
||||
|
||||
@ -75,6 +76,7 @@ export const engines: Engines = {
|
||||
readability,
|
||||
google,
|
||||
stackoverflow,
|
||||
searx,
|
||||
};
|
||||
|
||||
export const engineList: string[] = Object.keys(engines);
|
||||
@ -88,4 +90,8 @@ export const fallback: EnginesMatch = [
|
||||
pattern: StackOverflowDomains,
|
||||
engine: engines.stackoverflow,
|
||||
},
|
||||
{
|
||||
pattern: SearxDomains,
|
||||
engine: engines.searx,
|
||||
},
|
||||
];
|
||||
|
59
src/handlers/searx.ts
Normal file
59
src/handlers/searx.ts
Normal file
@ -0,0 +1,59 @@
|
||||
import { HandlerInput } from './handler-input';
|
||||
import { IHandlerOutput } from './handler.interface';
|
||||
|
||||
export default async function searx(
|
||||
input: HandlerInput
|
||||
): Promise<IHandlerOutput> {
|
||||
const document = input.parseDom().window.document;
|
||||
|
||||
const search = document.getElementById('q') as HTMLTextAreaElement;
|
||||
|
||||
const url = new URL(input.getUrl());
|
||||
|
||||
const page = parseInt(url.searchParams.get('pageno') || '1');
|
||||
|
||||
const page_footer = `${
|
||||
page !== 1
|
||||
? `<a href="${url.origin}${url.pathname}?q=${search.value}&pageno=${
|
||||
page - 1
|
||||
}">Previous </a>|`
|
||||
: ''
|
||||
}<a href="${url.origin}${url.pathname}?q=${search.value}&pageno=${
|
||||
page + 1
|
||||
}"> Next</a>`;
|
||||
|
||||
const articles = Array.from(document.querySelectorAll('.result'));
|
||||
|
||||
const articles_parsed = articles.map((a) => {
|
||||
const parsed = {
|
||||
url:
|
||||
(a.getElementsByClassName('url_wrapper')[0] as HTMLAnchorElement)
|
||||
.href || '',
|
||||
title:
|
||||
(a.getElementsByTagName('h3')[0] as HTMLHeadingElement).textContent ||
|
||||
'',
|
||||
content:
|
||||
(a.getElementsByClassName('content')[0] as HTMLDivElement)
|
||||
.textContent || '',
|
||||
};
|
||||
|
||||
return {
|
||||
html: `<a href="${parsed.url}">${parsed.title}</a><p>${parsed.content}</p><hr>`,
|
||||
text: `${parsed.title} (${parsed.url})\n${parsed.content}\n---\n\n`,
|
||||
};
|
||||
});
|
||||
|
||||
const content = `${articles_parsed
|
||||
.map((a) => a.html)
|
||||
.join('')}${page_footer}`;
|
||||
const textContent = articles_parsed.map((a) => a.text).join('');
|
||||
|
||||
return {
|
||||
content,
|
||||
textContent,
|
||||
title: `${search.value} - Searx - Page ${page}`,
|
||||
lang: document.documentElement.lang,
|
||||
};
|
||||
}
|
||||
|
||||
export const SearxDomains = ['searx.*'];
|
@ -1,5 +1,5 @@
|
||||
export default {
|
||||
version: '1.4.0',
|
||||
version: '1.5.0',
|
||||
description:
|
||||
'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts',
|
||||
};
|
||||
|
@ -4,6 +4,8 @@ import { GetSchema, IGetSchema } from '../../types/requests/browser';
|
||||
import handlePage from '../../handlers/main';
|
||||
import { generateRequestUrl } from '../../utils/generate';
|
||||
|
||||
import getConfig from '../../config/main';
|
||||
|
||||
export default async function getRoute(fastify: FastifyInstance) {
|
||||
fastify.get<IGetSchema>(
|
||||
'/get',
|
||||
@ -27,7 +29,11 @@ export default async function getRoute(fastify: FastifyInstance) {
|
||||
return parsed.textContent;
|
||||
} else {
|
||||
reply.type('text/html; charset=utf-8');
|
||||
return reply.view('/templates/get.ejs', { parsed, remoteUrl });
|
||||
return reply.view('/templates/get.ejs', {
|
||||
parsed,
|
||||
remoteUrl,
|
||||
config: getConfig(),
|
||||
});
|
||||
}
|
||||
}
|
||||
);
|
||||
|
@ -4,8 +4,14 @@ import publicConfig from '../../publicConfig';
|
||||
import { engineList } from '../../handlers/main';
|
||||
import { indexSchema } from '../../types/requests/browser';
|
||||
|
||||
import getConfig from '../../config/main';
|
||||
|
||||
export default async function indexRoute(fastify: FastifyInstance) {
|
||||
fastify.get('/', { schema: indexSchema }, async (_, reply) => {
|
||||
return reply.view('/templates/index.ejs', { publicConfig, engineList });
|
||||
return reply.view('/templates/index.ejs', {
|
||||
publicConfig,
|
||||
engineList,
|
||||
config: getConfig(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
24
src/routes/browser/search.ts
Normal file
24
src/routes/browser/search.ts
Normal file
@ -0,0 +1,24 @@
|
||||
import { FastifyInstance } from 'fastify';
|
||||
|
||||
import { searchSchema, ISearchSchema } from '../../types/requests/browser';
|
||||
|
||||
import getConfig from '../../config/main';
|
||||
|
||||
export default async function searchRoute(fastify: FastifyInstance) {
|
||||
fastify.get<ISearchSchema>(
|
||||
'/search',
|
||||
{ schema: searchSchema },
|
||||
async (request, reply) => {
|
||||
const query = request.query.q;
|
||||
|
||||
const config = getConfig();
|
||||
|
||||
if (config.search.enabled) {
|
||||
const searchUrl = `${config.search.searx_url}/search?q=${query}`;
|
||||
reply.redirect(`/get?url=${encodeURI(searchUrl)}`);
|
||||
} else {
|
||||
throw new Error('Search is not enabled');
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
@ -10,6 +10,22 @@ export interface IProxySchema {
|
||||
Querystring: IProxyQuerySchema;
|
||||
}
|
||||
|
||||
export interface ISearchSchema {
|
||||
Querystring: ISearchQuerySchema;
|
||||
}
|
||||
|
||||
export const searchQuerySchema = {
|
||||
type: 'object',
|
||||
required: ['q'],
|
||||
properties: {
|
||||
q: {
|
||||
type: 'string',
|
||||
description: 'Search query',
|
||||
},
|
||||
},
|
||||
} as const;
|
||||
export type ISearchQuerySchema = FromSchema<typeof searchQuerySchema>;
|
||||
|
||||
export const getQuerySchema = {
|
||||
type: 'object',
|
||||
required: ['url'],
|
||||
@ -48,6 +64,12 @@ export const indexSchema = {
|
||||
produces: ['text/html'],
|
||||
};
|
||||
|
||||
export const searchSchema: FastifySchema = {
|
||||
description: 'Search redirection page',
|
||||
hide: true,
|
||||
querystring: searchQuerySchema,
|
||||
};
|
||||
|
||||
export const GetSchema: FastifySchema = {
|
||||
description: 'Get page',
|
||||
hide: true,
|
||||
|
@ -29,7 +29,8 @@ label {
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
#url {
|
||||
#url,
|
||||
#search {
|
||||
width: 100%;
|
||||
height: 100%; /* shrink to #submit height */
|
||||
|
||||
|
25
static/search.css
Normal file
25
static/search.css
Normal file
@ -0,0 +1,25 @@
|
||||
.right {
|
||||
display: flex;
|
||||
margin-left: auto;
|
||||
}
|
||||
|
||||
#search {
|
||||
width: 100%;
|
||||
height: 100%; /* shrink to #submit height */
|
||||
|
||||
outline: none;
|
||||
border: 0;
|
||||
border-bottom: 0.125rem solid var(--fg2);
|
||||
|
||||
background: var(--bg);
|
||||
color: var(--fg);
|
||||
font-size: 1rem;
|
||||
|
||||
margin-right: 0.5rem;
|
||||
margin-left: 0.5rem;
|
||||
}
|
||||
|
||||
#search::placeholder {
|
||||
color: var(--fg2);
|
||||
opacity: 1;
|
||||
}
|
@ -8,12 +8,27 @@
|
||||
<title><%= parsed.title %></title>
|
||||
<link rel="stylesheet" href="/static/common.css">
|
||||
<link rel="stylesheet" href="/static/get.css">
|
||||
<%
|
||||
if (config.search.enabled) {
|
||||
%><link rel="stylesheet" href="/static/search.css"><%
|
||||
}
|
||||
%>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<div class="menu">
|
||||
<a class="button secondary" href="/">Home</a>
|
||||
<a class="button secondary" href="<%= remoteUrl %>">Original page</a>
|
||||
<%
|
||||
if (config.search.enabled) {
|
||||
%>
|
||||
<form class="right" action="/search" method="get">
|
||||
<input type="text" name="q" id="search" placeholder="Search">
|
||||
<input class="button" type="submit" value="Go"/>
|
||||
</form>
|
||||
<%
|
||||
}
|
||||
%>
|
||||
</div>
|
||||
<p class="title">
|
||||
<%= parsed.title %>
|
||||
|
@ -21,6 +21,23 @@
|
||||
</p>
|
||||
<p><%= publicConfig.description %></p>
|
||||
</header>
|
||||
<%
|
||||
if (config.search.enabled) {
|
||||
|
||||
%>
|
||||
<form action="/search" method="get" class="input-grid">
|
||||
<div class="input">
|
||||
<input type="text" name="q" id="search" placeholder="Search">
|
||||
</div>
|
||||
<div class="input">
|
||||
<input type="submit" id="submit" class="button" value="Go">
|
||||
</div>
|
||||
</form>
|
||||
<%
|
||||
|
||||
%><details style="margin-top: 1rem;"><summary>Advanced</summary><%
|
||||
}
|
||||
%>
|
||||
<form action="/get" method="get" class="input-grid">
|
||||
<div class="input">
|
||||
<input type="text" name="url" id="url" placeholder="URL">
|
||||
@ -52,6 +69,11 @@
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
<%
|
||||
if (config.search.enabled) {
|
||||
%></details><%
|
||||
}
|
||||
%>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
|
Loading…
Reference in New Issue
Block a user