Search by default (#71)

* feat: searx parser

* feat: search form in get page

* feat: search in main page

* fix: button naming

* chore: update version

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "txtdot",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "txtdot",
-      "version": "1.4.0",
+      "version": "1.5.0",
       "license": "MIT",
       "dependencies": {
         "@fastify/static": "^6.10.2",

package.json

@@ -1,6 +1,6 @@
 {
   "name": "txtdot",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "private": true,
   "description": "",
   "main": "dist/app.js",

src/app.ts

@@ -16,6 +16,7 @@ import rawHtml from './routes/api/raw-html';
 import publicConfig from './publicConfig';
 import errorHandler from './errors/handler';
 import getConfig from './config/main';
+import searchRoute from './routes/browser/search';
 
 class App {
   async init() {
@@ -54,6 +55,10 @@ class App {
     fastify.register(indexRoute);
     fastify.register(getRoute);
 
+    if (config.search.enabled) {
+      fastify.register(searchRoute);
+    }
+
     if (config.proxy_res) fastify.register(proxyRoute);
 
     fastify.register(parseRoute);

src/config/config.service.ts

@@ -7,6 +7,7 @@ export class ConfigService {
   public readonly reverse_proxy: boolean;
   public readonly proxy_res: boolean;
   public readonly swagger: boolean;
+  public readonly search: SearchConfig;
 
   constructor() {
     config();
@@ -20,6 +21,11 @@ export class ConfigService {
 
     this.proxy_res = this.parseBool(process.env.PROXY_RES, true);
     this.swagger = this.parseBool(process.env.SWAGGER, false);
+
+    this.search = {
+      enabled: this.parseBool(process.env.SEARCH_ENABLED, false),
+      searx_url: process.env.SEARX_URL,
+    };
   }
 
   parseBool(value: string | undefined, def: boolean): boolean {
@@ -27,3 +33,8 @@ export class ConfigService {
     return value === 'true' || value === '1';
   }
 }
+
+interface SearchConfig {
+  enabled: boolean;
+  searx_url?: string;
+}

src/handlers/main.ts

@@ -11,6 +11,7 @@ import { Readable } from 'stream';
 import readability from './readability';
 import google, { GoogleDomains } from './google';
 import stackoverflow, { StackOverflowDomains } from './stackoverflow/main';
+import searx, { SearxDomains } from './searx';
 
 import isLocalResource from '../utils/islocal';
 
@@ -75,6 +76,7 @@ export const engines: Engines = {
   readability,
   google,
   stackoverflow,
+  searx,
 };
 
 export const engineList: string[] = Object.keys(engines);
@@ -88,4 +90,8 @@ export const fallback: EnginesMatch = [
     pattern: StackOverflowDomains,
     engine: engines.stackoverflow,
   },
+  {
+    pattern: SearxDomains,
+    engine: engines.searx,
+  },
 ];

src/handlers/searx.ts

@@ -0,0 +1,59 @@
+import { HandlerInput } from './handler-input';
+import { IHandlerOutput } from './handler.interface';
+
+export default async function searx(
+  input: HandlerInput
+): Promise<IHandlerOutput> {
+  const document = input.parseDom().window.document;
+
+  const search = document.getElementById('q') as HTMLTextAreaElement;
+
+  const url = new URL(input.getUrl());
+
+  const page = parseInt(url.searchParams.get('pageno') || '1');
+
+  const page_footer = `${
+    page !== 1
+      ? `<a href="${url.origin}${url.pathname}?q=${search.value}&pageno=${
+          page - 1
+        }">Previous </a>|`
+      : ''
+  }<a href="${url.origin}${url.pathname}?q=${search.value}&pageno=${
+    page + 1
+  }"> Next</a>`;
+
+  const articles = Array.from(document.querySelectorAll('.result'));
+
+  const articles_parsed = articles.map((a) => {
+    const parsed = {
+      url:
+        (a.getElementsByClassName('url_wrapper')[0] as HTMLAnchorElement)
+          .href || '',
+      title:
+        (a.getElementsByTagName('h3')[0] as HTMLHeadingElement).textContent ||
+        '',
+      content:
+        (a.getElementsByClassName('content')[0] as HTMLDivElement)
+          .textContent || '',
+    };
+
+    return {
+      html: `<a href="${parsed.url}">${parsed.title}</a><p>${parsed.content}</p><hr>`,
+      text: `${parsed.title} (${parsed.url})\n${parsed.content}\n---\n\n`,
+    };
+  });
+
+  const content = `${articles_parsed
+    .map((a) => a.html)
+    .join('')}${page_footer}`;
+  const textContent = articles_parsed.map((a) => a.text).join('');
+
+  return {
+    content,
+    textContent,
+    title: `${search.value} - Searx - Page ${page}`,
+    lang: document.documentElement.lang,
+  };
+}
+
+export const SearxDomains = ['searx.*'];

src/publicConfig.ts

@@ -1,5 +1,5 @@
 export default {
-  version: '1.4.0',
+  version: '1.5.0',
   description:
     'txtdot is an HTTP proxy that parses only text, links and pictures from pages reducing internet bandwidth usage, removing ads and heavy scripts',
 };

src/routes/browser/get.ts

@@ -4,6 +4,8 @@ import { GetSchema, IGetSchema } from '../../types/requests/browser';
 import handlePage from '../../handlers/main';
 import { generateRequestUrl } from '../../utils/generate';
 
+import getConfig from '../../config/main';
+
 export default async function getRoute(fastify: FastifyInstance) {
   fastify.get<IGetSchema>(
     '/get',
@@ -27,7 +29,11 @@ export default async function getRoute(fastify: FastifyInstance) {
         return parsed.textContent;
       } else {
         reply.type('text/html; charset=utf-8');
-        return reply.view('/templates/get.ejs', { parsed, remoteUrl });
+        return reply.view('/templates/get.ejs', {
+          parsed,
+          remoteUrl,
+          config: getConfig(),
+        });
       }
     }
   );

src/routes/browser/index.ts

@@ -4,8 +4,14 @@ import publicConfig from '../../publicConfig';
 import { engineList } from '../../handlers/main';
 import { indexSchema } from '../../types/requests/browser';
 
+import getConfig from '../../config/main';
+
 export default async function indexRoute(fastify: FastifyInstance) {
   fastify.get('/', { schema: indexSchema }, async (_, reply) => {
-    return reply.view('/templates/index.ejs', { publicConfig, engineList });
+    return reply.view('/templates/index.ejs', {
+      publicConfig,
+      engineList,
+      config: getConfig(),
+    });
   });
 }

src/routes/browser/search.ts

@@ -0,0 +1,24 @@
+import { FastifyInstance } from 'fastify';
+
+import { searchSchema, ISearchSchema } from '../../types/requests/browser';
+
+import getConfig from '../../config/main';
+
+export default async function searchRoute(fastify: FastifyInstance) {
+  fastify.get<ISearchSchema>(
+    '/search',
+    { schema: searchSchema },
+    async (request, reply) => {
+      const query = request.query.q;
+
+      const config = getConfig();
+
+      if (config.search.enabled) {
+        const searchUrl = `${config.search.searx_url}/search?q=${query}`;
+        reply.redirect(`/get?url=${encodeURI(searchUrl)}`);
+      } else {
+        throw new Error('Search is not enabled');
+      }
+    }
+  );
+}

src/types/requests/browser.ts

@@ -10,6 +10,22 @@ export interface IProxySchema {
   Querystring: IProxyQuerySchema;
 }
 
+export interface ISearchSchema {
+  Querystring: ISearchQuerySchema;
+}
+
+export const searchQuerySchema = {
+  type: 'object',
+  required: ['q'],
+  properties: {
+    q: {
+      type: 'string',
+      description: 'Search query',
+    },
+  },
+} as const;
+export type ISearchQuerySchema = FromSchema<typeof searchQuerySchema>;
+
 export const getQuerySchema = {
   type: 'object',
   required: ['url'],
@@ -48,6 +64,12 @@ export const indexSchema = {
   produces: ['text/html'],
 };
 
+export const searchSchema: FastifySchema = {
+  description: 'Search redirection page',
+  hide: true,
+  querystring: searchQuerySchema,
+};
+
 export const GetSchema: FastifySchema = {
   description: 'Get page',
   hide: true,

static/form.css

@@ -29,7 +29,8 @@ label {
   font-size: 0.9rem;
 }
 
-#url {
+#url,
+#search {
   width: 100%;
   height: 100%; /* shrink to #submit height */
 

static/search.css

@@ -0,0 +1,25 @@
+.right {
+  display: flex;
+  margin-left: auto;
+}
+
+#search {
+  width: 100%;
+  height: 100%; /* shrink to #submit height */
+
+  outline: none;
+  border: 0;
+  border-bottom: 0.125rem solid var(--fg2);
+
+  background: var(--bg);
+  color: var(--fg);
+  font-size: 1rem;
+
+  margin-right: 0.5rem;
+  margin-left: 0.5rem;
+}
+
+#search::placeholder {
+  color: var(--fg2);
+  opacity: 1;
+}

templates/get.ejs

@@ -8,12 +8,27 @@
     <title><%= parsed.title %></title>
     <link rel="stylesheet" href="/static/common.css">
     <link rel="stylesheet" href="/static/get.css">
+    <% 
+      if (config.search.enabled) {
+        %><link rel="stylesheet" href="/static/search.css"><%
+      }
+    %>
   </head>
   <body>
     <main>
       <div class="menu">
         <a class="button secondary" href="/">Home</a>
         <a class="button secondary" href="<%= remoteUrl %>">Original page</a>
+        <% 
+          if (config.search.enabled) {
+            %>
+            <form class="right" action="/search" method="get">
+              <input type="text" name="q" id="search" placeholder="Search">
+              <input class="button" type="submit" value="Go"/>
+            </form>
+            <%
+          }
+        %>
       </div>
       <p class="title">
         <%= parsed.title %>

templates/index.ejs

@@ -21,6 +21,23 @@
         </p>
         <p><%= publicConfig.description %></p>
       </header>
+      <% 
+        if (config.search.enabled) {
+
+          %>
+          <form action="/search" method="get" class="input-grid">
+            <div class="input">
+              <input type="text" name="q" id="search" placeholder="Search">
+            </div>
+            <div class="input">
+              <input type="submit" id="submit" class="button" value="Go">
+            </div>
+          </form>
+          <%
+
+          %><details style="margin-top: 1rem;"><summary>Advanced</summary><%
+        }
+      %>
       <form action="/get" method="get" class="input-grid">
         <div class="input">
           <input type="text" name="url" id="url" placeholder="URL">
@@ -52,6 +69,11 @@
           </div>
         </div>
       </form>
+      <%
+        if (config.search.enabled) {
+          %></details><%
+        }
+      %>
     </main>
   </body>
 </html>