Local flexsearch (#160)

* Switch to using local search
* Switch from lunr to FlexSearch
* Cache searchindex in localstorage
* Link unresolved links to 404 page in build step

.eleventy.js

@@ -194,6 +194,11 @@ module.exports = function (eleventyConfig) {
 
   eleventyConfig.setLibrary("md", markdownLib);
 
+  eleventyConfig.addFilter("isoDate", function (date) {
+    return date && date.toISOString();
+  });
+
+
   eleventyConfig.addFilter("link", function (str) {
     return (
       str &&
@@ -240,9 +245,10 @@ module.exports = function (eleventyConfig) {
           deadLink = true;
         }
 
-        return `<a class="internal-link ${
-          deadLink ? "is-unresolved" : ""
-        }" ${deadLink ? "" : 'data-note-icon="' + noteIcon + '"'} href="${permalink}${headerLinkPath}">${title}</a>`;
+        if(deadLink){
+          return `<a class="internal-link is-unresolved" href="/404">${title}</a>`;
+        }
+        return `<a class="internal-link data-note-icon="${noteIcon}" href="${permalink}${headerLinkPath}">${title}</a>`;
       })
     );
   });

api/search.js

@@ -1,11 +0,0 @@
-//vercel search function
-const searchHandler = require('../netlify/functions/search/search.js').handler;
-async function vercelSearch(request, response) {
-    let event = {queryStringParameters: request.query};
-
-    let searchResponse = await searchHandler(event);
-
-    return response.status(200).json(JSON.parse(searchResponse.body));
-  }
-
-exports.default = vercelSearch;

netlify/functions/search/search.js

@@ -1,49 +0,0 @@
-const lunrjs = require("lunr");
-
-const handler = async (event) => {
-  try {
-    const search = event.queryStringParameters.term;
-    if (!search) throw "Missing term query parameter";
-
-    const data = require("./data.json");
-    const indexJson = require("./index.json");
-    const index = lunrjs.Index.load(indexJson);
-    console.log("index made");
-
-    let results =
-      search[0] == "#" && search.length > 1
-        ? index.search(`tags:${search.substring(1)}`)
-        : index.search(search + "*");
-
-    results.forEach((r) => {
-      r.title = data[r.ref].title;
-      r.content = truncate(data[r.ref].content, 400);
-      r.date = data[r.ref].date;
-      r.url = data[r.ref].url;
-      r.tags = data[r.ref].tags.filter(
-        (x) => x != "gardenEntry" && x != "note"
-      ); //Note is automatically added by 11ty. GardenEntry is used internally to mark the home page
-
-      delete r.ref;
-    });
-
-    return {
-      statusCode: 200,
-      body: JSON.stringify(results),
-      // // more keys you can return:
-      // headers: { "headerName": "headerValue", ... },
-      // isBase64Encoded: true,
-    };
-  } catch (error) {
-    return { statusCode: 500, body: error.toString() };
-  }
-};
-
-function truncate(str, size) {
-  //first, remove HTML
-  str = str.replace(/<.*?>/g, "");
-  if (str.length < size) return str;
-  return str.substring(0, size - 3) + "...";
-}
-
-module.exports = { handler };

package.json

@@ -10,8 +10,7 @@
         "build:eleventy": "cross-env ELEVENTY_ENV=prod NODE_OPTIONS=--max-old-space-size=4096 eleventy",
         "build:sass": "sass src/site/styles:dist/styles --style compressed",
         "get-theme": "node src/site/get-theme.js",
-        "build": "npm-run-all get-theme build:*",
-        "postbuild": "node src/site/lunr-index.js"
+        "build": "npm-run-all get-theme build:*"
     },
     "keywords": [],
     "author": "",

plugin-info.json

@@ -3,7 +3,15 @@
         "src/site/styles/style.css",
         "src/site/index.njk",
         "src/site/index.11tydata.js",
-        "src/site/_data/filetree.js"
+        "src/site/_data/filetree.js",
+        "api/search.js",
+        "netlify/functions/search/search.js",
+        "src/site/versionednote.njk",
+        "src/site/_includes/layouts/versionednote.njk",
+        "src/site/lunr-index.js",
+        "src/site/_data/versionednotes.js",
+        "src/site/lunr.njk"
+        
     ],
     "filesToAdd": [
         "src/site/styles/custom-style.scss",
@@ -27,7 +35,6 @@
         "src/site/404.njk",
         "src/site/sitemap.njk",
         "src/site/feed.njk",
-        "src/site/versionednote.njk",
         "src/site/styles/style.scss",
         "src/site/styles/digital-garden-base.scss",
         "src/site/styles/obsidian-base.scss",
@@ -35,7 +42,6 @@
         "src/site/notes/notes.11tydata.js",
         "src/site/_includes/layouts/note.njk",
         "src/site/_includes/layouts/index.njk",
-        "src/site/_includes/layouts/versionednote.njk",
         "src/site/_includes/components/notegrowthhistory.njk",
         "src/site/_includes/components/pageheader.njk",
         "src/site/_includes/components/linkPreview.njk",
@@ -51,9 +57,6 @@
         "src/site/_includes/components/calloutScript.njk",
         "src/site/_includes/components/lucideIcons.njk",
         "src/site/_includes/components/timestamps.njk",
-        "src/site/lunr-index.js",
-        "src/site/lunr.njk",
-        "src/site/_data/versionednotes.js",
         "src/site/_data/meta.js",
         "src/site/_data/dynamics.js",
         "src/site/img/outgoing.svg",
@@ -61,10 +64,9 @@
         "src/helpers/utils.js",
         "src/helpers/filetreeUtils.js",
         "src/helpers/linkUtils.js",
-        "netlify/functions/search/search.js",
         "src/site/get-theme.js",
-        "api/search.js",
         "src/site/_data/eleventyComputed.js",
-        "src/site/graph.njk"
+        "src/site/graph.njk",
+        "src/site/search-index.njk"
     ]
 }

src/site/_data/eleventyComputed.js

@@ -5,5 +5,5 @@ const { userComputed } = require("../../helpers/userUtils");
 module.exports = {
   graph: (data) => getGraph(data),
   filetree: (data) => getFileTree(data),
-  userComputed: (data) => userComputed(data),
+  userComputed: (data) => userComputed(data)
 };

src/site/_data/meta.js

@@ -4,7 +4,7 @@ const fs = require("fs");
 const crypto = require("crypto");
 const { globSync } = require("glob");
 
-module.exports = async () => {
+module.exports = async (data) => {
   let baseUrl = process.env.SITE_BASE_URL || "";
   if (baseUrl && !baseUrl.startsWith("http")) {
     baseUrl = "https://" + baseUrl;
@@ -67,7 +67,8 @@ module.exports = async () => {
     baseTheme: process.env.BASE_THEME || "dark",
     siteName: process.env.SITE_NAME_HEADER || "Digital Garden",
     siteBaseUrl: baseUrl,
-    styleSettingsCss
+    styleSettingsCss,
+    buildDate: new Date(),
   };
 
   return meta;

src/site/_data/versionednotes.js

@@ -1,66 +0,0 @@
-require('dotenv').config();
-const { Octokit } = require("@octokit/core");
-const githubToken = process.env.GH_TOKEN;
-const octokit = new Octokit({ auth: githubToken });
-const markdownIt = require("markdown-it");
-const md = markdownIt({
-    html: true,
-}).use(function(md) {
-    //https://github.com/DCsunset/markdown-it-mermaid-plugin
-    const origRule = md.renderer.rules.fence.bind(md.renderer.rules);
-    md.renderer.rules.fence = (tokens, idx, options, env, slf) => {
-        const token = tokens[idx];
-        if (token.info === 'mermaid') {
-            const code = token.content.trim();
-            return `<pre class="mermaid">${code}</pre>`;
-        }
-
-        // Other languages
-        return origRule(tokens, idx, options, env, slf);
-    };
-});
-
-module.exports = async function() {
-    if (!process.env.ENABLE_VERSION_HISTORY) {
-        return [];
-    }
-    //list all files
-    const noteFolder = 'src/site/notes';
-    const fs = require('fs');
-
-    const files = fs.readdirSync(noteFolder).filter(file => file.endsWith(".md")).map(file => noteFolder + '/' + file);
-    const notes = [];
-    for (const filePath of files) {
-        const fileCommits = await octokit.request(`GET /repos/{owner}/{repo}/commits?path=${encodeURI(filePath)}`, {
-            owner: process.env.GH_USERNAME,
-            repo: process.env.GH_REPO_NAME
-        })
-        if (filePath.indexOf("digital garden") > -1) {
-            console.log(fileCommits);
-        }
-        for (const commit of fileCommits.data) {
-
-            const sha = commit.sha
-            const fileData = await octokit.request(`GET /repos/{owner}/{repo}/contents/${encodeURI(filePath)}?ref=${sha}`, {
-                owner: process.env.GH_USERNAME,
-                repo: process.env.GH_REPO_NAME
-            });
-            const content = Buffer.from(fileData.data.content, 'base64').toString('utf8');
-            const segments = filePath.split("/");
-            const name = segments[segments.length - 1].replace(".md", "");
-            const date = commit.commit.author.date;
-            let markdown = ''
-            try {
-                markdown = md.render(content);
-            } catch (e) {
-                console.log(e);
-            }
-            const note = { content: markdown, title: name, fullTitle: name + " - " + sha, sha: sha, date: date };
-
-            notes.push(note);
-        }
-
-    }
-
-    return notes;
-}

src/site/_includes/components/searchScript.njk

@@ -1,21 +1,33 @@
+<script src="https://cdn.jsdelivr.net/npm/flexsearch@0.7.21/dist/flexsearch.bundle.js"></script>
 <script>
     document.addEventListener('DOMContentLoaded', init, false);
     document.addEventListener('DOMContentLoaded', setCorrectShortcut, false);
 
-    window.toggleSearch=function(){
-        if(document.getElementById('globalsearch').classList.contains('active')){
-            document.getElementById('globalsearch').classList.remove('active');
-        }else{
-            document.getElementById('globalsearch').classList.add('active');
-            document.getElementById('term').focus();
+    window.toggleSearch = function () {
+        if (document.getElementById('globalsearch').classList.contains('active')) {
+            document
+                .getElementById('globalsearch')
+                .classList
+                .remove('active');
+        } else {
+            document
+                .getElementById('globalsearch')
+                .classList
+                .add('active');
+            document
+                .getElementById('term')
+                .focus();
         }
     }
 
-    window.toggleTagSearch=function(evt) {
+    window.toggleTagSearch = function (evt) {
         console.log(evt.textContent);
         const term = evt.textContent;
-        if(term){
-            window.document.getElementById('term').value = term.trim();
+        if (term) {
+            window
+                .document
+                .getElementById('term')
+                .value = term.trim();
             window.toggleSearch();
             window.search();
         }
@@ -66,26 +78,83 @@
 
     function debounce(func, wait, immediate) {
         var timeout;
-        return function() {
-            var context = this, args = arguments;
-            var later = function() {
+        return function () {
+            var context = this,
+                args = arguments;
+            var later = function () {
                 timeout = null;
-                if (!immediate) func.apply(context, args);
-            };
+                if (!immediate) 
+                    func.apply(context, args);
+                };
             var callNow = immediate && !timeout;
             clearTimeout(timeout);
             timeout = setTimeout(later, wait);
-            if (callNow) func.apply(context, args);
-        };
+            if (callNow) 
+                func.apply(context, args);
+            };
     };
 
-    function setCorrectShortcut(){
-        if(navigator.platform.toUpperCase().indexOf('MAC')>=0){
-            document.querySelectorAll(".search-keys").forEach(x=>x.innerHTML = "⌘ + K");
+    function setCorrectShortcut() {
+        if (navigator.platform.toUpperCase().indexOf('MAC') >= 0) {
+            document
+                .querySelectorAll(".search-keys")
+                .forEach(x => x.innerHTML = "⌘ + K");
         }
     }
 
+    function createIndex(posts) {
+        const encoder = (str) => str
+            .toLowerCase()
+            .split(/([^a-z]|[^\x00-\x7F])/)
+        const contentIndex = new FlexSearch.Document({
+            cache: true,
+            charset: "latin:extra",
+            optimize: true,
+            index: [
+                {
+                    field: "content",
+                    tokenize: "reverse",
+                    encode: encoder
+                }, {
+                    field: "title",
+                    tokenize: "forward",
+                    encode: encoder
+                }, {
+                    field: "tags",
+                    tokenize: "forward",
+                    encode: encoder
+                }
+            ]
+        })
+        posts.forEach((p, idx) => {
+            contentIndex.add({
+                id: idx, title: p.title, content: p.content, //Change to removeHTML
+            })
+        });
+        return contentIndex;
+    }
+
     async function init() {
+        //init offline search index
+
+        const searchIndexDate = '{{meta.buildDate|isoDate}}';
+        let shouldFetch = true;
+        if(localStorage.getItem("searchIndex")) {
+            let {date, docs}= JSON.parse(localStorage.getItem('searchIndex'));
+            if(date === searchIndexDate){
+                shouldFetch = false;
+                let index = createIndex(docs);
+                window.docs = docs
+                window.index = index;
+            }        
+        }
+        if(shouldFetch){
+            let docs = await(await fetch('/searchIndex.json?v={{meta.buildDate|isoDate}}')).json();
+            let index = createIndex(docs);
+            localStorage.setItem("searchIndex", JSON.stringify({date: '{{meta.buildDate|isoDate}}', docs}));
+            window.docs = docs
+            window.index = index;
+        }
 
         //open searchmodal when ctrl + k is pressed, cmd + k on mac
         document.addEventListener('keydown', (e) => {
@@ -94,7 +163,10 @@
                 toggleSearch();
             }
             if (e.key === 'Escape') {
-                document.getElementById('globalsearch').classList.remove('active');
+                document
+                    .getElementById('globalsearch')
+                    .classList
+                    .remove('active');
             }
 
             //navigate search results with arrow keys
@@ -103,54 +175,70 @@
                     e.preventDefault();
                     let active = document.querySelector('.searchresult.active');
                     if (active) {
-                        active.classList.remove('active');
+                        active
+                            .classList
+                            .remove('active');
                         if (active.nextElementSibling) {
-                            active.nextElementSibling.classList.add('active');
+                            active
+                                .nextElementSibling
+                                .classList
+                                .add('active');
                         } else {
-                            document.querySelector('.searchresult').classList.add('active');
+                            document
+                                .querySelector('.searchresult')
+                                .classList
+                                .add('active');
                         }
                     } else {
-                        document.querySelector('.searchresult').classList.add('active');
+                        document
+                            .querySelector('.searchresult')
+                            .classList
+                            .add('active');
                     }
-                    
+
                     let currentActive = document.querySelector('.searchresult.active');
                     if (currentActive) {
-                        currentActive .scrollIntoView({
-                            behavior: 'smooth',
-                            block: 'nearest',
-                            inline: 'start',
-                        });
+                        currentActive.scrollIntoView({behavior: 'smooth', block: 'nearest', inline: 'start'});
                     }
                 }
                 if (e.key === 'ArrowUp') {
                     e.preventDefault();
                     let active = document.querySelector('.searchresult.active');
                     if (active) {
-                        active.classList.remove('active');
+                        active
+                            .classList
+                            .remove('active');
                         if (active.previousElementSibling) {
-                            active.previousElementSibling.classList.add('active');
+                            active
+                                .previousElementSibling
+                                .classList
+                                .add('active');
                         } else {
-                            document.querySelectorAll('.searchresult').forEach((el) => {
-                                if (!el.nextElementSibling) {
-                                    el.classList.add('active');
-                                }
-                            });
+                            document
+                                .querySelectorAll('.searchresult')
+                                .forEach((el) => {
+                                    if (!el.nextElementSibling) {
+                                        el
+                                            .classList
+                                            .add('active');
+                                    }
+                                });
                         }
                     } else {
-                        document.querySelectorAll('.searchresult').forEach((el) => {
-                            if (el.nextElementSibling) {
-                                el.classList.add('active');
-                            }
-                        });
+                        document
+                            .querySelectorAll('.searchresult')
+                            .forEach((el) => {
+                                if (el.nextElementSibling) {
+                                    el
+                                        .classList
+                                        .add('active');
+                                }
+                            });
                     }
 
                     let currentActive = document.querySelector('.searchresult.active');
                     if (currentActive) {
-                        currentActive .scrollIntoView({
-                            behavior: 'smooth',
-                            block: 'nearest',
-                            inline: 'start',
-                        });
+                        currentActive.scrollIntoView({behavior: 'smooth', block: 'nearest', inline: 'start'});
                     }
 
                 }
@@ -158,7 +246,9 @@
                     e.preventDefault();
                     let active = document.querySelector('.searchresult.active');
                     if (active) {
-                        window.location.href = active.querySelector("a").href;
+                        window.location.href = active
+                            .querySelector("a")
+                            .href;
                     }
                 }
             }
@@ -187,13 +277,15 @@
             .trim();
         if (!search) 
             return;
-        if(search == lastSearch) return;
+        if (search == lastSearch) 
+            return;
         console.log(`search for ${search}`);
         window.lastSearch = search;
 
         resultsDiv.innerHTML = loadingSvg;
-        let searchRequest = await fetch(`/api/search?term=${encodeURIComponent(search)}`);
-        let results = await searchRequest.json();
+        //let searchRequest = await fetch(`/api/search?term=${encodeURIComponent(search)}`);
+        //let results = await searchRequest.json();
+        let results = offlineSearch(search);
         let resultsHTML = '';
         if (!results.length) {
             let resultParagraph = document.createElement("p");
@@ -229,4 +321,59 @@
         resultsHTML += '</div>';
         resultsDiv.innerHTML = resultsHTML;
     }
-</script>
+
+    function truncate(str, size) {
+        //first, remove HTML
+        str = str.replaceAll(/<[^>]*>/g, '');
+        if (str.length < size) 
+            return str;
+        return str.substring(0, size - 3) + '...';
+    }
+
+    function offlineSearch(searchQuery) {
+        let data = window.docs;
+
+        let isTagSearch = search[0] === "#" && search.length > 1;
+
+        let searchResults = isTagSearch
+            ? index.search(searchQuery.substring(1), [
+                {
+                    field: "tags"
+                }
+            ])
+            : index.search(searchQuery, [
+                {
+                    field: "title",
+                    limit: 5
+                }, {
+                    field: "content",
+                    weight: 10
+                }
+            ]);
+
+        const getByField = (field) => {
+            const results = searchResults.filter((x) => x.field === field)
+            if (results.length === 0) {
+                return []
+            } else {
+                return [...results[0].result]
+            }
+        }
+        const allIds = new Set([
+            ...getByField("title"),
+            ...getByField("content")
+        ])
+        const dataIds = [...allIds];
+        const finalResults = dataIds.map((id) => {
+            let result = data[id];
+            result.content = truncate(result.content, 400);
+            result.tags = result
+                .tags
+                .filter((x) => x != "gardenEntry" && x != "note"); //Note is automatically added by 11ty. GardenEntry is used internally to mark the home page
+
+            return result;
+        })
+        return finalResults;
+
+    }
+</script>

src/site/_includes/layouts/versionednote.njk

@@ -1,12 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <title>{{ item.title }}</title>
-    {%include "components/pageheader.njk"%}
-  </head>
-  <body>
-    <div class="content">
-        {{ item.content | safe}}
-      </div>
-    </body>
-  </html>

src/site/lunr-index.js

@@ -1,22 +0,0 @@
-require("dotenv").config()
-const lunrjs = require('lunr');
-const path = require('path');
-
-function createIndex(posts) {
-  return lunrjs(function () {
-    this.ref('id');
-    this.field('title');
-    this.field('content');
-    this.field('date');
-    this.field("tags");
-
-    posts.forEach((p, idx) => {
-      p.id = idx;
-      this.add(p);
-    });
-  });
-}
-
-const data = require('../../netlify/functions/search/data.json');
-const index = createIndex(data);
-require('fs').writeFileSync(path.join(__dirname, '../../netlify/functions/search/index.json'), JSON.stringify(index));

src/site/search-index.njk

@@ -1,6 +1,5 @@
 ---
-permalink: netlify/functions/search/data.json
-permalinkBypassOutputDir: true
+permalink: /searchIndex.json
 eleventyExcludeFromCollections: true
 ---
 [{% for post in collections.note %}
@@ -11,4 +10,4 @@ eleventyExcludeFromCollections: true
 		"content": {{ post.templateContent | striptags(true) | link | jsonify | safe  }},
 		"tags": [{{post.templateContent | link | searchableTags | safe }} {% if post.data.tags %}{% for tag in post.data.tags %}"{{tag|validJson}}"{% if not loop.last %},{% endif %}{% endfor %}{% endif %}]
 }{% if not loop.last %},{% endif %}
-{% endfor %}]
+{% endfor %}]

src/site/versionednote.njk

@@ -1,10 +0,0 @@
----
-layout: layouts/versionednote.njk
-tags: versionednote
-pagination:
-    data: versionednotes
-    size: 1
-    alias: item
-    addAllPagesToCollections: true
-permalink: versionednotes/{{item.title}}/{{item.sha}}/index.html
----