From 48b716229849d108f19638406e6c8f276b718251 Mon Sep 17 00:00:00 2001 From: Ole Eskild Steensen Date: Tue, 23 May 2023 22:14:50 +0200 Subject: [PATCH] Local flexsearch (#160) * Switch to using local search * Switch from lunr to FlexSearch * Cache searchindex in localstorage * Link unresolved links to 404 page in build step --- .eleventy.js | 12 +- api/search.js | 11 - netlify/functions/search/search.js | 49 ---- package.json | 3 +- plugin-info.json | 20 +- src/site/_data/eleventyComputed.js | 2 +- src/site/_data/meta.js | 5 +- src/site/_data/versionednotes.js | 66 ----- .../_includes/components/searchScript.njk | 251 ++++++++++++++---- src/site/_includes/layouts/versionednote.njk | 12 - src/site/lunr-index.js | 22 -- src/site/{lunr.njk => search-index.njk} | 5 +- src/site/versionednote.njk | 10 - 13 files changed, 226 insertions(+), 242 deletions(-) delete mode 100644 api/search.js delete mode 100644 netlify/functions/search/search.js delete mode 100644 src/site/_data/versionednotes.js delete mode 100644 src/site/_includes/layouts/versionednote.njk delete mode 100644 src/site/lunr-index.js rename src/site/{lunr.njk => search-index.njk} (86%) delete mode 100644 src/site/versionednote.njk diff --git a/.eleventy.js b/.eleventy.js index f45e932..f3e0864 100644 --- a/.eleventy.js +++ b/.eleventy.js @@ -194,6 +194,11 @@ module.exports = function (eleventyConfig) { eleventyConfig.setLibrary("md", markdownLib); + eleventyConfig.addFilter("isoDate", function (date) { + return date && date.toISOString(); + }); + + eleventyConfig.addFilter("link", function (str) { return ( str && @@ -240,9 +245,10 @@ module.exports = function (eleventyConfig) { deadLink = true; } - return `${title}`; + if(deadLink){ + return `${title}`; + } + return `${title}`; }) ); }); diff --git a/api/search.js b/api/search.js deleted file mode 100644 index 9bd0ccc..0000000 --- a/api/search.js +++ /dev/null @@ -1,11 +0,0 @@ -//vercel search function -const searchHandler = require('../netlify/functions/search/search.js').handler; -async function vercelSearch(request, response) { - let event = {queryStringParameters: request.query}; - - let searchResponse = await searchHandler(event); - - return response.status(200).json(JSON.parse(searchResponse.body)); - } - -exports.default = vercelSearch; \ No newline at end of file diff --git a/netlify/functions/search/search.js b/netlify/functions/search/search.js deleted file mode 100644 index f4bb4e8..0000000 --- a/netlify/functions/search/search.js +++ /dev/null @@ -1,49 +0,0 @@ -const lunrjs = require("lunr"); - -const handler = async (event) => { - try { - const search = event.queryStringParameters.term; - if (!search) throw "Missing term query parameter"; - - const data = require("./data.json"); - const indexJson = require("./index.json"); - const index = lunrjs.Index.load(indexJson); - console.log("index made"); - - let results = - search[0] == "#" && search.length > 1 - ? index.search(`tags:${search.substring(1)}`) - : index.search(search + "*"); - - results.forEach((r) => { - r.title = data[r.ref].title; - r.content = truncate(data[r.ref].content, 400); - r.date = data[r.ref].date; - r.url = data[r.ref].url; - r.tags = data[r.ref].tags.filter( - (x) => x != "gardenEntry" && x != "note" - ); //Note is automatically added by 11ty. GardenEntry is used internally to mark the home page - - delete r.ref; - }); - - return { - statusCode: 200, - body: JSON.stringify(results), - // // more keys you can return: - // headers: { "headerName": "headerValue", ... }, - // isBase64Encoded: true, - }; - } catch (error) { - return { statusCode: 500, body: error.toString() }; - } -}; - -function truncate(str, size) { - //first, remove HTML - str = str.replace(/<.*?>/g, ""); - if (str.length < size) return str; - return str.substring(0, size - 3) + "..."; -} - -module.exports = { handler }; diff --git a/package.json b/package.json index d1f6b03..d9dfab5 100644 --- a/package.json +++ b/package.json @@ -10,8 +10,7 @@ "build:eleventy": "cross-env ELEVENTY_ENV=prod NODE_OPTIONS=--max-old-space-size=4096 eleventy", "build:sass": "sass src/site/styles:dist/styles --style compressed", "get-theme": "node src/site/get-theme.js", - "build": "npm-run-all get-theme build:*", - "postbuild": "node src/site/lunr-index.js" + "build": "npm-run-all get-theme build:*" }, "keywords": [], "author": "", diff --git a/plugin-info.json b/plugin-info.json index 677fd6b..6be8b21 100644 --- a/plugin-info.json +++ b/plugin-info.json @@ -3,7 +3,15 @@ "src/site/styles/style.css", "src/site/index.njk", "src/site/index.11tydata.js", - "src/site/_data/filetree.js" + "src/site/_data/filetree.js", + "api/search.js", + "netlify/functions/search/search.js", + "src/site/versionednote.njk", + "src/site/_includes/layouts/versionednote.njk", + "src/site/lunr-index.js", + "src/site/_data/versionednotes.js", + "src/site/lunr.njk" + ], "filesToAdd": [ "src/site/styles/custom-style.scss", @@ -27,7 +35,6 @@ "src/site/404.njk", "src/site/sitemap.njk", "src/site/feed.njk", - "src/site/versionednote.njk", "src/site/styles/style.scss", "src/site/styles/digital-garden-base.scss", "src/site/styles/obsidian-base.scss", @@ -35,7 +42,6 @@ "src/site/notes/notes.11tydata.js", "src/site/_includes/layouts/note.njk", "src/site/_includes/layouts/index.njk", - "src/site/_includes/layouts/versionednote.njk", "src/site/_includes/components/notegrowthhistory.njk", "src/site/_includes/components/pageheader.njk", "src/site/_includes/components/linkPreview.njk", @@ -51,9 +57,6 @@ "src/site/_includes/components/calloutScript.njk", "src/site/_includes/components/lucideIcons.njk", "src/site/_includes/components/timestamps.njk", - "src/site/lunr-index.js", - "src/site/lunr.njk", - "src/site/_data/versionednotes.js", "src/site/_data/meta.js", "src/site/_data/dynamics.js", "src/site/img/outgoing.svg", @@ -61,10 +64,9 @@ "src/helpers/utils.js", "src/helpers/filetreeUtils.js", "src/helpers/linkUtils.js", - "netlify/functions/search/search.js", "src/site/get-theme.js", - "api/search.js", "src/site/_data/eleventyComputed.js", - "src/site/graph.njk" + "src/site/graph.njk", + "src/site/search-index.njk" ] } \ No newline at end of file diff --git a/src/site/_data/eleventyComputed.js b/src/site/_data/eleventyComputed.js index 6e6919f..d23f2b5 100644 --- a/src/site/_data/eleventyComputed.js +++ b/src/site/_data/eleventyComputed.js @@ -5,5 +5,5 @@ const { userComputed } = require("../../helpers/userUtils"); module.exports = { graph: (data) => getGraph(data), filetree: (data) => getFileTree(data), - userComputed: (data) => userComputed(data), + userComputed: (data) => userComputed(data) }; diff --git a/src/site/_data/meta.js b/src/site/_data/meta.js index 6385008..d36143b 100644 --- a/src/site/_data/meta.js +++ b/src/site/_data/meta.js @@ -4,7 +4,7 @@ const fs = require("fs"); const crypto = require("crypto"); const { globSync } = require("glob"); -module.exports = async () => { +module.exports = async (data) => { let baseUrl = process.env.SITE_BASE_URL || ""; if (baseUrl && !baseUrl.startsWith("http")) { baseUrl = "https://" + baseUrl; @@ -67,7 +67,8 @@ module.exports = async () => { baseTheme: process.env.BASE_THEME || "dark", siteName: process.env.SITE_NAME_HEADER || "Digital Garden", siteBaseUrl: baseUrl, - styleSettingsCss + styleSettingsCss, + buildDate: new Date(), }; return meta; diff --git a/src/site/_data/versionednotes.js b/src/site/_data/versionednotes.js deleted file mode 100644 index e69b4fa..0000000 --- a/src/site/_data/versionednotes.js +++ /dev/null @@ -1,66 +0,0 @@ -require('dotenv').config(); -const { Octokit } = require("@octokit/core"); -const githubToken = process.env.GH_TOKEN; -const octokit = new Octokit({ auth: githubToken }); -const markdownIt = require("markdown-it"); -const md = markdownIt({ - html: true, -}).use(function(md) { - //https://github.com/DCsunset/markdown-it-mermaid-plugin - const origRule = md.renderer.rules.fence.bind(md.renderer.rules); - md.renderer.rules.fence = (tokens, idx, options, env, slf) => { - const token = tokens[idx]; - if (token.info === 'mermaid') { - const code = token.content.trim(); - return `
${code}
`; - } - - // Other languages - return origRule(tokens, idx, options, env, slf); - }; -}); - -module.exports = async function() { - if (!process.env.ENABLE_VERSION_HISTORY) { - return []; - } - //list all files - const noteFolder = 'src/site/notes'; - const fs = require('fs'); - - const files = fs.readdirSync(noteFolder).filter(file => file.endsWith(".md")).map(file => noteFolder + '/' + file); - const notes = []; - for (const filePath of files) { - const fileCommits = await octokit.request(`GET /repos/{owner}/{repo}/commits?path=${encodeURI(filePath)}`, { - owner: process.env.GH_USERNAME, - repo: process.env.GH_REPO_NAME - }) - if (filePath.indexOf("digital garden") > -1) { - console.log(fileCommits); - } - for (const commit of fileCommits.data) { - - const sha = commit.sha - const fileData = await octokit.request(`GET /repos/{owner}/{repo}/contents/${encodeURI(filePath)}?ref=${sha}`, { - owner: process.env.GH_USERNAME, - repo: process.env.GH_REPO_NAME - }); - const content = Buffer.from(fileData.data.content, 'base64').toString('utf8'); - const segments = filePath.split("/"); - const name = segments[segments.length - 1].replace(".md", ""); - const date = commit.commit.author.date; - let markdown = '' - try { - markdown = md.render(content); - } catch (e) { - console.log(e); - } - const note = { content: markdown, title: name, fullTitle: name + " - " + sha, sha: sha, date: date }; - - notes.push(note); - } - - } - - return notes; -} \ No newline at end of file diff --git a/src/site/_includes/components/searchScript.njk b/src/site/_includes/components/searchScript.njk index a70cc7e..69deb2c 100644 --- a/src/site/_includes/components/searchScript.njk +++ b/src/site/_includes/components/searchScript.njk @@ -1,21 +1,33 @@ + + + function truncate(str, size) { + //first, remove HTML + str = str.replaceAll(/<[^>]*>/g, ''); + if (str.length < size) + return str; + return str.substring(0, size - 3) + '...'; + } + + function offlineSearch(searchQuery) { + let data = window.docs; + + let isTagSearch = search[0] === "#" && search.length > 1; + + let searchResults = isTagSearch + ? index.search(searchQuery.substring(1), [ + { + field: "tags" + } + ]) + : index.search(searchQuery, [ + { + field: "title", + limit: 5 + }, { + field: "content", + weight: 10 + } + ]); + + const getByField = (field) => { + const results = searchResults.filter((x) => x.field === field) + if (results.length === 0) { + return [] + } else { + return [...results[0].result] + } + } + const allIds = new Set([ + ...getByField("title"), + ...getByField("content") + ]) + const dataIds = [...allIds]; + const finalResults = dataIds.map((id) => { + let result = data[id]; + result.content = truncate(result.content, 400); + result.tags = result + .tags + .filter((x) => x != "gardenEntry" && x != "note"); //Note is automatically added by 11ty. GardenEntry is used internally to mark the home page + + return result; + }) + return finalResults; + + } + \ No newline at end of file diff --git a/src/site/_includes/layouts/versionednote.njk b/src/site/_includes/layouts/versionednote.njk deleted file mode 100644 index a27ffdc..0000000 --- a/src/site/_includes/layouts/versionednote.njk +++ /dev/null @@ -1,12 +0,0 @@ - - - - {{ item.title }} - {%include "components/pageheader.njk"%} - - -
- {{ item.content | safe}} -
- - \ No newline at end of file diff --git a/src/site/lunr-index.js b/src/site/lunr-index.js deleted file mode 100644 index 03a84d2..0000000 --- a/src/site/lunr-index.js +++ /dev/null @@ -1,22 +0,0 @@ -require("dotenv").config() -const lunrjs = require('lunr'); -const path = require('path'); - -function createIndex(posts) { - return lunrjs(function () { - this.ref('id'); - this.field('title'); - this.field('content'); - this.field('date'); - this.field("tags"); - - posts.forEach((p, idx) => { - p.id = idx; - this.add(p); - }); - }); -} - -const data = require('../../netlify/functions/search/data.json'); -const index = createIndex(data); -require('fs').writeFileSync(path.join(__dirname, '../../netlify/functions/search/index.json'), JSON.stringify(index)); diff --git a/src/site/lunr.njk b/src/site/search-index.njk similarity index 86% rename from src/site/lunr.njk rename to src/site/search-index.njk index 8e79500..c587a5d 100644 --- a/src/site/lunr.njk +++ b/src/site/search-index.njk @@ -1,6 +1,5 @@ --- -permalink: netlify/functions/search/data.json -permalinkBypassOutputDir: true +permalink: /searchIndex.json eleventyExcludeFromCollections: true --- [{% for post in collections.note %} @@ -11,4 +10,4 @@ eleventyExcludeFromCollections: true "content": {{ post.templateContent | striptags(true) | link | jsonify | safe }}, "tags": [{{post.templateContent | link | searchableTags | safe }} {% if post.data.tags %}{% for tag in post.data.tags %}"{{tag|validJson}}"{% if not loop.last %},{% endif %}{% endfor %}{% endif %}] }{% if not loop.last %},{% endif %} -{% endfor %}] +{% endfor %}] \ No newline at end of file diff --git a/src/site/versionednote.njk b/src/site/versionednote.njk deleted file mode 100644 index 1646b24..0000000 --- a/src/site/versionednote.njk +++ /dev/null @@ -1,10 +0,0 @@ ---- -layout: layouts/versionednote.njk -tags: versionednote -pagination: - data: versionednotes - size: 1 - alias: item - addAllPagesToCollections: true -permalink: versionednotes/{{item.title}}/{{item.sha}}/index.html ---- \ No newline at end of file