Add offline search with Pagefind.app

Run `bb index` to build the search index & supporting files,
then serve `./search.html` + the generated `./pagefind/` to be able to search.

Uses the pagefind Node lib to be able to index .edn files as "custom records".
Written with Squint, so that I don't need to write JS :-)

Author: holyjak

.gitignore

@@ -8,4 +8,6 @@ nongit/
 .clj-kondo/
 .nrepl-port
 .portal/
-.vscode/
+.vscode/
+build-pagefind-index.mjs
+pagefind/

bb.edn

@@ -1,8 +1,11 @@
 {:tasks {run {:doc "Download posts added after the previous run" 
+              :override-builtin true
               :task (clojure "-M" "-m" "clj-tumblr-summarizer.main")}
          summarize {:doc "Create an HTML summary of (already downloaded) posts from the previous month" 
                     :task (clojure "-X:summarize")} ; also: clojure "-X:summarize" ":date" '"2022-12"'
          summarize-m {:doc "Create an HTML summary of (already downloaded) posts from AND month given as CLI arg like 2022-12"
                        :task (when-let [md (first *command-line-args*)]
                                (assert (re-matches (re-pattern "^\\d{4}-\\d{2}$") md) "The month arg must match yyyy-mm")
-                               (clojure (format "-X:summarize :date '\"%s\"'" md)))}}}
+                               (clojure (format "-X:summarize :date '\"%s\"'" md)))}
+         index {:doc "Build Pagefind.app index for static search of the records"
+                :task (shell "npx squint run build-pagefind-index.cljs")}}}

build-pagefind-index.cljs

@@ -0,0 +1,54 @@
+;; TODO: Add to the index incrementally, only new stuff => faster; currently takes 3 min
+(ns build-pagefind-index
+  "Build a pagefind search index from .edn Tumblr entries previously downloaded
+   into /data, together with the supporting .js, .css and other files.
+   Serve statically ./search.html and ./pagefind/ to use the search."
+  {:clj-kondo/ignore true}
+  (:require
+   ["fs" :as fs]
+   ["node:fs/promises" :refer [readdir readFile]]
+   ["url" :refer [fileURLToPath]]
+   ["pagefind" :as pagefind]
+   ["edn-data" :refer [parseEDNString]]))
+
+(defn entry->record [{:keys [tags summary content timestamp post_url date] :as _entry}]
+  (try
+   (let [date (when date (.substring date 0 10))
+         content (-> (.map content (fn [{:keys [text description]}] 
+                                    ;; type 'link' -> description, text -> text
+                                     (or text description)))
+                     (.join "\n"))]
+     {:url post_url
+      :content content
+      :language "en"
+     ;; optional:
+      :meta (cond-> {:title summary
+                     ;; These below will be displayed below the result
+                     :date date #_image}
+              (seq tags)
+              (assoc! :tags (-> tags (.sort) (.join ", "))))
+      :filters {:tags tags}
+      :sort {:date date}})
+    (catch :default err
+      (println "ERROR processing entry" _entry ":" (.-message err))
+      (throw err))))
+
+(defn ^:async index-all []
+  (let [entries (js-await (readdir "data"))
+        index (.-index (js-await (pagefind/createIndex)))]
+    ;; Add all entries to the index:
+    (->
+     (for [e entries
+           :when (.endsWith e ".edn")]
+       (-> (readFile (str "data/" e) {:encoding "utf8"})
+           (.then #(parseEDNString % {:mapAs "object" :keywordAs "string"}))
+           (.then entry->record)
+           (.then #(.addCustomRecord index %))))
+     (Promise/all)
+     (js-await)) 
+    ;; Save the index:
+    (js-await (.writeFiles index {:outputPath "pagefind"}))
+    (println "Index written with" (count entries) "entries")
+    (js-await (.close pagefind))))
+
+(js-await (index-all))

package.json

@@ -0,0 +1,17 @@
+{
+  "name": "clj-tumblr-summarizer",
+  "version": "1.0.0",
+  "description": "A tool / library that progressively backs up posts from a Tumblr.com micro-blog and can produce monthly summaries of the published posts in HTML. Can be run as a scheduled GitHub Action.",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "edn-data": "^1.0.0",
+    "pagefind": "^1.0.3",
+    "squint-cljs": "^0.3.35"
+  }
+}

search.html

@@ -0,0 +1,27 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Search entries</title>
+    <link href="/pagefind/pagefind-ui.css" rel="stylesheet">
+    <script src="/pagefind/pagefind-ui.js"></script>
+    <body>
+        <h1>Search entries</h1>    
+<div id="search"></div>
+<script>
+    window.addEventListener('DOMContentLoaded', (event) => {
+        new PagefindUI({ 
+            element: "#search", 
+            showSubResults: true, 
+            excerptLength: 100,
+            showEmptyFilters: false,
+            baseUrl: "/",
+            processResult: function (result) {
+                result.url = result.url.substring(1); // drop the leading / that pagefind adds
+                return result;
+            }
+        });
+    });
+</script>
+    </body>
+</html>