first commit

.devcontainer/Dockerfile

@@ -0,0 +1,19 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.194.3/containers/go/.devcontainer/base.Dockerfile
+
+# [Choice] Go version: 1, 1.16, 1.17
+ARG VARIANT="1.17"
+FROM mcr.microsoft.com/vscode/devcontainers/go:0-${VARIANT}
+
+# [Choice] Node.js version: none, lts/*, 16, 14, 12, 10
+ARG NODE_VERSION="none"
+RUN if [ "${NODE_VERSION}" != "none" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
+
+# [Optional] Uncomment this section to install additional OS packages.
+# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+#     && apt-get -y install --no-install-recommends <your-package-list-here>
+
+# [Optional] Uncomment the next line to use go get to install anything else you need
+# RUN go get -x <your-dependency-or-tool>
+
+# [Optional] Uncomment this line to install global node packages.
+# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1

.devcontainer/devcontainer.json

@@ -0,0 +1,35 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.194.3/containers/go
+{
+  "name": "Go",
+  "build": {
+    "dockerfile": "Dockerfile",
+    "args": {
+      // Update the VARIANT arg to pick a version of Go: 1, 1.16, 1.17
+      "VARIANT": "1.17",
+      // Options
+      "NODE_VERSION": "lts/*"
+    }
+  },
+  "runArgs": ["--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined"],
+
+  // Set *default* container specific settings.json values on container create.
+  "settings": {
+    "go.toolsManagement.checkForUpdates": "local",
+    "go.useLanguageServer": true,
+    "go.gopath": "/go",
+    "go.goroot": "/usr/local/go"
+  },
+
+  // Add the IDs of extensions you want installed when the container is created.
+  "extensions": ["golang.Go"],
+
+  // Use 'forwardPorts' to make a list of ports inside the container available locally.
+  // "forwardPorts": [],
+
+  // Use 'postCreateCommand' to run commands after the container is created.
+  // "postCreateCommand": "go version",
+
+  // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+  "remoteUser": "vscode"
+}

.gitignore

@@ -0,0 +1,11 @@
+secrets/*
+.DS_Store
+data
+build/
+static
+vendora
+*.plist
+appconfig.json
+source/instapaper-full-text/node_modules
+source/instapaper-full-text/tmp
+search/misou.bleve

LICENCE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 Adrian Stobbe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,5 @@
+# cfgPath=/Users/adria/Programming/misou/appconfig.json
+build:
+	wails build -p -ldflags="-X 'github.com/elchead/misou/integration.cfgPath=${misouCfg}'"
+index:
+	go run ./scripts/indexer.go

README.md

@@ -0,0 +1,94 @@
+# 🔎 Mi 搜 - a personal search engine
+
+![overview](sneek.png)
+
+Misou is a personal search engine very much inspired by [monocle](https://github.com/thesephist/monocle) that looks through my knowledge sources. It is written in Go and React and uses [wails](https://wails.app/) to build a native desktop app. The app is mainly build for my usage but I tried to make it generally usable. It's usable at this point but still early-stage.
+
+## Features
+
+- search and configure different [sources](#sources)
+- file links are opened in Obsidian
+- browser links are opened in Browser
+- shortcuts
+
+## Installation
+
+Words of caution: The installation is suited to my setup and has not been tested on other machines. If you encounter problems, please contact me :)
+
+0.  Install the dependencies:
+
+    ```
+    brew install ripgrep-all jq
+    ```
+
+1.  Configure paths inside `appconfig.json` and move to desired directory
+2.  Export config path:
+    ```
+    export misouCfg=/Users/adria/Programming/misou/appconfig.json
+    ```
+3.  Build app:
+    ```
+    make build
+    ```
+    Note that the config location is injected during build time and cannot be changed afterwards!
+4.  Generate search index (if local sources are used, see [Sources](#sources))
+
+    ```
+    make index
+    ```
+
+## Sources
+
+using external API (slow ~500ms):
+
+- local file folder
+- bookmarks and history (tested with Chrome)
+
+locally indexed (much faster ~30ms):
+
+- readwise (provided with exported csv)
+- instapaper
+- twitter
+- personal CRM (not public but based on [noahm's fork of mira](https://github.com/noahm/contactful))
+
+---
+
+The data of indexed sources need to be manually updated in the folder. Afterwards, the index can be rebuilt using `make index`
+
+### Readwise
+
+If you use Obsidian, it's better to use the Readwise-sync plugin and combine it with local file search.
+Otherwise export the csv manually from the UI.
+
+### Instapaper
+
+If you use Readwise, highlights are synced, but this plugin allows for full-text search in archive by locally downloading the file content.
+
+### Twitter
+
+To get the twitter archive check [here](https://help.twitter.com/en/managing-your-account/how-to-download-your-twitter-archive).
+The update of the twitter archive requires a conversion of the `tweet.js` to a json file. [This](https://www.convertonline.io/convert/js-to-json) converter can be used, saving the file in `tweet.json` to format the file in vscode. The latter ensures proper escaping of `"` in field text.
+
+## Future features
+
+- support exact search (indicated with surrounding `"`) not only for indexed sources
+- integrate Memex
+- Alfred workflow
+
+## Bugs
+
+I use `fzf` and the [bookmarks](https://github.com/junegunn/fzf/wiki/examples#bookmarks) and [history](https://github.com/junegunn/fzf/wiki/examples#bookmarks) function and found that after a system restart, I need to run these commands once before running my scripts for the corresponding sources.
+
+In `sources`, there is a Gdrive client which I don't support anymore because I don't need it. I remember that after a few days the token had to be deleted manually.
+
+## Dependencies
+
+- [wails](https://wails.app/): for native app build
+- [RGA](https://github.com/phiresky/ripgrep-all): for local file search
+- [Blevesearch](https://github.com/blevesearch/bleve): used as main indexer (alternative implementation based on [Apollo](https://github.com/amirgamil/apollo#design) can also be used)
+
+## Attributions
+
+Appicon (Sou): <a href="https://www.freepik.com" title="Freepik">Freepik</a> from <a href="https://www.flaticon.com/" title="Flaticon">www.flaticon.com</a>
+
+This project started on a [hackathon](https://devpost.com/software/gerstler) with the help of my friends from university.

cmd/cli/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path"
+
+	"github.com/elchead/misou/config"
+	"github.com/elchead/misou/search"
+	"github.com/urfave/cli/v2"
+)
+
+func prettyPrintResults(res []search.SearchResult) {
+	for _, s := range res {
+		fmt.Printf("Title: %s\nSource: %s\nContent: %s\nProvider: %s\n\n", s.Title, s.Link, s.Content, s.Provider)
+	}
+}
+
+func main() {
+	s := search.NewSearcher()
+	config := config.LoadConfig(path.Join(".", "appconfig.json"))
+	config.InitSources(s)
+
+	app := &cli.App{
+		Name:  "Misou",
+		Usage: "explore your knowledge universe",
+		Action: func(c *cli.Context) error {
+			query := c.Args().Get(0)
+			res, err := s.Search(query)
+			if err != nil {
+				fmt.Printf("Search %s failed: %v\n", query, err)
+			}
+			prettyPrintResults(res)
+			return nil
+		},
+	}
+	err := app.Run(os.Args)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

cmd/webserver/main.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+	"log"
+	"net/http"
+	"path"
+
+	"github.com/elchead/misou/config"
+	"github.com/elchead/misou/search"
+	"github.com/elchead/misou/webserver"
+)
+
+func main() {
+	searcher := search.NewSearcher()
+	config := config.LoadConfig(path.Join("..", "appconfig.json"))
+	config.InitSources(searcher)
+	server := webserver.NewSearchHandler(searcher)
+	log.Fatal(http.ListenAndServe("localhost:5000", server))
+}

config/appconfig.go

@@ -0,0 +1,142 @@
+package config
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/elchead/misou/filesystem"
+	"github.com/elchead/misou/search"
+	"github.com/elchead/misou/source"
+	"github.com/pkg/errors"
+)
+
+type AppConfig struct {
+	BookmarkBashSubPath string `json:"bookmarkBashSubPath"`
+	FileScrapeRgaPath string `json:"fileScrapeRgaPath"`
+	FileScrapePath      string `json:"fileScrapePath"`
+	HistoryBashSubPath  string `json:"historyBashSubPath"`
+	ReadwiseCsvPath     string `json:"readwiseCsvPath"`
+	PeoplePath string `json:"peoplePath"`
+	PeopleUrl string `json:"peopleUrl"`
+	TwitterPath string `json:"twitterPath"`
+	InstapaperPath     string `json:"instapaperPath"`
+	RepoPath            string `json:"repoPath"`
+	SecretsSubPath      string `json:"secretsSubPath"`
+	Sources             struct {
+		Bookmarks  bool `json:"bookmarks"`
+		Gdrive     bool `json:"gdrive"`
+		History    bool `json:"history"`
+		Localfiles bool `json:"localfiles"`
+		Readwise   bool `json:"readwise"`
+		Instapaper bool `json:"instapaper"`
+		People     bool `json:"people"`
+		Twitter     bool `json:"twitter"`
+	} `json:"sources"`
+}
+
+func (config AppConfig) InitSources(s search.SearcherI) {
+	if config.Sources.Localfiles {
+		fileSearcher := source.NewFileSearcher(config.FileScrapeRgaPath,config.FileScrapePath)
+		s.AddApiClient(fileSearcher)
+	}
+	if config.Sources.Readwise {
+		f, _ := filesystem.OpenFile(config.ReadwiseCsvPath)
+		defer f.Close()
+		readwiseClient := source.NewReadwiseCSVParser(f)
+		s.AddSrcToDb(readwiseClient)
+	}
+
+	if config.Sources.People {
+		f, err := filesystem.OpenFile(config.PeoplePath)
+		if err != nil {
+			log.Infof("Failed to open people file: %v", err)
+		}
+		peopleParser := source.NewPeopleParser(f)
+		peopleParser.Url = config.PeopleUrl
+		s.AddSrcToDb(peopleParser)
+	}
+	if config.Sources.Twitter {
+		f, err := filesystem.OpenFile(config.TwitterPath)
+		if err != nil {
+			log.Infof("Failed to open people file: %v", err)
+		}
+		peopleParser := &source.TwitterParser{File:f}
+		s.AddSrcToDb(peopleParser)
+	}
+
+
+	if config.Sources.Instapaper {
+		f, _ := filesystem.OpenFile(config.InstapaperPath)
+		defer f.Close()
+		instapaperClient := source.NewInstapaperParser(f)
+		s.AddSrcToDb(instapaperClient)
+	}
+
+	if config.Sources.Bookmarks {
+		bookmarkBashPath := path.Join(config.RepoPath, config.BookmarkBashSubPath)
+		bookmarkSearcher := source.NewBookmarkSearcher(bookmarkBashPath)
+		s.AddApiClient(bookmarkSearcher)
+	}
+
+	if config.Sources.History {
+		historyBashPath := path.Join(config.RepoPath, config.HistoryBashSubPath)
+		historySearcher := source.NewBrowserHistorySearcher(historyBashPath)
+		s.AddApiClient(historySearcher)
+	}
+
+
+	if config.Sources.Gdrive {
+		secretPath := path.Join(config.RepoPath, config.SecretsSubPath)
+		gdriveCredentialsPath := filepath.Join(secretPath, "gdrive.json")
+		tokenPath := filepath.Join(secretPath, "token.json")
+
+		credFile, err := os.Open(gdriveCredentialsPath)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer credFile.Close()
+
+		fmt.Printf("Saving credential file to: %s\n", tokenPath)
+		saveTokenFile, err := os.OpenFile(tokenPath, os.O_RDWR|os.O_CREATE, 0600)
+		if err != nil {
+			log.Fatalf("Unable to cache oauth token: %v", err)
+		}
+		defer saveTokenFile.Close()
+		client := source.NewGdriveClient(secretPath, credFile, saveTokenFile, saveTokenFile)
+		s.AddApiClient(client)
+	}
+}
+
+func LoadConfig(filePath string) AppConfig {
+	file, err := os.Open(filePath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	config, err := LoadConfigFromReader(file)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return config
+}
+
+func LoadConfigFromReader(file io.Reader) (AppConfig, error) {
+	var config AppConfig
+	err := filesystem.LoadVariable(file, &config)
+	if err != nil {
+		return AppConfig{}, errors.Wrapf(err, "failed to load app config from file")
+	}
+	return config, nil
+}
+
+func (config AppConfig) BookmarkPath() string {
+	return path.Join(config.RepoPath, config.BookmarkBashSubPath)
+}
+
+func (config AppConfig) HistoryPath() string {
+	return config.RepoPath + config.HistoryBashSubPath
+}