feat: Dev Container (#1149)

* temporary dvctr * start of work * fix: timeouts in windows * fish? * feat: better, working stuff * refactor: rename database-seeds to seeds/ * better * fdsaf * feat: don't let our container get owned by root * fix: better user handling * feat: falling asleep at my desk i yield to you what i have done * feat: autoadminify * docs: denote trickshot * feat: open in browser on launch * better seeds workings... * fix: dont kill shell * docs: mention dev/debs * fix: whoops * fix: make seeds load within the century * feat: rest of the containerisation * feat: new alias * style: fix linter errors * fix: bad call * fix: better justfile
zkrising · Aug 17, 2024 · 593adb2 · 593adb2
1 parent 14831a5
commit 593adb2
Show file tree

Hide file tree

Showing 305 changed files with 6,019 additions and 703 deletions.
diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -0,0 +1,29 @@
+{
+	"name": "Tachi",
+	"dockerComposeFile": [
+		"../docker-compose-dev.yml"
+	],
+	"service": "tachi-dev",
+	"workspaceFolder": "/tachi",
+	"postAttachCommand": "./dev/bootstrap.sh",
+	"updateRemoteUserUID": true,
+	"containerUser": "tachi",
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"typescript.tsdk": "node_modules/typescript/lib",
+				"[typescriptreact]": {
+					"editor.formatOnSave": true,
+					"editor.defaultFormatter": "dbaeumer.vscode-eslint"
+				},
+				"[typescript]": {
+					"editor.formatOnSave": true,
+					"editor.defaultFormatter": "dbaeumer.vscode-eslint"
+				}
+			},
+			"extensions": [
+				"dbaeumer.vscode-eslint"
+			]
+		}
+	}
+}
diff --git a/.dockerignore b/.dockerignore
@@ -5,5 +5,5 @@ node_modules
 **/.tsbuildinfo
 
 # Extracted data from iidx merge script
-database-seeds/scripts/rerunners/iidx/iidx-mdb-parse/ifs-output
-database-seeds/scripts/rerunners/iidx/iidx-mdb-parse/logs
+seeds/scripts/rerunners/iidx/iidx-mdb-parse/ifs-output
+seeds/scripts/rerunners/iidx/iidx-mdb-parse/logs
diff --git a/.github/workflows/database-seeds.yml b/.github/workflows/database-seeds.yml
@@ -1,17 +1,17 @@
-name: Database-Seeds CI/CD
+name: Seeds CI/CD
 
 on:
   push:
     branches:
       - "main"
     paths:
-      - "database-seeds/**"
+      - "seeds/**"
       - "common/**"
   pull_request:
     branches:
       - "main"
     paths:
-      - "database-seeds/**"
+      - "seeds/**"
       - "common/**"
   workflow_dispatch:
 
@@ -32,10 +32,10 @@ jobs:
           cache: pnpm
 
       - name: Install dependencies
-        run: pnpm --filter tachi-database-seeds-scripts... --filter . install
+        run: pnpm --filter tachi-seeds-scripts... --filter . install
 
       - name: Run Tests
-        run: pnpm --filter tachi-database-seeds-scripts test
+        run: pnpm --filter tachi-seeds-scripts test
         env:
           NODE_ENV: "test"
   deploy:

diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -0,0 +1,5 @@
+{
+	"recommendations": [
+		"ms-vscode-remote.remote-containers"
+	]
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -5,6 +5,5 @@
 	},
 	"[typescript]": {
 		"editor.defaultFormatter": "dbaeumer.vscode-eslint"
-	}
-}
+	},
 }
diff --git a/Dockerfile.bootstrap b/Dockerfile.bootstrap
diff --git a/Dockerfile.dev b/Dockerfile.dev
@@ -0,0 +1,55 @@
+# For use via `devcontainer.json`. The docker-compose-dev file sets some other
+# important variables.
+
+# we use ubuntu because 'just' isn't available on debian 13
+# haha.. haha...
+FROM ubuntu:24.10
+
+WORKDIR /tachi
+
+RUN apt update
+
+# essentials
+RUN apt install -y git npm locales sudo
+
+# setup locales
+# https://stackoverflow.com/questions/28405902/how-to-set-the-locale-inside-a-debian-ubuntu-docker-container
+RUN echo 'en_US.UTF-8' > /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+# nodeisms
+RUN npm install --silent -g [email protected]
+
+# docs pythonisms
+RUN apt install -y python-is-python3 pip mkdocs mkdocs-material
+
+# https://github.com/python-babel/babel/issues/990
+# it wouldn't be python without needing absurd global state manipulation to fix
+# an incoherent error message
+RUN rm -f /etc/localtime && ln -s /usr/share/zoneinfo/Etc/UTC /etc/localtime
+
+# Fix locale issue perl repeatedly complains about
+RUN echo "LC_ALL=en_US.UTF-8\nLANG=en_US.UTF-8" > /etc/default/locale
+
+# nice to haves
+RUN apt install -y gh fish just fzf curl wget parallel neovim fd-find bat
+
+# `fd` is called `find-fd` on ubuntu. Awesome.
+RUN ln -s $(which fdfind) /usr/bin/fd
+
+# rename ubuntu user to tachi and give them sudo
+RUN usermod -l tachi ubuntu          && \
+	usermod -d /home/tachi -m tachi  && \
+	echo "tachi ALL = NOPASSWD : ALL" >> /etc/sudoers
+
+# Docker volumes are mounted as root UNLESS the folder already exists inside the host
+# and has non-root ownership. This is the only way to declare a volume in docker has
+# non-root ownership. Unbelievably obscure.
+RUN mkdir node_modules && \
+	mkdir .pnpm-store  && \
+	chown tachi:1000 node_modules .pnpm-store
+
+# keep container alive indefinitely
+CMD ["/bin/fish"]
diff --git a/Dockerfile.seeds b/Dockerfile.seeds
diff --git a/Justfile b/Justfile
@@ -0,0 +1,48 @@
+mod seeds
+mod server
+mod client
+mod docs
+
+[private]
+interactive:
+	-@just --choose
+
+# Run the frontend and backend for Tachi.
+#
+# This is the main command you want to use to start up tachi. Go for it!
+start:
+	parallel --lb ::: 'FORCE_COLOR=1 just server start' 'FORCE_COLOR=1 just client start'
+
+# test everything
+test:
+	just server test
+	just client test
+	just seeds test
+
+latest_dataset := "2024-05"
+# Load the latest Kamaitachi dataset. This is put in the "anon-kamai" database. 
+load-kamai-dataset:
+	wget -O- https://cdn-kamai.tachi.ac/datasets/{{latest_dataset}}.dump | mongorestore --uri='mongodb://mongo' --gzip --archive
+
+	echo "Successfully loaded. You should change 'server/conf.json5' to use anon-kamai as the database."
+
+# Load the latest Bokutachi dataset. This is put in the "anon-boku" database. 
+load-boku-dataset:
+	wget -O- https://cdn-boku.tachi.ac/datasets/{{latest_dataset}}.dump | mongorestore --uri='mongodb://mongo' --gzip --archive
+
+	echo "Successfully loaded. You should change 'server/conf.json5' to use anon-boku as the database."
+
+# Check that the data in MongoDB makes any sense.
+validate-db:
+	cd server/ && pnpm validate-database
+
+# reload the shell setup
+setup-fish:
+	@fish dev/setup.fish
+
+	@exec fish
+
+# force a re-bootstrap
+bootstrap:
+	-@rm I_HAVE_BOOTSTRAPPED_OK
+	./dev/bootstrap.sh
diff --git a/README.md b/README.md
@@ -28,13 +28,28 @@ Check the [Documentation](https://docs.tachi.ac/contributing/setup) for how to s
 
 You can then check the component-specific guides to see how to run those components and contribute back!
 
-## Quick Setup For Nerds
+## Quick Setup For Experienced Programmers
 
-Already know what you're doing?
+Install VSCode and use the dev container extension.
+This is my the supported way of working and will ensure you have the correct versions of everything.
 
-Install docker + docker-compose and use `./run.sh start` or `./run.bat start` to start Tachi.
+### Unsupported stuff
 
-Other commands you might want to execute are included inside the `run` file.
+Tachi is intended to be developed inside a container. This ensures that you have the correct version of MongoDB, Redis, Typescript and all that jazz.
+
+VSCode has excellent native support for dev containers, and as such this is the only method of local development we officially support.
+
+Over the years we have had a *lot* of issues with people having subtle variations on their system (or on windows). Given the contributor-centricity of Tachi, it's untenable to expect every contributor to be an expert with local dev setup.
+
+The devcontainer provides us with the most simple, consistent experience, and allows us to put nice-to-haves inside the user's shell.
+
+That said, if you're ardently against using VSCode or Docker...
+
+**DO NOT REPORT ISSUES TO ME IF YOU DO THIS**
+
+You can run the docker-compose file and `dev/boostrap.sh` inside the container yourself. Work inside the container.
+
+Alternatively if you want to work outside of docker you're on your own. Figure out the correct versions for everything (npm, pnpm, mongo, redis, ts-node...) and `dev/bootstrap.sh`.
 
 ## Repository Info
 
@@ -48,7 +63,7 @@ The client and the server are fairly decoupled. Someone could trivially create t
 
 This contains all of our API calls, and interfaces with our database, and powers the actual score import engine.
 
-- `database-seeds/`, Which is a git-tracked set of data to be synced with Tachi. (unlicense)
+- `seeds/`, Which is a git-tracked set of data to be synced with Tachi. (unlicense)
 
 **This is the source of truth for the songs, charts, and more on the site!**
 By submitting PRs to this, you can fix bugs on the website, add new charts, and more.

diff --git a/_scripts/make-datadumps.sh b/_scripts/make-datadumps.sh
@@ -20,7 +20,7 @@ for kind in "kamai" "boku"; do
 
 	mongodump --archive --port=$remote_port --db=$kind | mongorestore --archive --nsFrom="$kind.*" --nsTo="anon-$kind.*"
 
-	pnpm ts-node src/scripts/anonymise-db 127.0.0.1:27017/anon-$kind
+	ts-node src/scripts/anonymise-db 127.0.0.1:27017/anon-$kind
 
 	TCHIS_CONF_LOCATION=$kind.dataset.conf.json5 pnpm set-indexes
 

diff --git a/client/Justfile b/client/Justfile
@@ -0,0 +1,12 @@
+[private]
+interactive:
+	-@cd ../ && just
+
+# Run the client interactively. Changes made to files will trigger reloads.
+start:
+	pnpm dev
+
+# Test that the client passes typechecking and linting.
+test:
+	pnpm typecheck
+	pnpm lint
diff --git a/client/package.json b/client/package.json
@@ -4,7 +4,7 @@
 	"private": true,
 	"license": "AGPL3",
 	"scripts": {
-		"start": "vite",
+		"dev": "vite --open",
 		"build": "vite build",
 		"lint": "eslint src",
 		"lint-fix": "eslint src --fix",

diff --git a/client/src/components/seeds/SeedsPicker.tsx b/client/src/components/seeds/SeedsPicker.tsx
@@ -271,9 +271,9 @@ function RevSelector({
 				const params = new URLSearchParams();
 
 				if (collection) {
-					params.set("path", `database-seeds/collections/${collection}`);
+					params.set("path", `seeds/collections/${collection}`);
 				} else {
-					params.set("path", "database-seeds/collections");
+					params.set("path", "seeds/collections");
 				}
 
 				const res = await fetch(

diff --git a/client/src/lib/config.ts b/client/src/lib/config.ts
@@ -36,25 +36,26 @@ try {
 		}
 	</style>
 	<div class="box">
-		<h1>Failed to connect!</h1>
-		<div>Welp. Looks like we're down. Sorry about that.</div>
-		<div>Chances are, this is just a temporary outage and will be fixed soon.</div>
-		<div style="font-size: 1.25rem; margin-top: 1rem; margin-bottom: 1rem;">
-			Please be patient, <a href="https://github.com/zkrising/Tachi">Tachi is maintained by a very small team.</a>
-		</div>
-		<div>An error message can be found in the console. (<code>Ctrl-Shift-I</code>)</div>
+
 		${
 			process.env.VITE_IS_LOCAL_DEV
 				? `
 			<hr />
-			<div><b>You're in local development mode.</b>
+			<h1><b>Couldn't connect to the server.</b></h1>
+			<h3>You are in local development mode.</h3>
 			<ul style="font-size: 2rem;">
 				<li>Have you accepted the HTTPS certificates for <a href="${ToAPIURL(
 					"/status"
 				)}">the server?</a>. If not, the site won't load.</li>
 			</ul>
 		`
-				: ""
+				: `<h1>Failed to connect!</h1>
+		<div>Welp. Looks like we're down. Sorry about that.</div>
+		<div>Chances are, this is just a temporary outage and will be fixed soon.</div>
+		<div style="font-size: 1.25rem; margin-top: 1rem; margin-bottom: 1rem;">
+			Please be patient, <a href="https://github.com/zkrising/Tachi">Tachi is maintained by a very small team.</a>
+		</div>
+		<div>An error message can be found in the console. (<code>Ctrl-Shift-I</code>)</div>`
 		}
 	</div>
 	`);

diff --git a/client/src/util/seeds.ts b/client/src/util/seeds.ts
@@ -30,7 +30,7 @@ import { CreateQuestMap, Dedupe, JSONAttributeDiff, JSONCompare } from "./misc";
 import { ValueGetterOrHybrid } from "./ztable/search";
 
 /**
- * Given a repo and a reference return the status of database-seeds/collections
+ * Given a repo and a reference return the status of seeds/collections
  * as of that commit.
  *
  * If "WORKING_DIRECTORY" is passed as a ref, and the repository is local, this will
@@ -63,7 +63,7 @@ export async function LoadSeeds(repo: string, ref: string): Promise<Partial<AllD
 				const res = await fetch(
 					`https://raw.githubusercontent.com/${repo.substring(
 						"GitHub:".length
-					)}/${ref}/database-seeds/collections/${file}`
+					)}/${ref}/seeds/collections/${file}`
 				);
 
 				if (res.status === 404) {
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,5 @@ @@
     	},
     	"[typescript]": {
     		"editor.defaultFormatter": "dbaeumer.vscode-eslint"
-    	}
-    }
+    	},
     }