Merge various work from Orchid Labs master branch.

README.md

@@ -10,7 +10,7 @@ Of course, you do need to have the requisite build tooling installed... in addit
 
 (At this point I will note, as this has come up multiple times: it is neither practical nor appropriate for Orchid's documentation to detail how to install any of these toolchains. The instructions are different for every operating system, are different for every single distribution of Linux, and are often even different for specific versions of a distribution. FWIW, developers already have most of this software installed; and, if you don't, these projects have their own documentation.)
 
-That said, the "usual algorithm" of "try to build it, and if you get an error saying you are missing X, just install X" should work, so I'd just dive in (like, honestly, I'm shocked you are reading this); that said, if you are "feeling lucky", you can run env/setup-mac.sh (if using macOS) or env/setup-lnx.sh (if using Ubuntu), which are scripts that install everything on either macOS or (specifically and only) Ubuntu (and thereby can serve as "documentation" if you refuse to just dive in).
+That said, the "usual algorithm" of "try to build it, and if you get an error saying you are missing X, just install X" should work, so I'd just dive in (like, honestly, I'm shocked you are reading this); that said, if you are "feeling lucky", you can run env/setup-mac.sh (if using macOS) or env/setup-lnx.sh (if using Ubuntu/Arch), which are scripts that install everything on either macOS or (specifically and only) Ubuntu (and thereby can serve as "documentation" if you refuse to just dive in).
 
 Given that you have a box capable of compiling other projects (as we aren't using anything "weird", really... meson is probably the rarest dependency we have, and it will be extremely obvious) you can then just go into any subfolder you want (such as app-{android,ios}, or cli-shared/srv-daemon and run "make". Seriously: it's that easy... if it breaks for some reason other than "you ran out of memory / disk space" or "command X not found" (which you can trivially solve), please file an issue.
 

app-flutter.mk

@@ -37,6 +37,8 @@ create: $(pwd/flutter)/packages/flutter/pubspec.lock
 
 builds := 
 builds += apk
+# XXX: I need to filter based on uname :/
+#builds += linux
 builds += macos
 builds += ios
 

app-linux/i3layout.json

@@ -0,0 +1,6 @@
+{
+  "border": "pixel", "current_border_width": 2,
+  "type": "con", "floating": "auto_off",
+  "percent": 0.393,
+  "name": "Orchid", "swallows": [{"class": "^Orchid$"}]
+}

app-linux/makefile

@@ -27,11 +27,17 @@ signed :=
 .PHONY: all
 all: $(output)/$(name).tgz
 
+.PHONY: app
+app: $(signed)
+
 source += $(wildcard $(pwd)/source/*.cpp)
 
 cflags/$(pwd)/source/main.cpp += -Wno-unused-function
+checks/$(pwd)/source/main.cpp += -bugprone-assignment-in-if-condition
 checks/$(pwd)/source/main.cpp += -clang-diagnostic-unused-function
 checks/$(pwd)/source/main.cpp += -cppcoreguidelines-pro-type-cstyle-cast
+checks/$(pwd)/source/main.cpp += -cppcoreguidelines-pro-type-member-init
+checks/$(pwd)/source/main.cpp += -cppcoreguidelines-pro-type-vararg
 checks/$(pwd)/source/main.cpp += -misc-const-correctness
 checks/$(pwd)/source/main.cpp += -misc-use-anonymous-namespace
 
@@ -56,9 +62,15 @@ lflags += $(output)/sysroot/usr/lib/$(host/x86_64)/libgtk-3.so
 lflags += $(output)/sysroot/usr/lib/$(host/x86_64)/libgio-2.0.so
 lflags += $(output)/sysroot/usr/lib/$(host/x86_64)/libgobject-2.0.so
 
+shareds := $(pwd/gui)/linux/flutter/ephemeral/.plugin_symlinks/*/linux/shared
+$(output)/package/lib/%: $$(wildcard $(shareds)/%)
+	@mkdir -p $(dir $@)
+	cp -af $< $@
+signed += $(patsubst %,$(output)/package/lib/%,$(notdir $(wildcard $(shareds)/*.so)))
+
 $(output)/package/$(name)$(exe): $(patsubst %,$(output)/$(machine)/%,$(object) $(linked))
 	@echo [LD] $@
-	@set -o pipefail; $(cxx) $(more/$(machine)) $(wflags) -o $@ $(filter %.o,$^) $(filter %.a,$^) $(filter %.lib,$^) $(lflags) -Wl,-rpath,. 2>&1 | nl
+	@set -o pipefail; $(cxx) $(more/$(machine)) $(wflags) -o $@ $(filter %.o,$^) $(filter %.a,$^) $(filter %.lib,$^) $(lflags) -Wl,-rpath,'$$ORIGIN' 2>&1 | nl
 	@openssl sha256 -r $@
 	@ls -la $@
 signed += $(output)/package/$(name)$(exe)
@@ -69,4 +81,6 @@ $(output)/$(name).tgz: $(signed)
 
 .PHONY: test
 test: $(signed)
+	xdotool windowkill "$$(i3-msg -t get_tree | jq 'recurse(.nodes[]?)|select(.name=="Orchid")|.window')"
+	i3-msg "append_layout ${PWD}/i3layout.json"
 	$(output)/package/$(name)$(exe)

app-linux/source/main.cpp

@@ -22,28 +22,46 @@
 
 #include <gtk/gtk.h>
 #include <flutter_linux/flutter_linux.h>
+#include <gdk/gdkx.h>
 #include "flutter/generated_plugin_registrant.h"
 
 G_DECLARE_FINAL_TYPE(MyApplication, my_application, MY, APPLICATION, GtkApplication)
 
 struct _MyApplication {
     GtkApplication parent_instance;
+    char **dart_entrypoint_arguments;
 };
 
 // NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,performance-no-int-to-ptr)
 G_DEFINE_TYPE(MyApplication, my_application, GTK_TYPE_APPLICATION)
 
 static void my_application_activate(GApplication *application) {
+    const auto self(MY_APPLICATION(application));
     const auto window(GTK_WINDOW(gtk_application_window_new(GTK_APPLICATION(application))));
-    const auto header(GTK_HEADER_BAR(gtk_header_bar_new()));
-    gtk_widget_show(GTK_WIDGET(header));
-    gtk_header_bar_set_title(header, "Orchid");
-    gtk_header_bar_set_show_close_button(header, TRUE);
-    gtk_window_set_titlebar(window, GTK_WIDGET(header));
+
+    bool use_header_bar(true);
+    if (const auto screen(gtk_window_get_screen(window)); GDK_IS_X11_SCREEN(screen)) {
+        const auto wm_name(gdk_x11_screen_get_window_manager_name(screen));
+        if (g_strcmp0(wm_name, "GNOME Shell") != 0)
+            use_header_bar = false;
+    }
+
+    if (use_header_bar) {
+        const auto header(GTK_HEADER_BAR(gtk_header_bar_new()));
+        gtk_widget_show(GTK_WIDGET(header));
+        gtk_header_bar_set_title(header, "Orchid");
+        gtk_header_bar_set_show_close_button(header, TRUE);
+        gtk_window_set_titlebar(window, GTK_WIDGET(header));
+    } else {
+        gtk_window_set_title(window, "Orchid");
+    }
+
     gtk_window_set_default_size(window, 360, 640);
     gtk_widget_show(GTK_WIDGET(window));
 
-    const auto project = fl_dart_project_new();
+    const auto project(fl_dart_project_new());
+    fl_dart_project_set_dart_entrypoint_arguments(project, self->dart_entrypoint_arguments);
+
     const auto view(fl_view_new(project));
     gtk_widget_show(GTK_WIDGET(view));
     gtk_container_add(GTK_CONTAINER(window), GTK_WIDGET(view));
@@ -52,15 +70,37 @@ static void my_application_activate(GApplication *application) {
     gtk_widget_grab_focus(GTK_WIDGET(view));
 }
 
+static gboolean my_application_local_command_line(GApplication* application, gchar*** arguments, int* exit_status) {
+    const auto self(MY_APPLICATION(application));
+    self->dart_entrypoint_arguments = g_strdupv(*arguments + 1);
+
+    g_autoptr(GError) error = nullptr;
+    if (g_application_register(application, nullptr, &error) == 0) {
+        g_warning("Failed to register: %s", error->message);
+        *exit_status = 1;
+        return TRUE;
+    }
+
+    g_application_activate(application);
+    *exit_status = 0;
+    return TRUE;
+}
+
+static void my_application_dispose(GObject *object) {
+    const auto self(MY_APPLICATION(object));
+    g_clear_pointer(&self->dart_entrypoint_arguments, g_strfreev);
+    G_OBJECT_CLASS(my_application_parent_class)->dispose(object);
+}
+
 static void my_application_class_init(MyApplicationClass *klass) {
     G_APPLICATION_CLASS(klass)->activate = my_application_activate;
+    G_OBJECT_CLASS(klass)->dispose = my_application_dispose;
 }
 
 static void my_application_init(MyApplication *self) {
 }
 
 MyApplication *my_application_new() {
-    // NOLINTNEXTLINE(cppcoreguidelines-pro-type-vararg)
     return MY_APPLICATION(g_object_new(my_application_get_type(), "application-id", "net.orchid.Orchid", nullptr));
 }
 

env/lnx-arch.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+pacman -Sy \
+    ubuntu-keyring \
+    bc tcl vim \
+    curl git rsync wget \
+    fakeroot talloc \
+    cpio rpm-tools unzip zstd \
+    clang lld llvm \
+    binutils aarch64-linux-gnu-binutils \
+    mingw-w64-binutils \
+    libc++ libc++abi \
+    python-pip python-setuptools \
+    jdk17-openjdk \
+    bison flex gperf \
+    gettext groff texinfo \
+    autoconf autoconf-archive automake \
+    libtool meson ninja pkgconf \
+    qemu-system-{x86,aarch64} \

env/setup-apt.sh → env/lnx-ubuntu.sh

@@ -5,20 +5,22 @@ export DEBIAN_FRONTEND=noninteractive
 apt-get update
 
 apt-get -y install \
+    ubuntu-keyring \
     bc tcl xxd \
     curl git-core rsync wget \
     fakeroot libtalloc-dev \
     cpio rpm unzip zstd \
     clang clang-tidy lld llvm \
-    binutils-{aarch64,x86-64}-linux-gnu \
+    binutils-{x86-64,aarch64}-linux-gnu \
+    binutils-mingw-w64-{i686,x86-64} \
     libc++-dev libc++abi-dev \
-    g++-multilib gcc-multilib \
     python3-pip python3-setuptools \
     openjdk-11-jre-headless \
     bison flex gperf \
     gettext groff texinfo \
     autoconf autoconf-archive automake \
     libtool ninja-build pkg-config \
+    qemu-system-{x86,arm} \
 
 function usable() {
     # Ubuntu bionic ships meson 0.45, which is too old to build glib

env/setup-lnx.sh

@@ -7,6 +7,7 @@ else
     sudo=(sudo -EH)
 fi
 
-"${sudo[@]}" env/setup-apt.sh
+. /etc/os-release
+"${sudo[@]}" "env/lnx-${ID}.sh"
 
 env/setup-all.sh

env/setup-mac.sh

@@ -1,6 +1,16 @@
 #!/bin/bash
 set -e
 which brew &>/dev/null || /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-brew install autoconf autoconf-archive automake binutils capnp fakeroot groff libtool meson rpm2cpio rustup-init zstd
+
+brew install \
+    fakeroot \
+    rpm2cpio zstd \
+    binutils \
+    mingw-w64 \
+    groff \
+    autoconf autoconf-archive automake \
+    libtool meson \
+    capnp rustup-init \
+
 rustup-init -y --no-modify-path --no-update-default-toolchain
 env/setup-all.sh

env/target-win.mk

@@ -39,7 +39,7 @@ include $(pwd)/target-gnu.mk
 define _
 more/$(1) := -target $(1)-pc-windows-gnu
 more/$(1) += --sysroot $(CURDIR)/$(output)/$(1)/mingw$(bits/$(1))
-temp := $(shell which $(1)-w64-mingw32-ld)
+temp := $$(shell which $(1)-w64-mingw32-ld)
 ifeq ($$(temp),)
 $$(error $(1)-w64-mingw32-ld must be on your path)
 endif
@@ -126,7 +126,7 @@ mingw := git-10.0.0.r258.g530c58e17
 msys2 := 
 msys2 += crt-$(mingw)-2
 msys2 += dlfcn-1.4.0-1
-msys2 += gcc-12.1.0-2
+msys2 += gcc-13.2.0-5
 msys2 += headers-$(mingw)-1
 msys2 += winpthreads-$(mingw)-1
 

env/uname.mk

@@ -13,6 +13,6 @@ uname-m := $(shell uname -m)
 uname-s := $(shell uname -s)
 uname-o := $(shell uname -o 2>/dev/null)
 
-objcopy = $(host/$*)-objcopy
+objcopy = $(shell which $(host/$*)-objcopy objcopy 2>&1 | head -n1)
 
 -include $(pwd)/uname-$(uname-s).mk

gui-orchid/.gitignore

@@ -18,4 +18,4 @@
 build
 test/receipt_local.txt
 *.log
-untransleted.txt
+untranslated.txt

lints.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 set -e
 set -o pipefail
-git grep 'NOLINT[A-Z]*(' | sed -e 's/.*(//;s/)//' | tr ',' $'\n' | sort | uniq -c
+git grep 'NOLINT\(NEXTLINE\|BEGIN\)(' | sed -e 's/.*(//;s/)//' | tr ',' $'\n' | sort | uniq -c

str-twincoding/README-in.md

@@ -5,7 +5,7 @@ _*Orchid Storage is a **work in progress***_
 
 Orchid is an open source project.  Help us in the effort to build a truly decentralized, incentive-aligned
 storage system.  
-Check out the [Storchid Litepaper](https://orchid.com/storchid-litepaper-latest.pdf)
+Check out the [Orchid Storage Litepaper](https://www.orchid.com/storage-litepaper-latest.pdf)
 and join the discussion on the [Orchid Subreddit](https://www.reddit.com/r/orchid).
 
 This repository contains work in progress on the file encoding CLI and server framework.

str-twincoding/README.md

@@ -5,7 +5,7 @@ _*Orchid Storage is a **work in progress***_
 
 Orchid is an open source project.  Help us in the effort to build a truly decentralized, incentive-aligned
 storage system.  
-Check out the [Storchid Litepaper](https://orchid.com/storchid-litepaper-latest.pdf)
+Check out the [Orchid Storage Litepaper](https://www.orchid.com/storage-litepaper-latest.pdf)
 and join the discussion on the [Orchid Subreddit](https://www.reddit.com/r/orchid).
 
 This repository contains work in progress on the file encoding CLI and server framework.
@@ -94,10 +94,21 @@ providers.sh list
 monitor.sh --update 1
 
 # Push the file by name
+# (Observe the availability of the file in the monitor)
 storage.sh push foo_file.dat
 
+# Delete a shard from one of the providers
+# (Observe the availability is reduced as a unique shard is lost)
+storage.sh delete_shard --provider 5001 foo_file.dat --node_type 0 --node_index 0
+
+# Request that the provider rebuild the lost node from specified other nodes in the cluster.
+storage.sh request_repair --to_provider 5001 foo_file.dat --node_type 0 --node_index 0 --from_providers 5002 5003 5004
+...
+
+
 # Shut down the servers
 examples/test-cluster.sh stop
+
 ```
 
 ## Encoding CLI Examples
@@ -330,6 +341,79 @@ options:
   --overwrite           Overwrite files on the server.
 None
 ```
+###`request_recovery_file`
+```
+usage: storage request_recovery_file [-h] [--repo REPO] --provider PROVIDER
+                                     [--overwrite] --recover_node_type
+                                     RECOVER_NODE_TYPE --recover_node_index
+                                     RECOVER_NODE_INDEX --source_node_index
+                                     SOURCE_NODE_INDEX
+                                     file
+
+positional arguments:
+  file                  Name of the file in the local and remote repositories.
+
+options:
+  -h, --help            show this help message and exit
+  --repo REPO           Path to the repository.
+  --provider PROVIDER   Provider from which to request the recovery file.
+  --overwrite           Overwrite any local file.
+  --recover_node_type RECOVER_NODE_TYPE
+                        The node type for the shard being recovered.
+  --recover_node_index RECOVER_NODE_INDEX
+                        The node index for the shard being recovered.
+  --source_node_index SOURCE_NODE_INDEX
+                        The source node index desired to be used to generate
+                        the recovery file.
+None
+```
+###`request_repair`
+```
+usage: storage request_repair [-h] [--repo REPO] --to_provider TO_PROVIDER
+                              --node_type NODE_TYPE --node_index NODE_INDEX
+                              [--from_providers [FROM_PROVIDERS ...]]
+                              [--dryrun] [--overwrite]
+                              file
+
+positional arguments:
+  file                  Name of the file in the repository.
+
+options:
+  -h, --help            show this help message and exit
+  --repo REPO           Path to the repository.
+  --to_provider TO_PROVIDER
+                        Provider to receive the repair request.
+  --node_type NODE_TYPE
+                        The node type for the shard being recovered.
+  --node_index NODE_INDEX
+                        The node index for the shard being recovered.
+  --from_providers [FROM_PROVIDERS ...]
+                        Optional list of provider names or urls for the
+                        repair.
+  --dryrun, -n          Show the plan without executing it.
+  --overwrite           Overwrite files on the server.
+None
+```
+###`request_delete_shard`
+```
+usage: storage request_delete_shard [-h] [--repo REPO] --provider PROVIDER
+                                    --node_type NODE_TYPE --node_index
+                                    NODE_INDEX
+                                    file
+
+positional arguments:
+  file                  Name of the file in the local and remote repositories.
+
+options:
+  -h, --help            show this help message and exit
+  --repo REPO           Path to the repository.
+  --provider PROVIDER   Provider to receive the deletion request.
+  --node_type NODE_TYPE
+                        The node type of the shard to be deleted.
+  --node_index NODE_INDEX
+                        The node index of the shard to be deleted.
+None
+```
 
 ## Server Docs
 ```