Merge pull request #23 from zefhub/dev

Initial merge for 0.15.8

.github/workflows/build-release.yml

@@ -23,7 +23,7 @@ jobs:
           MAYBE_DISPATCH: ${{ github.event.inputs.tag }}
           MAYBE_RELEASE: ${{ github.event.release.tag_name }}
         run: |
-          if [ -n $MAYBE_DISPATCH ] ; then
+          if [ -n "$MAYBE_DISPATCH" ] ; then
               echo ::set-output name=tag::${MAYBE_DISPATCH}
           else
               echo ::set-output name=tag::${MAYBE_RELEASE##pyzef-}

.github/workflows/pyzef-pypi.yml

@@ -1,11 +1,12 @@
 name: PyPI releases
 
 on:
- push:
-   tags:
-     - "pyzef-*"
-     - "!pyzef-*a*"
-     - "!pyzef-*dev*"
+  push:
+    tags:
+      - "pyzef-*"
+      - "!pyzef-*a*"
+      - "!pyzef-*dev*"
+  workflow_dispatch:
 
 jobs:
   check-license:
@@ -98,4 +99,63 @@ jobs:
         run: |
           ls -R wheelhouse
           pip3 install --upgrade twine
-          python3 -m twine upload -u __token__ -p "${PYPI_TOKEN}" --skip-existing wheelhouse/*
+          python3 -m twine upload -u __token__ -p "${PYPI_TOKEN}" --skip-existing wheelhouse/*
+
+  push_to_docker_hub:
+    name: Push Docker image to Docker Hub
+    environment: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: zefhub/zef
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          file: dockerfiles/Dockerfile.base
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  push_to_github_packages:
+    name: Push Docker image to Github Packages
+    environment: Release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.CR_PAT }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ghcr.io/${{ github.repository }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        with:
+          context: .
+          push: true
+          file: dockerfiles/Dockerfile.base
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

CHANGELOG.md

@@ -1,3 +1,39 @@
+pyzef-0.15.8
+============
+
+New features:
+-------------
+
+- Rich output in `zef.ui`, via `show`, `to_table` and `to_card`.
+- `splice`, `alias`, ... zefops.
+- Filegraphs and credentials are stored together in directory for ZefHub URL.
+  This may require a new login with your credentials.
+
+Breaking changes:
+-----------------
+
+- `ZEFHUB_AUTH_KEY` is repurposed for API keys for service jobs. API keys can be
+  created for your account at `console.zefhub.io`.
+
+Fixes:
+------
+
+- SimpleGQL graphs are kept alive by default.
+- Fixes in ordering of graph delta commands.
+- Zefops `ZefGenerator`s internally now for reliable iteration.
+
+Improvements:
+-------------
+
+- FlatGraphs:
+  - FlatGraphs can absorbed other FlatGraphs from FlatRefs
+- SimpleGQL:
+  - Faster queries/updates with explicit IDs.
+  - Queries work in consistent graph slice.
+  - `debug-level` for extra info
+- Core:
+  - GraphSlices now hold a reference to their corresponding Graph.
+
 pyzef-0.15.7
 ============
 

compile_for_local_dev.sh

@@ -83,3 +83,5 @@ python3 -m pip install -qr python/requirements.txt || exit 1
 
 ln -fs $(realpath python/pyzef/build/pyzef.* --relative-to=python/zef) python/zef/
 ln -fs $(realpath core/auth.html --relative-to=python/zef) python/zef/
+# Generate the rel_ent_instances.pyi file inplace
+python3 core/scripts/run_cog_gen.py python/zef/core/internals/ python/zef/core/internals/ python/pyzef/build/zef-build/

core/CMakeLists.txt

@@ -140,7 +140,6 @@ target_link_libraries(zef PUBLIC range-v3)
 target_link_libraries(zef PRIVATE doctest)
 target_link_libraries(zef PUBLIC websocketpp)
 # target_link_libraries(zef PRIVATE blake3)
-target_link_libraries(zef PUBLIC openssl)
 target_link_libraries(zef PRIVATE libcurl)
 target_link_libraries(zef PRIVATE jwt-cpp)
 # Until I fix up our header deps this will have to be public

core/include/butler/butler.h

@@ -190,6 +190,7 @@ namespace zefDB {
             void handle_incoming_message(json & j, std::vector<std::string> & rest);
 
             void handle_incoming_terminate(json & j);
+            void handle_incoming_force_unsubscribe(json & j);
             void handle_incoming_graph_update(json & j, std::vector<std::string> & rest);
             void handle_incoming_update_tag_list(json & j);
             void handle_incoming_merge_request(json & j);
@@ -210,11 +211,13 @@ namespace zefDB {
             std::atomic_bool no_credentials_warning = false;
             // The authentication string. This is set on connection.
             std::string refresh_token = "";
+            std::string api_key = "";
             // The protocol version chosen for communication. This may have to be autodetected in earlier versions.
             std::atomic_int zefdb_protocol_version = -1;
             constexpr static int zefdb_protocol_version_min = 4;
-            constexpr static int zefdb_protocol_version_max = 5;
+            constexpr static int zefdb_protocol_version_max = 6;
             AtomicLockWrapper auth_locker;
+            std::string upstream_layout();
 
             int chunked_transfer_size_user = -1;
             int chunked_transfer_size = 10*1024;
@@ -271,7 +274,7 @@ namespace zefDB {
             bool want_upstream_connection();
             // wait_for_auth: will start_connection if not already connected
             bool wait_for_auth(std::chrono::duration<double> timeout=std::chrono::seconds(-1));
-            void determine_refresh_token();
+            void determine_login_token();
             std::string who_am_i();
             // ensure_auth_credentials: if no credentials will pop up browser
             void ensure_auth_credentials();

core/include/butler/messages.h

@@ -164,6 +164,10 @@ namespace zefDB {
             UpdatePayload payload;
         };
 
+        struct ForceUnsubscribe {
+            std::string graph_uid;
+        };
+
         struct MergeRequest {
             struct PayloadGraphDelta {
                 // TODO: Currently this is got from python as a python object, but will later be a native C object.
@@ -248,6 +252,7 @@ namespace zefDB {
             NewTransactionCreated,
             Reconnected,
             Disconnected,
+            ForceUnsubscribe,
             NewGraph,
             LocalGraph,
             LoadGraph,
@@ -299,6 +304,7 @@ namespace zefDB {
         CREATE_NAME_STRING(TokenQuery)
         CREATE_NAME_STRING(Reconnected)
         CREATE_NAME_STRING(Disconnected)
+        CREATE_NAME_STRING(ForceUnsubscribe)
         CREATE_NAME_STRING(NewGraph)
         CREATE_NAME_STRING(LocalGraph)
         CREATE_NAME_STRING(LoadGraph)

core/include/verification.h

@@ -45,11 +45,11 @@ namespace zefDB {
         LIBZEF_DLL_EXPORTED void break_graph(Graph&g, blob_index index, int style);
 
         inline bool verify_graph(Graph&g) {
-            // This is to handle mixed kinds of return false/true or the exception throwing
             try {
-                return (verify_graph_double_linking(g)
-                        && verify_chronological_instantiation_order(g));
-            } catch(...) {
+            return (verify_graph_double_linking(g)
+                    && verify_chronological_instantiation_order(g));
+            } catch(const std::exception & e) {
+                std::cerr << "Verification failed with: " << e.what() << std::endl;
                 return false;
             }
         }

core/include/zwitch.h

@@ -122,4 +122,15 @@ namespace zefDB {
         std::chrono::duration<double> elapsed = std::chrono::steady_clock::now() - time_start_of_process;
         std::cerr << elapsed.count() << " seconds: " << msg << std::endl;
     }
+
+    inline void developer_output(std::string s) {
+        if(!zwitch.developer_output())
+            return;
+
+        std::time_t t = std::time(nullptr);
+        std::tm tm = *std::localtime(&t);
+
+        std::cerr << "DEV-" << std::put_time(&tm, "%T") << ": ";
+        std::cerr << s << std::endl;
+    }
 }

core/src/butler/butler.cpp

@@ -847,8 +847,8 @@ namespace zefDB {
                 return std::filesystem::path(std::string(env)) / upstream_name;
 
             std::filesystem::path path = zefdb_config_path();
-            path /= "graphs";
             path /= upstream_name;
+            path /= "graphs";
             return path;
         }
 
@@ -882,7 +882,17 @@ namespace zefDB {
 
         ////////////////////////////////////////////////////////
         // * External handlers
+        std::string Butler::upstream_layout() {
+            if(butler_is_master)
+                return data_layout_version;
 
+            if(zefdb_protocol_version == -1)
+                throw std::runtime_error("Shouldn't be asking for upstream layout when we haven't connected and done a handshake.");
+
+            if(zefdb_protocol_version <= 6)
+                return "0.2.0";
+            throw std::runtime_error("Don't know what the upstream layout should be for this protocol version");
+        }
 
         // ** Merge handler
         std::optional<std::function<merge_handler_t>> merge_handler;

core/src/butler/butler_connection.cpp

@@ -74,15 +74,16 @@ bool Butler::wait_for_auth(std::chrono::duration<double> timeout) {
     return true;
 }
 
-void Butler::determine_refresh_token() {
+void Butler::determine_login_token() {
     std::optional<std::string> key_string = load_forced_zefhub_key();
     if(key_string) {
+        refresh_token = "";
         if(*key_string == constants::zefhub_guest_key) {
             std::cerr << "Connecting as guest user" << std::endl;
-            refresh_token = "";
             have_logged_in_as_guest = true;
-        } else 
-            refresh_token = get_firebase_refresh_token_email(*key_string);
+        } else  {
+            api_key = *key_string;
+        }
         return;
     } else if(!have_auth_credentials()) {
         no_credentials_warning = true;
@@ -108,9 +109,7 @@ std::string Butler::who_am_i() {
         if(*key_string == constants::zefhub_guest_key) {
             name = "GUEST via forced key";
         } else {
-            // This is  duplicated with get_firebase_refresh_token_email which is annoying
-            int colon = key_string->find(':');
-            name = key_string->substr(0,colon) + " via forced key";
+            name = "API key beginning with " + key_string->substr(0, 4) + "...";
         }
     } else if(!have_auth_credentials()) {
         name = "";
@@ -147,10 +146,14 @@ std::string Butler::who_am_i() {
 }
 
 Butler::header_list_t Butler::prepare_send_headers(void) {
-    determine_refresh_token();
+    determine_login_token();
     decltype(network)::header_list_t headers;
-    std::string auth_token = get_firebase_token_refresh_token(refresh_token);
-    headers.emplace_back("X-Auth-Token", auth_token);
+    if(api_key != "") {
+        headers.emplace_back("X-API-Key", api_key);
+    } else {
+        std::string auth_token = get_firebase_token_refresh_token(refresh_token);
+        headers.emplace_back("X-Auth-Token", auth_token);
+    }
     headers.emplace_back("X-Requested-Version", to_str(zefdb_protocol_version_max));
     return headers;
 }
@@ -293,27 +296,14 @@ std::optional<std::string> Butler::load_forced_zefhub_key() {
         return output;
     }
 
-    // Old location for fallback
-#ifdef ZEF_WIN32
-    env = std::getenv("LOCALAPPDATA");
-#else
-    env = std::getenv("HOME");
-#endif
-    std::filesystem::path path2(env);
-    path2 /= ".zefdb";
-    path2 /= "zefhub.key";
-    if (std::filesystem::exists(path2)) {
-        std::ifstream file(path2);
-        std::string output;
-        std::getline(file, output);
-        return output;
-    }
-
     return {};
 }
 
 bool Butler::have_auth_credentials() {
     // TODO: This should become more sophisticated in the future
+    if(api_key != "" || refresh_token != "")
+        return true;
+
     if(load_forced_zefhub_key())
         return true;
 
@@ -358,7 +348,7 @@ void Butler::ensure_auth_credentials() {
         if(*forced_zefhub_key == constants::zefhub_guest_key)
             have_logged_in_as_guest = true;
         else
-            get_firebase_refresh_token_email(*forced_zefhub_key);
+            api_key = *forced_zefhub_key;
     } else {
         auto credentials_file = credentials_path();
         if(is_credentials_file_valid())
@@ -426,34 +416,11 @@ void Butler::user_login() {
 }
 
 void Butler::user_logout() {
-    // // Always remove any graphs that were being synced with ZefHub. Do
-    // // this first as the sync messages might require the right
-    // // credentials.
-    // std::vector<std::shared_ptr<Butler::GraphTrackingData>> saved_list;
-    // {
-    //     std::lock_guard lock(butler->graph_manager_list_mutex);
-    //     saved_list = butler->graph_manager_list;
-    // }
-    // for(auto data : butler->graph_manager_list) {
-    //     if(!data->gd->should_sync)
-    //         continue;
-
-    //     std::cerr << "Going to stop graph manager: " << uid(*(data->gd)) << std::endl;
-
-    //     try {
-    //         data->queue.set_closed(false);
-    //         long_wait_or_kill(*data->managing_thread, data->return_value, data->uid);
-    //     } catch(const std::exception & e) {
-    //         std::cerr << "Exception while trying to close queue for graph manager thread: " << e.what() << std::endl;
-    //     }
-    // }
-
-
     auto butler = get_butler();
     butler->session_auth_key.reset();
 
     // Now remove credentials
-    if(load_forced_zefhub_key())
+    if(api_key != "" || load_forced_zefhub_key())
         throw std::runtime_error("Can't logout when an explicit key is given in ZEFHUB_AUTH_KEY or zefhub.key");
     if(have_auth_credentials()) {
         auto credentials_file = credentials_path();