Console paging

core/bindings/ConsoleQueryParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ConsoleQueryParams { start_snowflake_id: bigint, count: number, }

core/src/db/read.rs

@@ -1,9 +1,11 @@
 use crate::{
-    error::Error, output_types::ClientEvent,
-    prelude::LODESTONE_EPOCH_MIL, events::EventQuery,
+    error::{Error, ErrorKind},
+    events::EventQuery,
+    output_types::ClientEvent,
+    prelude::LODESTONE_EPOCH_MIL,
 };
 
-use color_eyre::eyre::Context;
+use color_eyre::eyre::{eyre, Context};
 use sqlx::sqlite::SqlitePool;
 use tracing::error;
 

core/src/events.rs

@@ -217,12 +217,8 @@ pub enum ProgressionEndValue {
 #[ts(export)]
 #[serde(tag = "type")]
 pub enum ProgressionStartValue {
-    InstanceCreation {
-        instance_uuid: InstanceUuid,
-    },
-    InstanceDelete {
-        instance_uuid: InstanceUuid,
-    },
+    InstanceCreation { instance_uuid: InstanceUuid },
+    InstanceDelete { instance_uuid: InstanceUuid },
 }
 
 // the backend will keep exactly 1 copy of ProgressionStart, and 1 copy of ProgressionUpdate OR ProgressionEnd

core/src/handlers/console.rs

@@ -0,0 +1,103 @@
+use crate::error::Error;
+use crate::events::{EventQuery, EventType, InstanceEventKind};
+use crate::{
+    output_types::ClientEvent,
+    types::{InstanceUuid, TimeRange},
+    AppState,
+};
+use axum::{
+    extract::{Path, Query},
+    routing::get,
+    Json, Router,
+};
+use color_eyre::eyre::{eyre, Context};
+use serde::{Deserialize, Serialize};
+use sqlx::sqlite::SqlitePool;
+use tracing::error;
+use ts_rs::TS;
+
+#[derive(Debug, Serialize, Deserialize, TS)]
+#[ts(export)]
+pub struct ConsoleQueryParams {
+    start_snowflake_id: i64,
+    count: u32,
+}
+
+async fn get_console_messages(
+    axum::extract::State(state): axum::extract::State<AppState>,
+    Path(uuid): Path<String>,
+    Query(query_params): Query<ConsoleQueryParams>,
+) -> Result<Json<Vec<ClientEvent>>, Error> {
+    let time_range = TimeRange {
+        start: query_params.start_snowflake_id,
+        end: i64::MAX,
+    };
+
+    let pool = &state.sqlite_pool;
+
+    let mut connection = pool
+        .acquire()
+        .await
+        .context("Failed to aquire connection to db")?;
+
+    let limit_num = &query_params.count * 2 + 10;
+
+    let rows = sqlx::query!(
+        r#"
+SELECT
+event_value, details, snowflake, level, caused_by_user_id, instance_id
+FROM ClientEvents
+WHERE snowflake <= ($1) AND event_value IS NOT NULL
+ORDER BY snowflake DESC
+LIMIT $2
+"#,
+        query_params.start_snowflake_id,
+        limit_num, // hacky, but need more since filter
+    )
+    .fetch_all(&mut connection)
+    .await
+    .context("Failed to fetch events")?;
+
+    let mut parsed_client_events: Vec<ClientEvent> = Vec::new();
+    for row in rows {
+        if let Some(event_value) = &row.event_value {
+            if let Ok(client_event) = serde_json::from_str(event_value) {
+                parsed_client_events.push(client_event);
+            } else {
+                error!("Failed to parse client event: {}", event_value);
+            }
+        } else {
+            error!("Failed to parse row");
+        }
+    }
+
+    let event_query = EventQuery {
+        event_levels: None,
+        event_types: Some(vec![EventType::InstanceEvent]),
+        instance_event_types: Some(vec![InstanceEventKind::InstanceOutput]),
+        user_event_types: None,
+        event_user_ids: None,
+        event_instance_ids: Some(vec![InstanceUuid::from(uuid)]),
+        bearer_token: None,
+        time_range: None,
+    };
+
+    let filtered: Vec<ClientEvent> = parsed_client_events
+        .into_iter()
+        .filter(|client_event| event_query.filter(client_event))
+        .collect();
+
+    let filtered = if filtered.len() as u32 > query_params.count {
+        filtered[0..query_params.count as usize].to_vec()
+    } else {
+        filtered
+    };
+
+    return Ok(Json(filtered));
+}
+
+pub fn get_console_routes(state: AppState) -> Router {
+    Router::new()
+        .route("/instance/:uuid/console", get(get_console_messages))
+        .with_state(state)
+}

core/src/handlers/mod.rs

@@ -2,6 +2,7 @@
 // pub mod instance;
 // pub mod users;
 pub mod checks;
+pub mod console;
 pub mod core_info;
 pub mod events;
 pub mod gateway;

core/src/lib.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::comparison_chain, clippy::type_complexity)]
 
 use crate::event_broadcaster::EventBroadcaster;
+use crate::handlers::console::get_console_routes;
 use crate::migration::migrate;
 use crate::prelude::{
     init_app_state, init_paths, lodestone_path, path_to_global_settings, path_to_stores,
@@ -43,6 +44,7 @@ use prelude::GameInstance;
 use reqwest::{header, Method};
 use ringbuffer::{AllocRingBuffer, RingBufferWrite};
 
+use fs3::FileExt;
 use semver::Version;
 use sqlx::{sqlite::SqliteConnectOptions, Pool};
 use std::{
@@ -68,7 +70,6 @@ use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, EnvFilter}
 use traits::{t_configurable::TConfigurable, t_server::MonitorReport, t_server::TServer};
 use types::{DotLodestoneConfig, InstanceUuid};
 use uuid::Uuid;
-use fs3::FileExt;
 
 pub mod auth;
 pub mod db;
@@ -613,6 +614,7 @@ pub async fn run(
                     .merge(get_global_fs_routes(shared_state.clone()))
                     .merge(get_global_settings_routes(shared_state.clone()))
                     .merge(get_gateway_routes(shared_state.clone()))
+                    .merge(get_console_routes(shared_state.clone()))
                     .layer(cors)
                     .layer(trace);
                 let app = Router::new().nest("/api/v1", api_routes);
@@ -724,4 +726,4 @@ pub async fn run(
         guard,
         shutdown_tx,
     )
-}
+}

dashboard/src/bindings/ConsoleQueryParams.ts

@@ -0,0 +1,3 @@
+// This file was generated by [ts-rs](https://github.com/Aleph-Alpha/ts-rs). Do not edit this file manually.
+
+export interface ConsoleQueryParams { start_snowflake_id: bigint, count: number, }

dashboard/src/components/GameConsole.tsx

@@ -6,7 +6,7 @@ import { InstanceContext } from 'data/InstanceContext';
 import { CommandHistoryContext } from 'data/CommandHistoryContext';
 import { useUserAuthorized } from 'data/UserInfo';
 import Tooltip from 'rc-tooltip';
-import { useContext, useEffect } from 'react';
+import React, { useContext, useEffect } from 'react';
 import { useRef, useState } from 'react';
 import { usePrevious } from 'utils/hooks';
 import { DISABLE_AUTOFILL } from 'utils/util';
@@ -24,18 +24,21 @@ export default function GameConsole() {
     'can_access_instance_console',
     uuid
   );
-  const { consoleLog, consoleStatus } = useConsoleStream(uuid);
+  const { consoleLog, consoleStatus, fetchConsolePage } = useConsoleStream(uuid, undefined);
   const [command, setCommand] = useState('');
   const { commandHistory, appendCommandHistory } = useContext(
     CommandHistoryContext
   );
+  const [lastScrollPos, setLastScrollPos] = useState(0);
+  const [additionalLogs, setAdditionaLogs] = useState(0);
+  const [logsRecentlyIncreased, setLogsRecentlyIncreased] = useState(false);
   const [commandNav, setCommandNav] = useState(commandHistory.length);
   const listRef = useRef<HTMLOListElement>(null);
   const isAtBottom = listRef.current
     ? listRef.current.scrollHeight -
-        listRef.current.scrollTop -
-        listRef.current.clientHeight <
-      autoScrollThreshold
+    listRef.current.scrollTop -
+    listRef.current.clientHeight <
+    autoScrollThreshold
     : false;
   const oldIsAtBottom = usePrevious(isAtBottom);
 
@@ -129,7 +132,7 @@ export default function GameConsole() {
   };
 
   return (
-    <div className="relative flex h-full w-full grow flex-col rounded-lg border border-gray-faded/30">
+    <div className="border-gray-faded/30 relative flex h-full w-full grow flex-col rounded-lg border">
       <Tooltip
         overlay={<span>{consoleStatusMessage}</span>}
         placement="bottom"
@@ -139,41 +142,68 @@ export default function GameConsole() {
       >
         <FontAwesomeIcon
           icon={faCircle}
-          className={`absolute top-0 right-0 select-none p-1.5 text-small ${consoleStatusColor}`}
+          className={`text-small absolute top-0 right-0 select-none p-1.5 ${consoleStatusColor}`}
         />
       </Tooltip>
       {!canAccessConsole || consoleStatus === 'no-permission' ? (
         <ErrorGraphic
           icon={faServer}
           message="You don't have permission to access this console"
-          className="rounded-t-lg border-b border-gray-faded/30"
+          className="border-gray-faded/30 rounded-t-lg border-b"
           iconClassName="text-gray-400"
           messageClassName="text-white/50"
         />
       ) : consoleLog.length === 0 ? (
         <ErrorGraphic
           icon={faServer}
           message="No console messages yet"
-          className="rounded-t-lg border-b border-gray-faded/30"
+          className="border-gray-faded/30 rounded-t-lg border-b"
           iconClassName="text-gray-400"
           messageClassName="text-white/50"
         />
       ) : (
         <ol
-          className="font-light flex h-0 grow flex-col overflow-y-auto whitespace-pre-wrap break-words rounded-t-lg border-b border-gray-faded/30 bg-gray-900 py-3 font-mono text-small tracking-tight text-gray-300"
+          className="border-gray-faded/30 text-small flex h-0 grow flex-col overflow-y-auto whitespace-pre-wrap break-words rounded-t-lg border-b bg-gray-900 py-3 font-mono font-light tracking-tight text-gray-300"
           ref={listRef}
+          onScroll={(e: React.SyntheticEvent) => {
+            // check that position is close to top
+            if (!e.currentTarget || !e.currentTarget.scrollTop || logsRecentlyIncreased) {
+              return;
+            }
+            const prevScrollPos = lastScrollPos;
+            setLastScrollPos(e.currentTarget.scrollTop)
+
+            const scrollPosDiff = e.currentTarget.scrollTop - prevScrollPos; // should be negative to be considered proper trigger
+            const triggerThreshhold = 
+              e.currentTarget.scrollTop >= autoScrollThreshold * 15 && 
+              e.currentTarget.scrollTop <= autoScrollThreshold * 20; // consider range to factor in scroll speed
+
+            if (!triggerThreshhold || scrollPosDiff >= 0 || logsRecentlyIncreased) {
+              //allow new logs to be generated again if user scrolls down first
+              return;
+            }
+            fetchConsolePage(consoleLog[0].snowflake as unknown as bigint, additionalLogs + 40);
+            setAdditionaLogs(currNumLogs => currNumLogs + 40);
+            setLogsRecentlyIncreased(true);
+
+            // debounce log generation
+            setTimeout(() => {
+              setLogsRecentlyIncreased(false);
+            }, 1000)
+
+          }}
         >
           {consoleLog.map((line) => (
             <li
               key={line.snowflake}
-              className="py-[0.125rem] px-4 hover:bg-gray-800"
+              className="px-4 py-[0.125rem] hover:bg-gray-800"
             >
               {line.message}
             </li>
           ))}
         </ol>
       )}
-      <div className="font-mono text-small">
+      <div className="text-small font-mono">
         <form
           noValidate
           autoComplete={DISABLE_AUTOFILL}
@@ -186,7 +216,7 @@ export default function GameConsole() {
           }}
         >
           <input
-            className="w-full rounded-b-lg bg-gray-850 py-3 px-4 text-gray-300 outline-white/50 placeholder:text-gray-500 focus-visible:outline focus-visible:outline-2 disabled:placeholder:text-gray-500"
+            className="bg-gray-850 w-full rounded-b-lg px-4 py-3 text-gray-300 outline-white/50 placeholder:text-gray-500 focus-visible:outline focus-visible:outline-2 disabled:placeholder:text-gray-500"
             placeholder={consoleInputMessage || 'Enter command...'}
             value={command}
             onChange={(e) => setCommand(e.target.value)}

dashboard/src/data/ConsoleEvent.ts

@@ -0,0 +1,52 @@
+import { getSnowflakeTimestamp } from './../utils/util';
+import { InstanceEvent } from './../bindings/InstanceEvent';
+import { ConsoleQueryParams } from './../bindings/ConsoleQueryParams';
+import { match, otherwise } from 'variant';
+import { ClientEvent } from 'bindings/ClientEvent';
+import { getConsoleEvents } from 'utils/apis';
+
+// simplified version of a ClientEvent with just InstanceOutput
+export type ConsoleEvent = {
+  timestamp: number;
+  snowflake: string;
+  detail: string;
+  uuid: string;
+  name: string;
+  message: string;
+};
+
+// function to convert a ClientEvent to a ConsoleEvent
+export const toConsoleEvent = (event: ClientEvent): ConsoleEvent => {
+  const event_inner: InstanceEvent = match(
+    event.event_inner,
+    otherwise(
+      {
+        InstanceEvent: (instanceEvent) => instanceEvent,
+      },
+      () => {
+        throw new Error('Expected InstanceEvent');
+      }
+    )
+  );
+
+  const message = match(
+    event_inner.instance_event_inner,
+    otherwise(
+      {
+        InstanceOutput: (instanceOutput) => instanceOutput.message,
+      },
+      () => {
+        throw new Error('Expected InstanceOutput');
+      }
+    )
+  );
+
+  return {
+    timestamp: getSnowflakeTimestamp(event.snowflake),
+    snowflake: event.snowflake,
+    detail: event.details,
+    uuid: event_inner.instance_uuid,
+    name: event_inner.instance_name,
+    message: message,
+  };
+};