feat: Add full-text search on windmill service logs (#4576)

* Add service_log indexer file (ee) * Rename to completed_run * rename to service_logs * Update lib.rs * Make indexing of logs and simple frontend * Add common ee file * Update lib.rs * Make log search global to fix openapi unused path param * Prepare sqlx * Show last indexed also when no jobs are found * Adapt feature flags to service logs indexer * Update api and main * Build service log search page * scope query to the min and max time * Migrate modal, bug fixes * Remove unused import * Make the service logs page take the full screen * improve quick access menu to service_logs, fix date stuff, hide 0 matches --------- Co-authored-by: Ruben Fiszel <[email protected]>
windmill-labs · Nov 5, 2024 · 77735d8 · 77735d8
1 parent 8e27392
commit 77735d8
Show file tree

Hide file tree

Showing 23 changed files with 925 additions and 239 deletions.
diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json
diff --git a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json b/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json
diff --git a/backend/.sqlx/query-dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6.json b/backend/.sqlx/query-dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6.json
diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt
@@ -1 +1 @@
-f136a2f499e0fe7c10c54c79488851980d796eb2
+01f384348341c87c5a0b32c4be7c699975ee7d6d
diff --git a/backend/src/main.rs b/backend/src/main.rs
@@ -531,7 +531,7 @@ Windmill Community Edition {GIT_VERSION}
 
         #[cfg(feature = "tantivy")]
         let (index_reader, index_writer) = if should_index_jobs {
-            let (r, w) = windmill_indexer::indexer_ee::init_index(&db).await?;
+            let (r, w) = windmill_indexer::completed_runs_ee::init_index(&db).await?;
             (Some(r), Some(w))
         } else {
             (None, None)
@@ -543,26 +543,61 @@ Windmill Community Edition {GIT_VERSION}
             let index_writer2 = index_writer.clone();
             async {
                 if let Some(index_writer) = index_writer2 {
-                    windmill_indexer::indexer_ee::run_indexer(db.clone(), index_writer, indexer_rx)
-                        .await;
+                    windmill_indexer::completed_runs_ee::run_indexer(
+                        db.clone(),
+                        index_writer,
+                        indexer_rx,
+                    )
+                    .await;
+                }
+                Ok(())
+            }
+        };
+
+        #[cfg(all(feature = "tantivy", feature = "parquet"))]
+        let (log_index_reader, log_index_writer) = if should_index_jobs {
+            let (r, w) = windmill_indexer::service_logs_ee::init_index(&db).await?;
+            (Some(r), Some(w))
+        } else {
+            (None, None)
+        };
+
+        #[cfg(all(feature = "tantivy", feature = "parquet"))]
+        let log_indexer_f = {
+            let log_indexer_rx = killpill_rx.resubscribe();
+            let log_index_writer2 = log_index_writer.clone();
+            async {
+                if let Some(log_index_writer) = log_index_writer2 {
+                    windmill_indexer::service_logs_ee::run_indexer(
+                        db.clone(),
+                        log_index_writer,
+                        log_indexer_rx,
+                    )
+                    .await;
                 }
                 Ok(())
             }
         };
 
         #[cfg(not(feature = "tantivy"))]
-        let (index_reader, index_writer) = (None, None);
+        let index_reader = None;
 
         #[cfg(not(feature = "tantivy"))]
         let indexer_f = async { Ok(()) as anyhow::Result<()> };
 
+        #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
+        let log_index_reader = None;
+
+        #[cfg(not(all(feature = "tantivy", feature = "parquet")))]
+        let log_indexer_f = async { Ok(()) as anyhow::Result<()> };
+
         let server_f = async {
             if !is_agent {
                 windmill_api::run_server(
                     db.clone(),
                     rsmq2,
                     index_reader,
-                    index_writer,
+                    log_index_reader,
                     addr,
                     server_killpill_rx,
                     base_internal_tx,
@@ -842,7 +877,8 @@ Windmill Community Edition {GIT_VERSION}
             monitor_f,
             server_f,
             metrics_f,
-            indexer_f
+            indexer_f,
+            log_indexer_f
         )?;
     } else {
         tracing::info!("Nothing to do, exiting.");

diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml
@@ -9546,6 +9546,109 @@ paths:
                     items:
                       $ref: "#/components/schemas/JobSearchHit"
 
+  /srch/index/search/service_logs:
+    get:
+      summary: Search through service logs with a string query
+      operationId: searchLogsIndex
+      tags:
+        - indexSearch
+      parameters:
+        - name: search_query
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: mode
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: worker_group
+          in: query
+          required: false
+          schema:
+            type: string
+        - name: hostname
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: min_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+        - name: max_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+      responses:
+        "200":
+          description: search results
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  query_parse_errors:
+                    description: a list of the terms that couldn't be parsed (and thus ignored)
+                    type: array
+                    items:
+                      type: string
+                  hits:
+                    description: log files that matched the query
+                    type: array
+                    items:
+                      $ref: "#/components/schemas/LogSearchHit"
+
+  /srch/index/search/count_service_logs:
+    get:
+      summary: Search and count the log line hits on every provided host
+      operationId: countSearchLogsIndex
+      tags:
+        - indexSearch
+      parameters:
+        - name: search_query
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: hosts
+          in: query
+          required: true
+          schema:
+            type: string
+        - name: min_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+        - name: max_ts
+          in: query
+          required: false
+          schema:
+            type: string
+            format: date-time
+      responses:
+        "200":
+          description: search results
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  query_parse_errors:
+                    description: a list of the terms that couldn't be parsed (and thus ignored)
+                    type: array
+                    items:
+                      type: string
+                  count_per_host:
+                    description: count of log lines that matched the query per hostname
+                    type: object
 
 components:
   securitySchemes:
@@ -12500,6 +12603,12 @@ components:
         dancer:
           type: string
 
+    LogSearchHit:
+      type: object
+      properties:
+        dancer:
+          type: string
+
     AutoscalingEvent:
       type: object
       properties:
@@ -12517,3 +12626,4 @@ components:
         applied_at:
           type: string
           format: date-time
+
diff --git a/backend/windmill-api/src/indexer_ee.rs b/backend/windmill-api/src/indexer_ee.rs
@@ -3,3 +3,7 @@ use axum::Router;
 pub fn workspaced_service() -> Router {
     Router::new()
 }
+
+pub fn global_service() -> Router {
+    Router::new()
+}