diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json index 75b8108281532..1fa370e682ca6 100644 --- a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json +++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "?column?", + "name": "bool", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json b/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json new file mode 100644 index 0000000000000..ba9f3814e8de4 --- /dev/null +++ b/backend/.sqlx/query-b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a.json @@ -0,0 +1,65 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n WHERE log_ts > $1\n ORDER BY log_ts ASC LIMIT $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hostname", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "mode", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "worker_group", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "log_ts", + "type_info": "Timestamp" + }, + { + "ordinal": 4, + "name": "file_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "ok_lines", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "err_lines", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "json_fmt", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Timestamp", + "Int8" + ] + }, + "nullable": [ + false, + null, + true, + false, + false, + true, + true, + true + ] + }, + "hash": "b5c839baab25c4dcdd503d380cf7a886242277cd50555f20b2e22e13942d2a3a" +} diff --git a/backend/.sqlx/query-dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6.json b/backend/.sqlx/query-dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6.json new file mode 100644 index 0000000000000..b64db838d35cd --- /dev/null +++ b/backend/.sqlx/query-dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6.json @@ -0,0 +1,64 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n hostname,\n mode::text,\n worker_group,\n log_ts,\n file_path,\n ok_lines,\n err_lines,\n json_fmt\n FROM log_file\n ORDER BY log_ts ASC LIMIT $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hostname", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "mode", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "worker_group", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "log_ts", + "type_info": "Timestamp" + }, + { + "ordinal": 4, + "name": "file_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "ok_lines", + "type_info": "Int8" + }, + { + "ordinal": 6, + "name": "err_lines", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "json_fmt", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + null, + true, + false, + false, + true, + true, + true + ] + }, + "hash": "dd967c5983fa0ff05e2b320ad0e0b5a152784826cb8fb4381c1ffe228cb7feb6" +} diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index e7dd9365ecdd0..2104f8ba8ce15 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -f136a2f499e0fe7c10c54c79488851980d796eb2 \ No newline at end of file +01f384348341c87c5a0b32c4be7c699975ee7d6d \ No newline at end of file diff --git a/backend/src/main.rs b/backend/src/main.rs index 11d12f9dab3b0..61ee657d2c7bd 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -531,7 +531,7 @@ Windmill Community Edition {GIT_VERSION} #[cfg(feature = "tantivy")] let (index_reader, index_writer) = if should_index_jobs { - let (r, w) = windmill_indexer::indexer_ee::init_index(&db).await?; + let (r, w) = windmill_indexer::completed_runs_ee::init_index(&db).await?; (Some(r), Some(w)) } else { (None, None) @@ -543,26 +543,61 @@ Windmill Community Edition {GIT_VERSION} let index_writer2 = index_writer.clone(); async { if let Some(index_writer) = index_writer2 { - windmill_indexer::indexer_ee::run_indexer(db.clone(), index_writer, indexer_rx) - .await; + windmill_indexer::completed_runs_ee::run_indexer( + db.clone(), + index_writer, + indexer_rx, + ) + .await; + } + Ok(()) + } + }; + + #[cfg(all(feature = "tantivy", feature = "parquet"))] + let (log_index_reader, log_index_writer) = if should_index_jobs { + let (r, w) = windmill_indexer::service_logs_ee::init_index(&db).await?; + (Some(r), Some(w)) + } else { + (None, None) + }; + + #[cfg(all(feature = "tantivy", feature = "parquet"))] + let log_indexer_f = { + let log_indexer_rx = killpill_rx.resubscribe(); + let log_index_writer2 = log_index_writer.clone(); + async { + if let Some(log_index_writer) = log_index_writer2 { + windmill_indexer::service_logs_ee::run_indexer( + db.clone(), + log_index_writer, + log_indexer_rx, + ) + .await; } Ok(()) } }; #[cfg(not(feature = "tantivy"))] - let (index_reader, index_writer) = (None, None); + let index_reader = None; #[cfg(not(feature = "tantivy"))] let indexer_f = async { Ok(()) as anyhow::Result<()> }; + #[cfg(not(all(feature = "tantivy", feature = "parquet")))] + let log_index_reader = None; + + #[cfg(not(all(feature = "tantivy", feature = "parquet")))] + let log_indexer_f = async { Ok(()) as anyhow::Result<()> }; + let server_f = async { if !is_agent { windmill_api::run_server( db.clone(), rsmq2, index_reader, - index_writer, + log_index_reader, addr, server_killpill_rx, base_internal_tx, @@ -842,7 +877,8 @@ Windmill Community Edition {GIT_VERSION} monitor_f, server_f, metrics_f, - indexer_f + indexer_f, + log_indexer_f )?; } else { tracing::info!("Nothing to do, exiting."); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index fc39c5abf8e85..0949843665574 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9546,6 +9546,109 @@ paths: items: $ref: "#/components/schemas/JobSearchHit" + /srch/index/search/service_logs: + get: + summary: Search through service logs with a string query + operationId: searchLogsIndex + tags: + - indexSearch + parameters: + - name: search_query + in: query + required: true + schema: + type: string + - name: mode + in: query + required: true + schema: + type: string + - name: worker_group + in: query + required: false + schema: + type: string + - name: hostname + in: query + required: true + schema: + type: string + - name: min_ts + in: query + required: false + schema: + type: string + format: date-time + - name: max_ts + in: query + required: false + schema: + type: string + format: date-time + responses: + "200": + description: search results + content: + application/json: + schema: + type: object + properties: + query_parse_errors: + description: a list of the terms that couldn't be parsed (and thus ignored) + type: array + items: + type: string + hits: + description: log files that matched the query + type: array + items: + $ref: "#/components/schemas/LogSearchHit" + + /srch/index/search/count_service_logs: + get: + summary: Search and count the log line hits on every provided host + operationId: countSearchLogsIndex + tags: + - indexSearch + parameters: + - name: search_query + in: query + required: true + schema: + type: string + - name: hosts + in: query + required: true + schema: + type: string + - name: min_ts + in: query + required: false + schema: + type: string + format: date-time + - name: max_ts + in: query + required: false + schema: + type: string + format: date-time + responses: + "200": + description: search results + content: + application/json: + schema: + type: object + properties: + query_parse_errors: + description: a list of the terms that couldn't be parsed (and thus ignored) + type: array + items: + type: string + count_per_host: + description: count of log lines that matched the query per hostname + type: object components: securitySchemes: @@ -12500,6 +12603,12 @@ components: dancer: type: string + LogSearchHit: + type: object + properties: + dancer: + type: string + AutoscalingEvent: type: object properties: @@ -12517,3 +12626,4 @@ components: applied_at: type: string format: date-time + diff --git a/backend/windmill-api/src/indexer_ee.rs b/backend/windmill-api/src/indexer_ee.rs index 61a946bb84a9b..2ccca92c27f9b 100644 --- a/backend/windmill-api/src/indexer_ee.rs +++ b/backend/windmill-api/src/indexer_ee.rs @@ -3,3 +3,7 @@ use axum::Router; pub fn workspaced_service() -> Router { Router::new() } + +pub fn global_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 3ea78ffd6da49..5a7f7fcded243 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -154,18 +154,18 @@ pub async fn add_webhook_allowed_origin( type IndexReader = (); #[cfg(not(feature = "tantivy"))] -type IndexWriter = (); +type ServiceLogIndexReader = (); #[cfg(feature = "tantivy")] -type IndexReader = windmill_indexer::indexer_ee::IndexReader; +type IndexReader = windmill_indexer::completed_runs_ee::IndexReader; #[cfg(feature = "tantivy")] -type IndexWriter = windmill_indexer::indexer_ee::IndexWriter; +type ServiceLogIndexReader = windmill_indexer::service_logs_ee::ServiceLogIndexReader; pub async fn run_server( db: DB, rsmq: Option, - index_reader: Option, - index_writer: Option, + job_index_reader: Option, + log_index_reader: Option, addr: SocketAddr, mut rx: tokio::sync::broadcast::Receiver<()>, port_tx: tokio::sync::oneshot::Sender, @@ -205,8 +205,9 @@ pub async fn run_server( .layer(Extension(rsmq.clone())) .layer(Extension(user_db.clone())) .layer(Extension(auth_cache.clone())) - .layer(Extension(index_reader)) - .layer(Extension(index_writer)) + .layer(Extension(job_index_reader)) + .layer(Extension(log_index_reader)) + // .layer(Extension(index_writer)) .layer(CookieManagerLayer::new()) .layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone()))) .layer(DefaultBodyLimit::max( @@ -322,6 +323,10 @@ pub async fn run_server( "/srch/w/:workspace_id/index", indexer_ee::workspaced_service(), ) + .nest( + "/srch/index", + indexer_ee::global_service(), + ) .nest("/oidc", oidc_ee::global_service()) .nest( "/saml", diff --git a/backend/windmill-indexer/src/completed_runs_ee.rs b/backend/windmill-indexer/src/completed_runs_ee.rs new file mode 100644 index 0000000000000..79bcbcff778a6 --- /dev/null +++ b/backend/windmill-indexer/src/completed_runs_ee.rs @@ -0,0 +1,21 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::error::Error; +use anyhow::anyhow; + +#[derive(Clone)] +pub struct IndexReader; + +#[derive(Clone)] +pub struct IndexWriter; + +pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> { + Err(anyhow!("Cannot initialize index: not in EE").into()) +} + +pub async fn run_indexer( + _db: Pool, + mut _index_writer: IndexWriter, + mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + tracing::error!("Cannot run indexer: not in EE"); +} diff --git a/backend/windmill-indexer/src/indexer_ee.rs b/backend/windmill-indexer/src/indexer_ee.rs index da92ff0ef82fd..e69de29bb2d1d 100644 --- a/backend/windmill-indexer/src/indexer_ee.rs +++ b/backend/windmill-indexer/src/indexer_ee.rs @@ -1,21 +0,0 @@ -use anyhow::anyhow; -use sqlx::{Pool, Postgres}; -use windmill_common::error::Error; - -#[derive(Clone)] -pub struct IndexReader; - -#[derive(Clone)] -pub struct IndexWriter; - -pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> { - Err(anyhow!("Cannot initialize index: not in EE").into()) -} - -pub async fn run_indexer( - _db: Pool, - mut _index_writer: IndexWriter, - mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, -) { - tracing::error!("Cannot run indexer: not in EE"); -} diff --git a/backend/windmill-indexer/src/lib.rs b/backend/windmill-indexer/src/lib.rs index c1bf4eea0c11d..14c6e7906d10b 100644 --- a/backend/windmill-indexer/src/lib.rs +++ b/backend/windmill-indexer/src/lib.rs @@ -1 +1,3 @@ +pub mod completed_runs_ee; +pub mod service_logs_ee; pub mod indexer_ee; diff --git a/backend/windmill-indexer/src/service_logs_ee.rs b/backend/windmill-indexer/src/service_logs_ee.rs new file mode 100644 index 0000000000000..79bcbcff778a6 --- /dev/null +++ b/backend/windmill-indexer/src/service_logs_ee.rs @@ -0,0 +1,21 @@ +use sqlx::{Pool, Postgres}; +use windmill_common::error::Error; +use anyhow::anyhow; + +#[derive(Clone)] +pub struct IndexReader; + +#[derive(Clone)] +pub struct IndexWriter; + +pub async fn init_index() -> Result<(IndexReader, IndexWriter), Error> { + Err(anyhow!("Cannot initialize index: not in EE").into()) +} + +pub async fn run_indexer( + _db: Pool, + mut _index_writer: IndexWriter, + mut _killpill_rx: tokio::sync::broadcast::Receiver<()>, +) { + tracing::error!("Cannot run indexer: not in EE"); +} diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 0fde0100912fc..a5c5e883d1e9d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -43,7 +43,7 @@ "graphql": "^16.7.1", "hash-sum": "^2.0.0", "highlight.js": "^11.8.0", - "lucide-svelte": "^0.293.0", + "lucide-svelte": "^0.399.0", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~8.0.2", "monaco-editor-wrapper": "^5.5.2", @@ -8607,11 +8607,11 @@ "optional": true }, "node_modules/lucide-svelte": { - "version": "0.293.0", - "resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.293.0.tgz", - "integrity": "sha512-nQ6QO6anjjiVoipw3dNptzI6NWuiArlhysVtRpoAWICSQVMR3ybwVf6piUUqbjGe+RfGufLzCl93C/yhsS+p4w==", + "version": "0.399.0", + "resolved": "https://registry.npmjs.org/lucide-svelte/-/lucide-svelte-0.399.0.tgz", + "integrity": "sha512-NQ8AxNMKbIJsx7HV//gnAsIY1wJfb3rbXSK2S/ZDjIldvAEdzGngpUT8T8Q8zHYUuii0bavAmVARN8giR4vvpA==", "peerDependencies": { - "svelte": ">=3 <5" + "svelte": "^3 || ^4 || ^5.0.0-next.42" } }, "node_modules/magic-string": { diff --git a/frontend/package.json b/frontend/package.json index de2a4b629bfbd..1716d5546ab4d 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -116,7 +116,7 @@ "graphql": "^16.7.1", "hash-sum": "^2.0.0", "highlight.js": "^11.8.0", - "lucide-svelte": "^0.293.0", + "lucide-svelte": "^0.399.0", "minimatch": "^10.0.1", "monaco-editor": "npm:@codingame/monaco-vscode-editor-api@~8.0.2", "monaco-editor-wrapper": "^5.5.2", diff --git a/frontend/src/lib/components/LogSnippetViewer.svelte b/frontend/src/lib/components/LogSnippetViewer.svelte new file mode 100644 index 0000000000000..693522ed33720 --- /dev/null +++ b/frontend/src/lib/components/LogSnippetViewer.svelte @@ -0,0 +1,38 @@ + + + diff --git a/frontend/src/lib/components/LogViewer.svelte b/frontend/src/lib/components/LogViewer.svelte index de78eefc5237d..d04b7f9c566cd 100644 --- a/frontend/src/lib/components/LogViewer.svelte +++ b/frontend/src/lib/components/LogViewer.svelte @@ -245,7 +245,7 @@ : 'top-2'} left-36">mem peak: {(mem / 1024).toPrecision(4)}MB {/if} -
{#if content}{@const len =
 					(content?.length ?? 0) +
 					(loadedFromObjectStore?.length ?? 0)}{#if downloadStartUrl}
+		
+		
+
...
{#each content.split('\n') as line, index}
{@html line}
{/each}...
+
+ + + +