-
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
1 parent
d5baa20
commit 5e6f1a8
Showing
7 changed files
with
253 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,3 +1,3 @@ | ||
[workspace] | ||
resolver = "2" | ||
members = ["skystreamer", "skystreamer-bin"] | ||
members = ["skystreamer", "skystreamer-bin", "skystreamer-prometheus-exporter"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
FROM rust:latest | ||
LABEL org.opencontainers.image.source = "https://github.com/FyraLabs/skystreamer" | ||
WORKDIR /usr/src/app | ||
COPY . . | ||
|
||
RUN cargo install --path skystreamer-prometheus-exporter | ||
|
||
WORKDIR / | ||
RUN rm -rf /usr/src/app | ||
|
||
|
||
CMD ["skystreamer-prometheus-exporter"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
[package] | ||
name = "skystreamer-prometheus-exporter" | ||
version = "0.1.0" | ||
edition = "2021" | ||
|
||
[dependencies] | ||
skystreamer = { path = "../skystreamer" } | ||
color-eyre = "0.6.3" | ||
prometheus_exporter = "0.8.5" | ||
tokio = { version = "1.42.0", features = ["full"] } | ||
tokio-stream = { version = "0.1.16", features = ["full"] } | ||
tracing = { version = "0.1.41", features = ["log"] } | ||
tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } | ||
futures = "0.3.31" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
# SkyStreamer Prometheus Exporter | ||
|
||
This is a simple implementation of a Prometheus exporter for SkyStreamer analytics. It exports a single `posts` metric that counts the number of posts collected by SkyStreamer. | ||
|
||
The counter loops back to 0 when the counter reaches 10000. | ||
|
||
To query using Prometheus, use the following query: | ||
|
||
```promql | ||
irate(skystreamer_bsky_posts[1m]) | ||
``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,64 @@ | ||
use color_eyre::Result; | ||
use futures::StreamExt; | ||
use prometheus_exporter::{self, prometheus::register_counter}; | ||
use skystreamer::{stream::PostStream, RepoSubscription}; | ||
use tracing::level_filters::LevelFilter; | ||
use tracing_subscriber::EnvFilter; | ||
|
||
fn default_level_filter() -> LevelFilter { | ||
#[cfg(debug_assertions)] | ||
return LevelFilter::DEBUG; | ||
#[cfg(not(debug_assertions))] | ||
return LevelFilter::INFO; | ||
} | ||
#[tokio::main] | ||
async fn main() -> Result<()> { | ||
let env_filter = EnvFilter::builder() | ||
.with_default_directive(default_level_filter().into()) | ||
.from_env()?; | ||
|
||
color_eyre::install()?; | ||
|
||
tracing_subscriber::fmt() | ||
.with_target(false) | ||
.with_thread_ids(true) | ||
.with_level(true) | ||
.with_file(false) | ||
.compact() | ||
.with_line_number(false) | ||
.with_env_filter(env_filter) | ||
.init(); | ||
|
||
let binding = "0.0.0.0:9100".parse()?; | ||
let _exporter = prometheus_exporter::start(binding)?; | ||
let counter = register_counter!( | ||
"skystreamer_bsky_posts", | ||
"Number of posts from bsky.network" | ||
)?; | ||
|
||
const MAX_SAMPLE_SIZE: usize = 10000; | ||
|
||
loop { | ||
let subscription = RepoSubscription::new("bsky.network").await.unwrap(); | ||
let post_stream = PostStream::new(subscription); | ||
let mut post_stream = post_stream.await; | ||
let stream = post_stream.stream().await?; | ||
|
||
futures::pin_mut!(stream); | ||
// let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); | ||
// interval.tick().await; | ||
|
||
// let mut last_tick = tokio::time::Instant::now(); | ||
|
||
while let Some(_post) = stream.next().await { | ||
if counter.get() > MAX_SAMPLE_SIZE as f64 { | ||
counter.reset(); | ||
} | ||
|
||
counter.inc(); | ||
// println!("Rate: {}", counter.get()); | ||
} | ||
} | ||
|
||
// Ok(()) | ||
} |