Skip to content

Commit

Permalink
add example prometheus exporter
Browse files Browse the repository at this point in the history
  • Loading branch information
korewaChino committed Dec 4, 2024
1 parent d5baa20 commit 5e6f1a8
Show file tree
Hide file tree
Showing 7 changed files with 253 additions and 5 deletions.
80 changes: 80 additions & 0 deletions .github/workflows/docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,87 @@ on:
push:
workflow_dispatch:

env:
REGISTRY: ghcr.io
# IMAGE_NAME: fyralabs/skystreamer-prometheus-exporter

jobs:
prometheus-exporter:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
# This is used to complete the identity challenge
# with sigstore/fulcio when running outside of PRs.
id-token: write

steps:
- name: Checkout repository
uses: actions/checkout@v4

# Install the cosign tool except on PR
# https://github.com/sigstore/cosign-installer
- name: Install cosign
uses: sigstore/cosign-installer@v3

- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: arm64

# Workaround: https://github.com/docker/build-push-action/issues/461
- name: Setup Docker buildx
uses: docker/setup-buildx-action@v3
with:
platforms: linux/amd64,linux/arm64

# Login against a Docker registry except on PR
# https://github.com/docker/login-action
- name: Log into registry ${{ env.REGISTRY }}
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}

# Extract metadata (tags, labels) for Docker
# https://github.com/docker/metadata-action
- name: Extract Docker metadata
id: meta
uses: docker/metadata-action@v5
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
images: ${{ env.REGISTRY }}/fyralabs/skystreamer-prometheus-exporter

# Build and push Docker image with Buildx (don't push on PR)
# https://github.com/docker/build-push-action
- name: Build and push Docker image
id: build-and-push
uses: docker/build-push-action@v5
with:
context: .
file: ./exporter.Dockerfile
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=gha
cache-to: type=gha,mode=max
platforms: linux/amd64,linux/arm64

# Sign the resulting Docker image digest except on PRs.
# This will only write to the public Rekor transparency log when the Docker
# repository is public to avoid leaking data. If you would like to publish
# transparency data even for private images, pass --force to cosign below.
# https://github.com/sigstore/cosign
- name: Sign the published Docker image
env:
# https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
TAGS: ${{ steps.meta.outputs.tags }}
DIGEST: ${{ steps.build-and-push.outputs.digest }}
# This step uses the identity token to provision an ephemeral certificate
# against the sigstore community Fulcio instance.
run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}

docker:
uses: FyraLabs/actions/.github/workflows/docker.yml@main
with:
Expand Down
75 changes: 71 additions & 4 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
[workspace]
resolver = "2"
members = ["skystreamer", "skystreamer-bin"]
members = ["skystreamer", "skystreamer-bin", "skystreamer-prometheus-exporter"]
12 changes: 12 additions & 0 deletions exporter.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
FROM rust:latest
LABEL org.opencontainers.image.source = "https://github.com/FyraLabs/skystreamer"
WORKDIR /usr/src/app
COPY . .

RUN cargo install --path skystreamer-prometheus-exporter

WORKDIR /
RUN rm -rf /usr/src/app


CMD ["skystreamer-prometheus-exporter"]
14 changes: 14 additions & 0 deletions skystreamer-prometheus-exporter/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
[package]
name = "skystreamer-prometheus-exporter"
version = "0.1.0"
edition = "2021"

[dependencies]
skystreamer = { path = "../skystreamer" }
color-eyre = "0.6.3"
prometheus_exporter = "0.8.5"
tokio = { version = "1.42.0", features = ["full"] }
tokio-stream = { version = "0.1.16", features = ["full"] }
tracing = { version = "0.1.41", features = ["log"] }
tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
futures = "0.3.31"
11 changes: 11 additions & 0 deletions skystreamer-prometheus-exporter/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# SkyStreamer Prometheus Exporter

This is a simple implementation of a Prometheus exporter for SkyStreamer analytics. It exports a single `posts` metric that counts the number of posts collected by SkyStreamer.

The counter loops back to 0 when the counter reaches 10000.

To query using Prometheus, use the following query:

```promql
irate(skystreamer_bsky_posts[1m])
```
64 changes: 64 additions & 0 deletions skystreamer-prometheus-exporter/src/main.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
use color_eyre::Result;
use futures::StreamExt;
use prometheus_exporter::{self, prometheus::register_counter};
use skystreamer::{stream::PostStream, RepoSubscription};
use tracing::level_filters::LevelFilter;
use tracing_subscriber::EnvFilter;

fn default_level_filter() -> LevelFilter {
#[cfg(debug_assertions)]
return LevelFilter::DEBUG;
#[cfg(not(debug_assertions))]
return LevelFilter::INFO;
}
#[tokio::main]
async fn main() -> Result<()> {
let env_filter = EnvFilter::builder()
.with_default_directive(default_level_filter().into())
.from_env()?;

color_eyre::install()?;

tracing_subscriber::fmt()
.with_target(false)
.with_thread_ids(true)
.with_level(true)
.with_file(false)
.compact()
.with_line_number(false)
.with_env_filter(env_filter)
.init();

let binding = "0.0.0.0:9100".parse()?;
let _exporter = prometheus_exporter::start(binding)?;
let counter = register_counter!(
"skystreamer_bsky_posts",
"Number of posts from bsky.network"
)?;

const MAX_SAMPLE_SIZE: usize = 10000;

loop {
let subscription = RepoSubscription::new("bsky.network").await.unwrap();
let post_stream = PostStream::new(subscription);
let mut post_stream = post_stream.await;
let stream = post_stream.stream().await?;

futures::pin_mut!(stream);
// let mut interval = tokio::time::interval(std::time::Duration::from_secs(1));
// interval.tick().await;

// let mut last_tick = tokio::time::Instant::now();

while let Some(_post) = stream.next().await {
if counter.get() > MAX_SAMPLE_SIZE as f64 {
counter.reset();
}

counter.inc();
// println!("Rate: {}", counter.get());
}
}

// Ok(())
}

0 comments on commit 5e6f1a8

Please sign in to comment.