diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8654a25..fe61467 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -4,7 +4,87 @@ on: push: workflow_dispatch: +env: + REGISTRY: ghcr.io + # IMAGE_NAME: fyralabs/skystreamer-prometheus-exporter + jobs: + prometheus-exporter: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # This is used to complete the identity challenge + # with sigstore/fulcio when running outside of PRs. + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install the cosign tool except on PR + # https://github.com/sigstore/cosign-installer + - name: Install cosign + uses: sigstore/cosign-installer@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: arm64 + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v3 + with: + platforms: linux/amd64,linux/arm64 + + # Login against a Docker registry except on PR + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + images: ${{ env.REGISTRY }}/fyralabs/skystreamer-prometheus-exporter + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v5 + with: + context: . + file: ./exporter.Dockerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: linux/amd64,linux/arm64 + + # Sign the resulting Docker image digest except on PRs. + # This will only write to the public Rekor transparency log when the Docker + # repository is public to avoid leaking data. If you would like to publish + # transparency data even for private images, pass --force to cosign below. + # https://github.com/sigstore/cosign + - name: Sign the published Docker image + env: + # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable + TAGS: ${{ steps.meta.outputs.tags }} + DIGEST: ${{ steps.build-and-push.outputs.digest }} + # This step uses the identity token to provision an ephemeral certificate + # against the sigstore community Fulcio instance. + run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST} + docker: uses: FyraLabs/actions/.github/workflows/docker.yml@main with: diff --git a/Cargo.lock b/Cargo.lock index 828d4c5..15314e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -218,6 +218,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "ascii-canvas" version = "3.0.0" @@ -794,6 +800,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "chunked_transfer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e4de3bc4ea267985becf712dc6d9eed8b04c953b3fcfb339ebc87acd9804901" + [[package]] name = "ciborium" version = "0.2.2" @@ -3102,6 +3114,34 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "thiserror 1.0.69", +] + +[[package]] +name = "prometheus_exporter" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf17cbebe0bfdf4f279ef84eeefe0d50468b0b7116f078acf41d456e48fe81a" +dependencies = [ + "ascii", + "lazy_static", + "log", + "prometheus", + "thiserror 1.0.69", + "tiny_http", +] + [[package]] name = "psl-types" version = "2.0.11" @@ -4111,6 +4151,20 @@ dependencies = [ "url", ] +[[package]] +name = "skystreamer-prometheus-exporter" +version = "0.1.0" +dependencies = [ + "color-eyre", + "futures", + "prometheus_exporter", + "skystreamer", + "tokio", + "tokio-stream", + "tracing", + "tracing-subscriber", +] + [[package]] name = "slab" version = "0.4.9" @@ -4658,6 +4712,19 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tiny_http" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f8734c6d6943ad6df6b588d228a87b4af184998bcffa268ceddf05c2055a8c" +dependencies = [ + "ascii", + "chunked_transfer", + "log", + "time", + "url", +] + [[package]] name = "tinystr" version = "0.7.6" @@ -4685,9 +4752,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.41.1" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", @@ -4887,9 +4954,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index 35d0dfb..855be97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,3 @@ [workspace] resolver = "2" -members = ["skystreamer", "skystreamer-bin"] +members = ["skystreamer", "skystreamer-bin", "skystreamer-prometheus-exporter"] diff --git a/exporter.Dockerfile b/exporter.Dockerfile new file mode 100644 index 0000000..916c47a --- /dev/null +++ b/exporter.Dockerfile @@ -0,0 +1,12 @@ +FROM rust:latest +LABEL org.opencontainers.image.source = "https://github.com/FyraLabs/skystreamer" +WORKDIR /usr/src/app +COPY . . + +RUN cargo install --path skystreamer-prometheus-exporter + +WORKDIR / +RUN rm -rf /usr/src/app + + +CMD ["skystreamer-prometheus-exporter"] \ No newline at end of file diff --git a/skystreamer-prometheus-exporter/Cargo.toml b/skystreamer-prometheus-exporter/Cargo.toml new file mode 100644 index 0000000..912eb7b --- /dev/null +++ b/skystreamer-prometheus-exporter/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "skystreamer-prometheus-exporter" +version = "0.1.0" +edition = "2021" + +[dependencies] +skystreamer = { path = "../skystreamer" } +color-eyre = "0.6.3" +prometheus_exporter = "0.8.5" +tokio = { version = "1.42.0", features = ["full"] } +tokio-stream = { version = "0.1.16", features = ["full"] } +tracing = { version = "0.1.41", features = ["log"] } +tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } +futures = "0.3.31" diff --git a/skystreamer-prometheus-exporter/README.md b/skystreamer-prometheus-exporter/README.md new file mode 100644 index 0000000..e51ec75 --- /dev/null +++ b/skystreamer-prometheus-exporter/README.md @@ -0,0 +1,11 @@ +# SkyStreamer Prometheus Exporter + +This is a simple implementation of a Prometheus exporter for SkyStreamer analytics. It exports a single `posts` metric that counts the number of posts collected by SkyStreamer. + +The counter loops back to 0 when the counter reaches 10000. + +To query using Prometheus, use the following query: + +```promql +irate(skystreamer_bsky_posts[1m]) +``` diff --git a/skystreamer-prometheus-exporter/src/main.rs b/skystreamer-prometheus-exporter/src/main.rs new file mode 100644 index 0000000..d9ec3de --- /dev/null +++ b/skystreamer-prometheus-exporter/src/main.rs @@ -0,0 +1,64 @@ +use color_eyre::Result; +use futures::StreamExt; +use prometheus_exporter::{self, prometheus::register_counter}; +use skystreamer::{stream::PostStream, RepoSubscription}; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::EnvFilter; + +fn default_level_filter() -> LevelFilter { + #[cfg(debug_assertions)] + return LevelFilter::DEBUG; + #[cfg(not(debug_assertions))] + return LevelFilter::INFO; +} +#[tokio::main] +async fn main() -> Result<()> { + let env_filter = EnvFilter::builder() + .with_default_directive(default_level_filter().into()) + .from_env()?; + + color_eyre::install()?; + + tracing_subscriber::fmt() + .with_target(false) + .with_thread_ids(true) + .with_level(true) + .with_file(false) + .compact() + .with_line_number(false) + .with_env_filter(env_filter) + .init(); + + let binding = "0.0.0.0:9100".parse()?; + let _exporter = prometheus_exporter::start(binding)?; + let counter = register_counter!( + "skystreamer_bsky_posts", + "Number of posts from bsky.network" + )?; + + const MAX_SAMPLE_SIZE: usize = 10000; + + loop { + let subscription = RepoSubscription::new("bsky.network").await.unwrap(); + let post_stream = PostStream::new(subscription); + let mut post_stream = post_stream.await; + let stream = post_stream.stream().await?; + + futures::pin_mut!(stream); + // let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + // interval.tick().await; + + // let mut last_tick = tokio::time::Instant::now(); + + while let Some(_post) = stream.next().await { + if counter.get() > MAX_SAMPLE_SIZE as f64 { + counter.reset(); + } + + counter.inc(); + // println!("Rate: {}", counter.get()); + } + } + + // Ok(()) +}