openshift · red-hat-konflux · Sep 10, 2024 · Sep 11, 2024 · Sep 11, 2024 · Sep 12, 2024
diff --git a/.tekton/trustee-pull-request.yaml b/.tekton/trustee-pull-request.yaml
diff --git a/.tekton/trustee-push.yaml b/.tekton/trustee-push.yaml
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,7 +5,6 @@ members = [
     "rvps",
     "tools/kbs-client",
     "deps/verifier",
-    "integration-tests",
 ]
 resolver = "2"
 
@@ -33,7 +32,7 @@ hex = "0.4.3"
 jwt-simple = { version = "0.12", default-features = false, features = [
     "pure-rust",
 ] }
-kbs_protocol = { git = "https://github.com/confidential-containers/guest-components.git", rev = "27b8245", default-features = false }
+kbs_protocol = { git = "https://github.com/confidential-containers/guest-components.git", rev = "e6999a3c0fd877dae9e68ea78b8b483062db32b8", default-features = false }
 kbs-types = "0.10.0"
 kms = { git = "https://github.com/confidential-containers/guest-components.git", rev = "e6999a3c0fd877dae9e68ea78b8b483062db32b8", default-features = false }
 jsonwebtoken = { version = "9", default-features = false }

diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml
diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs
diff --git a/kbs/Cargo.toml b/kbs/Cargo.toml
@@ -60,6 +60,7 @@ lazy_static = "1.4.0"
 log.workspace = true
 mobc = { version = "0.8.5", optional = true }
 p256 = { workspace = true, features = ["ecdh"] }
+prometheus = "0.13.4"
 prost = { workspace = true, optional = true }
 rand = "0.8.5"
 regex = "1.11.1"
@@ -103,7 +104,20 @@ josekit = "0.10.0"
 tempfile.workspace = true
 rstest.workspace = true
 reference-value-provider-service.path = "../rvps"
-serial_test = "3.0"
+serial_test.workspace = true
+
+[target.'cfg(not(any(target_arch = "s390x", target_arch = "aarch64")))'.dev-dependencies]
+kbs-client = { path = "../tools/kbs-client" }
+
+[target.'cfg(target_arch = "s390x")'.dev-dependencies]
+kbs-client = { path = "../tools/kbs-client", default-features = false, features = [
+    "se-attester",
+] }
+
+[target.'cfg(target_arch = "aarch64")'.dev-dependencies]
+kbs-client = { path = "../tools/kbs-client", default-features = false, features = [
+    "cca-attester",
+] }
 
 [build-dependencies]
 tonic-build = { workspace = true, optional = true }
diff --git a/kbs/Makefile b/kbs/Makefile
@@ -115,7 +115,7 @@ uninstall:
 	rm -rf $(INSTALL_DESTDIR)/kbs $(INSTALL_DESTDIR)/kbs-client $(INSTALL_DESTDIR)/issuer-kbs $(INSTALL_DESTDIR)/resource-kbs
 
 check:
-	cargo test -p kbs -p kbs-client -p integration-tests $(TEST_ARGUMENTS)
+	cargo test -p kbs -p kbs-client $(TEST_ARGUMENTS)
 
 lint:
 	cargo clippy -p kbs -p kbs-client $(TEST_ARGUMENTS) -- -D warnings

diff --git a/kbs/docker/kbs-client-image/Dockerfile b/kbs/docker/kbs-client-image/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.io/library/rust:1.80.0 AS builder
+FROM docker.io/library/rust:1.89.0 AS builder
 
 WORKDIR /usr/src/kbs
 COPY . .

diff --git a/kbs/docker/kbs-client/Dockerfile b/kbs/docker/kbs-client/Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.io/library/rust:1.80.0 AS builder
+FROM docker.io/library/rust:1.89.0 AS builder
 ARG ARCH=x86_64
 
 WORKDIR /usr/src/kbs

diff --git a/kbs/docker/rhel-ubi/Dockerfile b/kbs/docker/rhel-ubi/Dockerfile
@@ -1,34 +1,95 @@
-# Use CentOS Stream to build.
-FROM quay.io/centos/centos:stream9 AS builder
+# Use UBI to build.
+FROM registry.access.redhat.com/ubi9/ubi:9.6-1755678605 as builder
+ARG ALIYUN=false
 
-# Install build dependencies from CentOS repos.
-RUN dnf -y --setopt=install_weak_deps=0 --enablerepo=crb install \
-cargo pkg-config perl-FindBin openssl-devel perl-lib perl-IPC-Cmd perl-File-Compare perl-File-Copy tpm2-tss-devel clang-devel protobuf-compiler \
-tar gzip
-
-# Install build dependencies from Intel repo.
-WORKDIR /root
-RUN curl -O https://download.01.org/intel-sgx/sgx-linux/2.24/distro/centos-stream9/sgx_rpm_local_repo.tgz && \
-tar -xaf sgx_rpm_local_repo.tgz && \
-dnf -y install --nogpgcheck --repofrompath "sgx,file:///root/sgx_rpm_local_repo" libsgx-dcap-quote-verify-devel
+# Install build dependencies from CentOS or RHEL repos.
+RUN \
+# Update packages. Get CVE fixes sooner.
+dnf -y update && \
+# Enable additional repositories for CentOS or RHEL.
+#if command -v subscription-manager; then \
+#  REPO_ARCH=$(uname -m) && \
+#  subscription-manager register --org "$(cat /activation-key/org)" --activationkey "$(cat /activation-key/activationkey)" && \
+#  subscription-manager repos --enable rhel-9-for-${REPO_ARCH}-appstream-rpms --enable codeready-builder-for-rhel-9-${REPO_ARCH}-rpms; \
+#else \
+#  dnf -y install 'dnf-command(config-manager)' && dnf config-manager --enable crb; \
+#fi && \
+# Install packages.
+dnf -y --setopt=install_weak_deps=0 install \
+  cargo pkg-config perl-FindBin openssl-devel perl-lib perl-IPC-Cmd perl-File-Compare perl-File-Copy clang-devel \
+  rust gcc gcc-c++ \ 
+  cmake glibc-static perl device-mapper-devel \
+  # These two are only available in the CodeReady Builder repo.
+  tpm2-tss-devel protobuf-compiler \
+  # This one is needed to build the stub.
+  meson
 
 # Build.
 WORKDIR /usr/src/kbs
 COPY . .
 ARG KBS_FEATURES=coco-as-builtin
 RUN \
-cargo install --locked --root /usr/local/ --path kbs --bin kbs --no-default-features --features ${KBS_FEATURES} && \
-# Collect linked files necessary for the binary to run.
-mkdir -p /root/trustee/lib64 && \
-ldd /usr/local/bin/kbs | sed 's@.*\s/@/@' | sed 's/\s.*//' | xargs -I {} cp {} /root/trustee/lib64
+# Build sgx_dcap_quoteverify stub.
+pushd sgx_dcap_quoteverify_stubs && \
+meson setup build --prefix=/usr && \
+meson compile -C build && \
+meson install -C build && \
+popd
+
+# Build KBS.
+RUN ARCH=$(uname -m) && \
+if [ ${ARCH} = "s390x" ]; then \
+  export OPENSSL_NO_VENDOR=1; \
+fi && \
+pushd kbs && make AS_FEATURE=coco-as-builtin ALIYUN=${ALIYUN} ARCH=${ARCH} && make ARCH=${ARCH} install-kbs && popd
+
+
+# Check the sha256sum of the Intel provided RPMs on x86_64.
+RUN if [ $(uname -m) = "x86_64" ]; then \
+  pushd sgx_dcap_quoteverify_stubs && \
+  echo "2621eac23cb756bc238f88d6db5401f7efed55d87855fc2b7e446ddfc1bd37ca" libsgx-dcap-default-qpl-1.21.100.3-1.el9.x86_64.rpm | sha256sum --check && \
+  echo "57da5fb2253a99bb2483d19b6f30d1170ebc384e2891937e2c89fa55886b7034" libsgx-dcap-quote-verify-1.21.100.3-1.el9.x86_64.rpm | sha256sum --check && \
+  popd; \
+fi
 
 # Package UBI image.
 FROM registry.access.redhat.com/ubi9
 
-# Install runtime dependencies from Intel repo.
-COPY --from=builder /root/sgx_rpm_local_repo /root/sgx_rpm_local_repo
-RUN dnf -y install --nogpgcheck --setopt=install_weak_deps=0 --repofrompath "sgx,file:///root/sgx_rpm_local_repo" \
-libsgx-dcap-default-qpl libsgx-dcap-quote-verify && \
-rm -rf /root/sgx_rpm_local_repo
+# Update packages. Get CVE fixes sooner.
+RUN dnf -y update && dnf clean all
 
 COPY --from=builder /usr/local/bin/kbs /usr/local/bin/kbs
+COPY --from=builder /usr/src/kbs/sgx_dcap_quoteverify_stubs/libsgx-dcap-quote-verify-1.21.100.3-1.el9.x86_64.rpm /tmp/libsgx-dcap-quote-verify-1.21.100.3-1.el9.x86_64.rpm
+COPY --from=builder /usr/src/kbs/sgx_dcap_quoteverify_stubs/libsgx-dcap-default-qpl-1.21.100.3-1.el9.x86_64.rpm /tmp/libsgx-dcap-default-qpl-1.21.100.3-1.el9.x86_64.rpm
+
+# Install Intel binaries
+RUN \
+if [ $(uname -m) = "x86_64" ]; then \
+  dnf -y --nogpgcheck --setopt=install_weak_deps=0 localinstall \
+    /tmp/libsgx-dcap-quote-verify-1.21.100.3-1.el9.x86_64.rpm \
+    /tmp/libsgx-dcap-default-qpl-1.21.100.3-1.el9.x86_64.rpm; \
+fi && \
+rm -f /tmp/libsgx-dcap-quote-verify-1.21.100.3-1.el9.x86_64.rpm /tmp/libsgx-dcap-default-qpl-1.21.100.3-1.el9.x86_64.rpm
+
+# Declare build-time variables.
+ARG NAME="trustee"
+ARG DESCRIPTION="The Trustee server."
+
+# Red Hat labels.
+LABEL com.redhat.component=$NAME
+LABEL description=$DESCRIPTION
+LABEL io.k8s.description=$DESCRIPTION
+LABEL io.k8s.display-name=$NAME
+LABEL name=$NAME
+LABEL summary=$DESCRIPTION
+LABEL distribution-scope=public
+LABEL release="1"
+LABEL url="https://access.redhat.com/"
+LABEL vendor="Red Hat, Inc."
+LABEL version="1"
+LABEL maintainer="Red Hat"
+# Reset labels inherited from base image.
+LABEL io.openshift.tags=""
+
+# Licenses
+COPY LICENSE /licenses/LICENSE
diff --git a/kbs/src/api_server.rs b/kbs/src/api_server.rs
@@ -11,8 +11,14 @@ use anyhow::Context;
 use log::info;
 
 use crate::{
-    admin::Admin, config::KbsConfig, jwe::jwe, plugins::PluginManager, policy_engine::PolicyEngine,
-    token::TokenVerifier, Error, Result,
+    admin::Admin,
+    config::KbsConfig,
+    jwe::jwe,
+    plugins::PluginManager,
+    policy_engine::PolicyEngine,
+    prometheus::{REQUEST_DURATION, REQUEST_SIZES, REQUEST_TOTAL},
+    token::TokenVerifier,
+    Error, Result,
 };
 
 const KBS_PREFIX: &str = "/kbs/v0";
@@ -101,11 +107,14 @@ impl ApiServer {
         );
 
         let http_config = self.config.http_server.clone();
+
+        #[allow(clippy::redundant_closure)]
         let http_server = HttpServer::new({
             move || {
                 let api_server = self.clone();
                 App::new()
                     .wrap(middleware::Logger::default())
+                    .wrap(middleware::from_fn(prometheus_metrics_middleware))
                     .app_data(web::Data::new(api_server))
                     .app_data(web::PayloadConfig::new(
                         (1024 * 1024 * http_config.payload_request_size) as usize,
@@ -115,6 +124,11 @@ impl ApiServer {
                             .route(web::get().to(api))
                             .route(web::post().to(api)),
                     )
+                    .service(
+                        web::resource("/metrics")
+                            .route(web::get().to(prometheus_metrics_handler))
+                            .route(web::post().to(|| HttpResponse::MethodNotAllowed())),
+                    )
             }
         });
 
@@ -258,3 +272,56 @@ pub(crate) async fn api(
         }
     }
 }
+
+pub(crate) async fn prometheus_metrics_handler(
+    _request: HttpRequest,
+    _core: web::Data<ApiServer>,
+) -> Result<HttpResponse> {
+    let report =
+        crate::prometheus::export_metrics().map_err(|e| Error::PrometheusError { source: e })?;
+    Ok(HttpResponse::Ok().body(report))
+}
+
+use actix_web::body::MessageBody;
+use actix_web::dev::{ServiceRequest, ServiceResponse};
+use actix_web::middleware::Next;
+
+async fn prometheus_metrics_middleware(
+    req: ServiceRequest,
+    next: Next<impl MessageBody>,
+) -> std::result::Result<ServiceResponse<impl MessageBody>, actix_web::Error> {
+    let start = actix::clock::Instant::now();
+
+    // Ignore requests like /metrics for metrics collection, they can make
+    // metrics weirdly not add up and distort metrics in odd ways.  They
+    // arguably are not very interesting either to a user of KBS metrics.
+    let is_kbs_req = req.request().path().starts_with("/kbs");
+    if is_kbs_req {
+        REQUEST_TOTAL.inc();
+
+        // Consider requests lacking a "content-length" header to be of zero
+        // size as this seems to be the usual case with KBS.  (Streamed
+        // requests would also lack "content-length" but they don't seem too
+        // relevant with KBS.)
+        if let Some(len) = req.headers().get("content-length") {
+            if let Ok(Ok(len)) = len.to_str().map(|l| l.parse::<u64>()) {
+                REQUEST_SIZES.observe(len as f64);
+            }
+        } else {
+            REQUEST_SIZES.observe(0_f64);
+        }
+    }
+
+    // This is the actual request handling.
+    let res = next.call(req).await?;
+
+    if is_kbs_req {
+        REQUEST_DURATION.observe(start.elapsed().as_secs_f64());
+
+        if let actix_web::body::BodySize::Sized(len) = res.response().body().size() {
+            REQUEST_SIZES.observe(len as f64);
+        }
+    }
+
+    Ok(res)
+}
diff --git a/kbs/src/error.rs b/kbs/src/error.rs
@@ -75,6 +75,12 @@ pub enum Error {
 
     #[error("Token Verifier error")]
     TokenVerifierError(#[from] crate::token::Error),
+
+    #[error("Prometheus error")]
+    PrometheusError {
+        #[from]
+        source: prometheus::Error,
+    },
 }
 
 impl ResponseError for Error {

diff --git a/kbs/src/lib.rs b/kbs/src/lib.rs
@@ -25,3 +25,5 @@ pub use error::*;
 pub mod admin;
 pub mod http;
 pub mod jwe;
+
+pub mod prometheus;
diff --git a/kbs/src/plugins/implementations/resource/backend.rs b/kbs/src/plugins/implementations/resource/backend.rs
@@ -9,6 +9,8 @@ use regex::Regex;
 use serde::Deserialize;
 use std::fmt;
 
+use crate::prometheus::{RESOURCE_READS_TOTAL, RESOURCE_WRITES_TOTAL};
+
 use super::local_fs;
 
 type RepositoryInstance = Arc<dyn StorageBackend>;
@@ -114,12 +116,18 @@ impl ResourceStorage {
         resource_desc: ResourceDesc,
         data: &[u8],
     ) -> Result<()> {
+        RESOURCE_WRITES_TOTAL
+            .with_label_values(&[&format!("{}", resource_desc)])
+            .inc();
         self.backend
             .write_secret_resource(resource_desc, data)
             .await
     }
 
     pub(crate) async fn get_secret_resource(&self, resource_desc: ResourceDesc) -> Result<Vec<u8>> {
+        RESOURCE_READS_TOTAL
+            .with_label_values(&[&format!("{}", resource_desc)])
+            .inc();
         self.backend.read_secret_resource(resource_desc).await
     }
 }