resolve conflicts

.github/workflows/server-integration.yml

@@ -0,0 +1,49 @@
+name: Server Integration Test
+on:
+  push:
+    branches: 
+      - 'main'
+      - 'cp2-server*'
+  pull_request:
+    branches: 
+      - 'main'
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    defaults:
+      run:
+        shell: bash
+        working-directory: ./server
+    runs-on: ubuntu-latest
+    env:
+      TEST_ROOT: ${{ github.workspace }}/server/tests
+      SERVER_ROOT: ${{ github.workspace }}/server
+    steps:
+    - uses: actions/checkout@v4
+    - name: Setup BATS testing framework
+      uses: mig4/[email protected]
+    - name: Run mock S3 server
+      run: docker run -p 6333:80 -v ${{ github.workspace }}/server/tests/test_s3_files:/usr/share/nginx/html -d nginx
+    - name: setup-redis
+      uses: shogo82148/actions-setup-redis@v1
+      with:
+        auto-start: false
+    - name: start redis nodes
+      run: redis-server ${{ github.workspace }}/server/redis.conf --port 6379 --cluster-config-file node1.conf&
+    - name: start redis nodes
+      run: redis-server ${{ github.workspace }}/server/redis.conf --port 6380 --cluster-config-file node2.conf&
+    - name: start redis nodes
+      run: redis-server ${{ github.workspace }}/server/redis.conf --port 6381 --cluster-config-file node3.conf&
+    - name: Create redis cluster
+      run: redis-cli --cluster create localhost:6379 localhost:6380 localhost:6381 --cluster-replicas 0 --cluster-yes
+    - name: Build
+      run: cargo build --verbose
+    - name: Run 
+      run: |
+        REDIS_PORT=6379 cargo run &
+        REDIS_PORT=6380 cargo run &
+        REDIS_PORT=6381 cargo run &
+    - name: Test get file
+      run: bats ${{ github.workspace }}/server/tests/server-integration.bats

.gitignore

@@ -19,4 +19,5 @@ Cargo.lock
 # local parquet file location for client
 client/parquet_files/
 
-bench_files
+bench_files
+.vscode

Cargo.toml

@@ -1,8 +1,7 @@
-[package]
-name = "cache2"
-version = "0.1.0"
-edition = "2021"
+[workspace]
+members = [
+    "server",
+    "client"
+]
 
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
+resolver = "2"

README.md

@@ -1,4 +1,93 @@
 # 15721-s24-cache2
 15-721 Spring 2024 - Cache #2
 
-To run benchmark, simple run bench.sh
+# LRU Cache Server
+
+This server implements a Least Recently Used (LRU) caching mechanism, providing a simple interface for fetching files from a simulated S3 storage and managing them within an LRU cache. The server is built using Rust and the Rocket framework.
+
+## Features
+
+- **Health Check**: Verify the server's health.
+- **Fetch File**: Retrieve files, either served from the cache or fetched from "S3" and then cached.
+- **Cache Stats**: Get statistics about the current state of the cache.
+- **Set Cache Size**: Adjust the maximum size of the cache dynamically.
+
+## Getting Started
+
+### Prerequisites
+
+- Rust and Cargo (latest stable version recommended)
+- Rocket Framework
+- Docker
+
+### Installation
+
+1. Clone the repository:
+    ```sh
+    git clone [email protected]:cmu-db15721-s24-cache2.git
+    cd 15721-s24-cache2/server
+    ```
+
+2. Build the project:
+    ```sh
+    docker build -t istziio .
+    ```
+
+3. Run the server:
+    ```sh
+    docker compose up -d
+    ```
+
+> [!IMPORTANT]
+> Under development stage, the server cluster can be access ONLY within the specific Docker network. Client side needs to be in the same Docker bridge network for the correct redirection.
+
+### Example
+
+```sh
+$ docker exec -it server-servernode_1-1 /bin/bash
+root@node1:/data> apt-get update && apt-get install curl -y
+root@node1:/data> curl -L http://node2:8000/s3/test1.txt # make sure -L flag is set for auto redirect to the correct node
+```
+
+
+## Usage
+
+### Health Check
+
+- **Endpoint**: `GET /`
+- **Description**: Checks if the server is running.
+- **CURL Command**:
+    ```sh
+    curl http://localhost:8000/
+    ```
+
+### Fetch File
+
+- **Endpoint**: `GET /s3/<path>`
+- **Description**: Retrieves a file from the cache or fetches it from the simulated S3 storage if not present in the cache. Error reports if file not existed.
+- **CURL Command**:
+    ```sh
+    curl http://localhost:8000/s3/<path-to-file>
+    ```
+
+### Cache Stats
+
+- **Endpoint**: `GET /stats`
+- **Description**: Returns statistics about the cache, such as current size, maximum size, and number of entries.
+- **CURL Command**:
+    ```sh
+    curl http://localhost:8000/stats
+    ```
+
+### Set Cache Size
+
+- **Endpoint**: `POST /size/<new_size>`
+- **Description**: Adjusts the maximum size of the cache.
+- **CURL Command**:
+    ```sh
+    curl -X POST http://localhost:8000/size/<new-size-in-bytes>
+    ```
+
+## Benchmark
+
+To run benchmark, simple run `bench.sh`

client/src/kv_store.rs

@@ -1,11 +1,11 @@
 // In src/kv_store.rs
 
-use tokio::net::TcpStream;
-use tokio::io::{self, AsyncWriteExt, BufReader};
-use tokio::io::AsyncBufReadExt;
-use std::sync::Mutex;
 use anyhow::Result;
 use log::info;
+use std::sync::Mutex;
+use tokio::io::AsyncBufReadExt;
+use tokio::io::{self, AsyncWriteExt, BufReader};
+use tokio::net::TcpStream;
 pub struct KVStore {
     address: String,
     port: u16,
@@ -43,7 +43,7 @@ impl KVStore {
         if let Some(stream) = conn.as_mut() {
             stream.write_all(message.as_bytes()).await?;
             stream.flush().await?;
-    
+
             // Use a loop to read lines from the server.
             let mut response = String::new();
             let mut reader = BufReader::new(stream);

server/.gitignore

@@ -1,2 +1,5 @@
 /target/
-Cargo.lock
+Cargo.lock
+cache
+output.log
+*.swp

server/Cargo.toml

@@ -1,12 +1,24 @@
 [package]
-name = "echo_server"
+name = "istziio_server_node"
 version = "0.1.0"
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+path = "src/lib.rs"
+
+[[bin]]
+name = "istziio_server_node"
+path = "src/main.rs"
 
 [dependencies]
 tokio = { version = "1", features = ["full"] }
 log = "0.4"
 env_logger = "0.11.1"
+fern = "0.5"
 clap = "3"
+rocket = { version = "0.5.0", features = ["json"] }
+redis = { version = "0.24.0", features = ["cluster"] }
+reqwest = "0.11"
+chrono = "0.4"
+url = "2.5"

server/Dockerfile

@@ -0,0 +1,30 @@
+FROM rust:1.76.0-bookworm AS chef
+# We only pay the installation cost once,
+# it will be cached from the second build onwards
+RUN cargo install cargo-chef
+
+WORKDIR app
+
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --recipe-path recipe.json
+
+FROM chef AS builder
+COPY --from=planner /app/recipe.json recipe.json
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Build application
+COPY . .
+ENV ROCKET_PROFILE=development
+RUN cargo install --path .
+
+# We do not need the Rust toolchain to run the binary!
+FROM redis:7.2.4
+RUN apt-get update && apt-get -y install curl
+COPY --from=builder /usr/local/cargo/bin/* /usr/local/bin
+COPY docker-entrypoint.sh redis.conf Rocket.toml /usr/local/bin/
+RUN chmod 755 /usr/local/bin/docker-entrypoint.sh
+
+CMD ["/usr/local/bin/docker-entrypoint.sh"]
+

server/Rocket.toml

@@ -0,0 +1,29 @@
+[default]
+address = "0.0.0.0"
+port = 8000
+workers = 2
+keep_alive = 5
+read_timeout = 5
+write_timeout = 5
+log = "normal"
+limits = { forms = 32768 }
+
+[staging]
+address = "0.0.0.0"
+port = 8000
+workers = 2
+keep_alive = 5
+read_timeout = 5
+write_timeout = 5
+log = "normal"
+limits = { forms = 32768 }
+
+[release]
+address = "0.0.0.0"
+port = 8000
+workers = 2
+keep_alive = 5
+read_timeout = 5
+write_timeout = 5
+log = "critical"
+limits = { forms = 32768 }

server/compose.yaml

@@ -0,0 +1,74 @@
+services:
+  servernode_1:
+    hostname: node1
+    image: istziio
+    build: .
+    ports:
+      - "8000:8000"
+      - "6379:6379"
+      - "16379:16379"
+    networks:
+      - cluster-network
+    environment:
+      - REDIS_PORT=6379
+      - S3_ENDPOINT=http://mocks3
+      - ROCKET_CONFIG=/usr/local/bin/Rocket.toml
+      - ROCKET_ENV=development
+
+  servernode_2:
+    hostname: node2
+    image: istziio
+    build: .
+    ports:
+      - "8001:8000"
+      - "6380:6379"
+      - "16380:16379"
+    networks:
+      - cluster-network
+    environment:
+      - REDIS_PORT=6379
+      - S3_ENDPOINT=http://mocks3
+      - ROCKET_CONFIG=/usr/local/bin/Rocket.toml
+      - ROCKET_ENV=development
+
+  servernode_3:
+    hostname: node3
+    image: istziio
+    build: .
+    ports:
+      - "8002:8000"
+      - "6381:6379"
+      - "16381:16379"
+    networks:
+      - cluster-network
+    environment:
+      - REDIS_PORT=6379
+      - S3_ENDPOINT=http://mocks3
+      - ROCKET_CONFIG=/usr/local/bin/Rocket.toml
+      - ROCKET_ENV=development
+
+  clustersetup:
+    image: redis
+    depends_on:
+      - servernode_1
+      - servernode_2
+      - servernode_3
+    restart: "no"
+    entrypoint: [ "bash", "-c", "yes \"yes\" | redis-cli --cluster create node1:6379 node2:6379 node3:6379 --cluster-replicas 0" ]
+    networks:
+      - cluster-network
+
+  mocks3:
+    image: nginx
+    ports:
+      - 8080:80
+    networks:
+      - cluster-network
+    volumes:
+      - type: bind
+        source: ./S3
+        target: /usr/share/nginx/html
+networks:
+  # The presence of these objects is sufficient to define them
+  cluster-network:
+    driver: bridge

server/debug.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Stop and remove containers, networks
+docker-compose down || { echo "docker-compose down failed"; exit 1; }
+
+# Build your Rust project
+cargo build || { echo "cargo build failed"; exit 1; }
+
+# Build your Docker image
+docker build -t istziio . || { echo "docker build failed"; exit 1; }
+
+# Start up your Docker containers in the background
+docker-compose up -d || { echo "docker-compose up -d failed"; exit 1; }
+
+# Execute a bash shell inside your specified container
+# Note: You might need to adjust the container name based on your docker-compose settings
+docker exec -it server-servernode_1-1 /bin/bash || { echo "docker exec failed"; exit 1; }

server/docker-entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+echo "port ${REDIS_PORT}" >> /usr/local/bin/redis.conf
+redis-server /usr/local/bin/redis.conf &
+/usr/local/bin/istziio_server_node