diff --git a/.github/workflows/ingestion.yml b/.github/workflows/ingestion.yml index 5766f17..ff38cb9 100644 --- a/.github/workflows/ingestion.yml +++ b/.github/workflows/ingestion.yml @@ -1,5 +1,7 @@ name: Ingestion +name: API + on: push: branches: @@ -8,27 +10,91 @@ on: branches: - main +env: + CARGO_TERM_COLOR: always + MANIFEST_PATH: --manifest-path ingestion/Cargo.toml + jobs: - build: + test: + name: Test runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + args: ${{ env.MANIFEST_PATH }} + fmt: + name: Format + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add rustfmt - - name: Set up JDK 11 - uses: actions/setup-java@v2 + - uses: actions-rs/cargo@v1 with: - java-version: 11 - distribution: "temurin" + command: fmt + args: ${{ env.MANIFEST_PATH }} --all -- --check - - name: Set up Scala - uses: olafurpg/setup-scala@v13 + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add clippy + - uses: actions-rs/cargo@v1 + with: + command: clippy + args: ${{ env.MANIFEST_PATH }} --all-targets --all-features -- -D warnings - - name: Run Tests - working-directory: ingestion - run: sbt test + check: + name: Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v1 + with: + key: ${{ runner.os }}-check + - uses: actions-rs/cargo@v1 + with: + command: check + args: ${{ env.MANIFEST_PATH }} + env: + RUSTFLAGS: -D warnings - - name: Upload Dependency Graph - uses: scalacenter/sbt-dependency-submission@ab086b50c947c9774b70f39fc7f6e20ca2706c91 + coverage: + name: Code Coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: cargo install cargo-tarpaulin + - uses: actions-rs/cargo@v1 with: - working-directory: ingestion + command: tarpaulin + args: ${{ env.MANIFEST_PATH }} --verbose --workspace diff --git a/.metals/metals.lock.db b/.metals/metals.lock.db index 15ae9c8..2c55dc9 100644 --- a/.metals/metals.lock.db +++ b/.metals/metals.lock.db @@ -1,5 +1,5 @@ #FileLock -#Mon Oct 09 15:38:57 EDT 2023 +#Tue Oct 10 09:45:31 EDT 2023 hostName=localhost id=18b15f323aa867ddcf2644b78a3028f68fadaed9c7f method=file diff --git a/.metals/metals.mv.db b/.metals/metals.mv.db deleted file mode 100644 index 20d3a52..0000000 Binary files a/.metals/metals.mv.db and /dev/null differ diff --git a/.scala-build/.bloop/project_bd2c96d2de-test.json b/.scala-build/.bloop/project_bd2c96d2de-test.json deleted file mode 100644 index 954293a..0000000 --- a/.scala-build/.bloop/project_bd2c96d2de-test.json +++ /dev/null @@ -1 +0,0 @@ -{"version":"1.4.0","project":{"name":"project_bd2c96d2de-test","directory":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build","workspaceDir":"/Users/brianreicher/Documents/GitHub/turbine","sources":[],"dependencies":["project_bd2c96d2de"],"classpath":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Documents/GitHub/turbine/.scala-build/project_bd2c96d2de/classes/main","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/sourcegraph/semanticdb-javac/0.7.4/semanticdb-javac-0.7.4.jar"],"out":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/.bloop/project_bd2c96d2de-test","classesDir":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/project_bd2c96d2de/classes/test","scala":{"organization":"org.scala-lang","name":"scala-compiler","version":"3.2.2","options":["-Xsemanticdb","-sourceroot","/Users/brianreicher/Documents/GitHub/turbine","-release","8"],"jars":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.2/scala3-compiler_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-interfaces/3.2.2/scala3-interfaces-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/tasty-core_3/3.2.2/tasty-core_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-asm/9.3.0-scala-1/scala-asm-9.3.0-scala-1.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/compiler-interface/1.3.5/compiler-interface-1.3.5.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-reader/3.19.0/jline-reader-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal/3.19.0/jline-terminal-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal-jna/3.19.0/jline-terminal-jna-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.7.0/protobuf-java-3.7.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/util-interface/1.3.0/util-interface-1.3.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.3.1/jna-5.3.1.jar"]},"java":{"options":["--release","8","-Xplugin:semanticdb -sourceroot:/Users/brianreicher/Documents/GitHub/turbine -targetroot:javac-classes-directory","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"]},"test":{"frameworks":[{"names":["com.novocode.junit.JUnitFramework"]},{"names":["org.scalatest.tools.Framework","org.scalatest.tools.ScalaTestFramework"]},{"names":["org.scalacheck.ScalaCheckFramework"]},{"names":["org.specs.runner.SpecsFramework","org.specs2.runner.Specs2Framework","org.specs2.runner.SpecsFramework"]},{"names":["utest.runner.Framework"]},{"names":["munit.Framework"]}],"options":{"excludes":[],"arguments":[]}},"platform":{"name":"jvm","config":{"options":[]},"mainClass":[]},"resolution":{"modules":[{"organization":"org.scala-lang","name":"scala3-library_3","version":"3.2.2","artifacts":[{"name":"scala3-library_3","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar"},{"name":"scala3-library_3","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2-sources.jar"}]},{"organization":"org.scala-lang","name":"scala-library","version":"2.13.10","artifacts":[{"name":"scala-library","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar"},{"name":"scala-library","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10-sources.jar"}]}]},"tags":["test"]}} \ No newline at end of file diff --git a/.scala-build/.bloop/project_bd2c96d2de.json b/.scala-build/.bloop/project_bd2c96d2de.json deleted file mode 100644 index ecf1ec5..0000000 --- a/.scala-build/.bloop/project_bd2c96d2de.json +++ /dev/null @@ -1 +0,0 @@ -{"version":"1.4.0","project":{"name":"project_bd2c96d2de","directory":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build","workspaceDir":"/Users/brianreicher/Documents/GitHub/turbine","sources":[],"dependencies":[],"classpath":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/sourcegraph/semanticdb-javac/0.7.4/semanticdb-javac-0.7.4.jar"],"out":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/.bloop/project_bd2c96d2de","classesDir":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/project_bd2c96d2de/classes/main","scala":{"organization":"org.scala-lang","name":"scala-compiler","version":"3.2.2","options":["-Xsemanticdb","-sourceroot","/Users/brianreicher/Documents/GitHub/turbine","-release","8"],"jars":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.2/scala3-compiler_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-interfaces/3.2.2/scala3-interfaces-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/tasty-core_3/3.2.2/tasty-core_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-asm/9.3.0-scala-1/scala-asm-9.3.0-scala-1.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/compiler-interface/1.3.5/compiler-interface-1.3.5.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-reader/3.19.0/jline-reader-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal/3.19.0/jline-terminal-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal-jna/3.19.0/jline-terminal-jna-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.7.0/protobuf-java-3.7.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/util-interface/1.3.0/util-interface-1.3.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.3.1/jna-5.3.1.jar"]},"java":{"options":["--release","8","-Xplugin:semanticdb -sourceroot:/Users/brianreicher/Documents/GitHub/turbine -targetroot:javac-classes-directory","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"]},"platform":{"name":"jvm","config":{"options":[]},"mainClass":[]},"resolution":{"modules":[{"organization":"org.scala-lang","name":"scala3-library_3","version":"3.2.2","artifacts":[{"name":"scala3-library_3","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar"},{"name":"scala3-library_3","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2-sources.jar"}]},{"organization":"org.scala-lang","name":"scala-library","version":"2.13.10","artifacts":[{"name":"scala-library","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar"},{"name":"scala-library","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10-sources.jar"}]}]},"tags":["library"]}} \ No newline at end of file diff --git a/ingestion/src/downloader.rs b/ingestion/src/downloader.rs index 8cd27f9..73456e7 100644 --- a/ingestion/src/downloader.rs +++ b/ingestion/src/downloader.rs @@ -24,7 +24,6 @@ impl GitHubDownloader { pub async fn download_git_zips( &self, urls: Vec<&str>, - repo_name: &str, ) -> Result<(), Box> { for url in urls { let response = self.client.get(url).send().await?; diff --git a/ingestion/src/main.rs b/ingestion/src/main.rs index 53f6f52..dd24c5d 100644 --- a/ingestion/src/main.rs +++ b/ingestion/src/main.rs @@ -1,18 +1,21 @@ use ingestion::mongo_utils::MongoDriver; -use dpw +use ingestion::downloader::GitHubDownloader; + + #[tokio::main] async fn main() -> std::io::Result<()> { let mongo: MongoDriver = MongoDriver::new("localhost", 8080, "turbine"); mongo.connect().await?; // create/connect if exists, and flush - mongo.create_collection("github_data").await?; - mongo.flush_collection("github_data").await?; + let collection = "github_data" + mongo.create_collection(collection).await?; + mongo.flush_collection(collection).await?; - // set remote urls and download + // set download links and ingest let urls = vec!["https://github.com/user/repo/raw/master/file1.zip", "https://github.com/user/repo/raw/master/file2.zip"]; - github_downloader.download_git_zips(urls, "my_repo").await?; - - + let downloader: GithubDownloader = GithubDownloader::new(mongo, collection); + downloader.download_git_zips(urls).await?; + Ok(()) } \ No newline at end of file diff --git a/ingestion/tests/ingestion/ing_tests.rs b/ingestion/tests/ingestion/ing_tests.rs new file mode 100644 index 0000000..7f2a19d --- /dev/null +++ b/ingestion/tests/ingestion/ing_tests.rs @@ -0,0 +1,5 @@ +#[tokio::test] +async fn mongo_connection() { + + assert_eq!(Some(0), Some(0)); +} diff --git a/ingestion/tests/ingestion/main.rs b/ingestion/tests/ingestion/main.rs new file mode 100644 index 0000000..0e9a27f --- /dev/null +++ b/ingestion/tests/ingestion/main.rs @@ -0,0 +1 @@ +mod ing_tests;