From 8268d98b996c5eb4249acd76ebfd1f197bb03561 Mon Sep 17 00:00:00 2001 From: brianreicher Date: Tue, 10 Oct 2023 09:53:07 -0400 Subject: [PATCH] Ingestion flow worked out --- .github/workflows/ingestion.yml | 94 +++++++++++++++--- .metals/metals.lock.db | 2 +- .metals/metals.mv.db | Bin 36864 -> 0 bytes .../.bloop/project_bd2c96d2de-test.json | 1 - .scala-build/.bloop/project_bd2c96d2de.json | 1 - ingestion/src/downloader.rs | 1 - ingestion/src/main.rs | 17 ++-- ingestion/tests/ingestion/ing_tests.rs | 5 + ingestion/tests/ingestion/main.rs | 1 + 9 files changed, 97 insertions(+), 25 deletions(-) delete mode 100644 .metals/metals.mv.db delete mode 100644 .scala-build/.bloop/project_bd2c96d2de-test.json delete mode 100644 .scala-build/.bloop/project_bd2c96d2de.json create mode 100644 ingestion/tests/ingestion/ing_tests.rs create mode 100644 ingestion/tests/ingestion/main.rs diff --git a/.github/workflows/ingestion.yml b/.github/workflows/ingestion.yml index 5766f17..ff38cb9 100644 --- a/.github/workflows/ingestion.yml +++ b/.github/workflows/ingestion.yml @@ -1,5 +1,7 @@ name: Ingestion +name: API + on: push: branches: @@ -8,27 +10,91 @@ on: branches: - main +env: + CARGO_TERM_COLOR: always + MANIFEST_PATH: --manifest-path ingestion/Cargo.toml + jobs: - build: + test: + name: Test runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + args: ${{ env.MANIFEST_PATH }} + fmt: + name: Format + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add rustfmt - - name: Set up JDK 11 - uses: actions/setup-java@v2 + - uses: actions-rs/cargo@v1 with: - java-version: 11 - distribution: "temurin" + command: fmt + args: ${{ env.MANIFEST_PATH }} --all -- --check - - name: Set up Scala - uses: olafurpg/setup-scala@v13 + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add clippy + - uses: actions-rs/cargo@v1 + with: + command: clippy + args: ${{ env.MANIFEST_PATH }} --all-targets --all-features -- -D warnings - - name: Run Tests - working-directory: ingestion - run: sbt test + check: + name: Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: Swatinem/rust-cache@v1 + with: + key: ${{ runner.os }}-check + - uses: actions-rs/cargo@v1 + with: + command: check + args: ${{ env.MANIFEST_PATH }} + env: + RUSTFLAGS: -D warnings - - name: Upload Dependency Graph - uses: scalacenter/sbt-dependency-submission@ab086b50c947c9774b70f39fc7f6e20ca2706c91 + coverage: + name: Code Coverage + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: cargo install cargo-tarpaulin + - uses: actions-rs/cargo@v1 with: - working-directory: ingestion + command: tarpaulin + args: ${{ env.MANIFEST_PATH }} --verbose --workspace diff --git a/.metals/metals.lock.db b/.metals/metals.lock.db index 15ae9c8..2c55dc9 100644 --- a/.metals/metals.lock.db +++ b/.metals/metals.lock.db @@ -1,5 +1,5 @@ #FileLock -#Mon Oct 09 15:38:57 EDT 2023 +#Tue Oct 10 09:45:31 EDT 2023 hostName=localhost id=18b15f323aa867ddcf2644b78a3028f68fadaed9c7f method=file diff --git a/.metals/metals.mv.db b/.metals/metals.mv.db deleted file mode 100644 index 20d3a5252591724bb4a79dcc24f64119703ef1d0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36864 zcmeHQUu@&Zc_(Sv=kwkscbC0OvYXxGYQ49si=vW4QX+*QU0R}S-Py8Fl)bxYT9%@y zJ$G%%wdA{X(1#RATWpX+fd=g%L3#+>!|m* ztmM5z-yQm$vaGjdt*anKHuBy6;l4YBvTyr`gWY})vUT_Tq4$=5SnlXuMevFa#I^ z3;~7!Lx3T`5V%ML$RWQ>Z2(Yy&+nDx{JwioHsJcCtjbcp=RX<30sn#f$RCu|{9(U8 zEURE4AW7YlRLl=|_kH*|v;>gz!@gHmQBge44;T~hkNAJ$gavLcqL$`@KhD29(GEjjs3GHqZmi*>De*SN8htrr-4s{hsG92lRd+ zO_2?M=;GW!?XsZp^E}6MH^o;U*j7udG%d4bIV;=s^{R+u6JRL!Omnr ziVofOo?nnYiL(s-{R032ihlw>AN4x@_5P!Rj9V)^xT12hBAmB8%KK_iv`p*GKlYEy zp(y3wcK6Dmq}|5ZfO2SUGFK>TIa1k(uF0ICz>eyJ4&z**+>%zQpcCRKUb8FX5pVEC zUgs5F<4Zj7iF(OcxIe)z2_A*ktV z9QRrSIDe1hzAnHsr0+ftDSZBCfu!up5MT%}1Q-Gg0fqoWfFZyTU*-bIh> z>28TNj~fJCo3)WAf6aK+40UPT(v$mNPZF9XP-s$*m*DzqVHPjOP$#=vx(_RzrprY;< zwYK55b+@Q%nyPi&cB!NlkuD+BDRm@CZmUQ!NE(I%5 zcDFYex_f(m#{re%SCeuy zQ^y~8hr0)XR&rWKBuTksy@5J+dfthwr0!{%1K+%f?9f~9@!)8G)Fn#fxWnz^8I?`c zH7*nMtX1Ez?3TH)C04DPxeZ*dY&V-$qvec$NxyOHbVrG+|S(>Z(sf{ z4}uH)O(#YmhE=h$*|1wp6J+PKHk|{Iu?YQlCrAJZtXdBu^~C%2D&WwgI3#mU?WHqt zb~{f};xS@CAo{jgt2gad?hcHxZ-=9O#u%$wTNcpWs62G+&FyBz3ddK=deCyJo0aX2 zKtQ&e^(itpVlx#{e5~NK%;u_fYLTVfT|nfQQ%i=CKw=8v*-4snIS`EBo*@{MRm>KO zv7|UaNlXx?w0fc}7}?i=G2c6dG50r{Z`f-V069q}odPEto}A^9EE-pP3QL!Ap9lW@ zN^1Vp68srAG==_YoXP{IzCD9eCsme;XF%OY;#n>G1>o6lp29PG1-MmTg-JFXMH^LZ z%DAD!1?J+`wjIpN4Ooz1*@`b+_t&h3)3hp^&8k(6%-&160?_hXr_fTZ+Z%P;wyF+P zTd&nCW~;v02=QF#NgIOg(A&Gf43fN)teS3nJy(uxt{}K94aO=CYj3GrVV|e z(*rCKAsLZxP7vLtT;_R@#V@Cp#iE`N#wi43BW>q0FM#kpogsXag;R+fEM+l~i?(;- z`~cKvUIbA8;1mwW=z@2-9jh7QI$i9N0%lw57Q7Kv9a?yj1ySBlmBh@afHB*t8Dl6X zF(&LoHXuKjxdJ@-jWj$7$)CBRM<;KfAjTHl)`|Q8qBAc6KYnluKk%v!XufWa_z)Rp z;;PlKnsE4D6;~dPUcgLSti$Tts<$4Bw)N&VEX2__VEdunvNnJ}l{$eY;VIu1>Hc!a$6&Vb!848tE1S6n zL*l!q42jm}*1EN0tvmL^jg|4VPoE~|o22M&nXR=c>t`}sFycnS7D_+J_2&TgU!bsK zK1YOlLZp&Y(**5V+)PZat^*hTZpIXqET2lafQZ^=)2gpFf=$w0+++$7gJMmqX2D*r zV!#{JXSQ-)M2cgbzw5H zU!v_jRZiVMA=f3zwv4&M}qK#E>;u=xGiP*MIq`{=bFMX+w7xF4s zv&~C?|NhVa$3OhnRZcW-TJoC}9B_&h7_8cFu0Pz79fy4D;A#ei!LRORzV(AY`_Fk= z{MWw^#a}hE7HZl4At>;~&hEhL!_9F}!&i2Y|-kCfZ( zb^IpI=wx5K1hr@=?P`5oR+2>Ime9kuQZ<(KYe<+*>qAojp^k8&)~3hW_xG`AL8wi1KVi-&_0)Pg z$_`~FtoUR+M()py)%6WLPlbl$6O1WFrm+<0cd6k3DMqAZ<{-yr2r<;-#7wjoB_@@q zlOf4*LRIt@c##kjy)A)Pe$|!DX}Ll_M}Ij6e;$EVow_J1fU2- zAN2EbFCguakt0)D=o9S&eNtmNX^-e{v`6F#QIzOYDeBD-t5Q*-Kb1~#bcp*sp!lN_ z6{XqpRA2`a6D1xAvMk7&AeRK$5M)=7JwfgWvM(Ts_xSGE*+7sbL5A##Agh906l7hH z+k9J)ySyJbBJeKXfw#sNZy!Iy!5_9cD##}~B@o|)Hc7ni@JTx<;-|E?`!;yJJSj{0 z;BE^$CKzRr#wi#vkvnrZUi#rB62~|lc{b(yo_BQM_Ppb=)CKnmYzkHM zT~#5s5^{|qxs^*L*{2zFIPVPeB2U8aZ`^CYhn*_;cW%Ha`+!^c=09>AShg(u+XwLb zJ#OJg6_Vojy?Y$I@4++2J$n_LG1!M8zz|>vFa#I^3;~7!L*OS4fu#L?_^ye@@$?N1 z?Em!(wf|Q}{{vv`|Bn#1|L=f~0tWyn#RC9%f%!l9+Jkoh4gi4K>d5|I!S?@d*E>J+ ze^Ctnoj{iR21hiYA8?%V0KWNv>#tyyVW0tBU3kj#*MGbkP^=Y2MDL2C-tja2 z;xYMA9VpX&CLQpp7e`bbCC^lhVE2yLZkCh__vF*HXE7~UOiN3398CLJ4*hGQd_c3c z3O}pb&$u2gsr^RrpOZR6G+lT4(-5K}s(_r(PEMZbd_-+%w6(LTJGJ?&z`w%Oopi#C z=_cimnY#1Gt=|-oEFcJx!vDqM|ITrXWBn%4bE2OTJ?DE=2b=U24Rx?`*Nv)S)eq}L zn$*9J|HstWtu&&5?Mk44#V@0vb@k4wfV~G%ZT=cwBO%4FllOwxNq7cbY;y~z?7|RW z2rvXL2m!MHAMd}z{eR^D>9rB;|2r4D|5xe$zl8n&gN^?|neP9Q4AVMR02ovOpb`K7 zzz|2odfaN0uWw+2Q!NRK%+~h z;}?qnK*DRX2mmzB=tqtK@Ggr0z#;%Z$O9GuV8Z>LMF5~77@`hJECN85MF0q0YC>n; zQD6ZU0U#FNV$#np>L11;0N@A#ECN6)h+mM(Ba1}H6*9L>+ zv9}+MRep9`<=J$Ntd}EJCXybrGPy(!KPA+bi}W5G{S2Sp7j=kT8{YehN?rXv|1x#; zd!M-ay$ddUfvevS@8QGcg&+O%1syW8YCpCJkn{iX`FnW&ua52iJ*6Z0(gmIW|35+= BG7SI# diff --git a/.scala-build/.bloop/project_bd2c96d2de-test.json b/.scala-build/.bloop/project_bd2c96d2de-test.json deleted file mode 100644 index 954293a..0000000 --- a/.scala-build/.bloop/project_bd2c96d2de-test.json +++ /dev/null @@ -1 +0,0 @@ -{"version":"1.4.0","project":{"name":"project_bd2c96d2de-test","directory":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build","workspaceDir":"/Users/brianreicher/Documents/GitHub/turbine","sources":[],"dependencies":["project_bd2c96d2de"],"classpath":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Documents/GitHub/turbine/.scala-build/project_bd2c96d2de/classes/main","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/sourcegraph/semanticdb-javac/0.7.4/semanticdb-javac-0.7.4.jar"],"out":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/.bloop/project_bd2c96d2de-test","classesDir":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/project_bd2c96d2de/classes/test","scala":{"organization":"org.scala-lang","name":"scala-compiler","version":"3.2.2","options":["-Xsemanticdb","-sourceroot","/Users/brianreicher/Documents/GitHub/turbine","-release","8"],"jars":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.2/scala3-compiler_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-interfaces/3.2.2/scala3-interfaces-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/tasty-core_3/3.2.2/tasty-core_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-asm/9.3.0-scala-1/scala-asm-9.3.0-scala-1.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/compiler-interface/1.3.5/compiler-interface-1.3.5.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-reader/3.19.0/jline-reader-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal/3.19.0/jline-terminal-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal-jna/3.19.0/jline-terminal-jna-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.7.0/protobuf-java-3.7.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/util-interface/1.3.0/util-interface-1.3.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.3.1/jna-5.3.1.jar"]},"java":{"options":["--release","8","-Xplugin:semanticdb -sourceroot:/Users/brianreicher/Documents/GitHub/turbine -targetroot:javac-classes-directory","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"]},"test":{"frameworks":[{"names":["com.novocode.junit.JUnitFramework"]},{"names":["org.scalatest.tools.Framework","org.scalatest.tools.ScalaTestFramework"]},{"names":["org.scalacheck.ScalaCheckFramework"]},{"names":["org.specs.runner.SpecsFramework","org.specs2.runner.Specs2Framework","org.specs2.runner.SpecsFramework"]},{"names":["utest.runner.Framework"]},{"names":["munit.Framework"]}],"options":{"excludes":[],"arguments":[]}},"platform":{"name":"jvm","config":{"options":[]},"mainClass":[]},"resolution":{"modules":[{"organization":"org.scala-lang","name":"scala3-library_3","version":"3.2.2","artifacts":[{"name":"scala3-library_3","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar"},{"name":"scala3-library_3","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2-sources.jar"}]},{"organization":"org.scala-lang","name":"scala-library","version":"2.13.10","artifacts":[{"name":"scala-library","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar"},{"name":"scala-library","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10-sources.jar"}]}]},"tags":["test"]}} \ No newline at end of file diff --git a/.scala-build/.bloop/project_bd2c96d2de.json b/.scala-build/.bloop/project_bd2c96d2de.json deleted file mode 100644 index ecf1ec5..0000000 --- a/.scala-build/.bloop/project_bd2c96d2de.json +++ /dev/null @@ -1 +0,0 @@ -{"version":"1.4.0","project":{"name":"project_bd2c96d2de","directory":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build","workspaceDir":"/Users/brianreicher/Documents/GitHub/turbine","sources":[],"dependencies":[],"classpath":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/sourcegraph/semanticdb-javac/0.7.4/semanticdb-javac-0.7.4.jar"],"out":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/.bloop/project_bd2c96d2de","classesDir":"/Users/brianreicher/Documents/GitHub/turbine/.scala-build/project_bd2c96d2de/classes/main","scala":{"organization":"org.scala-lang","name":"scala-compiler","version":"3.2.2","options":["-Xsemanticdb","-sourceroot","/Users/brianreicher/Documents/GitHub/turbine","-release","8"],"jars":["/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-compiler_3/3.2.2/scala3-compiler_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-interfaces/3.2.2/scala3-interfaces-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/tasty-core_3/3.2.2/tasty-core_3-3.2.2.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/modules/scala-asm/9.3.0-scala-1/scala-asm-9.3.0-scala-1.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/compiler-interface/1.3.5/compiler-interface-1.3.5.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-reader/3.19.0/jline-reader-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal/3.19.0/jline-terminal-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/jline/jline-terminal-jna/3.19.0/jline-terminal-jna-3.19.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.7.0/protobuf-java-3.7.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-sbt/util-interface/1.3.0/util-interface-1.3.0.jar","/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/net/java/dev/jna/jna/5.3.1/jna-5.3.1.jar"]},"java":{"options":["--release","8","-Xplugin:semanticdb -sourceroot:/Users/brianreicher/Documents/GitHub/turbine -targetroot:javac-classes-directory","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED","-J--add-exports","-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"]},"platform":{"name":"jvm","config":{"options":[]},"mainClass":[]},"resolution":{"modules":[{"organization":"org.scala-lang","name":"scala3-library_3","version":"3.2.2","artifacts":[{"name":"scala3-library_3","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2.jar"},{"name":"scala3-library_3","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala3-library_3/3.2.2/scala3-library_3-3.2.2-sources.jar"}]},{"organization":"org.scala-lang","name":"scala-library","version":"2.13.10","artifacts":[{"name":"scala-library","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10.jar"},{"name":"scala-library","classifier":"sources","path":"/Users/brianreicher/Library/Caches/Coursier/v1/https/repo1.maven.org/maven2/org/scala-lang/scala-library/2.13.10/scala-library-2.13.10-sources.jar"}]}]},"tags":["library"]}} \ No newline at end of file diff --git a/ingestion/src/downloader.rs b/ingestion/src/downloader.rs index 8cd27f9..73456e7 100644 --- a/ingestion/src/downloader.rs +++ b/ingestion/src/downloader.rs @@ -24,7 +24,6 @@ impl GitHubDownloader { pub async fn download_git_zips( &self, urls: Vec<&str>, - repo_name: &str, ) -> Result<(), Box> { for url in urls { let response = self.client.get(url).send().await?; diff --git a/ingestion/src/main.rs b/ingestion/src/main.rs index 53f6f52..dd24c5d 100644 --- a/ingestion/src/main.rs +++ b/ingestion/src/main.rs @@ -1,18 +1,21 @@ use ingestion::mongo_utils::MongoDriver; -use dpw +use ingestion::downloader::GitHubDownloader; + + #[tokio::main] async fn main() -> std::io::Result<()> { let mongo: MongoDriver = MongoDriver::new("localhost", 8080, "turbine"); mongo.connect().await?; // create/connect if exists, and flush - mongo.create_collection("github_data").await?; - mongo.flush_collection("github_data").await?; + let collection = "github_data" + mongo.create_collection(collection).await?; + mongo.flush_collection(collection).await?; - // set remote urls and download + // set download links and ingest let urls = vec!["https://github.com/user/repo/raw/master/file1.zip", "https://github.com/user/repo/raw/master/file2.zip"]; - github_downloader.download_git_zips(urls, "my_repo").await?; - - + let downloader: GithubDownloader = GithubDownloader::new(mongo, collection); + downloader.download_git_zips(urls).await?; + Ok(()) } \ No newline at end of file diff --git a/ingestion/tests/ingestion/ing_tests.rs b/ingestion/tests/ingestion/ing_tests.rs new file mode 100644 index 0000000..7f2a19d --- /dev/null +++ b/ingestion/tests/ingestion/ing_tests.rs @@ -0,0 +1,5 @@ +#[tokio::test] +async fn mongo_connection() { + + assert_eq!(Some(0), Some(0)); +} diff --git a/ingestion/tests/ingestion/main.rs b/ingestion/tests/ingestion/main.rs new file mode 100644 index 0000000..0e9a27f --- /dev/null +++ b/ingestion/tests/ingestion/main.rs @@ -0,0 +1 @@ +mod ing_tests;