-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathbuild.sbt
88 lines (77 loc) · 3.99 KB
/
build.sbt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
/**
* Copyright (C) 2021 Ferlab Ste-Justine contributors.
* See the LICENCE.txt file distributed with this work for additional
* information regarding copyright ownership.
*/
lazy val scala212 = "2.12.18"
lazy val supportedScalaVersions = List(scala212)
scalacOptions ++= Seq("-deprecation", "-Ypartial-unification")
javacOptions ++= Seq("-source", "11", "-target", "11", "-Xlint")
val spark3Version = "3.5.1"
val catsVersion = "2.10.0"
val scalatestVersion = "3.2.17"
val pureconfigVersion = "0.17.6"
val elasticsearchVersion = "7.15.0"
val deltaVersion = "3.1.0"
val glowVersion = "2.0.0"
updateOptions := updateOptions.value.withGigahorse(false)
lazy val `datalake-commons` = (project in file("datalake-commons"))
.settings(
scalaVersion := scala212,
libraryDependencies += "com.github.pureconfig" %% "pureconfig" % pureconfigVersion,
libraryDependencies += "org.scala-lang.modules" %% "scala-collection-compat" % "2.11.0",
libraryDependencies += "org.typelevel" %% "cats-core" % catsVersion,
libraryDependencies += "org.scalatest" %% "scalatest" % scalatestVersion,
libraryDependencies += "io.projectglow" %% "glow-spark3" % glowVersion % Provided exclude("org.apache.hadoop", "hadoop-client"),
libraryDependencies += "com.outr" %% "hasher" % "1.2.2",
libraryDependencies += "com.lihaoyi"%% "mainargs" % "0.6.1",
libraryDependencies += "org.apache.spark" %% "spark-core" % spark3Version % Provided,
libraryDependencies += "org.apache.spark" %% "spark-sql" % spark3Version % Provided,
dependencyOverrides ++= Seq(
"org.apache.commons" % "commons-lang3" % "3.12.0"
)
)
lazy val `datalake-test-utils` = (project in file("datalake-test-utils"))
.settings(
scalaVersion := scala212,
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % spark3Version % Provided,
"org.apache.spark" %% "spark-sql" % spark3Version % Provided,
"io.delta" %% "delta-spark" % deltaVersion % Provided,
"org.scalatest" %% "scalatest" % scalatestVersion % Test
)
)
.dependsOn(`datalake-commons`)
lazy val `datalake-spark3` = (project in file("datalake-spark3"))
.settings(
scalaVersion := scala212,
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % spark3Version % Provided,
"org.apache.spark" %% "spark-sql" % spark3Version % Provided,
"io.delta" %% "delta-spark" % deltaVersion % Provided,
"org.elasticsearch" %% "elasticsearch-spark-30" % elasticsearchVersion % Provided,
"com.github.pureconfig" %% "pureconfig" % pureconfigVersion,
"org.typelevel" %% "cats-core" % catsVersion,
"io.projectglow" %% "glow-spark3" % glowVersion % Provided exclude("org.apache.hadoop", "hadoop-client") exclude("io.netty","netty-all") exclude("io.netty","netty-handler") exclude("io.netty","netty-transport-native-epoll"),
"com.microsoft.sqlserver" % "mssql-jdbc" % "8.4.1.jre11" % Provided,
"com.microsoft.aad" % "adal4j" % "0.0.2" % Provided,
"com.microsoft.azure" % "spark-mssql-connector_2.12" % "1.2.0" % Provided,
"com.crealytics" %% "spark-excel" % "3.5.0_0.20.3" % Provided,
//Use by ElasticsearchClient
"com.softwaremill.sttp.client3" %% "core" % "3.9.2",
"com.softwaremill.sttp.client3" %% "json4s" % "3.9.2" exclude("org.json4s", "json4s-core_2.12"), //Exclusion because json4s is used in spark
"com.softwaremill.sttp.client3" %% "slf4j-backend" % "3.9.2",
"com.dimafeng" %% "testcontainers-scala-scalatest" % "0.41.0" % Test,
"com.dimafeng" %% "testcontainers-scala-elasticsearch" % "0.41.2" % Test,
"com.dimafeng" %% "testcontainers-scala-postgresql" % "0.41.3" % Test,
"org.postgresql" % "postgresql" % "42.5.1" % Test,
"org.scalatest" %% "scalatest" % scalatestVersion % Test,
"org.apache.spark" %% "spark-hive" % spark3Version % Test,
),
dependencyOverrides ++= Seq(
"org.apache.commons" % "commons-lang3" % "3.12.0"
),
)
.dependsOn(`datalake-commons`)
.dependsOn(`datalake-test-utils` % "test->compile")
Test / fork := true