Skip to content

Commit

Permalink
Merge pull request #126 from Opetushallitus/OK-636-europass-publisher…
Browse files Browse the repository at this point in the history
…-build

OK-636, OK-758: europass-publisher docker image build
  • Loading branch information
pkalliok authored Nov 28, 2024
2 parents aa7fd1b + 1432a54 commit 5d86822
Show file tree
Hide file tree
Showing 11 changed files with 150 additions and 30 deletions.
55 changes: 46 additions & 9 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,8 @@ jobs:
run: |
git clone https://github.com/Opetushallitus/ci-tools.git
source ci-tools/common/setup-tools.sh
cd postgresql/docker
docker build --tag koutaexternal-postgres .
cd -
(cd postgresql/docker && docker build --tag koutaexternal-postgres .)
sudo apt install docker-compose
- name: Build with Maven
env:
Expand All @@ -45,10 +44,12 @@ jobs:
- uses: actions/cache@v3
id: restore-build
with:
path: kouta-external/target
path: |
kouta-external/target
europass-publisher/target
key: ${{ github.sha }}

deploy-container:
deploy-kouta-container:
needs: test-and-build
runs-on: ubuntu-latest

Expand All @@ -60,6 +61,7 @@ jobs:
with:
path: |
kouta-external/target
europass-publisher/target
key: ${{ github.sha }}

- name: Build and deploy Docker container
Expand All @@ -70,10 +72,45 @@ jobs:
run: |
git clone https://github.com/Opetushallitus/ci-tools.git
source ci-tools/common/setup-tools.sh
export ARTIFACT_NAME="kouta-external"
mv kouta-external/target/kouta-external-*-jar-with-dependencies.jar $DOCKER_BUILD_DIR/artifact/${ARTIFACT_NAME}.jar
export BASE_IMAGE="baseimage-fatjar-openjdk11:master"
export KOUTA_ARTIFACT_NAME="kouta-external"
mv kouta-external/target/kouta-external-*-jar-with-dependencies.jar \
$DOCKER_BUILD_DIR/artifact/${KOUTA_ARTIFACT_NAME}.jar
cp -vr kouta-external/src/main/resources/* $DOCKER_BUILD_DIR/config/
./ci-tools/common/pull-image.sh
./ci-tools/github-build/build-fatjar.sh $KOUTA_ARTIFACT_NAME
./ci-tools/github-build/upload-image.sh $KOUTA_ARTIFACT_NAME
deploy-europass-container:
needs: test-and-build
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v3

- uses: actions/cache@v3
id: restore-build
with:
path: |
kouta-external/target
europass-publisher/target
key: ${{ github.sha }}

- name: Build and deploy Docker container
shell: bash
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
run: |
git clone https://github.com/Opetushallitus/ci-tools.git
source ci-tools/common/setup-tools.sh
export BASE_IMAGE="baseimage-fatjar-openjdk11:master"
export EUROPASS_ARTIFACT_NAME="kouta-external-europass"
mv europass-publisher/target/europass-publisher-*-jar-with-dependencies.jar \
$DOCKER_BUILD_DIR/artifact/${EUROPASS_ARTIFACT_NAME}.jar
cp -vr europass-publisher/src/main/resources/* $DOCKER_BUILD_DIR/config/
./ci-tools/common/pull-image.sh
./ci-tools/github-build/build-fatjar.sh $ARTIFACT_NAME
./ci-tools/github-build/upload-image.sh $ARTIFACT_NAME
./ci-tools/github-build/build-fatjar.sh $EUROPASS_ARTIFACT_NAME
./ci-tools/github-build/upload-image.sh $EUROPASS_ARTIFACT_NAME
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,15 @@ Jos Maven on asennettuna, voi testit ajaa myös komentoriviltä `mvn test` komen
ajettavien testejä `mvn test -Dsuites="<testiluokan nimet pilkulla erotettuna>"`.
Esimerkiksi `mvn test -Dsuites="fi.oph.kouta.external.integration.HakukohdeSpec"`

Jos ajat testejä monta kertaa peräkkäin, ajamista voi nopeuttaa
pitämällä testidatalla täytetyn ElasticSearchin koko ajan käynnissä.
Tämä onnistuu seuraavilla komennoilla:

``` shell
$ docker-compose up -d kouta-elastic
$ docker-compose up elasticdump-loader
$ export TEST_USE_PRERUN_ELASTIC=true
```

### 3.3. Migraatiot

Expand Down
11 changes: 11 additions & 0 deletions europass-publisher/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,18 @@
<dependency>
<groupId>fi.oph.kouta</groupId>
<artifactId>kouta-common</artifactId>
<exclusions>
<exclusion>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-xml_2.12</artifactId>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
europass-publisher.elasticsearch.url = {{kouta_external_elasticsearch7_url}}
europass-publisher.elasticsearch.username = {{kouta_external_elasticsearch_username}}
europass-publisher.elasticsearch.password = {{kouta_external_elasticsearch_password}}
europass-publisher.s3.region = eu-west-1
europass-publisher.s3.bucketname = europass-publish-{{ env("ENV_NAME") }}
europass-publisher.s3.keyname = europass-export-1.xml
europass-publisher.s3.endpoint =
Original file line number Diff line number Diff line change
@@ -1,8 +1,32 @@
package fi.oph.kouta.europass

import com.typesafe.config.ConfigFactory
import fi.oph.kouta.logging.Logging
import com.typesafe.config.{Config, ConfigFactory}
import java.io.File

object EuropassConfiguration {
lazy val config = ConfigFactory.load("europass-publisher")
.withFallback(ConfigFactory.load("default"))
object EuropassConfiguration extends Logging {
def classPathUrls(cl: ClassLoader): Array[java.net.URL] = cl match {
case null => Array()
case u: java.net.URLClassLoader => u.getURLs() ++ classPathUrls(cl.getParent)
case _ => classPathUrls(cl.getParent)
}

def createConfig(): Config = {
logger.info("Reading configuration from classpath: " +
classPathUrls(getClass.getClassLoader).mkString(", "))

val configFile = new File(System.getProperty("user.home") +
"/oph-configuration/kouta-external-europass.properties")
val templatedConfig = if (configFile.exists()) {
ConfigFactory.load(ConfigFactory.parseFile(configFile))
} else {
logger.warn("Actual config file kouta-external-europass.properties not found")
ConfigFactory.load()
}
val localConfig = ConfigFactory.load("europass-publisher")
val fallbackConfig = ConfigFactory.load("default")
return templatedConfig.withFallback(localConfig).withFallback(fallbackConfig)
}

lazy val config = createConfig()
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import scala.concurrent.Await

import fi.oph.kouta.europass.ElasticClient

class ElasticClientSpec extends ScalatraFlatSpec {
class ElasticClientSpec extends ScalatraFlatSpec with ElasticFixture {
implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global
implicit val formats = DefaultFormats

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package fi.oph.kouta.europass.test

import org.scalatest.{BeforeAndAfterAll, Suite}
import scala.sys.process.Process

trait ElasticFixture extends BeforeAndAfterAll { this: Suite =>

private def useFixture(): Boolean =
!(System.getenv("TEST_USE_PRERUN_ELASTIC") == "true")

override def beforeAll() {
if (useFixture()) {
Process("docker-compose up -d kouta-elastic europass-s3").!
Process("docker-compose up elasticdump-loader s3-configurator").!
}
}

override def afterAll() {
if (useFixture()) {
Process("docker-compose down").!
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import java.io._

import fi.oph.kouta.europass.Publisher

class PublisherSpec extends ScalatraFlatSpec {
class PublisherSpec extends ScalatraFlatSpec with ElasticFixture {

"toteutusToFile" should "create correct toteutusXml from ElasticSearch" in {
val writer = new StringWriter()
Expand Down
34 changes: 22 additions & 12 deletions kouta-external/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
</dependency>

<dependency>
<groupId>fi.oph.kouta</groupId>
<artifactId>kouta-common</artifactId>
Expand All @@ -28,20 +29,12 @@
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>fi.oph.kouta</groupId>
<artifactId>kouta-common</artifactId>
<exclusions>
<exclusion>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</exclusion>
</exclusions>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.json4s</groupId>
Expand Down Expand Up @@ -180,6 +173,23 @@
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>fi.oph.kouta</groupId>
<artifactId>kouta-common</artifactId>
<exclusions>
<exclusion>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</exclusion>
<exclusion>
<groupId>jakarta.xml.bind</groupId>
<artifactId>jakarta.xml.bind-api</artifactId>
</exclusion>
</exclusions>
<classifier>tests</classifier>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.scalatra</groupId>
<artifactId>scalatra-scalatest_2.12</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import org.testcontainers.elasticsearch.ElasticsearchContainer
import org.testcontainers.utility.DockerImageName

object TempElasticClient {
val useTestContainersElastic = true // Aseta falseksi, jos haluat ajaa testejä jo käynnissä olevaa elasticsearchia vasten
val useTestContainersElastic = !(System.getenv("TEST_USE_PRERUN_ELASTIC") == "true")
var url = ""
if (useTestContainersElastic) {
url = s"http://localhost:${TempElastic.start()}"
Expand Down
3 changes: 1 addition & 2 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,7 @@

<modules>
<module>kouta-external</module>
<!-- Disabloitu koska testejä ei pysty ajamaan pelkällä komennolla: mvn clean test -->
<!--module>europass-publisher</module-->
<module>europass-publisher</module>
</modules>

<properties>
Expand Down

0 comments on commit 5d86822

Please sign in to comment.