diff --git a/.github/workflows/cron-cicd.yaml b/.github/workflows/cron-cicd.yaml new file mode 100644 index 000000000..ac4cc16e2 --- /dev/null +++ b/.github/workflows/cron-cicd.yaml @@ -0,0 +1,66 @@ +name: backend CI + +on: + push: + paths: + - "cron/**" + pull_request: + paths: + - "cron/**" + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Make ssh file + run: | + mkdir -p ~/.ssh + echo '${{ secrets.SSH_CONFIG }}' | base64 -d > ~/.ssh/id_rsa + chmod 400 ~/.ssh/id_rsa + - name: Checkout + uses: actions/checkout@v3 + with: + submodule: true + - name: Update submodule + run: | + git submodule update --init config/ + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: "17" + distribution: "corretto" + - name: Cache Gradle packages + uses: actions/cache@v3 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} + restore-keys: | + ${{ runner.os }}-gradle- + - name: Build with Gradle + run: | + cd cron + chmod +x gradlew + ./gradlew jar + shell: bash + - name: Main S3에 업로드 + if: ${{ github.ref == 'refs/heads/main' }} + run: | + cp cron/build/libs/cron-*.jar deploy-cron/build + mkdir -p deploy && cp deploy-cron/* deploy/ + zip -r deploy.zip deploy + + aws s3 cp deploy.zip s3://${{ secrets.AWS_S3_MAIN_BUCKET_NAME }}/deploy.zip + + aws deploy create-deployment \ + --application-name ${{ secrets.AWS_CODEDEPLOY_MAIN_APP_NAME }} \ + --deployment-config-name CodeDeployDefault.AllAtOnce \ + --deployment-group-name ${{ secrets.AWS_CODEDEPLOY_MAIN_GROUP_NAME }} \ + --file-exists-behavior OVERWRITE \ + --s3-location bucket=${{ secrets.AWS_S3_MAIN_BUCKET_NAME }},bundleType=zip,key=deploy.zip diff --git a/cron/.gitignore b/cron/.gitignore new file mode 100644 index 000000000..a57c536e4 --- /dev/null +++ b/cron/.gitignore @@ -0,0 +1,44 @@ +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store + +**/resources/config/** diff --git a/cron/build.gradle b/cron/build.gradle new file mode 100644 index 000000000..fbbc690c1 --- /dev/null +++ b/cron/build.gradle @@ -0,0 +1,68 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.9.0' + id 'application' +} +ext { + jacksonVersion = '2.15.2' + okhttpVersion = '2.7.5' + exposedVersion = '0.41.1' + mariadbVersion = "3.1.4" + h2Version = "2.2.220" +} + +group = 'org.example' +version = '1.0-SNAPSHOT' + +repositories { + mavenCentral() +} + +dependencies { + implementation "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" + implementation "com.squareup.okhttp:okhttp:${okhttpVersion}" + implementation "org.jetbrains.exposed:exposed-core:${exposedVersion}" + implementation "org.jetbrains.exposed:exposed-dao:${exposedVersion}" + implementation "org.jetbrains.exposed:exposed-jdbc:${exposedVersion}" + implementation "org.jetbrains.exposed:exposed-java-time:${exposedVersion}" + runtimeOnly "org.mariadb.jdbc:mariadb-java-client:${mariadbVersion}" + implementation "com.zaxxer:HikariCP:5.0.1" + implementation "io.github.microutils:kotlin-logging-jvm:2.0.10" + implementation "org.slf4j:slf4j-api:1.7.30" + implementation "org.slf4j:slf4j-simple:1.7.25" + + testImplementation "org.jetbrains.kotlin:kotlin-test" + testRuntimeOnly "com.h2database:h2:${h2Version}" +} + +tasks.register('copyMainConfig', Copy) { + from '../config/cron/src/resources/config' + into 'src/main/resources/config' +} + +tasks.named('processResources') { + dependsOn 'copyMainConfig' +} + +test { + useJUnitPlatform() +} + +kotlin { + jvmToolchain(17) +} + +application { + mainClass.set("MainKt") +} + +jar { + manifest { + attributes 'Main-Class': 'MainKt' + } + from { + configurations.runtimeClasspath.collect { + it.isDirectory() ? it : zipTree(it) + } + } + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} \ No newline at end of file diff --git a/cron/gradle.properties b/cron/gradle.properties new file mode 100644 index 000000000..7fc6f1ff2 --- /dev/null +++ b/cron/gradle.properties @@ -0,0 +1 @@ +kotlin.code.style=official diff --git a/cron/gradle/wrapper/gradle-wrapper.jar b/cron/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 000000000..249e5832f Binary files /dev/null and b/cron/gradle/wrapper/gradle-wrapper.jar differ diff --git a/cron/gradle/wrapper/gradle-wrapper.properties b/cron/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 000000000..06febab41 --- /dev/null +++ b/cron/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists \ No newline at end of file diff --git a/cron/gradlew b/cron/gradlew new file mode 100755 index 000000000..1b6c78733 --- /dev/null +++ b/cron/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/cron/gradlew.bat b/cron/gradlew.bat new file mode 100644 index 000000000..107acd32c --- /dev/null +++ b/cron/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/cron/settings.gradle b/cron/settings.gradle new file mode 100644 index 000000000..da01875bb --- /dev/null +++ b/cron/settings.gradle @@ -0,0 +1,12 @@ +pluginManagement { + repositories { + mavenCentral() + gradlePluginPortal() + } +} + +plugins { + id 'org.gradle.toolchains.foojay-resolver-convention' version '0.5.0' +} + +rootProject.name = 'cron' \ No newline at end of file diff --git a/cron/src/main/kotlin/Main.kt b/cron/src/main/kotlin/Main.kt new file mode 100644 index 000000000..1b2925439 --- /dev/null +++ b/cron/src/main/kotlin/Main.kt @@ -0,0 +1,40 @@ +import jobs.FtTokenFetcher +import jobs.blackhole.BlackholeUpdater +import mu.KotlinLogging +import kotlin.math.log +import kotlin.system.exitProcess + +/** + * command list + * format: command to creator function + */ +private val mapping = mapOf( + "update-blackhole" to BlackholeUpdater::create, + "get-ft-token" to FtTokenFetcher::create, +) + +private val log = KotlinLogging.logger {} + +/** + * main function + * @param args command list + * + * 인자가 하나가 아니라면 에러 메시지를 로깅하고 종료한다. + * 잘못된 인자가 들어왔다면 에러 메시지를 로깅하고 종료한다. + * 실행 중 에러가 발생하면 에러 메시지를 로깅하고 종료한다. + */ +fun main(args: Array) { + if (args.size != 1) { + log.warn { "실행하려는 명령어 하나를 입력하세요" } + exitProcess(1) + } + try { + mapping[args[0]]?.let { + val result = it().sprint() + if (result !is Unit) print(result) + }?: throw Exception("해당 명령어 ${args[0]}는 존재하지 않습니다.") + } catch (e: Exception) { + log.warn { e.message } + exitProcess(1) + } +} \ No newline at end of file diff --git a/cron/src/main/kotlin/http/OkhttpExtension.kt b/cron/src/main/kotlin/http/OkhttpExtension.kt new file mode 100644 index 000000000..b01524218 --- /dev/null +++ b/cron/src/main/kotlin/http/OkhttpExtension.kt @@ -0,0 +1,46 @@ +package http + +import com.squareup.okhttp.Request +import com.squareup.okhttp.Response + +// request +private val AUTHORIZATION = "Authorization" +private val BEARER_PREFIX = "Bearer " + +// response +private val TOTAL_INDEX = "X-Total" +private val PAGE_INDEX = "X-Page" +private val PAGE_SIZE_INDEX = "X-Per-Page" + +/** + * OkhttpExtension + * - token을 Authorization header에 추가한다. + */ +fun Request.Builder.addToken(token: String): Request.Builder = + this.header(AUTHORIZATION, BEARER_PREFIX + token) + +/** + * OkhttpExtension + * - response header에서 total page를 가져온다. (X-Total header를 이용) + */ +fun Response.ftGetTotalPages(): Int { + val total = this.header(TOTAL_INDEX)?.toInt() ?: throw Exception("total page is not found") + val perPage = this.header(PAGE_SIZE_INDEX)?.toInt() ?: throw Exception("per page is not found") + return total / perPage + 1 +} + +/** + * OkhttpExtension + * - response header에서 current page를 가져온다. (X-Page header를 이용) + */ +fun Response.ftGetCurrentPage(): Int { + return this.header(PAGE_INDEX)?.toInt() ?: throw Exception("current page is not found") +} + +/** + * OkhttpExtension + * - response가 401(Unauthorized)인지 확인한다. + */ +fun Response.isUnauthorized(): Boolean { + return this.code() == 401 +} diff --git a/cron/src/main/kotlin/jobs/FtTokenFetcher.kt b/cron/src/main/kotlin/jobs/FtTokenFetcher.kt new file mode 100644 index 000000000..a778b5686 --- /dev/null +++ b/cron/src/main/kotlin/jobs/FtTokenFetcher.kt @@ -0,0 +1,104 @@ +package jobs + +import com.fasterxml.jackson.annotation.JsonSetter +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import com.squareup.okhttp.MultipartBuilder +import com.squareup.okhttp.OkHttpClient +import com.squareup.okhttp.Request +import com.squareup.okhttp.RequestBody +import mu.KotlinLogging +import utils.ConfigLoader +import utils.Configuration +import utils.Sprinter + +private val log = KotlinLogging.logger {} + +/** + * FtTokenFetcher + * ft token을 가져온다. + * 1. ft token을 가져온다. + * 2. 가져온 ft token를 저장해놓는다 (캐싱) + * 3. 가져온 ft token을 반환한다. + * 4. refresh()를 사용하여 캐싱된 ft token을 갱신할 수 있다. + */ +sealed interface FtTokenFetcher: Sprinter { + companion object { + @JvmStatic fun create(): FtTokenFetcher { + val config = ConfigLoader.create(FtTokenConfig::class) + return FtTokenFetcherImpl(config); + } + } + + /** + * ft token을 다시 가져온다.. + * + */ + fun refresh(): String +} + +private val clientSecret = "client_secret" +private val clientId = "client_id" +private val grantType = "grant_type" +private val scope = "scope" +private val accessTokenKey = "access_token" + +internal class FtTokenFetcherImpl(val FtTokenConfig: FtTokenConfig): FtTokenFetcher { + private val client = OkHttpClient() + private var token: String? = null + + override fun sprint(): String { + log.info { "token fetcher start" } + token = token ?: fetchToken() + return token!! + } + + override fun refresh(): String { + log.info { "refreshing token start" } + token = fetchToken() + return token!! + } + + private fun fetchToken(): String { + log.info("fetching token start") + val request = generateRequest(generateBody()) + val response = client.newCall(request).execute() + if (response.code() != 200) { + log.info { "fetching token fail code: ${response.code()}" } + throw Exception("server is not connected: ${response.code()})") + } + val valueMap: Map = ObjectMapper() + .readValue(response.body().string(), object: TypeReference>(){}) + log.info { "fetching token success" } + return valueMap[accessTokenKey] ?: throw Exception("access token is not found") + } + + private fun generateBody(): RequestBody { + return MultipartBuilder().type(MultipartBuilder.FORM) + .addFormDataPart(clientSecret, FtTokenConfig.clientSecret) + .addFormDataPart(clientId, FtTokenConfig.clientId) + .addFormDataPart(grantType, FtTokenConfig.grantType) + .addFormDataPart(scope, FtTokenConfig.scope) + .build() + } + + private fun generateRequest(body: RequestBody): Request { + return Request.Builder() + .url(FtTokenConfig.url) + .post(body) + .build() + } +} + +internal data class FtTokenConfig +constructor( + @JsonSetter("url") + val url: String, + @JsonSetter("clientSecret") + val clientSecret: String, + @JsonSetter("clientId") + val clientId: String, + @JsonSetter("grantType") + val grantType: String, + @JsonSetter("scope") + val scope: String): Configuration diff --git a/cron/src/main/kotlin/jobs/blackhole/BlackholeDbManager.kt b/cron/src/main/kotlin/jobs/blackhole/BlackholeDbManager.kt new file mode 100644 index 000000000..1ae6f0737 --- /dev/null +++ b/cron/src/main/kotlin/jobs/blackhole/BlackholeDbManager.kt @@ -0,0 +1,64 @@ +package jobs.blackhole + +import org.jetbrains.exposed.sql.* +import org.jetbrains.exposed.sql.javatime.datetime +import org.jetbrains.exposed.sql.transactions.transaction +import utils.DbManager +import java.time.LocalDateTime + +private object Users : Table("user") { + val userId: Column = long("user_id").autoIncrement() + val name: Column = varchar("name", 100) + val email: Column = varchar("email", 100) + val role: Column = varchar("role", 100) + val blackholedAt: Column = datetime("blackholed_at").nullable() + val deletedAt: Column = datetime("deleted_at").nullable() +} + +private fun ResultRow.toUserProfile() = + UserProfile( + name = this[Users.name], + email = this[Users.email], + blackholedAt = this[Users.blackholedAt], + ) + +/** + * BlackholeDbManager + */ +internal class BlackholeDbManager { + private val dbManager: DbManager = DbManager() + + /** + * 블랙홀 업데이트가 필요한 유저들을 필터링한다. (db select) + */ + fun filterRequiredUpdate(users: List): List { + dbManager.connect() + val queryRst: ArrayList = ArrayList() + transaction { + addLogger(StdOutSqlLogger) + Users.select { Users.email inList users.map { it.email } } + .forEach { queryRst.add(it.toUserProfile()) } + } + return users.filter { user -> + queryRst.stream().anyMatch { + (it.email == user.email) + .and(it.name == user.name) + .and(it.blackholedAt != user.blackholedAt) + }} + } + + /** + * 블랙홀 업데이트가 필요한 유저들을 업데이트한다. (db set) + */ + fun updateBlackholedUsers(users: List) { + dbManager.connect() + transaction { + addLogger(StdOutSqlLogger) + users.forEach { user -> + Users.update({ (Users.email eq user.email) and (Users.name eq user.name) }) { + it[Users.blackholedAt] = user.blackholedAt + } + } + } + } +} diff --git a/cron/src/main/kotlin/jobs/blackhole/BlackholeUpdater.kt b/cron/src/main/kotlin/jobs/blackhole/BlackholeUpdater.kt new file mode 100644 index 000000000..98f7b446d --- /dev/null +++ b/cron/src/main/kotlin/jobs/blackhole/BlackholeUpdater.kt @@ -0,0 +1,124 @@ +package jobs.blackhole + +import com.fasterxml.jackson.annotation.JsonSetter +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.squareup.okhttp.OkHttpClient +import com.squareup.okhttp.Request +import com.squareup.okhttp.Response +import http.addToken +import http.ftGetCurrentPage +import http.ftGetTotalPages +import http.isUnauthorized +import utils.Configuration +import jobs.FtTokenFetcher +import utils.Sprinter +import jobs.blackhole.CursusUsersDeserializer.toUsers +import mu.KotlinLogging +import utils.ConfigLoader +import java.util.stream.Stream + +private val log = KotlinLogging.logger {} + +private val DEFAULT_INDEX = 1 +private val SLEEP_TIME = 60 * 5 * 1000L + +/** + * BlackholeUpdater + * 블랙홀 업데이트를 수행한다. + * 1. ft token을 가져온다. + * 2. ft api를 호출하여 블랙홀 유저들을 가져온다. + * 3. db에 저장된 블랙홀 유저들과 비교하여 업데이트가 필요한 유저들을 필터링한다. + * 4. 업데이트가 필요한 유저들을 업데이트한다. + * + * 3번까지 시도해보고 실패하면 서버가 잘못된 것으로 판단하여 종료한다. + */ +sealed interface BlackholeUpdater: Sprinter { +companion object { + @JvmStatic fun create(): BlackholeUpdater { + val config = ConfigLoader.create(BlackholeUpdaterConfig::class) + return BlackholeUpdaterImpl(config) + } + } +} + +private data class ResponseUsers( + val totalPages: Int, + val currentPage: Int, + val users: List +) { + fun isLastPage() = totalPages + 1 == currentPage +} + +internal class BlackholeUpdaterImpl(private val config: BlackholeUpdaterConfig): BlackholeUpdater { + private val client = OkHttpClient() + private val mapper = ObjectMapper() + private val ftTokenFetcher: FtTokenFetcher = FtTokenFetcher.create() + private val dbManager: BlackholeDbManager = BlackholeDbManager() + + override fun sprint() { + log.info { "Blackhole update start" } + ftTokenFetcher.sprint() + Stream.iterate(DEFAULT_INDEX) { it + 1 } + .map { requestUsers(it) } // request + .takeWhile { !it.isLastPage() } // check last page + .map { it.users } // get users + .map { toUsers(it) } // get profile + .map(::filterUpdateUsers) // update + .forEach(::updateUsers) + } + + private fun filterUpdateUsers(users: List): List { + val filtered = dbManager.filterRequiredUpdate(users) + log.info { "filterUpdateUsers filtered: ${filtered.size}, total: ${users.size}" } + return filtered + } + + private fun updateUsers(users: List) { + if (users.isEmpty()) return + log.info { "updateUsers size: ${users.size}" } + dbManager.updateBlackholedUsers(users) + } + + private fun requestUsers(page: Int): ResponseUsers { + log.info{ "request users page: $page" } + val request = Request.Builder() + .url(formatUrl(page)) + .addToken(ftTokenFetcher.sprint()) + .get().build() + return executeRequest(request) + } + + private fun executeRequest(request: Request, count: Int = 0): ResponseUsers { + log.info { "execute request url: ${request.url()} fail count: $count" } + if (count == 3) throw Exception("server is not connected: ${request.url()}") + val response = client.newCall(request).execute() + if (response.isSuccessful.not()) { + log.info { "execute request fail code: ${response.code()}, msg: ${response.body()}" } + if (response.isUnauthorized()) ftTokenFetcher.refresh() + Thread.sleep(SLEEP_TIME) + return executeRequest(request, count + 1) + } + return parseResponseUsers(response) + } + + private fun parseResponseUsers(response: Response): ResponseUsers { + val values = mapper.readValue(response.body().string(), Array::class.java) + val rst = ResponseUsers( + totalPages = response.ftGetTotalPages(), + currentPage = response.ftGetCurrentPage(), + users = values.toList() + ) + log.info { "parse response currentPage: ${rst.currentPage}, totalPage: ${rst.totalPages}" } + return rst + } + + private fun formatUrl(page: Int): String { + return config.formatUrl.format(page) + } +} + +internal data class BlackholeUpdaterConfig( + @JsonSetter("formatUrl") + val formatUrl: String, +): Configuration \ No newline at end of file diff --git a/cron/src/main/kotlin/jobs/blackhole/CursusUsersDeserializer.kt b/cron/src/main/kotlin/jobs/blackhole/CursusUsersDeserializer.kt new file mode 100644 index 000000000..bdf196dd9 --- /dev/null +++ b/cron/src/main/kotlin/jobs/blackhole/CursusUsersDeserializer.kt @@ -0,0 +1,46 @@ +package jobs.blackhole + +import com.fasterxml.jackson.databind.JsonNode +import mu.KotlinLogging +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.time.format.DateTimeParseException + +private val log = KotlinLogging.logger {} + +private const val PROFILE = "user" +private const val USERNAME = "login" +private const val EMAIL = "email" +private const val BLACKHOLED_AT = "blackholed_at" +private const val DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'" // 2023-08-29T23:58:16.887Z + +internal object CursusUsersDeserializer { + val formatter = DateTimeFormatter.ofPattern(DATE_FORMAT) + + fun toUsers(node: List): List { + log.info { "called toUsers" } + return node.map { toUser(it) } + } + + fun toUser(node: JsonNode): UserProfile { + log.info { "called toUser username: ${node[PROFILE][USERNAME]}" } + return UserProfile( + name = node[PROFILE][USERNAME].asText(), + email = node[PROFILE][EMAIL].asText(), + blackholedAt = parseDate(node[BLACKHOLED_AT].asText()) + ) + } + + private fun parseDate(date: String): LocalDateTime? = + try { + LocalDateTime.parse(date, formatter) + } catch (e: DateTimeParseException) { + null + } +} + +internal data class UserProfile( + val name: String, + val email: String, + val blackholedAt: LocalDateTime? +) \ No newline at end of file diff --git a/cron/src/main/kotlin/utils/ConfigLoader.kt b/cron/src/main/kotlin/utils/ConfigLoader.kt new file mode 100644 index 000000000..8aa144fcf --- /dev/null +++ b/cron/src/main/kotlin/utils/ConfigLoader.kt @@ -0,0 +1,53 @@ +package utils + +import com.fasterxml.jackson.databind.ObjectMapper +import kotlin.reflect.KClass + +/** + * A configuration interface. + * ConfigLoader와 함께 사용한다. + * - 해당 인터페이스를 구현한 클래스의 맴버변수들은 "config/className.json"에 값이 있어야한다. (className은 해당 클래스의 이름) + * - 해당 인터페이스를 구현한 클래스는 Jacson의 ObjectMapper를 이용하여 값을 집어넣을 수 있어야한다. + * + * @see ConfigLoader + */ +interface Configuration + +/** + * Configuration을 생성하는 클래스. + * create()를 통해 Configuration을 생성한다. + * + * @see Configuration + */ +object ConfigLoader { + /** + * Configuration을 생성한다. + * - 해당 클래스의 이름으로 config/className.json 파일을 찾는다. + * + * @param configClass Configuration을 생성할 클래스 + * @see Configuration + * @throws NoSuchElementException 파일이 없을 경우 + * @throws IllegalArgumentException 파일 포멧이 잘못된 경우 + */ + fun create(configClass: KClass): T { + return create(configClass, "config/${configClass.simpleName}.json") + } + + /** + * Configuration을 생성한다. + * + * @param configClass Configuration을 생성할 클래스 + * @param filePath Configuration을 생성할 파일의 경로 + * @see Configuration + * @throws NoSuchElementException 파일이 없을 경우 + * @throws IllegalArgumentException 파일 포멧이 잘못된 경우 + */ + fun create(configClass: KClass, filePath: String): T { + val url = configClass::class.java.classLoader.getResource(filePath) + ?: throw NoSuchElementException("No such file: $filePath") + return ObjectMapper() + .readerFor(configClass.java) + .readValue(url, configClass.java) + ?: throw IllegalArgumentException("Failed to read config file: $filePath") + } +} \ No newline at end of file diff --git a/cron/src/main/kotlin/utils/DbManager.kt b/cron/src/main/kotlin/utils/DbManager.kt new file mode 100644 index 000000000..4f5200aee --- /dev/null +++ b/cron/src/main/kotlin/utils/DbManager.kt @@ -0,0 +1,57 @@ +package utils + +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonSetter +import com.zaxxer.hikari.HikariConfig +import com.zaxxer.hikari.HikariDataSource +import org.jetbrains.exposed.sql.Database + +/** + * hikari cp를 사용하는 db manager. + * 쿼리 전에 connect()를 호출해야 한다. + * @see connect + */ +class DbManager { + private val dataSource: HikariDataSource + + init { + val dbConfig = ConfigLoader.create(DbManagerConfig::class) + val config = HikariConfig().apply { + setDriverClassName(dbConfig.driverClassName) + jdbcUrl = dbConfig.url + username = dbConfig.username + password = dbConfig.password + } + dataSource = HikariDataSource(config) + } + + /** + * Database.connect()를 호출한다. + * - db query가 필요하다면 실행하기 전에 반드시 호출해야 한다. + * @see Database.connect + */ + fun connect() { + Database.connect(dataSource) + } +} + +/** + * A configuration of DbManager. + * + * @param url The database url. + * @param driverClassName The driver class name. + * @param username The username. + * @param password The password. + */ +data class DbManagerConfig +@JsonCreator(mode = JsonCreator.Mode.PROPERTIES) +constructor( + @JsonSetter("url") + val url: String, + @JsonSetter("driverClassName") + val driverClassName: String, + @JsonSetter("username") + val username: String, + @JsonSetter("password") + val password: String +): Configuration diff --git a/cron/src/main/kotlin/utils/Sprinter.kt b/cron/src/main/kotlin/utils/Sprinter.kt new file mode 100644 index 000000000..8e6a96cf6 --- /dev/null +++ b/cron/src/main/kotlin/utils/Sprinter.kt @@ -0,0 +1,15 @@ +package utils + +/** + * job sprinter + * main 함수에서 job을 실행할 때 사용한다. + * @param T + */ +fun interface Sprinter { + /** + * job sprint + * @return T + * @throws Exception + */ + fun sprint(): T +} \ No newline at end of file diff --git a/deploy-cron/Dockerfile b/deploy-cron/Dockerfile new file mode 100644 index 000000000..16a636c2f --- /dev/null +++ b/deploy-cron/Dockerfile @@ -0,0 +1,10 @@ +FROM openjdk:17-jdk-slim + +WORKDIR /app + +RUN apt-get update +RUN apt-get -y install cron + +ADD build/cron-1.0-SNAPSHOT.jar cron.jar + +CMD tail -f /var/log/cron.log diff --git a/deploy-cron/appspec.yml b/deploy-cron/appspec.yml new file mode 100644 index 000000000..daba31f09 --- /dev/null +++ b/deploy-cron/appspec.yml @@ -0,0 +1,18 @@ +version: 0.0 +os: linux +files: + - source: / + destination: /home/ec2-user/cron/zip/ + overwrite: yes + +permissions: + - object: / + pattern: "**" + owner: ec2-user + group: ec2-user + +hooks: + ApplicationStart: + - location: deploy.sh + timeout: 60 + runas: ec2-user diff --git a/deploy-cron/deploy.sh b/deploy-cron/deploy.sh new file mode 100755 index 000000000..a23024628 --- /dev/null +++ b/deploy-cron/deploy.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +mkdir -p /home/ec2-user/cron/zip +cd /home/ec2-user/cron/zip/ + +docker compose down --rmi all +docker compose up -d diff --git a/deploy-cron/docker-compose.yml b/deploy-cron/docker-compose.yml new file mode 100644 index 000000000..d1e726724 --- /dev/null +++ b/deploy-cron/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3.6" + +services: + cabi-cron: + build: + context: . + dockerfile: Dockerfile + container_name: "cabi-cron" + image: "cabi/cron" + volumes: + - ./log:/var/log/ diff --git a/deploy-cron/kotlin-cron b/deploy-cron/kotlin-cron new file mode 100644 index 000000000..f341a61af --- /dev/null +++ b/deploy-cron/kotlin-cron @@ -0,0 +1,2 @@ +* * 1 * * /usr/bin/java -jar /app/cron.jar get-ft-token &>> /var/log/cron.log +