From aa8de911fd9a95ee7781126c3e5f8bfa4325383a Mon Sep 17 00:00:00 2001 From: Carlton N Hanna Date: Tue, 24 Oct 2023 09:48:37 -0600 Subject: [PATCH] update historical service to pull from multiple tickers, update start time calculation --- .../explorer/domain/entities/ExplorerCache.kt | 17 ++++-- .../explorer/service/TokenService.kt | 57 +++---------------- .../explorer/service/async/AsyncService.kt | 43 +++++++++----- 3 files changed, 50 insertions(+), 67 deletions(-) diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt index ece23e6f..fe6efe1d 100644 --- a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt +++ b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt @@ -90,7 +90,8 @@ class ValidatorMarketRateStatsRecord(id: EntityID) : IntEntity(id) { } fun findByAddress(address: String, fromDate: DateTime?, toDate: DateTime?, count: Int) = transaction { - val query = ValidatorMarketRateStatsTable.select { ValidatorMarketRateStatsTable.operatorAddress eq address } + val query = + ValidatorMarketRateStatsTable.select { ValidatorMarketRateStatsTable.operatorAddress eq address } if (fromDate != null) { query.andWhere { ValidatorMarketRateStatsTable.date greaterEq fromDate } } @@ -214,7 +215,7 @@ class ChainAumHourlyRecord(id: EntityID) : IntEntity(id) { fun getAumForPeriod(fromDate: DateTime, toDate: DateTime) = transaction { ChainAumHourlyRecord.find { (ChainAumHourlyTable.datetime greaterEq fromDate) and - (ChainAumHourlyTable.datetime lessEq toDate.plusDays(1)) + (ChainAumHourlyTable.datetime lessEq toDate.plusDays(1)) } .orderBy(Pair(ChainAumHourlyTable.datetime, SortOrder.ASC)) .toList() @@ -272,6 +273,14 @@ class TokenHistoricalDailyRecord(id: EntityID) : Entity(id) .orderBy(Pair(TokenHistoricalDailyTable.timestamp, SortOrder.DESC)) .firstOrNull()?.data?.quote?.get(USD_UPPER)?.close ?: BigDecimal.ZERO } + + fun getLatestDateEntry(): TokenHistoricalDailyRecord? = transaction { + return@transaction TokenHistoricalDailyRecord + .all() + .orderBy(Pair(TokenHistoricalDailyTable.timestamp, SortOrder.DESC)) + .limit(1) + .firstOrNull() + } } var timestamp by TokenHistoricalDailyTable.timestamp @@ -292,7 +301,7 @@ class ProcessQueueRecord(id: EntityID) : IntEntity(id) { fun findByType(processType: ProcessQueueType) = transaction { ProcessQueueRecord.find { (ProcessQueueTable.processType eq processType.name) and - (ProcessQueueTable.processing eq false) + (ProcessQueueTable.processing eq false) }.toList() } @@ -305,7 +314,7 @@ class ProcessQueueRecord(id: EntityID) : IntEntity(id) { fun delete(processType: ProcessQueueType, value: String) = transaction { ProcessQueueTable.deleteWhere { (ProcessQueueTable.processType eq processType.name) and - (ProcessQueueTable.processValue eq value) + (ProcessQueueTable.processValue eq value) } } diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt index d299b9d4..b08f32bc 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt @@ -223,10 +223,10 @@ class TokenService(private val accountClient: AccountGrpcClient) { } } - fun getHistoricalFromDlob(startTime: DateTime): DlobHistBase? = runBlocking { + fun getHistoricalFromDlob(startTime: DateTime, tickerId: String): DlobHistBase? = runBlocking { try { - KTOR_CLIENT_JAVA.get("https://www.dlob.io:443/gecko/external/api/v1/exchange/historical_trades") { - parameter("ticker_id", "HASH_USD") + KTOR_CLIENT_JAVA.get("https://test.dlob.io:443/gecko/external/api/v1/exchange/historical_trades") { + parameter("ticker_id", tickerId) parameter("type", "buy") parameter("start_time", DateTimeFormat.forPattern("dd-MM-yyyy").print(startTime)) accept(ContentType.Application.Json) @@ -240,53 +240,14 @@ class TokenService(private val accountClient: AccountGrpcClient) { } } - suspend fun getHistoricalFromDlobNew(startTime: DateTime): DlobHistBase? { - val dlobContractUrls = listOf( - "https://www.dlob.io/gecko/external/api/v1/order-books/pb1w6ul64t5fjcg65mmscec758dgyml6xmmw5fy2vyxxc9dhq3tmhusyzcj3r/aggregate?unit=YEAR", - "https://www.dlob.io/gecko/external/api/v1/order-books/pb18vd8fpwxzck93qlwghaj6arh4p7c5n894vnu5g/aggregate?unit=YEAR" - ) - var dlobHistorBase: DlobHistBase? = null - for (url in dlobContractUrls) { - try { - val data = fetchDataFromUrl(url, startTime) - if (data != null) { - if (dlobHistorBase == null) { - dlobHistorBase = data - }else { - val combinedBuy = dlobHistorBase.buy + data.buy - dlobHistorBase = dlobHistorBase.copy(buy = combinedBuy) - } - } - } catch (e: ResponseException) { - logger.error("Error fetching from Dlob: ${e.response}") - } catch (e: Exception) { - logger.error("Error fetching from Dlob: ${e.message}") - } catch (e: Throwable) { - logger.error("Error fetching from Dlob: ${e.message}") - } - } + fun getHistoricalFromDlob(startTime: DateTime): DlobHistBase { + val tickerIds = listOf("HASH_USD", "HASH_USDOMNI") - return dlobHistorBase - } + val dlobHistorical = tickerIds + .mapNotNull { getHistoricalFromDlob(startTime, it)?.buy } + .flatten() - private suspend fun fetchDataFromUrl(url: String, startTime: DateTime): DlobHistBase? { - return try { - KTOR_CLIENT_JAVA.get(url) { - parameter("ticker_id", "HASH_USD") - parameter("type", "buy") - parameter("start_time", DateTimeFormat.forPattern("dd-MM-yyyy").print(startTime)) - accept(ContentType.Application.Json) - }.body() - } catch (e: ResponseException) { - logger.error("Error fetching from Dlob: ${e.response}") - null - } catch (e: Exception) { - logger.error("Error fetching from Dlob: ${e.message}") - null - } catch (e: Throwable) { - logger.error("Error fetching from Dlob: ${e.message}") - null - } + return DlobHistBase(dlobHistorical) } fun getTokenHistorical(fromDate: DateTime?, toDate: DateTime?) = diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt index 5c68cd75..45f259d6 100644 --- a/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt +++ b/service/src/main/kotlin/io/provenance/explorer/service/async/AsyncService.kt @@ -104,7 +104,7 @@ class AsyncService( protected val logger = logger(AsyncService::class) protected var collectHistorical = true - @Scheduled(initialDelay = 0L, fixedDelay = 5000L) + // @Scheduled(initialDelay = 0L, fixedDelay = 5000L) fun updateLatestBlockHeightJob() { val index = getBlockIndex() val startHeight = blockService.getLatestBlockHeight() @@ -277,10 +277,15 @@ class AsyncService( @Scheduled(cron = "0 0 0/1 * * ?") // Every hour fun saveChainAum() = explorerService.saveChainAum() - @Scheduled(cron = "0 0 1 * * ?") // Every day at 1 am + @Scheduled(cron = "0 0 1 * * *") // Every day at 1 am fun updateTokenHistorical() { val today = DateTime.now().startOfDay() - val startDate = today.minusMonths(1) + var startDate = today.minusMonths(1) + var latest = TokenHistoricalDailyRecord.getLatestDateEntry() + if (latest != null) { + startDate = latest.timestamp.minusDays(1) + } + val dlobRes = tokenService.getHistoricalFromDlob(startDate) ?: return val baseMap = Interval(startDate, today) .let { int -> generateSequence(int.start) { dt -> dt.plusDays(1) }.takeWhile { dt -> dt < int.end } } @@ -306,22 +311,26 @@ class AsyncService( time_low = if (low != null) DateTime(low.trade_timestamp * 1000) else k, quote = mapOf( USD_UPPER to - CmcQuote( - open = open, - high = high?.price ?: prevPrice, - low = low?.price ?: prevPrice, - close = close, - volume = usdVolume, - market_cap = close.multiply(tokenService.totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER)), - timestamp = closeDate - ) + CmcQuote( + open = open, + high = high?.price ?: prevPrice, + low = low?.price ?: prevPrice, + close = close, + volume = usdVolume, + market_cap = close.multiply( + tokenService.totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER) + ), + timestamp = closeDate + ) ) ).also { prevPrice = close } TokenHistoricalDailyRecord.save(record.time_open.startOfDay(), record) } } - @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes + // @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes + @Scheduled(initialDelay = 0L, fixedDelay = 5000L) + fun updateTokenLatest() { val today = DateTime.now().withZone(DateTimeZone.UTC) val startDate = today.minusDays(7) @@ -346,7 +355,10 @@ class AsyncService( today, mapOf(USD_UPPER to CmcLatestQuoteAbbrev(price, percentChg, vol24Hr, marketCap, today)) ) - CacheUpdateRecord.updateCacheByKey(CacheKeys.UTILITY_TOKEN_LATEST.key, VANILLA_MAPPER.writeValueAsString(rec)) + CacheUpdateRecord.updateCacheByKey( + CacheKeys.UTILITY_TOKEN_LATEST.key, + VANILLA_MAPPER.writeValueAsString(rec) + ) } } @@ -443,7 +455,8 @@ class AsyncService( try { transaction { it.apply { this.processing = true } } send(it.processValue) - } catch (_: Exception) { } + } catch (_: Exception) { + } } } }