diff --git a/src-tauri/migrations/init/asset_prices_up.sql b/src-tauri/migrations/init/asset_prices_up.sql index 17fd5d6..fff7d03 100644 --- a/src-tauri/migrations/init/asset_prices_up.sql +++ b/src-tauri/migrations/init/asset_prices_up.sql @@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS asset_prices ( symbol TEXT NOT NULL, -- if amount > 0, price is cost price, else price is sell price price REAL NOT NULL, - -- createdAt in assets_v2 - createdAt DATETIME NOT NULL + -- assetCreatedAt in assets_v2 + assetCreatedAt DATETIME NOT NULL, + updatedAt DATETIME NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS unique_uuid_asset_id ON asset_prices (uuid, assetID); diff --git a/src-tauri/migrations/v03t04/asset_prices_up.sql b/src-tauri/migrations/v03t04/asset_prices_up.sql index 17fd5d6..fff7d03 100644 --- a/src-tauri/migrations/v03t04/asset_prices_up.sql +++ b/src-tauri/migrations/v03t04/asset_prices_up.sql @@ -6,7 +6,8 @@ CREATE TABLE IF NOT EXISTS asset_prices ( symbol TEXT NOT NULL, -- if amount > 0, price is cost price, else price is sell price price REAL NOT NULL, - -- createdAt in assets_v2 - createdAt DATETIME NOT NULL + -- assetCreatedAt in assets_v2 + assetCreatedAt DATETIME NOT NULL, + updatedAt DATETIME NOT NULL ); CREATE UNIQUE INDEX IF NOT EXISTS unique_uuid_asset_id ON asset_prices (uuid, assetID); diff --git a/src/middlelayers/charts.ts b/src/middlelayers/charts.ts index 31bb197..1f7937b 100644 --- a/src/middlelayers/charts.ts +++ b/src/middlelayers/charts.ts @@ -37,7 +37,7 @@ export async function loadAllAssetActionsBySymbol(symbol: string): Promise { return queryAssetPrices() } -async function queryAssetPrices(symbol?: string): Promise { +export function queryAssetPricesAfterAssetCreatedAt(createdAt?: number): Promise { + const ts = createdAt ? new Date(createdAt).toISOString() : undefined + return queryAssetPrices(undefined, ts) +} + +export function queryAssetPricesAfterUpdatedAt(updatedAt?: number): Promise { + const ts = updatedAt ? new Date(updatedAt).toISOString() : undefined + return queryAssetPrices(undefined, undefined, ts) +} + +async function queryAssetPrices(symbol?: string, assetCreated?: string, updatedAt?: string): Promise { const db = await getDatabase() const params = symbol ? [symbol] : [] - const prices = await db.select(`SELECT * FROM ${ASSETS_PRICE_TABLE_NAME} WHERE 1=1 ${symbol ? 'and symbol = ?' : ''}`, params) + if (assetCreated) { + params.push(assetCreated) + } + if (updatedAt) { + params.push(updatedAt) + } + const prices = await db.select(`SELECT * FROM ${ASSETS_PRICE_TABLE_NAME} WHERE 1=1 ${symbol ? 'and symbol = ?' : ''} ${assetCreated ? 'and assetCreatedAt > ?' : ''} ${updatedAt ? 'and updatedAt > ?' : ''}`, params) + return prices } @@ -215,6 +232,12 @@ export async function queryAssetsAfterCreatedAt(createdAt?: number): Promise { + const db = await getDatabase() + const assets = await db.select(`SELECT * FROM ${ASSETS_TABLE_NAME} WHERE id in (${ids.join(",")})`) + return assets +} + async function queryAssetByUUID(id: string): Promise { const db = await getDatabase() const assets = await db.select(`SELECT * FROM ${ASSETS_TABLE_NAME} WHERE uuid = ?`, [id]) diff --git a/src/middlelayers/cloudsync.ts b/src/middlelayers/cloudsync.ts index 4a58c58..9e466cf 100644 --- a/src/middlelayers/cloudsync.ts +++ b/src/middlelayers/cloudsync.ts @@ -1,8 +1,8 @@ import { Polybase } from '@polybase/client' import { getDatabase, saveModelsToDatabase } from './database' import { v4 as uuidv4 } from 'uuid' -import { ASSETS_TABLE_NAME, queryAssetsAfterCreatedAt } from './charts' -import { AssetModel, CloudAssetModel, CloudSyncConfiguration } from './types' +import { ASSETS_PRICE_TABLE_NAME, ASSETS_TABLE_NAME, queryAssetPricesAfterAssetCreatedAt, queryAssetPricesAfterUpdatedAt, queryAssetsAfterCreatedAt, queryAssetsByIDs } from './charts' +import { AssetModel, AssetPriceModel, CloudAssetModel, CloudSyncConfiguration, ExportAssetModel } from './types' import { getCloudSyncConfiguration as getCloudSyncConfigurationModel, saveCloudSyncConfiguration as saveCloudSyncConfigurationModel } from './configuration' import _ from 'lodash' import Database from 'tauri-plugin-sql-api' @@ -185,14 +185,14 @@ export async function getPublicKey() { } // list all assets from cloud -async function dumpAssetsFromCloudAfterCreatedAt(createdAt?: number): Promise { +async function dumpAssetsFromCloudAfterCreatedAt(createdAt?: number): Promise { const p = await getPolybaseDB() // filter assets > createdAt from cloud, if createdAt is not provided, get all assets const records = await p.collection(RECORD_COLLECTION_NAME).where("createdAt", ">=", createdAt || 0).sort("createdAt", "desc").get() - const needSyncedAssets = _(records.data). + return _(records.data). map('data'). - map(record => record.records ? JSON.parse(record.records) as AssetModel[] : []). + map(record => record.records ? JSON.parse(record.records) as ExportAssetModel[] : []). flatten(). compact(). map((record) => ({ @@ -204,22 +204,37 @@ async function dumpAssetsFromCloudAfterCreatedAt(createdAt?: number): Promise ({ - id: asset.id, - uuid: asset.uuid, - createdAt: asset.createdAt, - symbol: asset.symbol, - amount: asset.amount, - value: asset.value, - price: asset.price, - wallet: asset.wallet, - })).value() } -async function dumpAssetsFromDBAfterCreatedAt(createdAt?: number): Promise { - return queryAssetsAfterCreatedAt(createdAt) +async function dumpAssetsFromDBAfterCreatedAt(createdAt?: number): Promise { + const models = await queryAssetsAfterCreatedAt(createdAt) + const prices = await queryAssetPricesAfterAssetCreatedAt(createdAt) + + + const getExportAssetMode = (ms: AssetModel[], ps: AssetPriceModel[]): ExportAssetModel[] => { + const priceMap = _(ps).mapKeys('assetID').mapValues('price').value() + return _(ms).map((model) => { + const costPrice = priceMap[model.id] + return { + ...model, + costPrice, + } as ExportAssetModel + }).value() + } + + const afterCreatedAt = getExportAssetMode(models, prices) + + const afterAssetIds = _(afterCreatedAt).map('id').value() + + const afterUpdatedAt = await queryAssetPricesAfterUpdatedAt(createdAt) + const updatedAssetIds = _(afterUpdatedAt).filter(a => !afterAssetIds.includes(a.assetID)).map('assetID').value() + const updatedAssets = await queryAssetsByIDs(updatedAssetIds) + const afterUpdatedAtAssets = getExportAssetMode(updatedAssets, afterUpdatedAt) + + return [...afterCreatedAt, ...afterUpdatedAtAssets] } export async function syncAssetsToCloudAndLocal(publicKey: string, createdAt?: number): Promise { @@ -241,12 +256,10 @@ export async function syncAssetsToCloudAndLocal(publicKey: string, createdAt?: n const needSyncedAssetsToDB = _(cloudAssets).differenceBy(localAssets, 'uuid').value() if (needSyncedAssetsToDB.length > 0) { // write data to local - console.log('needSyncedAssetsToDB', needSyncedAssetsToDB) - synced += await writeAssetsToDB(d, needSyncedAssetsToDB) } - await updateLastSyncTime(d, publicKey) + await markAsSynced(d, publicKey) return synced } @@ -259,24 +272,48 @@ export async function forceSyncAssetsToCloudFromLocal(publicKey: string): Promis const cloudAssets = await dumpAssetsFromCloudAfterCreatedAt() let synced = 0 + // data in cloud not equal to local, need to be removed + const changedDataInLocal = _(localAssets).filter(a => { + const dataInCloud = _(cloudAssets).find(c => c.uuid === a.uuid && c.symbol === a.symbol && c.wallet === a.wallet) + if (!dataInCloud) { + return false + } + if (dataInCloud.amount !== a.amount) { + return true + } + if (dataInCloud.value !== a.value) { + return true + } - // add data to cloud if not in cloud - const needSyncedAssetsToCloud = _(localAssets).differenceBy(cloudAssets, 'uuid').value() - if (needSyncedAssetsToCloud.length > 0) { - // write data to cloud - synced += await writeAssetsToCloud(publicKey, needSyncedAssetsToCloud) - } + if (dataInCloud.price !== a.price) { + return true + } + if (dataInCloud.costPrice !== a.costPrice) { + return true + } + return false + }).value() + const getModelKey = (a: ExportAssetModel) => a.uuid + a.symbol + a.wallet // remove data in cloud if not in local const needRemovedInCloud = _(cloudAssets).differenceBy(localAssets, 'uuid').value() if (needRemovedInCloud.length > 0) { synced += await removeAssetsInCloud(needRemovedInCloud) } + // add data to cloud if not in cloud + const needSyncedAssetsToCloud = _(localAssets).differenceBy(cloudAssets, 'uuid').value() + if (needSyncedAssetsToCloud.length > 0 || changedDataInLocal.length > 0) { + // write data to cloud + synced += await writeAssetsToCloud(publicKey, _([...needSyncedAssetsToCloud, ...changedDataInLocal]) + .uniqBy(a => getModelKey(a)) + .value() + ) + } - await updateLastSyncTime(d, publicKey) + await markAsSynced(d, publicKey) return synced } -async function writeAssetsToCloud(publicKey: string, assets: AssetModel[]): Promise { +async function writeAssetsToCloud(publicKey: string, assets: ExportAssetModel[]): Promise { const gas = _(assets).groupBy('uuid').value() const p = await getPolybaseDB() const res = await bluebird.map(Object.keys(gas), async (uuid) => { @@ -298,11 +335,9 @@ async function writeAssetsToCloud(publicKey: string, assets: AssetModel[]): Prom return _(res).sum() } -async function removeAssetsInCloud(assets: AssetModel[]): Promise { +async function removeAssetsInCloud(assets: ExportAssetModel[]): Promise { const p = await getPolybaseDB() const res = await bluebird.map(_(assets).map('uuid').uniq().value(), async (uid: string) => { - console.log('removeAssetsInCloud', uid); - const records = await p.collection(RECORD_COLLECTION_NAME).where("uuid", "==", uid).get() if (records.data.length === 0) { return 0 @@ -317,12 +352,27 @@ async function removeAssetsInCloud(assets: AssetModel[]): Promise { } // return updated how many records -async function writeAssetsToDB(d: Database, assets: AssetModel[]): Promise { - const res = await saveModelsToDatabase(ASSETS_TABLE_NAME, _.map(assets, (obj) => _.omit(obj, "id"))) +async function writeAssetsToDB(d: Database, assets: ExportAssetModel[]): Promise { + const res = await saveModelsToDatabase(ASSETS_TABLE_NAME, _(assets).map(a => _(a).omit("id").omit("costPrice").value()).value()) + const f = _(assets).find(a => a.costPrice !== undefined) + if (!f) { + // not asset_price need to be saved + return res.length + } + const priceMOdels = _(assets).filter(a => a.costPrice !== undefined).map(a => ({ + uuid: a.uuid, + assetID: a.id, + symbol: a.symbol, + price: a.costPrice, + assetCreatedAt: a.createdAt, + updatedAt: new Date().toISOString(), + } as AssetPriceModel)).value() + + await saveModelsToDatabase(ASSETS_PRICE_TABLE_NAME, priceMOdels) return res.length } -async function updateLastSyncTime(d: Database, publicKey: string) { +async function markAsSynced(d: Database, publicKey: string) { await d.execute(`INSERT INTO ${CLOUD_SYNC_TABLE_NAME} (publicKey, updatedAt) VALUES (?, ?) ON CONFLICT(publicKey) DO UPDATE SET updatedAt = ?`, [publicKey, new Date().toISOString(), new Date().toISOString()]) } diff --git a/src/middlelayers/data.ts b/src/middlelayers/data.ts index fdebe5f..bf27082 100644 --- a/src/middlelayers/data.ts +++ b/src/middlelayers/data.ts @@ -12,8 +12,8 @@ import { ASSETS_PRICE_TABLE_NAME, ASSETS_TABLE_NAME, queryAllAssetPrices, queryH import _ from 'lodash' import { save, open } from "@tauri-apps/api/dialog" import { writeTextFile, readTextFile } from "@tauri-apps/api/fs" -import { AssetModel, AssetPriceModel, HistoricalData } from './types' -import { getDatabase, saveModelsToDatabase } from './database' +import { AssetPriceModel, ExportAssetModel, HistoricalData } from './types' +import { saveModelsToDatabase } from './database' import { exportConfigurationString, importRawConfiguration } from './configuration' type ExportData = { @@ -159,7 +159,7 @@ export async function importHistoricalData(): Promise { } // import historicalData from file -async function saveHistoricalDataAssets(assets: (AssetModel & { costPrice?: number })[]) { +async function saveHistoricalDataAssets(assets: ExportAssetModel[]) { const requiredKeys = ["uuid", "createdAt", "symbol", "amount", "value", "price"] _(assets).forEach((asset) => { _(requiredKeys).forEach(k => { @@ -173,10 +173,12 @@ async function saveHistoricalDataAssets(assets: (AssetModel & { costPrice?: numb // import asset prices const importedAssets = _(await queryHistoricalData(-1, false)).map(d => d.assets).flatten().value() + const importAssetsMap = _(importedAssets).mapKeys(a => `${a.uuid}/${a.symbol}/${a.wallet}`).value() + const assetPriceModels = _(assets).filter(a => a.costPrice !== undefined).map(a => { - console.log(a); - - const f = _(importedAssets).find(ia => ia.uuid === a.uuid && ia.symbol === a.symbol && ia.wallet === a.wallet) + const key = `${a.uuid}/${a.symbol}/${a.wallet}` + + const f = importAssetsMap[key] if (!f) { return } @@ -185,7 +187,8 @@ async function saveHistoricalDataAssets(assets: (AssetModel & { costPrice?: numb assetID: f.id, symbol: a.symbol, price: a.costPrice, - createdAt: a.createdAt + assetCreatedAt: a.createdAt, + updatedAt: new Date().toISOString(), } as AssetPriceModel }).compact().value() diff --git a/src/middlelayers/database.ts b/src/middlelayers/database.ts index c16729a..4256af6 100644 --- a/src/middlelayers/database.ts +++ b/src/middlelayers/database.ts @@ -2,8 +2,8 @@ import _ from "lodash" import Database from "tauri-plugin-sql-api" import { v4 as uuidv4 } from 'uuid' import { CoinModel } from './datafetch/types' -import { AssetPriceModel, AssetModel, WalletCoinUSD } from './types' -import { ASSETS_PRICE_TABLE_NAME, ASSETS_TABLE_NAME } from './charts' +import { AssetModel, WalletCoinUSD } from './types' +import { ASSETS_TABLE_NAME } from './charts' import md5 from 'md5' export const databaseName = "track3.db" diff --git a/src/middlelayers/types.d.ts b/src/middlelayers/types.d.ts index a6fc036..092740d 100644 --- a/src/middlelayers/types.d.ts +++ b/src/middlelayers/types.d.ts @@ -10,6 +10,9 @@ export type AssetModel = { wallet?: string } +// type when exporting assets to json or cloud database +export type ExportAssetModel = AssetModel & { costPrice?: number } + // asset_prices table // to record the cost price or sell price of each coins, can be updated by users export type AssetPriceModel = { @@ -23,7 +26,8 @@ export type AssetPriceModel = { price: number // createdAt in assets_v2 table - createdAt: string + assetCreatedAt: string + updatedAt: string } export type CloudAssetModel = { @@ -33,7 +37,7 @@ export type CloudAssetModel = { uuid: string - // json stringify from AssetModel[] + // json stringify from ExportAssetModel[] records: string createdAt: number @@ -169,7 +173,7 @@ export type HistoricalData = { id: string createdAt: string // costPrice only exists when exporting historical data - assets: (AssetModel & { costPrice?: number })[] + assets: ExportAssetModel[] total: number }