Skip to content

Commit

Permalink
fix: dont use missing/stale related asset data when generating the re…
Browse files Browse the repository at this point in the history 
…lated asset index (#8424)

* fix: dont use stale related asset data when generating the related asset index

* Revert "fix: dont use stale related asset data when generating the related asset index"

This reverts commit bd04c84.

* fix: dont use stale related asset data when generating the related asset index

* chore regenerate related asset index

* fix: dont choose a relatedAssetKey for an asset we dont have data for

* chore: regenerate relatedAssetIndex.json
  • Loading branch information
@woodenfurniture
woodenfurniture authored Dec 20, 2024
1 parent 9bef22e commit 31baf7b
Show file tree
Hide file tree
Showing 3 changed files with 147 additions and 369 deletions.
86 changes: 44 additions & 42 deletions scripts/generateAssetData/generateRelatedAssetIndex/generateRelatedAssetIndex.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,28 +123,20 @@ const getZerionRelatedAssetIds = async (
if (firstEntry === undefined) return

const implementations = firstEntry.attributes.implementations
const primaryImplementationId = firstEntry.id

const primaryImplementation = implementations?.find(
implementation => implementation.address === primaryImplementationId,
)

const relatedAssetKey = primaryImplementation
? zerionImplementationToMaybeAssetId(primaryImplementation)
: undefined

const relatedAssetIds = implementations
// Use all assetIds actually present in the dataset
const allRelatedAssetIds = implementations
?.map(zerionImplementationToMaybeAssetId)
.filter(isSome)
.filter(
relatedAssetId =>
relatedAssetId !== relatedAssetKey && assetData[relatedAssetId] !== undefined,
)
.filter(relatedAssetId => assetData[relatedAssetId] !== undefined)

if (!relatedAssetKey || !relatedAssetIds || relatedAssetIds.length === 0) {
if (!allRelatedAssetIds || allRelatedAssetIds.length <= 1) {
return
}

const relatedAssetKey = allRelatedAssetIds[0]
const relatedAssetIds = allRelatedAssetIds.filter(assetId => assetId !== relatedAssetKey)

return { relatedAssetIds, relatedAssetKey }
}

Expand All @@ -161,24 +153,20 @@ const getCoingeckoRelatedAssetIds = async (
const { data } = await axios.get<CoingeckoAssetDetails>(`${coingeckoBaseUrl}/coins/${coinUri}`)

const platforms = data.platforms
const primaryPlatform = Object.entries(data.platforms)[0]

const relatedAssetKey = primaryPlatform
? coingeckoPlatformDetailsToMaybeAssetId(primaryPlatform)
: undefined

const relatedAssetIds = Object.entries(platforms)
// Use all assetIds actually present in the dataset
const allRelatedAssetIds = Object.entries(platforms)
?.map(coingeckoPlatformDetailsToMaybeAssetId)
.filter(isSome)
.filter(
relatedAssetId =>
relatedAssetId !== relatedAssetKey && assetData[relatedAssetId] !== undefined,
)
.filter(relatedAssetId => assetData[relatedAssetId] !== undefined)

if (!relatedAssetKey || !relatedAssetIds || relatedAssetIds.length === 0) {
if (allRelatedAssetIds.length <= 1) {
return
}

const relatedAssetKey = allRelatedAssetIds[0]
const relatedAssetIds = allRelatedAssetIds.filter(assetId => assetId !== relatedAssetKey)

return { relatedAssetIds, relatedAssetKey }
}

Expand Down Expand Up @@ -222,9 +210,11 @@ const processRelatedAssetIds = async (
})

const zerionRelatedAssetsResult = await getZerionRelatedAssetIds(
// DO NOT REMOVE ME - reuse the relatedAssetKey if found with coingecko fetch. cg may not have all related assetIds for a given asset, and Zerion may not have
// any at all the same asset. e.g USDC.SOL is found on Coingecko but with only USDC.ETH as a relatedAssetId, and is not present at all under the USDC.SOL umbrella in Zerion.
// Using the primary implementation ensures we use a reliable identifier for the related assets, not a more obscure one.
// DO NOT REMOVE ME - reuse the relatedAssetKey if found with coingecko fetch. cg may not have
// all related assetIds for a given asset, and Zerion may not have any at all the same asset.
// e.g USDC.SOL is found on Coingecko but with only USDC.ETH as a relatedAssetId, and is not
// present at all under the USDC.SOL umbrella in Zerion. Using the primary implementation
// ensures we use a reliable identifier for the related assets, not a more obscure one.
coingeckoRelatedAssetsResult?.relatedAssetKey ?? assetId,
assetData,
)
Expand Down Expand Up @@ -299,14 +289,31 @@ export const generateRelatedAssetIndex = async (rebuildAll: boolean = false) =>
const relatedAssetIndex: Record<AssetId, AssetId[]> = JSON.parse(
await fs.promises.readFile(relatedAssetIndexPath, 'utf8'),
)
const assetDataWithRelatedAssetKeys: Record<AssetId, PartialFields<Asset, 'relatedAssetKey'>> = {
...generatedAssetData,
}

if (rebuildAll) {
// Remove stale related asset data from the assetData where:
// a) rebuildAll is set
// b) the primary related asset no longer exists in the dataset
Object.values(generatedAssetData).forEach(asset => {
const relatedAssetKey = asset.relatedAssetKey

if (!relatedAssetKey) return

const primaryRelatedAsset = generatedAssetData[relatedAssetKey]

// remove relatedAssetKey from the existing data to ensure the related assets get updated
Object.values(assetDataWithRelatedAssetKeys).forEach(asset => delete asset.relatedAssetKey)
}
if (rebuildAll || primaryRelatedAsset === undefined) {
delete relatedAssetIndex[relatedAssetKey]
delete asset.relatedAssetKey
}
})

// Remove stale related asset data from the relatedAssetIndex where:
// a) a related assetId no longer exists in the dataset
Object.entries(relatedAssetIndex).forEach(([relatedAssetKey, relatedAssetIds]) => {
relatedAssetIndex[relatedAssetKey] = relatedAssetIds.filter(
assetId => generatedAssetData[assetId] !== undefined,
)
})

const { throttle, clear: clearThrottleInterval } = createThrottle({
capacity: 50, // Reduced initial capacity to allow for a burst but not too high
Expand All @@ -319,12 +326,7 @@ export const generateRelatedAssetIndex = async (rebuildAll: boolean = false) =>
console.log(`Processing chunk: ${i} of ${chunks.length}`)
await Promise.all(
batch.map(async assetId => {
await processRelatedAssetIds(
assetId,
assetDataWithRelatedAssetKeys,
relatedAssetIndex,
throttle,
)
await processRelatedAssetIds(assetId, generatedAssetData, relatedAssetIndex, throttle)
return
}),
)
Expand All @@ -335,7 +337,7 @@ export const generateRelatedAssetIndex = async (rebuildAll: boolean = false) =>
await fs.promises.writeFile(
generatedAssetsPath,
// beautify the file for github diff.
JSON.stringify(assetDataWithRelatedAssetKeys, null, 2),
JSON.stringify(generatedAssetData, null, 2),
)

await fs.promises.writeFile(
Expand Down
38 changes: 19 additions & 19 deletions src/lib/asset-service/service/generatedAssetData.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9977,7 +9977,7 @@
"explorer": "https://etherscan.io",
"explorerAddressLink": "https://etherscan.io/address/",
"explorerTxLink": "https://etherscan.io/tx/",
"relatedAssetKey": "eip155:56/bep20:0x16594930d16f3970e1a4317c6016555cb2e7b7fc"
"relatedAssetKey": null
},
"eip155:1/erc20:0x167478921b907422f8e88b43c4af2b8bea278d3a": {
"explorer": "https://etherscan.io",
Expand Down Expand Up @@ -23608,7 +23608,7 @@
"explorer": "https://etherscan.io",
"explorerAddressLink": "https://etherscan.io/address/",
"explorerTxLink": "https://etherscan.io/tx/",
"relatedAssetKey": "eip155:137/erc20:0x7d4d68f18d1be3410ab8d827fb7ebc690f938d2d"
"relatedAssetKey": null
},
"eip155:1/erc20:0x37a2f8701856a78de92dbe35df2200c355eae090": {
"assetId": "eip155:1/erc20:0x37a2f8701856a78de92dbe35df2200c355eae090",
Expand Down Expand Up @@ -46195,7 +46195,7 @@
"explorer": "https://etherscan.io",
"explorerAddressLink": "https://etherscan.io/address/",
"explorerTxLink": "https://etherscan.io/tx/",
"relatedAssetKey": "eip155:42161/erc20:0x6c249b6f6492864d914361308601a7abb32e68f8"
"relatedAssetKey": "eip155:43114/erc20:0xb418417374fca27bb54169d3c777492e6fe17ee7"
},
"eip155:1/erc20:0x6c280db098db673d30d5b34ec04b6387185d3620": {
"explorer": "https://etherscan.io",
Expand Down Expand Up @@ -94078,7 +94078,7 @@
"explorer": "https://etherscan.io",
"explorerAddressLink": "https://etherscan.io/address/",
"explorerTxLink": "https://etherscan.io/tx/",
"relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
"relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
},
"eip155:1/erc20:0xde654f497a563dd7a121c176a125dd2f11f13a83": {
"assetId": "eip155:1/erc20:0xde654f497a563dd7a121c176a125dd2f11f13a83",
Expand Down Expand Up @@ -123808,7 +123808,7 @@
"explorer": "https://polygonscan.com/",
"explorerAddressLink": "https://polygonscan.com/address/",
"explorerTxLink": "https://polygonscan.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0x9d0b65a76274645b29e4cc41b8f23081fa09f4a3"
"relatedAssetKey": "eip155:56/bep20:0x7bc75e291e656e8658d66be1cc8154a3769a35dd"
},
"eip155:137/erc20:0x7f792db54b0e580cdc755178443f0430cf799aca": {
"assetId": "eip155:137/erc20:0x7f792db54b0e580cdc755178443f0430cf799aca",
Expand Down Expand Up @@ -130082,7 +130082,7 @@
"explorer": "https://polygonscan.com/",
"explorerAddressLink": "https://polygonscan.com/address/",
"explorerTxLink": "https://polygonscan.com/tx/",
"relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
"relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
},
"eip155:137/erc20:0xdf7837de1f2fa4631d716cf2502f8b230f1dcc32": {
"assetId": "eip155:137/erc20:0xdf7837de1f2fa4631d716cf2502f8b230f1dcc32",
Expand Down Expand Up @@ -146484,7 +146484,7 @@
"explorer": "https://arbiscan.io",
"explorerAddressLink": "https://arbiscan.io/address/",
"explorerTxLink": "https://arbiscan.io/tx/",
"relatedAssetKey": "eip155:1/erc20:0xb4371da53140417cbb3362055374b10d97e420bb"
"relatedAssetKey": "eip155:42161/erc20:0xf763fa322dc58dee588252fafee5f448e863b633"
},
"eip155:42161/erc20:0xf7693c6fd9a7172d537fa75d133d309501cbd657": {
"assetId": "eip155:42161/erc20:0xf7693c6fd9a7172d537fa75d133d309501cbd657",
Expand Down Expand Up @@ -146549,7 +146549,7 @@
"explorer": "https://arbiscan.io",
"explorerAddressLink": "https://arbiscan.io/address/",
"explorerTxLink": "https://arbiscan.io/tx/",
"relatedAssetKey": "eip155:1/erc20:0xdd50c053c096cb04a3e3362e2b622529ec5f2e8a"
"relatedAssetKey": null
},
"eip155:42161/erc20:0xf890360473c12d8015da8dbf7af11da87337a065": {
"explorer": "https://arbiscan.io",
Expand Down Expand Up @@ -152692,7 +152692,7 @@
"explorer": "https://snowtrace.dev",
"explorerAddressLink": "https://snowtrace.dev/address/",
"explorerTxLink": "https://snowtrace.dev/tx/",
"relatedAssetKey": "eip155:42161/erc20:0x6c249b6f6492864d914361308601a7abb32e68f8"
"relatedAssetKey": "eip155:43114/erc20:0xb418417374fca27bb54169d3c777492e6fe17ee7"
},
"eip155:43114/erc20:0xb44a9b6905af7c801311e8f4e76932ee959c663c": {
"assetId": "eip155:43114/erc20:0xb44a9b6905af7c801311e8f4e76932ee959c663c",
Expand Down Expand Up @@ -166312,7 +166312,7 @@
"explorer": "https://bscscan.com",
"explorerAddressLink": "https://bscscan.com/address/",
"explorerTxLink": "https://bscscan.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0xcc1a8bd438bebc4b2a885a34475bb974f2124317"
"relatedAssetKey": null
},
"eip155:56/bep20:0x366d71ab095735b7dae83ce2b82d5262ef655f10": {
"assetId": "eip155:56/bep20:0x366d71ab095735b7dae83ce2b82d5262ef655f10",
Expand Down Expand Up @@ -166882,7 +166882,7 @@
"explorer": "https://bscscan.com",
"explorerAddressLink": "https://bscscan.com/address/",
"explorerTxLink": "https://bscscan.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0xd9ec3ff1f8be459bb9369b4e79e9ebcf7141c093"
"relatedAssetKey": null
},
"eip155:56/bep20:0x39bff8613defd221b0410ed3d4e5c07512d55f2d": {
"assetId": "eip155:56/bep20:0x39bff8613defd221b0410ed3d4e5c07512d55f2d",
Expand Down Expand Up @@ -180795,7 +180795,7 @@
"explorer": "https://bscscan.com",
"explorerAddressLink": "https://bscscan.com/address/",
"explorerTxLink": "https://bscscan.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0x9d0b65a76274645b29e4cc41b8f23081fa09f4a3"
"relatedAssetKey": "eip155:56/bep20:0x7bc75e291e656e8658d66be1cc8154a3769a35dd"
},
"eip155:56/bep20:0x7bd6fabd64813c48545c9c0e312a0099d9be2540": {
"assetId": "eip155:56/bep20:0x7bd6fabd64813c48545c9c0e312a0099d9be2540",
Expand Down Expand Up @@ -192496,7 +192496,7 @@
"explorer": "https://bscscan.com",
"explorerAddressLink": "https://bscscan.com/address/",
"explorerTxLink": "https://bscscan.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0x9fc8f0ca1668e87294941b7f627e9c15ea06b459"
"relatedAssetKey": null
},
"eip155:56/bep20:0xb350aebaedb1ed3269b0e25d5e593a9bb4b9f9d5": {
"assetId": "eip155:56/bep20:0xb350aebaedb1ed3269b0e25d5e593a9bb4b9f9d5",
Expand Down Expand Up @@ -195469,7 +195469,7 @@
"explorer": "https://bscscan.com",
"explorerAddressLink": "https://bscscan.com/address/",
"explorerTxLink": "https://bscscan.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0xb4371da53140417cbb3362055374b10d97e420bb"
"relatedAssetKey": "eip155:42161/erc20:0xf763fa322dc58dee588252fafee5f448e863b633"
},
"eip155:56/bep20:0xc0eff7749b125444953ef89682201fb8c6a917cd": {
"assetId": "eip155:56/bep20:0xc0eff7749b125444953ef89682201fb8c6a917cd",
Expand Down Expand Up @@ -201045,7 +201045,7 @@
"explorer": "https://bscscan.com",
"explorerAddressLink": "https://bscscan.com/address/",
"explorerTxLink": "https://bscscan.com/tx/",
"relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
"relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
},
"eip155:56/bep20:0xde619a9e0eeeaa9f8cd39522ed788234837f3b26": {
"assetId": "eip155:56/bep20:0xde619a9e0eeeaa9f8cd39522ed788234837f3b26",
Expand Down Expand Up @@ -223429,7 +223429,7 @@
"explorer": "https://basescan.org",
"explorerAddressLink": "https://basescan.org/address/",
"explorerTxLink": "https://basescan.org/tx/",
"relatedAssetKey": "eip155:1/erc20:0xacdf0dba4b9839b96221a8487e9ca660a48212be"
"relatedAssetKey": null
},
"eip155:8453/erc20:0xcd2f22236dd9dfe2356d7c543161d4d260fd9bcb": {
"assetId": "eip155:8453/erc20:0xcd2f22236dd9dfe2356d7c543161d4d260fd9bcb",
Expand Down Expand Up @@ -224603,7 +224603,7 @@
"explorer": "https://basescan.org",
"explorerAddressLink": "https://basescan.org/address/",
"explorerTxLink": "https://basescan.org/tx/",
"relatedAssetKey": "eip155:42161/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
"relatedAssetKey": "eip155:1/erc20:0xde5ed76e7c05ec5e4572cfc88d1acea165109e44"
},
"eip155:8453/erc20:0xde66c35e01ed8e619bf092352338ef94f2327337": {
"explorer": "https://basescan.org",
Expand Down Expand Up @@ -231732,7 +231732,7 @@
"explorer": "https://explorer.solana.com",
"explorerAddressLink": "https://explorer.solana.com/address/",
"explorerTxLink": "https://explorer.solana.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0xdd50c053c096cb04a3e3362e2b622529ec5f2e8a"
"relatedAssetKey": null
},
"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:4MshgHvWGvxDs8mtFqPGKC8kX6kuhniWSYPguBb1p1bh": {
"assetId": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:4MshgHvWGvxDs8mtFqPGKC8kX6kuhniWSYPguBb1p1bh",
Expand Down Expand Up @@ -253988,7 +253988,7 @@
"explorer": "https://explorer.solana.com",
"explorerAddressLink": "https://explorer.solana.com/address/",
"explorerTxLink": "https://explorer.solana.com/tx/",
"relatedAssetKey": "eip155:1/erc20:0xb6667b04cb61aa16b59617f90ffa068722cf21da"
"relatedAssetKey": "eip155:1/erc20:0x7cd017ca5ddb86861fa983a34b5f495c6f898c41"
},
"solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:Ed1yisBEAo8UXToSswvVFgzyJKpF48HEdq5kvz2zpump": {
"assetId": "solana:5eykt4UsFv8P8NJdTREpY1vzqKqZKvdp/token:Ed1yisBEAo8UXToSswvVFgzyJKpF48HEdq5kvz2zpump",
Expand Down
Loading

0 comments on commit 31baf7b

Please sign in to comment.