From 8b0fcdff1c0aaae0835601d4c5655ff10d1aea59 Mon Sep 17 00:00:00 2001 From: Karl Prieb Date: Fri, 27 Sep 2024 15:33:30 -0300 Subject: [PATCH] fix error handling and upgrade duckdb --- package.json | 2 +- src/workers/parquet-exporter.ts | 53 +++++++++++++++++++++------------ yarn.lock | 18 +++++------ 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/package.json b/package.json index b35efd18..3259c07d 100644 --- a/package.json +++ b/package.json @@ -21,7 +21,7 @@ "cors": "^2.8.5", "crypto": "^1.0.1", "dotenv": "^16.3.1", - "duckdb-async": "^1.0.0", + "duckdb-async": "^1.1.0", "express": "^4.18.1", "express-async-handler": "^1.2.0", "express-openapi-validator": "^5.1.2", diff --git a/src/workers/parquet-exporter.ts b/src/workers/parquet-exporter.ts index a6f71c2b..5b07de2c 100644 --- a/src/workers/parquet-exporter.ts +++ b/src/workers/parquet-exporter.ts @@ -121,8 +121,10 @@ export class ParquetExporter { try { await this.db.exec(`INSERT INTO blocks ${query}`); log.info('Blocks inserted into DuckDB'); - } catch (error) { - throw `Error importing blocks: ${error}`; + } catch (error: any) { + const newError = new Error('Error importing blocks'); + newError.stack = error.stack; + throw newError; } } @@ -156,9 +158,9 @@ export class ParquetExporter { NULL AS data_offset, NULL AS owner_offset, NULL AS owner_size, - CASE - WHEN octet_length(w.public_modulus) <= 64 THEN w.public_modulus - ELSE NULL + CASE + WHEN octet_length(w.public_modulus) <= 64 THEN w.public_modulus + ELSE NULL END AS owner, NULL AS signature_offset, NULL AS signature_size, @@ -176,8 +178,10 @@ export class ParquetExporter { try { await this.db.exec(`INSERT INTO transactions ${query}`); log.info('Transactions inserted into DuckDB'); - } catch (error) { - throw `Error importing transactions: ${error}`; + } catch (error: any) { + const newError = new Error('Error importing transactions'); + newError.stack = error.stack; + throw newError; } } @@ -211,9 +215,9 @@ export class ParquetExporter { sdi.data_offset, sdi.owner_offset, sdi.owner_size, - CASE - WHEN octet_length(w.public_modulus) <= 64 THEN w.public_modulus - ELSE NULL + CASE + WHEN octet_length(w.public_modulus) <= 64 THEN w.public_modulus + ELSE NULL END AS owner, sdi.signature_offset, sdi.signature_size, @@ -231,8 +235,10 @@ export class ParquetExporter { try { await this.db.exec(`INSERT INTO transactions ${query}`); log.info('Data items inserted into DuckDB'); - } catch (error) { - throw `Error importing data items: ${error}`; + } catch (error: any) { + const newError = new Error('Error importing data items'); + newError.stack = error.stack; + throw newError; } } @@ -268,8 +274,10 @@ export class ParquetExporter { try { await this.db.exec(`INSERT INTO tags ${query}`); log.info('Transaction tags inserted into DuckDB'); - } catch (error) { - throw `Error importing transaction tags: ${error}`; + } catch (error: any) { + const newError = new Error('Error importing transaction tags'); + newError.stack = error.stack; + throw newError; } } @@ -307,8 +315,10 @@ export class ParquetExporter { try { await this.db.exec(`INSERT INTO tags ${query}`); log.info('Data item tags inserted into DuckDB'); - } catch (error) { - throw `Error importing data item tags: ${error}`; + } catch (error: any) { + const newError = new Error('Error importing data item tags'); + newError.stack = error.stack; + throw newError; } } @@ -361,8 +371,12 @@ export class ParquetExporter { minHeight = height + 1n; rowCount = 0n; - } catch (error) { - throw `Error exporting Parquet file ${fileName}: ${error}`; + } catch (error: any) { + const newError = new Error( + `Error exporting Parquet file ${fileName}`, + ); + newError.stack = error.stack; + throw newError; } } } @@ -438,7 +452,7 @@ export class ParquetExporter { log.info('Parquet export complete'); } catch (error) { - log.error('Error exporting Parquet files:', { + log.error('Error exporting Parquet files', { error: error instanceof Error ? error.message : String(error), stack: error instanceof Error ? error.stack : undefined, }); @@ -448,6 +462,7 @@ export class ParquetExporter { // Delete the duckdb file try { rmSync(this.duckDbPath, { recursive: true, force: true }); + rmSync(`${this.duckDbPath}.wal`, { force: true }); } catch (error) { log.error(`Error deleting duckdb file ${this.duckDbPath}:`, error); } diff --git a/yarn.lock b/yarn.lock index fab12a87..c437e25f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4211,17 +4211,17 @@ dotenv@^16.3.1: resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg== -duckdb-async@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/duckdb-async/-/duckdb-async-1.0.0.tgz#0dd51f03404cec4a1ab0e8fa5941c756479122f4" - integrity sha512-1UTAZ2LVk9s4NPMiLnB9L2DI716I/LqtDYL8I7RFEP981/fo0SaqcRQFwtgPPz8MyEPcx8QPPmAWEsal5q68xQ== +duckdb-async@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/duckdb-async/-/duckdb-async-1.1.0.tgz#d01b4bcefb47138cbfe4216b20f935b74c0a444f" + integrity sha512-UQybqQMn0KGZkmCMtDq7Eg2z5LICPR0pHTihdvzbVwhNJHknnVIMI8U21Cd0zbwFsewQVfvYwZRxyDxb0F5F7g== dependencies: - duckdb "1.0.0" + duckdb "1.1.0" -duckdb@1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/duckdb/-/duckdb-1.0.0.tgz#de81d9b93311bb816901b582de27ae75fa0e20e2" - integrity sha512-QwpcIeN42A2lL19S70mUFibZgRcEcZpCkKHdzDgecHaYZhXj3+1i2cxSDyAk/RVg5CYnqj1Dp4jAuN4cc80udA== +duckdb@1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/duckdb/-/duckdb-1.1.0.tgz#d3897d4e75ff40476471936a6c39c42845977f69" + integrity sha512-T814+D3H8y1bCidJP5QxnIoE0UC6ETbEctFPyIEkCt1byPioKpiuNy8ieY2GqG5ZXTMLkaynNo1JZ7pev+B0Rg== dependencies: "@mapbox/node-pre-gyp" "^1.0.0" node-addon-api "^7.0.0"