From 7ebcf7db970606a8e6e5b06e533e120f8cd72db7 Mon Sep 17 00:00:00 2001 From: zederer Date: Thu, 26 Sep 2024 10:27:33 +0800 Subject: [PATCH] =?UTF-8?q?=E9=BB=98=E8=AE=A4=E8=AF=BB=E5=8F=96URL?= =?UTF-8?q?=E5=AF=B9=E5=BA=94=E7=9A=84s3=E6=A1=B6=E5=8F=8A=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E4=B8=8A=E7=9A=84=E6=96=87=E4=BB=B6=EF=BC=8C=20?= =?UTF-8?q?=E5=90=8E=E7=BB=AD=E6=8A=8A=E4=B8=8A=E4=BC=A0=E4=B9=9F=E5=AF=B9?= =?UTF-8?q?=E6=8E=A5s3=E3=80=82=20=E5=BD=93=E5=89=8D=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=9A=84minio=20=E6=9B=BF=E4=BB=A3=20s3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 17 ++ docker-compose.yml | 4 + package-lock.json | 297 ++++++++++++++++++++++-- package.json | 9 +- src/app/components/GraphDataHandler.tsx | 68 +++++- src/app/components/GraphViewer.tsx | 2 +- src/app/hooks/useFileHandler.ts | 12 +- src/app/layout/App.tsx | 10 +- src/app/utils/parquet-utils.ts | 11 +- 9 files changed, 394 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index bcb4087..1fd164c 100644 --- a/README.md +++ b/README.md @@ -46,3 +46,20 @@ The logic for creating relationships for text units, documents, communities, and | Text Unit | `HAS_COVARIATE` | Covariate | | Community | `HAS_FINDING` | Finding | | Entity | `IN_COMMUNITY` | Community | + + +## + + +```bash + +docker compose build + +docker compose up -d + +images=$(grep 'image:' docker-compose.yml | awk '{print $2}') + +docker save -o graphrag-visualizer.tar $images + + +``` \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index aa3a616..b2972d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,6 +10,10 @@ services: - "3004:3004" environment: NODE_ENV: development + MINIO_ACCESS_KEY: minio // 设置环境变量或直接传值 + MINIO_SECRET_KEY: minio@123 + MINIO_ENDPOINT: http://192.168.1.182:9080 // Minio 的 S3 兼容地址 + volumes: - /opt/dintal/work/graphrag-visualizer:/app/graphrag-visualizer/data networks: diff --git a/package-lock.json b/package-lock.json index cf2f24e..ec2407b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@types/node": "^16.18.104", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", + "aws-sdk": "^2.1691.0", "axios": "^1.7.2", "fuse.js": "^7.0.0", "hyparquet": "^1.1.0", @@ -6188,6 +6189,48 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-sdk": { + "version": "2.1691.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1691.0.tgz", + "integrity": "sha512-/F2YC+DlsY3UBM2Bdnh5RLHOPNibS/+IcjUuhP8XuctyrN+MlL+fWDAiela32LTDk7hMy4rx8MTgvbJ+0blO5g==", + "hasInstallScript": true, + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "util": "^0.12.4", + "uuid": "8.0.0", + "xml2js": "0.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-sdk/node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/aws-sdk/node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" + }, + "node_modules/aws-sdk/node_modules/uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/axe-core": { "version": "4.10.0", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.0.tgz", @@ -6537,6 +6580,25 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -6739,12 +6801,27 @@ "node-int64": "^0.4.0" } }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", "license": "MIT" }, + "node_modules/buffer/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, "node_modules/builtin-modules": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", @@ -8323,15 +8400,6 @@ "tslib": "^2.0.3" } }, - "node_modules/dotenv": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", - "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=10" - } - }, "node_modules/dotenv-expand": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz", @@ -11008,6 +11076,11 @@ "node": ">=4" } }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -13901,6 +13974,14 @@ "jiti": "bin/jiti.js" } }, + "node_modules/jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -16769,6 +16850,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", @@ -17167,11 +17257,6 @@ "react": "*" } }, - "node_modules/react-ga4": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/react-ga4/-/react-ga4-2.1.0.tgz", - "integrity": "sha512-ZKS7PGNFqqMd3PJ6+C2Jtz/o1iU9ggiy8Y8nUeksgVuvNISbmrQtJiZNvC/TjDsqD0QlU5Wkgs7i+w9+OjHhhQ==" - }, "node_modules/react-i18next": { "version": "12.3.1", "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-12.3.1.tgz", @@ -17295,6 +17380,14 @@ } } }, + "node_modules/react-scripts/node_modules/dotenv": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", + "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==", + "engines": { + "node": ">=10" + } + }, "node_modules/react-table": { "version": "7.8.0", "resolved": "https://registry.npmjs.org/react-table/-/react-table-7.8.0.tgz", @@ -19798,6 +19891,15 @@ "punycode": "^2.1.0" } }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, "node_modules/url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -19808,6 +19910,23 @@ "requires-port": "^1.0.0" } }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" + }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -20831,6 +20950,26 @@ "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==", "license": "Apache-2.0" }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", @@ -24837,6 +24976,40 @@ "possible-typed-array-names": "^1.0.0" } }, + "aws-sdk": { + "version": "2.1691.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1691.0.tgz", + "integrity": "sha512-/F2YC+DlsY3UBM2Bdnh5RLHOPNibS/+IcjUuhP8XuctyrN+MlL+fWDAiela32LTDk7hMy4rx8MTgvbJ+0blO5g==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "util": "^0.12.4", + "uuid": "8.0.0", + "xml2js": "0.6.2" + }, + "dependencies": { + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" + }, + "uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==" + } + } + }, "axe-core": { "version": "4.10.0", "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.10.0.tgz", @@ -25089,6 +25262,11 @@ "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" + }, "batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -25228,6 +25406,23 @@ "node-int64": "^0.4.0" } }, + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + }, + "dependencies": { + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + } + } + }, "buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -26276,11 +26471,6 @@ "tslib": "^2.0.3" } }, - "dotenv": { - "version": "10.0.0", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", - "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==" - }, "dotenv-expand": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz", @@ -28091,6 +28281,11 @@ "harmony-reflect": "^1.4.6" } }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -30004,6 +30199,11 @@ "resolved": "https://registry.npmjs.org/jiti/-/jiti-1.21.6.tgz", "integrity": "sha512-2yTgeWTWzMWkHu6Jp9NKgePDaYHbntiwvYuuJLbbN9vl7DC9DvXKOB2BC3ZZ92D3cvV/aflH0osDfwpHepQ53w==" }, + "jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==" + }, "js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -31764,6 +31964,11 @@ "side-channel": "^1.0.6" } }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==" + }, "querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", @@ -32027,11 +32232,6 @@ "react-kapsule": "2" } }, - "react-ga4": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/react-ga4/-/react-ga4-2.1.0.tgz", - "integrity": "sha512-ZKS7PGNFqqMd3PJ6+C2Jtz/o1iU9ggiy8Y8nUeksgVuvNISbmrQtJiZNvC/TjDsqD0QlU5Wkgs7i+w9+OjHhhQ==" - }, "react-i18next": { "version": "12.3.1", "resolved": "https://registry.npmjs.org/react-i18next/-/react-i18next-12.3.1.tgz", @@ -32113,6 +32313,13 @@ "webpack-dev-server": "^4.6.0", "webpack-manifest-plugin": "^4.0.2", "workbox-webpack-plugin": "^6.4.1" + }, + "dependencies": { + "dotenv": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-10.0.0.tgz", + "integrity": "sha512-rlBi9d8jpv9Sf1klPjNfFAuWDjKLwTIJJ/VxtoTwIR6hnZxcEOQCZg2oIL3MWBYw5GpUDKOEnND7LXTbIpQ03Q==" + } } }, "react-table": { @@ -33808,6 +34015,22 @@ "punycode": "^2.1.0" } }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + }, + "dependencies": { + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" + } + } + }, "url-parse": { "version": "1.5.10", "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", @@ -33817,6 +34040,18 @@ "requires-port": "^1.0.0" } }, + "util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "requires": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -34552,6 +34787,20 @@ "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz", "integrity": "sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==" }, + "xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + } + }, + "xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==" + }, "xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", diff --git a/package.json b/package.json index 1c9bb0e..b44fd75 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,11 @@ "@types/node": "^16.18.104", "@types/react": "^18.3.3", "@types/react-dom": "^18.3.0", + "aws-sdk": "^2.1691.0", "axios": "^1.7.2", "fuse.js": "^7.0.0", "hyparquet": "^1.1.0", + "i18next": "^22.4.13", "material-react-table": "^2.13.1", "react": "^18.3.1", "react-app-rewired": "^2.2.1", @@ -25,19 +27,18 @@ "react-dropzone": "^14.2.3", "react-force-graph-2d": "^1.25.5", "react-force-graph-3d": "^1.24.3", + "react-i18next": "^12.2.0", "react-scripts": "^5.0.1", "react-table": "^7.8.0", "three": "^0.167.1", "three-spritetext": "^1.8.2", "typescript": "^4.9.5", - "web-vitals": "^2.1.4", - "i18next": "^22.4.13", - "react-i18next": "^12.2.0" + "web-vitals": "^2.1.4" }, "scripts": { "predeploy": "npm run build", "deploy": "gh-pages -d build", - "start": "PORT=3004 react-app-rewired start", + "start": "react-app-rewired start", "build": "react-app-rewired build", "test": "react-app-rewired test", "eject": "react-scripts eject" diff --git a/src/app/components/GraphDataHandler.tsx b/src/app/components/GraphDataHandler.tsx index 5a52d3f..f0b5b73 100644 --- a/src/app/components/GraphDataHandler.tsx +++ b/src/app/components/GraphDataHandler.tsx @@ -8,8 +8,19 @@ import useFileHandler from "../hooks/useFileHandler"; import useGraphData from "../hooks/useGraphData"; import DataTableContainer from "./DataTableContainer"; import { useTranslation } from 'react-i18next'; +import AWS from 'aws-sdk'; -const GraphDataHandler: React.FC = () => { +// 配置 Minio 客户端 +const s3 = new AWS.S3({ + accessKeyId: process.env.REACT_APP_MINIO_ACCESS_KEY, // 设置环境变量或直接传值 + secretAccessKey: process.env.REACT_APP_MINIO_SECRET_KEY, + endpoint: process.env.REACT_APP_MINIO_ENDPOINT, // Minio 的 S3 兼容地址 + s3ForcePathStyle: true, + signatureVersion: 'v4', +}); + + +const GraphDataHandler: React.FC<{ categary: string|null }> = ({ categary }) => { const { t } = useTranslation('layout'); const [tabIndex, setTabIndex] = useState(0); const [graphType, setGraphType] = useState<"2d" | "3d">("2d"); @@ -64,6 +75,61 @@ const GraphDataHandler: React.FC = () => { } }, [entities]); + useEffect(() => { + if (categary) { + // 从 Minio 加载数据 + loadParquetFilesFromMinio(categary); + } + }, [categary]); + + // 加载 Parquet 文件 + async function loadParquetFilesFromMinio(categary: string) { + const bucketName = process.env.REACT_APP_MINIO_BUCKET || 'graphrag' + try { + const params = { + Bucket: bucketName, + Prefix: categary, // categary作为目录路径前缀 + }; + + const objects = await s3.listObjectsV2(params).promise(); + const buffers: { name: string, buffer: ArrayBuffer }[] = []; + for (const object of objects.Contents || []) { + if (object.Key && object.Key.endsWith(".parquet")) { + const getObjectParams = { + Bucket: bucketName, + Key: object.Key!, + }; + + const data = await s3.getObject(getObjectParams).promise(); + + let arrayBuffer; + if (data.Body instanceof Uint8Array) { + arrayBuffer = data.Body.buffer; // 转换为 ArrayBuffer + } else if (data.Body instanceof ArrayBuffer) { + arrayBuffer = data.Body; // 已经是 ArrayBuffer + } else { + console.error("Unknown data type from S3"); + continue; // 跳过未知类型 + } + + const lastSlashIndex = object.Key.lastIndexOf("/"); + const fileName = lastSlashIndex !== -1 + ? object.Key.slice(lastSlashIndex + 1) // 仅保留文件名部分 + : object.Key; // 如果没有斜杠，则整个就是文件名 + // 将数据添加到数组中 + buffers.push({ + name: fileName, + buffer: arrayBuffer, + }); + } + } + await handleFilesRead(buffers); + + } catch (error) { + console.error('Error loading parquet files from Minio:', error); + } + } + const onDrop = (acceptedFiles: File[]) => { handleFilesRead(acceptedFiles); }; diff --git a/src/app/components/GraphViewer.tsx b/src/app/components/GraphViewer.tsx index fdacc73..6d7b5ef 100644 --- a/src/app/components/GraphViewer.tsx +++ b/src/app/components/GraphViewer.tsx @@ -106,7 +106,7 @@ const GraphViewer: React.FC = ({ const [linkedRelationships, setLinkedRelationships] = useState( [] ); - const [showLabels, setShowLabels] = useState(false); + const [showLabels, setShowLabels] = useState(true); const [showLinkLabels, setShowLinkLabels] = useState(false); const [showHighlight, setShowHighlight] = useState(true); const graphRef = useRef(); diff --git a/src/app/hooks/useFileHandler.ts b/src/app/hooks/useFileHandler.ts index 5f5cf32..69c09fd 100644 --- a/src/app/hooks/useFileHandler.ts +++ b/src/app/hooks/useFileHandler.ts @@ -6,7 +6,7 @@ import { TextUnit } from "../models/text-unit"; import { Community } from "../models/community"; import { CommunityReport } from "../models/community-report"; import { Covariate } from "../models/covariate"; -import { readParquetFile } from "../utils/parquet-utils"; +import { readParquetFile, readParquetBuffer } from "../utils/parquet-utils"; const fileSchemas: { [key: string]: string } = { "create_final_entities.parquet": "entity", @@ -27,7 +27,7 @@ const useFileHandler = () => { const [covariates, setCovariates] = useState([]); const [communityReports, setCommunityReports] = useState([]); - const handleFilesRead = async (files: File[]) => { + const handleFilesRead = async (files: File[] | { name: string, buffer: ArrayBuffer}[]) => { const entitiesArray: Entity[][] = []; const relationshipsArray: Relationship[][] = []; const documentsArray: Document[][] = []; @@ -35,10 +35,14 @@ const useFileHandler = () => { const communitiesArray: Community[][] = []; const communityReportsArray: CommunityReport[][] = []; const covariatesArray: Covariate[][] = []; - for (const file of files) { const schema = fileSchemas[file.name]; - const data = await readParquetFile(file, schema); + let data = []; + if (file instanceof File) { + data = await readParquetFile(file, schema); + } else if (file.buffer instanceof ArrayBuffer) { + data = await readParquetBuffer(file.buffer, schema); + } if (schema === "entity") { entitiesArray.push(data); diff --git a/src/app/layout/App.tsx b/src/app/layout/App.tsx index c3b4174..f59e85f 100644 --- a/src/app/layout/App.tsx +++ b/src/app/layout/App.tsx @@ -20,6 +20,7 @@ import { useTranslation } from 'react-i18next'; const App: React.FC = () => { const { t } = useTranslation('layout'); const [darkMode, setDarkMode] = useState(true); + const [categary, setCategary] = useState(null); const paletteType = darkMode ? "dark" : "light"; const theme = createTheme({ @@ -57,6 +58,13 @@ const App: React.FC = () => { useEffect(() => { const currentTheme = localStorage.getItem("theme"); setDarkMode(currentTheme === "dark"); + + // 获取 URL 中的 categary 参数 + const queryParams = new URLSearchParams(window?.location?.search); + const categaryParam = queryParams.get('categary'); + if (categaryParam) { + setCategary(categaryParam); + } }, []); return ( @@ -100,7 +108,7 @@ const App: React.FC = () => { )} - + ); diff --git a/src/app/utils/parquet-utils.ts b/src/app/utils/parquet-utils.ts index 2225a72..63032f5 100644 --- a/src/app/utils/parquet-utils.ts +++ b/src/app/utils/parquet-utils.ts @@ -31,10 +31,19 @@ const parseValue = (value: any, type: 'number' | 'bigint'): any => { } return type === 'bigint' ? BigInt(value) : Number(value); }; - export const readParquetFile = async (file: File, schema?: string): Promise => { try { const arrayBuffer = await file.arrayBuffer(); + return readParquetBuffer(arrayBuffer, schema) + }catch (err) { + console.error("Error reading Parquet file", err); + return []; + } +} + +export const readParquetBuffer = async (arrayBuffer: ArrayBuffer, schema?: string): Promise => { + + try { const asyncBuffer = new AsyncBuffer(arrayBuffer); return new Promise((resolve, reject) => {