getObject、selectObjectContent 支持 stream

tencentyun · Oct 30, 2020 · 1a40126 · 1a40126
1 parent 13cd713
commit 1a40126
Show file tree

Hide file tree

Showing 7 changed files with 301 additions and 33 deletions.
diff --git a/README.md b/README.md
@@ -19,7 +19,7 @@ npm i cos-nodejs-sdk-v5 --save
 var COS = require('cos-nodejs-sdk-v5');
 // 创建实例
 var cos = new COS({
-    SecretId: 'AKIDxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
+    SecretId: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
     SecretKey: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx',
 });
 // 分片上传

diff --git a/demo/demo.js b/demo/demo.js
@@ -723,17 +723,22 @@ function putObjectCopy() {
 
 function getObject() {
     var filepath1 = path.resolve(__dirname, '1mb.out1.zip');
-    var filepath2 = path.resolve(__dirname, '123/1mb.out2.zip');
-    // cos.getObject({
-    //     Bucket: config.Bucket, // Bucket 格式：test-1250000000
-    //     Region: config.Region,
-    //     Key: '1mb.zip',
-    //     onProgress: function (progressData) {
-    //         console.log(JSON.stringify(progressData));
-    //     }
-    // }, function (err, data) {
-    //     fs.writeFileSync(filepath1, data.Body);
-    // });
+    var filepath2 = path.resolve(__dirname, '1mb.out2.zip');
+    var filepath3 = path.resolve(__dirname, '1mb.out3.zip');
+
+    // file1 获取对象字节到内存变量
+    cos.getObject({
+        Bucket: config.Bucket, // Bucket 格式：test-1250000000
+        Region: config.Region,
+        Key: '1mb.zip',
+        onProgress: function (progressData) {
+            console.log(JSON.stringify(progressData));
+        }
+    }, function (err, data) {
+        fs.writeFileSync(filepath1, data.Body);
+    });
+
+    // file2 获取对象到本地文件
     cos.getObject({
         Bucket: config.Bucket, // Bucket 格式：test-1250000000
         Region: config.Region,
@@ -745,6 +750,19 @@ function getObject() {
     }, function (err, data) {
         console.log(err || data);
     });
+
+    // file3 pipe 格式获取对象到本地文件
+    var stream = cos.getObjectStream({
+        Bucket: config.Bucket, // Bucket 格式：test-1250000000
+        Region: config.Region,
+        Key: '1mb.zip',
+        onProgress: function (progressData) {
+            console.log(JSON.stringify(progressData));
+        }
+    }, function (err, data) {
+        console.log(err || data);
+    });
+    stream.pipe(fs.createWriteStream(filepath3))
 }
 
 function headObject() {
@@ -845,6 +863,92 @@ function restoreObject() {
     });
 }
 
+var selectCsvOpt = {
+    Bucket: config.Bucket, // Bucket 格式：test-1250000000
+    Region: config.Region,
+    Key: '1.csv',
+    SelectType: 2,
+    SelectRequest: {
+        // Expression: "select * from cosobject s limit 100",
+        Expression: "Select * from COSObject",
+        ExpressionType: "SQL",
+        InputSerialization: {
+            CSV: {
+                FileHeaderInfo: "IGNORE",
+                RecordDelimiter: "\\n",
+                FieldDelimiter: ",",
+                QuoteCharacter: "\"",
+                QuoteEscapeCharacter: "\"",
+                Comments: "#",
+                AllowQuotedRecordDelimiter: "FALSE"
+            }
+        },
+        OutputSerialization: {
+            CSV: {
+                QuoteFields: "ASNEEDED",
+                RecordDelimiter: "\\n",
+                FieldDelimiter: ",",
+                QuoteCharacter: "\"",
+                QuoteEscapeCharacter: "\""
+            }
+        },
+        RequestProgress: {
+            Enabled: "FALSE"
+        }
+    },
+};
+
+var selectJsonOpt = {
+    Bucket: config.Bucket, // Bucket 格式：test-1250000000
+    Region: config.Region,
+    Key: '1.json',
+    SelectType: 2,
+    SelectRequest: {
+        Expression: "Select * from COSObject",
+        ExpressionType: "SQL",
+        InputSerialization: {
+            JSON: {
+                Type: "DOCUMENT",
+            },
+        },
+        OutputSerialization: {
+            JSON: {
+                RecordDelimiter: "\n"
+            },
+        },
+        RequestProgress: {
+            Enabled: "FALSE"
+        }
+    },
+};
+
+function selectObjectContentStream() {
+    // 查询 JSON
+    var selectStream = cos.selectObjectContentStream({
+        ...selectJsonOpt,
+        // DataType: 'raw',
+    }, function (err, data) {
+        console.log(err || data);
+    });
+    var outFile = './result.txt';
+    selectStream.pipe(fs.createWriteStream(outFile));
+    selectStream.on('end', () => console.log(fs.readFileSync(outFile).toString()))
+}
+
+selectObjectContent()
+function selectObjectContent() {
+    // // 如果返回结果很大，可以用 selectObjectContentStream 处理
+    // // 查询 CSV
+    // cos.selectObjectContent(selectCsvOpt, function (err, data) {
+    //     console.log(err || data);
+    // });
+
+    // 查询 JSON
+    cos.selectObjectContent(selectJsonOpt, function (err, data) {
+        console.log(err || data);
+    });
+}
+
 function abortUploadTask() {
     cos.abortUploadTask({
         Bucket: config.Bucket, /* 必须 */ // Bucket 格式：test-1250000000
@@ -1113,9 +1217,9 @@ function deleteFolder() {
 // getBucketLifecycle();
 // putBucketLifecycle();
 // deleteBucketLifecycle();
+// putBucketVersioning();
 // getBucketVersioning();
 // listObjectVersions();
-// putBucketVersioning();
 // getBucketReplication();
 // putBucketReplication();
 // deleteBucketReplication();
@@ -1124,9 +1228,19 @@ function deleteFolder() {
 // deleteBucketWebsite();
 // putBucketReferer();
 // getBucketReferer();
+// putBucketDomain()
+// getBucketDomain()
+// deleteBucketDomain()
+// putBucketLogging()
+// getBucketLogging()
+// deleteBucketLogging()
+// putBucketInventory()
+// getBucketInventory()
+// deleteBucketInventory()
+// listBucketInventory()
 // deleteBucket();
-// putObject();
 // putObjectCopy();
+// getObjectStream();
 // getObject();
 // headObject();
 // putObjectAcl();
@@ -1135,11 +1249,14 @@ function deleteFolder() {
 // deleteMultipleObject();
 // restoreObject();
 // abortUploadTask();
+// selectObjectContentStream();
+// selectObjectContent();
 // sliceUploadFile();
+// uploadFiles();
 // cancelTask();
 // pauseTask();
 // restartTask();
-// uploadFiles();
+// putObject();
 // sliceCopyFile();
 // uploadFolder();
 // listFolder();

diff --git a/demo/util.js b/demo/util.js
@@ -1,7 +1,7 @@
 var os = require('os');
 var fs = require('fs');
-var platform = os.platform();
 
+var platform = os.platform();
 var createFile = function (filepath, size, callback) {
     var cb = function (err) {
         callback && callback();

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cos-nodejs-sdk-v5",
-  "version": "2.8.2",
+  "version": "2.8.3",
   "description": "cos nodejs sdk v5",
   "main": "index.js",
   "scripts": {

diff --git a/sdk/advance.js b/sdk/advance.js
@@ -277,6 +277,7 @@ function getUploadIdAndPartList(params, callback) {
             Region: Region,
             Key: Key,
             Headers: util.clone(params.Headers),
+            Query: util.clone(params.Query),
             StorageClass: StorageClass,
         }, params);
         self.multipartInit(_params, function (err, data) {