#42 add new resources.

services/cloud-functions/java/test1/cloudfntest1/pom.xml

@@ -1,17 +1,29 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <groupId>cloudfn</groupId>
   <artifactId>cloudfntest1</artifactId>
   <packaging>jar</packaging>
   <version>1.0-SNAPSHOT</version>
   <name>cloudfntest1</name>
-  
+
   <properties>
     <maven.compiler.target>11</maven.compiler.target>
     <maven.compiler.source>11</maven.compiler.source>
   </properties>
 
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>com.google.cloud</groupId>
+        <artifactId>libraries-bom</artifactId>
+        <version>26.20.0</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
   <dependencies>
     <dependency>
       <groupId>junit</groupId>
@@ -26,6 +38,46 @@
       <version>1.1.0</version>
       <scope>provided</scope>
     </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloud-storage</artifactId>
+      <version>2.32.1</version>
+    </dependency>
+    <dependency>
+      <groupId>com.google.cloud</groupId>
+      <artifactId>google-cloudevent-types</artifactId>
+      <version>0.14.0</version>
+    </dependency>
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-core</artifactId>
+      <version>2.16.1</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-databind -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.core</groupId>
+      <artifactId>jackson-databind</artifactId>
+      <version>2.16.1</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-csv -->
+    <dependency>
+      <groupId>com.fasterxml.jackson.dataformat</groupId>
+      <artifactId>jackson-dataformat-csv</artifactId>
+      <version>2.16.1</version>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/org.projectlombok/lombok -->
+    <dependency>
+      <groupId>org.projectlombok</groupId>
+      <artifactId>lombok</artifactId>
+      <version>1.18.30</version>
+      <scope>provided</scope>
+    </dependency>
+    <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+    <dependency>
+      <groupId>org.apache.commons</groupId>
+      <artifactId>commons-lang3</artifactId>
+      <version>3.14.0</version>
+    </dependency>
   </dependencies>
 
   <build>

services/cloud-functions/java/test1/cloudfntest1/src/main/java/cloudfn/App.java

@@ -1,13 +1,42 @@
 package cloudfn;
 
+import java.io.File;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvGenerator;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema.Builder;
+
 /**
  * Hello world!
  *
  */
 public class App 
 {
-    public static void main( String[] args )
-    {
-        System.out.println( "Hello World!" );
+    public static void main( String[] args ) throws Exception {
+
+        String jsn = "[{\"name\": \"mosh\", \"job\" : \"Officer\"},{\"name\": \"pp\", \"job\" : \"oo\"}]";
+        JsonNode jsonTree = new ObjectMapper().readTree(jsn);
+        Builder csvBuilder = CsvSchema.builder();
+        JsonNode firstObject = jsonTree.elements().next();
+        firstObject.fieldNames().forEachRemaining(fieldName -> {csvBuilder.addColumn(fieldName);});
+        CsvSchema csvSchema = csvBuilder.build().withQuoteChar('"').withHeader();
+
+        String csv = "";
+        CsvMapper csvMapper = new CsvMapper();
+        csvMapper.configure(CsvGenerator.Feature.ALWAYS_QUOTE_STRINGS, true);
+        csvMapper.writerFor(JsonNode.class)
+        .with(csvSchema)
+        .writeValue(new File("test.csv"), jsonTree);
+
+        String jsn2 = "{\"name\": \"mosh\", \"job\" : \" Officer\"},{\\\"name\\\": \\\"henrig\\\", \\\"job\\\" : \\\" pop\\\"}";
+        ObjectMapper mapper = new ObjectMapper();
+        JsonNode root = mapper.readTree(jsn2);
+
+        System.out.println(root.get(0).toString());
+        System.out.println(root.get(1).toString());
+
+
     }
 }

services/cloud-functions/java/test1/cloudfntest1/src/main/java/cloudfn/App2.java

@@ -0,0 +1,64 @@
+package cloudfn;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.StringReader;
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvGenerator;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+
+import cloudfn.elements.StudentProfile;
+
+public class App2 {
+
+    private static final String HEADER = "\"name\",\"age\",\"sex\",\"address\",\"school\"";
+
+    public static void main(String[] args) throws IOException {
+        Storage storage = StorageOptions.newBuilder().setProjectId("dataflowtest002").build().getService();
+
+        byte[] content = storage.readAllBytes("testcfneventjp20240129", "test.json");
+        BufferedReader reader = new BufferedReader(new StringReader(new String(content)));
+
+        StringBuilder sb = new StringBuilder();
+
+        sb.append(HEADER);
+        ObjectMapper oMapper = new ObjectMapper();
+        StudentProfile student = new StudentProfile();
+        CsvMapper mapper = new CsvMapper();
+        mapper.configure(CsvGenerator.Feature.ALWAYS_QUOTE_STRINGS, true);
+        CsvSchema schema = mapper.schemaFor(StudentProfile.class)
+            .sortedBy("name", "age","sex","address","school")
+            .withQuoteChar('"') .withoutHeader();
+
+        String line;
+        while ((line = reader.readLine()) != null) {
+            // Object Mapper convert Json to Object.
+            try {
+                student = oMapper.readValue(line, StudentProfile.class);
+            } catch (Exception e) {
+                System.err.println("error");
+            }
+
+            // CSV Mapper conver Object to CSV.
+            try {
+                sb.append(mapper.writer(schema).writeValueAsString(student));
+            } catch (Exception e) {
+                System.err.println("error");
+            }
+        }
+
+        String uploadFileName = StringUtils.replace("test.json", ".json", ".csv");
+        BlobId blobId = BlobId.of("testcfneventjp20240129", uploadFileName);
+        BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
+        storage.createFrom(blobInfo, new ByteArrayInputStream(sb.toString().getBytes()));
+    }
+
+}

services/cloud-functions/java/test1/cloudfntest1/src/main/java/cloudfn/GCSEventFunction.java

@@ -0,0 +1,37 @@
+package cloudfn;
+
+import java.nio.charset.StandardCharsets;
+import java.util.logging.Logger;
+
+import com.google.events.cloud.storage.v1.StorageObjectData;
+import com.google.protobuf.util.JsonFormat;
+import com.google.cloud.functions.CloudEventsFunction;
+
+import io.cloudevents.CloudEvent;
+
+public class GCSEventFunction implements CloudEventsFunction {
+
+    private static final Logger logger = Logger.getLogger(GCSEventFunction.class.getName());
+
+    @Override
+    public void accept(CloudEvent event) throws Exception {
+        logger.info("Event" + event.getId());
+        logger.info("Event Type:" + event.getType());
+
+        if (event.getData() == null) {
+            logger.warning("No data found in cloud event payload.");
+        }
+
+        String cloudEventData = new String(event.getData().toBytes(), StandardCharsets.UTF_8);
+        StorageObjectData.Builder builder = StorageObjectData.newBuilder();
+        JsonFormat.parser().merge(cloudEventData, builder);
+        StorageObjectData data = builder.build();
+
+        logger.info("Bucket: " + data.getBucket());
+        logger.info("File: " + data.getName());
+        logger.info("Metageneration: " + data.getMetageneration());
+        logger.info("Created: " + data.getTimeCreated());
+        logger.info("Updated: " + data.getUpdated());
+    }
+
+}

services/cloud-functions/java/test1/cloudfntest1/src/main/java/cloudfn/JsonEventConverterFn.java

@@ -0,0 +1,112 @@
+package cloudfn;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.util.logging.Logger;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.google.events.cloud.storage.v1.StorageObjectData;
+import com.google.protobuf.util.JsonFormat;
+
+import cloudfn.elements.StudentProfile;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvGenerator;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvSchema;
+import com.google.cloud.functions.CloudEventsFunction;
+import com.google.cloud.storage.BlobId;
+import com.google.cloud.storage.BlobInfo;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageOptions;
+
+import io.cloudevents.CloudEvent;
+import lombok.Data;
+
+public class JsonEventConverterFn implements CloudEventsFunction {
+
+    private static final Logger logger = Logger.getLogger(GCSEventFunction.class.getName());
+
+    private static final String HEADER_RECORD = "\"name\",\"age\",\"sex\",\"address\",\"school\"";
+
+    private static final String EXT_JSON = ".json";
+    private static final String EXT_CSV = ".csv";
+
+    private static final char QUOTE_CHAR = '"';
+
+    @Override
+    public void accept(CloudEvent event) throws Exception {
+        logger.info("Event" + event.getId());
+        logger.info("Event Type:" + event.getType());
+
+        if (event.getData() == null) {
+            logger.warning("No data found in cloud event payload.");
+            return;
+        }
+
+        String cloudEventData = new String(event.getData().toBytes(), StandardCharsets.UTF_8);
+        StorageObjectData.Builder builder = StorageObjectData.newBuilder();
+        JsonFormat.parser().merge(cloudEventData, builder);
+        StorageObjectData data = builder.build();
+
+        logger.info("Bucket: " + data.getBucket());
+        logger.info("File: " + data.getName());
+        logger.info("Metageneration: " + data.getMetageneration());
+        logger.info("Created: " + data.getTimeCreated());
+        logger.info("Updated: " + data.getUpdated());
+
+        // ファイル名を作成
+        String uploadFileName = StringUtils.replace(data.getName(), EXT_JSON, EXT_CSV);
+
+        // Storageオブジェクト作成
+        Storage storage = StorageOptions.newBuilder()
+                .setProjectId(System.getenv("PROJECTID"))
+                .build().getService();
+
+        // 対象ファイルダウンロード
+        byte[] fileContent = storage.readAllBytes(data.getBucket(), data.getName());
+        // ファイルはLF改行されているため、BufferedReaderで読み取り
+        BufferedReader reader = new BufferedReader(new StringReader(new String(fileContent)));
+
+        // CSVファイル構築用StringBuilder(ヘッダレコード追加)
+        StringBuilder outputFileBuilder = new StringBuilder();
+        outputFileBuilder.append(HEADER_RECORD);
+
+        // ObjectMapper作成
+        ObjectMapper objectMapper = new ObjectMapper();
+        // CsvMapper作成(囲い文字設定有効化)
+        CsvMapper csvMapper = new CsvMapper();
+        csvMapper.configure(CsvGenerator.Feature.ALWAYS_QUOTE_STRINGS, true);
+        // CsvSchema作成
+        CsvSchema csvSchema = csvMapper.schemaFor(StudentProfile.class)
+        .sortedBy("name", "age","sex","address","school")
+        .withQuoteChar(QUOTE_CHAR)
+        .withoutHeader();
+
+        // JSON→CSV変換処理
+        StudentProfile lineObject = new StudentProfile();
+        String line;
+        while ((line = reader.readLine()) != null) {
+            try {
+                // 行データをオブジェクトに変換
+                lineObject = objectMapper.readValue(line, StudentProfile.class);
+                // オブジェクトをCSV行に変換してStringBuilderにCSV行を追加
+                outputFileBuilder.append(csvMapper.writer(csvSchema).writeValueAsString(lineObject));
+            } catch (Exception e) {
+                logger.warning("Error happen when convert json to csv.");
+                throw e;
+            }
+        }
+
+        // ファイルを指定のバケットにアップロード
+        BlobId blobId = BlobId.of(System.getenv("UPLOADBKT"), uploadFileName);
+        BlobInfo blobInfo = BlobInfo.newBuilder(blobId).build();
+        storage.createFrom(blobInfo, new ByteArrayInputStream(outputFileBuilder.toString().getBytes()));
+
+        logger.info("Success convert json to csv.");
+    }
+
+}

services/cloud-functions/java/test1/cloudfntest1/src/main/java/cloudfn/elements/StudentProfile.java

@@ -0,0 +1,16 @@
+package cloudfn.elements;
+
+import lombok.Data;
+
+@Data
+public class StudentProfile {
+
+    public StudentProfile() {
+    }
+
+    private String name;
+    private String age;
+    private String sex;
+    private String address;
+    private String school;
+}

services/cloud-functions/java/test1/cloudfntest1/src/main/java/cloudfn/gcsEventFunction.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+gsutil mb -c standard -l asia-northeast1 gs://testcfneventjp20240129
+
+gcloud functions deploy java-finalize-function \
+--gen2 \
+--runtime=java17 \
+--region=asia-northeast1 \
+--source=. \
+--entry-point=cloudfn.GCSEventFunction \
+--memory=512MB \
+--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
+--trigger-event-filters="bucket=testcfneventjp20240129"

services/cloud-functions/java/test1/cloudfntest1/src/main/java/cloudfn/jsonEventConverterFn.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+gsutil mb -c standard -l asia-northeast1 gs://testcfneventjp20240129
+
+gsutil mb -c standard -l asia-northeast1 gs://testuploadcfneventjp20240130
+
+gcloud functions deploy json-conv-csv-fn \
+--gen2 \
+--runtime=java17 \
+--region=asia-northeast1 \
+--service-account=437530287615-compute@developer.gserviceaccount.com \
+--source=. \
+--entry-point=cloudfn.JsonEventConverterFn \
+--memory=512MB \
+--trigger-event-filters="type=google.cloud.storage.object.v1.finalized" \
+--trigger-event-filters="bucket=testcfneventjp20240129" \
+--set-env-vars="UPLOADBKT=testuploadcfneventjp20240130" \
+--set-env-vars="PROJECTID=dataflowtest002"