added google cloud tests

neo4j-contrib · Nov 19, 2024 · b8c681d · b8c681d
1 parent 1342ac1
commit b8c681d
Show file tree

Hide file tree

Showing 23 changed files with 469 additions and 295 deletions.
diff --git a/extended-it/build.gradle b/extended-it/build.gradle
@@ -57,6 +57,7 @@ dependencies {
     testImplementation group: 'org.testcontainers', name: 'milvus', version: '1.20.2'
     testImplementation group: 'org.apache.poi', name: 'poi', version: '5.1.0'
     testImplementation group: 'org.apache.poi', name: 'poi-ooxml', version: '5.1.0'
+    testImplementation 'com.azure:azure-storage-blob:12.22.0'
     configurations.all {
         exclude group: 'org.slf4j', module: 'slf4j-nop'
         exclude group: 'ch.qos.logback', module: 'logback-classic'

diff --git a/extended-it/src/test/java/apoc/azure/AzureStorageBaseTest.java b/extended-it/src/test/java/apoc/azure/AzureStorageBaseTest.java
@@ -0,0 +1,74 @@
+package apoc.azure;
+
+import com.azure.core.util.Context;
+import com.azure.storage.blob.BlobClient;
+import com.azure.storage.blob.BlobContainerClient;
+import com.azure.storage.blob.BlobContainerClientBuilder;
+import com.azure.storage.blob.sas.BlobSasPermission;
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
+import org.apache.commons.io.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.utility.DockerImageName;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.util.UUID;
+
+public class AzureStorageBaseTest {
+
+    public static GenericContainer<?> azuriteContainer;
+    public static BlobContainerClient containerClient;
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        DockerImageName azuriteImg = DockerImageName.parse("mcr.microsoft.com/azure-storage/azurite");
+        azuriteContainer = new GenericContainer<>(azuriteImg)
+                .withExposedPorts(10000);
+
+        azuriteContainer.start();
+
+        var accountName = "devstoreaccount1";
+        var accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+        var blobEndpoint = "http://%s:%d/%s".formatted(azuriteContainer.getHost(), azuriteContainer.getMappedPort(10000), accountName);
+        var connectionString = "DefaultEndpointsProtocol=http;AccountName=%s;AccountKey=%s;BlobEndpoint=%s;"
+                .formatted(accountName, accountKey, blobEndpoint);
+
+        containerClient = new BlobContainerClientBuilder()
+                .connectionString(connectionString)
+                .containerName("test-container")
+                .buildClient();
+        containerClient.create();
+    }    
+
+    @AfterClass
+    public static void teardown() {
+        azuriteContainer.close();
+    }
+
+    public static String putToAzureStorageAndGetUrl(String url) {
+        try {
+            File file = new File(url);
+            byte[] content = FileUtils.readFileToByteArray(file);
+
+            var blobClient = getBlobClient(content);
+            BlobSasPermission permission = new BlobSasPermission().setReadPermission(true);
+            OffsetDateTime expiryTime = OffsetDateTime.now().plusHours(1);
+            String sasToken = blobClient.generateSas(new BlobServiceSasSignatureValues(expiryTime, permission), new Context("Azure-Storage-Log-String-To-Sign", "true"));
+            return blobClient.getBlobUrl() + "?" + sasToken;
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    public static BlobClient getBlobClient(byte[] content) {
+        var blobName = "blob-" + UUID.randomUUID();
+        var blobClient = containerClient.getBlobClient(blobName);
+        blobClient.upload(new ByteArrayInputStream(content));
+        return blobClient;
+    }
+
+}
diff --git a/extended-it/src/test/java/apoc/azure/ImportAzureStorageTest.java b/extended-it/src/test/java/apoc/azure/ImportAzureStorageTest.java
@@ -0,0 +1,63 @@
+package apoc.azure;
+
+import apoc.export.arrow.ExportArrow;
+import apoc.export.arrow.ImportArrow;
+import apoc.export.arrow.ImportArrowTestUtil;
+import apoc.load.Gexf;
+import apoc.meta.Meta;
+import apoc.util.TestUtil;
+import apoc.util.s3.S3BaseTest;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import org.neo4j.test.rule.DbmsRule;
+import org.neo4j.test.rule.ImpermanentDbmsRule;
+
+import java.io.File;
+import java.util.Map;
+
+import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED;
+import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
+import static apoc.ApocConfig.apocConfig;
+import static apoc.export.arrow.ImportArrowTestUtil.ARROW_BASE_FOLDER;
+import static apoc.export.arrow.ImportArrowTestUtil.MAPPING_ALL;
+import static apoc.export.arrow.ImportArrowTestUtil.prepareDbForArrow;
+import static apoc.export.arrow.ImportArrowTestUtil.testImportCommon;
+import static apoc.util.ExtendedITUtil.EXTENDED_PATH;
+import static apoc.util.ExtendedTestUtil.clearDb;
+import static apoc.util.GexfTestUtil.testImportGexfCommon;
+
+public class ImportAzureStorageTest extends AzureStorageBaseTest {
+    private static File directory = new File(ARROW_BASE_FOLDER);
+    static { //noinspection ResultOfMethodCallIgnored
+        directory.mkdirs();
+    }
+
+    @ClassRule
+    public static DbmsRule db = new ImpermanentDbmsRule();
+
+    @BeforeClass
+    public static void beforeClass() {
+        TestUtil.registerProcedure(db, ExportArrow.class, ImportArrow.class, Meta.class, Gexf.class);
+        prepareDbForArrow(db);
+        apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
+        apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true);
+    }
+
+    @Test
+    public void testImportArrow() {
+        String fileWithPath = ARROW_BASE_FOLDER + File.separator + "test_all.arrow";
+        String url = putToAzureStorageAndGetUrl(fileWithPath);
+
+        testImportCommon(db, url, MAPPING_ALL);
+    }
+
+    @Test
+    public void testImportGexf() {
+        clearDb(db);
+        String filename = EXTENDED_PATH + "src/test/resources/gexf/data.gexf";
+        String url = putToAzureStorageAndGetUrl(filename);
+        testImportGexfCommon(db, url);
+    }
+}
diff --git a/extended-it/src/test/java/apoc/azure/LoadAzureStorageTest.java b/extended-it/src/test/java/apoc/azure/LoadAzureStorageTest.java
@@ -0,0 +1,87 @@
+package apoc.azure;
+
+import apoc.load.LoadCsv;
+import apoc.load.LoadDirectory;
+import apoc.load.LoadHtml;
+import apoc.load.LoadJson;
+import apoc.load.Xml;
+import apoc.load.xls.LoadXls;
+import apoc.util.TestUtil;
+import com.azure.core.util.Context;
+import com.azure.storage.blob.*;
+import com.azure.storage.blob.sas.BlobSasPermission;
+import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
+import org.apache.commons.io.FileUtils;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.neo4j.test.rule.DbmsRule;
+import org.neo4j.test.rule.ImpermanentDbmsRule;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.IOException;
+import java.time.OffsetDateTime;
+import java.util.UUID;
+
+import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED;
+import static apoc.ApocConfig.APOC_IMPORT_FILE_USE_NEO4J_CONFIG;
+import static apoc.ApocConfig.apocConfig;
+import static apoc.load.LoadCsvTest.commonTestLoadCsv;
+import static apoc.load.LoadHtmlTest.testLoadHtmlWithGetLinksCommon;
+import static apoc.load.xls.LoadXlsTest.testLoadXlsCommon;
+import static apoc.util.ExtendedITUtil.EXTENDED_PATH;
+import static apoc.util.ExtendedITUtil.testLoadJsonCommon;
+import static apoc.util.ExtendedITUtil.testLoadXmlCommon;
+import static apoc.util.TestUtil.testResult;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+
+public class LoadAzureStorageTest extends AzureStorageBaseTest {
+
+
+    @ClassRule
+    public static DbmsRule db = new ImpermanentDbmsRule();
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        AzureStorageBaseTest.setUp();
+
+        TestUtil.registerProcedure(db, LoadCsv.class, LoadDirectory.class, LoadJson.class, LoadHtml.class, LoadXls.class, Xml.class);
+        apocConfig().setProperty(APOC_IMPORT_FILE_ENABLED, true);
+        apocConfig().setProperty(APOC_IMPORT_FILE_USE_NEO4J_CONFIG, false);
+    }
+
+
+    @Test
+    public void testLoadCsv() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_PATH + "src/test/resources/test.csv");
+        commonTestLoadCsv(db, url);
+    }
+
+    @Test
+    public void testLoadJson() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_PATH + "src/test/resources/map.json");
+        testLoadJsonCommon(db, url);
+    }
+
+    @Test
+    public void testLoadXml() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_PATH + "src/test/resources/xml/books.xml");
+        testLoadXmlCommon(db, url);
+    }
+
+    @Test
+    public void testLoadXls() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_PATH + "src/test/resources/load_test.xlsx");
+        testLoadXlsCommon(db, url);
+    }
+
+    @Test
+    public void testLoadHtml() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_PATH + "src/test/resources/wikipedia.html");
+        testLoadHtmlWithGetLinksCommon(db, url);
+    }
+
+}
diff --git a/extended-it/src/test/java/apoc/azure/ParquetAzureStorageTest.java b/extended-it/src/test/java/apoc/azure/ParquetAzureStorageTest.java
@@ -0,0 +1,61 @@
+package apoc.azure;
+
+import apoc.export.parquet.ParquetTestUtil;
+import apoc.util.collection.Iterators;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.neo4j.test.rule.DbmsRule;
+import org.neo4j.test.rule.ImpermanentDbmsRule;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+import static apoc.export.parquet.ParquetTest.MAPPING_ALL;
+import static apoc.export.parquet.ParquetTestUtil.beforeClassCommon;
+import static apoc.export.parquet.ParquetTestUtil.beforeCommon;
+import static apoc.export.parquet.ParquetTestUtil.testImportAllCommon;
+import static apoc.util.ExtendedITUtil.EXTENDED_PATH;
+import static apoc.util.GoogleCloudStorageContainerExtension.gcsUrl;
+import static apoc.util.TestUtil.testCall;
+import static apoc.util.TestUtil.testResult;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class ParquetAzureStorageTest extends AzureStorageBaseTest {
+
+    private final String EXPORT_FILENAME = "test_all.parquet";
+
+    @ClassRule
+    public static DbmsRule db = new ImpermanentDbmsRule();
+
+    @BeforeClass
+    public static void beforeClass() {
+        beforeClassCommon(db);
+    }
+
+    @Before
+    public void before() {
+        beforeCommon(db);
+    }
+
+    @Test
+    public void testLoadParquet() {
+        String query = "CALL apoc.load.parquet($url, $config) YIELD value " +
+                       "RETURN value";
+
+        String url = putToAzureStorageAndGetUrl(EXTENDED_PATH + "src/test/resources/" + EXPORT_FILENAME);
+        testResult(db, query, Map.of("url", url,  "config", MAPPING_ALL),
+                ParquetTestUtil::roundtripLoadAllAssertions);
+    }
+
+    @Test
+    public void testImportParquet() {
+        String url = putToAzureStorageAndGetUrl(EXTENDED_PATH + "src/test/resources/" + EXPORT_FILENAME);
+
+        Map<String, Object> params = Map.of("file", url, "config", MAPPING_ALL);
+        testImportAllCommon(db, params);
+    }
+}
diff --git a/.../export/ImportGoogleCloudStorageTest.java → ...apoc/gc/ImportGoogleCloudStorageTest.java b/.../export/ImportGoogleCloudStorageTest.java → ...apoc/gc/ImportGoogleCloudStorageTest.java
@@ -1,4 +1,4 @@
-package apoc.export;
+package apoc.gc;
 
 import apoc.export.arrow.ExportArrow;
 import apoc.export.arrow.ImportArrow;
@@ -41,14 +41,14 @@ public static void setUp() throws Exception {
 
     @Test
     public void testImportArrow() {
-        String url = gcsUrl(gcs, "b/folder/o/test_all.arrow?alt=media");
+        String url = gcsUrl(gcs, "test_all.arrow");
         testImportCommon(db, url, MAPPING_ALL);
     }
 
     @Test
     public void testImportGexf() {
         clearDb(db);
-        String url = gcsUrl(gcs, "b/folder/o/data.gexf?alt=media");
+        String url = gcsUrl(gcs, "data.gexf");
         testImportGexfCommon(db, url);
     }
 

diff --git a/...apoc/load/LoadGoogleCloudStorageTest.java → ...a/apoc/gc/LoadGoogleCloudStorageTest.java b/...apoc/load/LoadGoogleCloudStorageTest.java → ...a/apoc/gc/LoadGoogleCloudStorageTest.java
@@ -1,5 +1,9 @@
-package apoc.load;
+package apoc.gc;
 
+import apoc.load.LoadCsv;
+import apoc.load.LoadHtml;
+import apoc.load.LoadJson;
+import apoc.load.Xml;
 import apoc.load.xls.LoadXls;
 import apoc.util.GoogleCloudStorageContainerExtension;
 import apoc.util.TestUtil;
@@ -56,7 +60,7 @@ public static void tearDown() {
 
     @Test
     public void testLoadCsv() {
-        String url = gcsUrl(gcs, "b/folder/o/test.csv?alt=media");
+        String url = gcsUrl(gcs, "test.csv");
 
         testResult(db, "CALL apoc.load.csv($url)", map("url", url), (r) -> {
             assertRow(r, "Selma", "8", 0L);
@@ -68,15 +72,15 @@ public void testLoadCsv() {
 
     @Test
     public void testLoadJSON() {
-        String url = gcsUrl(gcs, "b/folder/o/map.json?alt=media");
+        String url = gcsUrl(gcs, "map.json");
         testCall(db, "CALL apoc.load.jsonArray($url, '$.foo')", map("url", url), (r) -> {
             assertEquals(asList(1L,2L,3L), r.get("value"));
         });
     }
 
     @Test
     public void testLoadXml() {
-        String url = gcsUrl(gcs, "b/folder/o/books.xml?alt=media");
+        String url = gcsUrl(gcs, "books.xml");
         testCall(db, "CALL apoc.load.xml($url,'/catalog/book[title=\"Maeve Ascendant\"]/.',{failOnError:false}) yield value as result", Util.map("url", url), (r) -> {
             Object value = Iterables.single(r.values());
             Assert.assertEquals(XmlTestUtils.XML_XPATH_AS_NESTED_MAP, value);
@@ -85,7 +89,7 @@ public void testLoadXml() {
 
     @Test
     public void testLoadXls() {
-        String url = gcsUrl(gcs, "b/folder/o/load_test.xlsx?alt=media");
+        String url = gcsUrl(gcs, "load_test.xlsx");
         testResult(db, "CALL apoc.load.xls($url,'Full',{mapping:{Integer:{type:'int'}, Array:{type:'int',array:true,arraySep:';'}}})", map("url",url), // 'file:load_test.xlsx'
                 (r) -> {
                     assertXlsRow(r,0L,"String","Test","Boolean",true,"Integer",2L,"Float",1.5d,"Array",asList(1L,2L,3L));
@@ -95,7 +99,7 @@ public void testLoadXls() {
 
     @Test
     public void testLoadHtml() {
-        String url = gcsUrl(gcs, "b/folder/o/wikipedia.html?alt=media");
+        String url = gcsUrl(gcs, "wikipedia.html");
 
         Map<String, Object> query = map("links", "a[href]");