Add Python Interface

.gitignore

@@ -19,3 +19,6 @@ bld*/
 
 # Vim
 *.swp
+
+# Python
+python/**/__pycache__/

Doxyfile

@@ -112,7 +112,7 @@ WARN_LOGFILE           =
 #---------------------------------------------------------------------------
 # Configuration options related to the input files
 #---------------------------------------------------------------------------
-INPUT                  = include lib
+INPUT                  = include lib python
 INPUT_ENCODING         = UTF-8
 FILE_PATTERNS          = *.c \
                          *.cc \

python/benchmark.py

@@ -0,0 +1,31 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from julea.benchmark import (
+    distributed_object_bench, item_bench, kv_bench, object_bench)
+from julea import juleadir
+import subprocess
+
+setupfile = juleadir.joinpath('scripts', 'setup.sh')
+
+subprocess.run([setupfile, "start"])
+
+kv_bench()
+distributed_object_bench()
+object_bench()
+item_bench()
+
+subprocess.run([setupfile, "stop"])

python/hello_world.py

@@ -0,0 +1,36 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from julea.object import JObject, JBatch, J_SEMANTICS_TEMPLATE_DEFAULT
+
+
+def main():
+    object = JObject("hello", "world")
+    hello_world = "Hello World!"
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        object.create(batch)
+        bw = object.write(hello_world.encode('utf-8'), 0, batch)
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        out = object.read(bw.value, 0, batch)
+
+    print("Object contains: {buffer} ({length} bytes)".format(
+        buffer=out[0].raw.decode('utf-8'), length=out[1].value))
+
+
+if __name__ == "__main__":
+    main()

python/julea/__init__.py

@@ -0,0 +1,26 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import ctypes
+import pathlib
+
+juleadir = pathlib.Path(__file__).parent.parent.parent
+libdir = juleadir.joinpath('bld')
+
+JULEA = ctypes.CDLL(libdir.joinpath('libjulea.so'))
+JULEA_ITEM = ctypes.CDLL(libdir.joinpath('libjulea-item.so'))
+JULEA_KV = ctypes.CDLL(libdir.joinpath('libjulea-kv.so'))
+JULEA_OBJECT = ctypes.CDLL(libdir.joinpath('libjulea-object.so'))

python/julea/benchmark/__init__.py

@@ -0,0 +1,20 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .bench_distributed_object import distributed_object_bench
+from .bench_item import item_bench
+from .bench_kv import kv_bench
+from .bench_object import object_bench

python/julea/benchmark/bench_distributed_object.py

@@ -0,0 +1,199 @@
+# JULEA - Flexible storage framework
+# Copyright (C) 2019-2020 Johannes Coym
+# Copyright (C) 2017-2020 Michael Kuhn
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from ..object import (JDistributedObject, JBatch, JDistribution,
+                      J_SEMANTICS_TEMPLATE_DEFAULT, J_DISTRIBUTION_ROUND_ROBIN)
+from ..lib.common import format_benchmark
+import time
+
+
+def bench_create(use_batch):
+    if use_batch:
+        n = 100000
+    else:
+        n = 1000
+
+    distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as delete_batch:
+        start = time.time()
+
+        with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+            for i in range(0, n):
+                name = "benchmark-" + str(i)
+                o = JDistributedObject("python", name, distribution)
+                o.create(batch)
+                o.delete(delete_batch)
+                if not use_batch:
+                    batch.execute()
+
+        end = time.time()
+
+    if use_batch:
+        return "/object/distributed-object/create-batch", end - start, n
+    else:
+        return "/object/distributed-object/create", end - start, n
+
+
+def bench_delete(use_batch):
+    n = 10000
+    distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        for i in range(0, n):
+            name = "benchmark-" + str(i)
+            o = JDistributedObject("python", name, distribution)
+            o.create(batch)
+
+    start = time.time()
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        for i in range(0, n):
+            name = "benchmark-" + str(i)
+            o = JDistributedObject("python", name, distribution)
+            o.delete(batch)
+            if not use_batch:
+                batch.execute()
+
+    end = time.time()
+
+    if use_batch:
+        return "/object/distributed-object/delete-batch", end - start, n
+    else:
+        return "/object/distributed-object/delete", end - start, n
+
+
+def bench_status(use_batch):
+    n = 1000
+    distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+    dummy = bytes(1)
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        o = JDistributedObject("python", "benchmark", distribution)
+        o.delete(batch)
+        o.create(batch)
+        o.write(dummy, 0, batch)
+
+    start = time.time()
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        for _ in range(0, n):
+            o.status(batch)
+            if not use_batch:
+                batch.execute()
+
+    end = time.time()
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        o.delete(batch)
+
+    if use_batch:
+        return "/object/distributed-object/status-batch", end - start, n
+    else:
+        return "/object/distributed-object/status", end - start, n
+
+
+def bench_read(use_batch, block_size):
+    n = 25000
+    distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+    dummy = bytes(block_size)
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        o = JDistributedObject("python", "benchmark", distribution)
+        o.delete(batch)
+        o.create(batch)
+
+        for i in range(0, n):
+            o.write(dummy, i * block_size, batch)
+
+    start = time.time()
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        for i in range(0, n):
+            o.read(block_size, i * block_size, batch)
+            if not use_batch:
+                batch.execute()
+
+    end = time.time()
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        o.delete(batch)
+
+    if use_batch:
+        name = "/object/distributed-object/read-batch"
+        return name, end - start, n, n * block_size
+    else:
+        name = "/object/distributed-object/read"
+        return name, end - start, n, n * block_size
+
+
+def bench_write(use_batch, block_size):
+    n = 25000
+    distribution = JDistribution(J_DISTRIBUTION_ROUND_ROBIN)
+    dummy = bytes(block_size)
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        o = JDistributedObject("python", "benchmark", distribution)
+        o.delete(batch)
+        o.create(batch)
+
+    start = time.time()
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        for i in range(0, n):
+            o.write(dummy, i * block_size, batch)
+            if not use_batch:
+                batch.execute()
+
+    end = time.time()
+
+    with JBatch(J_SEMANTICS_TEMPLATE_DEFAULT) as batch:
+        o.delete(batch)
+
+    if use_batch:
+        name = "/object/distributed-object/write-batch"
+        return name, end - start, n, n * block_size
+    else:
+        name = "/object/distributed-object/write"
+        return name, end - start, n, n * block_size
+
+
+def distributed_object_bench():
+    create = bench_create(False)
+    print(format_benchmark(create))
+    create_batch = bench_create(True)
+    print(format_benchmark(create_batch))
+
+    delete = bench_delete(False)
+    print(format_benchmark(delete))
+    delete_batch = bench_delete(True)
+    print(format_benchmark(delete_batch))
+
+    status = bench_status(False)
+    print(format_benchmark(status))
+    status_batch = bench_status(True)
+    print(format_benchmark(status_batch))
+
+    read = bench_read(False, 4 * 1024)
+    print(format_benchmark(read))
+    read_batch = bench_read(True, 4 * 1024)
+    print(format_benchmark(read_batch))
+
+    write = bench_write(False, 4 * 1024)
+    print(format_benchmark(write))
+    write_batch = bench_write(True, 4 * 1024)
+    print(format_benchmark(write_batch))