V0.11.21 (#88)

* Prepare next release * Dash: Imports adjusted for current version * Demo notebook extended * Docs: JOSS reviews * Docs: DBMS infos cleaned * Start improving docs in code * Docs: JOSS more details
Beuth-Erdelt · Jun 1, 2022 · eda33d5 · eda33d5
1 parent d5fd774
commit eda33d5
Show file tree

Hide file tree

Showing 11 changed files with 2,887 additions and 413 deletions.
diff --git a/Evaluation-Demo.html b/Evaluation-Demo.html
diff --git a/Evaluation-Demo.ipynb b/Evaluation-Demo.ipynb
diff --git a/dashboard.py b/dashboard.py
@@ -171,6 +171,8 @@ def get_connections_by_filter(filter_by: str, e: inspector.inspector) -> dict:
         connections_by_filter = e.get_experiment_list_connections_by_hostsystem('CPU')
     elif filter_by == 'CPU Limit':
         connections_by_filter = e.get_experiment_list_connections_by_hostsystem('limits_cpu')
+    elif filter_by == 'Docker Image':
+        connections_by_filter = e.get_experiment_list_connections_by_parameter('dockerimage')
     else:
         raise KeyError('filter_by')
 

diff --git a/dbmsbenchmarker/__pycache__/.gitignore b/dbmsbenchmarker/__pycache__/.gitignore
@@ -1,3 +1,4 @@
 *.cpython-36.pyc
 *.cpython-37.pyc
 *.cpython-38.pyc
+*.cpython-39.pyc
diff --git a/dbmsbenchmarker/inspector.py b/dbmsbenchmarker/inspector.py
@@ -324,6 +324,14 @@ def get_aggregated_experiment_statistics(self, type='timer', name='run', dbms_fi
             #print(df_result)
             return df_result
     def get_aggregated_by_connection(self, dataframe, list_connections=[], connection_aggregate='Mean'):
+        """
+        Calculate the connection aggregate
+        
+        :param dataframe: DataFrame 
+        :param list_connections:
+        :param connection_aggregate: 
+        :return: 
+        """
         df_stats = pd.DataFrame()
         if len(list_connections) > 0:
             for i, l2 in list_connections.items():

diff --git a/dbmsbenchmarker/layout.py b/dbmsbenchmarker/layout.py
@@ -427,7 +427,7 @@ def serve_layout(preview) -> html.Div:
                     html.Label('Color by', id='label_graph_colorby'),
                     dcc.Dropdown(
                         id='dd_graph_colorby',
-                        options=[{'label': x, 'value': x} for x in ['DBMS', 'Node', 'Script', 'CPU Limit', 'Client', 'GPU', 'CPU']],
+                        options=[{'label': x, 'value': x} for x in ['DBMS', 'Node', 'Script', 'CPU Limit', 'Client', 'GPU', 'CPU', 'Docker Image']],
                     ),
 
                     html.Label('Boxpoints', id='label_boxpoints'),

diff --git a/dbmsbenchmarker/scripts/dashboardcli.py b/dbmsbenchmarker/scripts/dashboardcli.py
@@ -173,6 +173,8 @@ def get_connections_by_filter(filter_by: str, e: inspector.inspector) -> dict:
         connections_by_filter = e.get_experiment_list_connections_by_hostsystem('CPU')
     elif filter_by == 'CPU Limit':
         connections_by_filter = e.get_experiment_list_connections_by_hostsystem('limits_cpu')
+    elif filter_by == 'Docker Image':
+        connections_by_filter = e.get_experiment_list_connections_by_parameter('dockerimage')
     else:
         raise KeyError('filter_by')
 

diff --git a/docs/DBMS.md b/docs/DBMS.md
@@ -12,21 +12,6 @@ Make sure to adjust the
 
 
 
-    {
-        'name': "Vertica",
-        'info': "This is Vertica on dsm server SF=1",
-        'active': True,
-        'JDBC': {
-            'driver': "com.mysql.cj.jdbc.Driver",
-            'url': 'jdbc:vertica://dsm.beuth-hochschule.de:5433/',
-            'auth': ["dbadmin", ""],
-            'jar': "jars/vertica-jdbc-11.1.0-0.jar"
-        },
-    },
-
-
-
-
 ## Citus Data
 
 https://www.citusdata.com/

diff --git a/layout.py b/layout.py
@@ -427,7 +427,7 @@ def serve_layout(preview) -> html.Div:
                     html.Label('Color by', id='label_graph_colorby'),
                     dcc.Dropdown(
                         id='dd_graph_colorby',
-                        options=[{'label': x, 'value': x} for x in ['DBMS', 'Node', 'Script', 'CPU Limit', 'Client', 'GPU', 'CPU']],
+                        options=[{'label': x, 'value': x} for x in ['DBMS', 'Node', 'Script', 'CPU Limit', 'Client', 'GPU', 'CPU', 'Docker Image']],
                     ),
 
                     html.Label('Boxpoints', id='label_boxpoints'),

diff --git a/paper.md b/paper.md
@@ -25,7 +25,7 @@ Queries can be parametrized and randomized.
 Results and evaluations are available via a Python interface and can be inspected with standard Python tools like pandas DataFrames.
 An interactive visual dashboard assists in multi-dimensional analysis of the results.
 
-This module has been tested with Clickhouse, Exasol, Citus Data (Hyperscale), IBM DB2, Kinetica, MariaDB, MariaDB Columnstore, MemSQL (SingleStore), MariaDB, MariaDB Columnstore, MonetDB, MySQL, OmniSci (HEAVY.AI) , Oracle DB, PostgreSQL, SQL Server, SAP HANA and Vertica.
+This module has been tested with Clickhouse, Exasol, Citus Data (Hyperscale), IBM DB2, MariaDB, MariaDB Columnstore, MemSQL (SingleStore), MonetDB, MySQL, OmniSci (HEAVY.AI) , Oracle DB, PostgreSQL, SQL Server, SAP HANA and Vertica.
 
 See the [homepage](https://github.com/Beuth-Erdelt/DBMS-Benchmarker) and the [documentation](https://dbmsbenchmarker.readthedocs.io/en/latest/Docs.html).
 
@@ -186,7 +186,7 @@ A session starts with establishing a connection and ends when the connection is
 
 We also measure and store the **total time** of the benchmark of the query, since for parallel execution this differs from the **sum of times** based on *timerRun*. Total time means measurement starts before first benchmark run and stops after the last benchmark run has been finished. Thus total time also includes some overhead (for spawning a pool of subprocesses, compute size of result sets and joining results of subprocesses).
 Thus the sum of times is more of an indicator for performance of the server system, the total time is more of an indicator for the performance the client user receives.
-We also compute for each query and DBMS **latency** (measured time) and **throughput** (number of parallel clients per mean time).
+We also compute for each query and DBMS **latency** (measured time) and **throughput** (number of parallel clients per measured time).
 Additionally error messages and timestamps of begin and end of benchmarking a query are stored.
 
 
@@ -222,13 +222,15 @@ We also can have various hardware metrics like CPU and GPU utilization, CPU thro
 These are also described in three dimensions:
 Second of query execution time, number of query and number of configuration.
 
-All these metrics can be sliced or diced, rolled-up or drilled-down into the various dimensions using several aggregation functions for evaluation.
-
 # Evaluation
 
 ## Python - Pandas
 
-There are DataFrames covering errors and warnings, that have occured, and timing and hardware metrics that have been collected or derived.
+All metrics can be sliced or diced, rolled-up or drilled-down into the various dimensions using several aggregation functions for evaluation:
+First, last, minimum, maximum, arithmetic and geometric mean, range and interquartile range, standard deviation, median, some quantiles, coefficient of variation and quartile coefficient of dispersion.
+This helps in univariate analysis of center and dispersion of the metrics to evaluate measures and stability.
+
+The package includes tools to convert the three dimensional results into pandas DataFrames, like covering errors and warnings, that have occured, and timing and hardware metrics, that have been collected or derived.
 
 For example the latency of execution, aggregated in the query dimension by computing the mean value, can be obtained as:
 
@@ -241,8 +243,8 @@ df = evaluate.get_aggregated_query_statistics(
 
 ## GUI - Dashboard
 
-The dashboard helps in interactive evaluation of experiment results.
-It shows plots of various types, that can be customized and filtered by DBMS configuration and query.
+The package includes a dashboard helping in interactive evaluation of experiment results.
+It shows predefined plots of various types, that can be customized and filtered by DBMS configuration and query.
 
 ![screenshot of dashboard.\label{fig:dashboard}](docs/dashboard.png){ width=1440}
 

diff --git a/setup.py b/setup.py
@@ -8,7 +8,7 @@
 
 setuptools.setup(
     name="dbmsbenchmarker",
-    version="0.11.20",
+    version="0.11.21",
     author="Patrick Erdelt",
     author_email="[email protected]",
     description="DBMS-Benchmarker is a Python-based application-level blackbox benchmark tool for Database Management Systems (DBMS). It connects to a given list of DBMS (via JDBC) and runs a given list of parametrized and randomized (SQL) benchmark queries. Evaluations are available via Python interface, in reports and at an interactive multi-dimensional dashboard.",