Upgrade/0.14.0 (#13)

* Updated README with instructions detailing proto client file generation.

* Updated proto client.

Updated the gRPC Python client files and adjusted the Cottontail DB Python client and affected tests.

* Bumped versions.

Author: Spiess

.github/workflows/python-package.yml

@@ -20,7 +20,7 @@ jobs:
     # Setup Cottontail DB service container
     services:
       cottontail:
-        image: vitrivr/cottontaildb:0.14.1
+        image: vitrivr/cottontaildb:0.15.0
         ports:
           - 1865:1865
         options: -it

README.md

@@ -4,7 +4,7 @@
 [![Python package workflow](https://github.com/Spiess/cottontaildb-python-client/actions/workflows/python-package.yml/badge.svg)](https://github.com/Spiess/cottontaildb-python-client/actions/workflows/python-package.yml)
 
 A Cottontail DB gRPC client for Python. Built with [Cottontail DB Proto](https://github.com/vitrivr/cottontaildb-proto)
-version `0.13.0`. Comes with an interactive CLI for remote DB access.
+version `0.14.0`. Comes with an interactive CLI for remote DB access.
 
 ## Installation
 
@@ -41,3 +41,24 @@ with CottontailDBClient('localhost', 1865) as client:
     ]
     client.insert_batch('example_schema', 'example_entity', columns, values)
 ```
+
+## Developing
+
+To update the gRPC client, regenerate `cottontaildb_pb2.py` and `cottontaildb_pb2_grpc.py` from the proto definitions
+file in the [Cottontail DB Proto](https://github.com/vitrivr/cottontaildb-proto) repository.
+
+The following is an approximate guide on how to do so from a terminal:
+
+```bash
+# Get the latest version of the Cottontail DB proto
+wget https://github.com/vitrivr/cottontaildb-proto/raw/master/src/main/protobuf/cottontail.proto
+# Install necessary python packages
+pip install grpcio grpcio-tools
+# Generate the gRPC client
+python -m grpc_tools.protoc -I. --python_out=. --grpc_python_out=. cottontail.proto
+# Update the client files (where PYTHON_CLIENT_PATH is the path to this repository)
+mv cottontail_pb2.py cottontail_pb2_grpc.py $PYTHON_CLIENT_PATH/cottontaildb_client/
+```
+
+Finally, the line `import cottontail_pb2 as cottontail__pb2` in the file `cottontail_pb2_grpc.py` must be replaced
+with `from . import cottontail_pb2 as cottontail__pb2`.

cottontaildb_client/cottontail_pb2.py

@@ -633,11 +633,6 @@ def __init__(self, channel):
                 request_serializer=cottontail__pb2.QueryMessage.SerializeToString,
                 response_deserializer=cottontail__pb2.QueryResponseMessage.FromString,
                 )
-        self.BatchQuery = channel.unary_stream(
-                '/org.vitrivr.cottontail.grpc.DQL/BatchQuery',
-                request_serializer=cottontail__pb2.BatchedQueryMessage.SerializeToString,
-                response_deserializer=cottontail__pb2.QueryResponseMessage.FromString,
-                )
         self.Ping = channel.unary_unary(
                 '/org.vitrivr.cottontail.grpc.DQL/Ping',
                 request_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString,
@@ -664,13 +659,6 @@ def Query(self, request, context):
         context.set_details('Method not implemented!')
         raise NotImplementedError('Method not implemented!')
 
-    def BatchQuery(self, request, context):
-        """* Executes a batched query through Cottontail DB (not supported yet). 
-        """
-        context.set_code(grpc.StatusCode.UNIMPLEMENTED)
-        context.set_details('Method not implemented!')
-        raise NotImplementedError('Method not implemented!')
-
     def Ping(self, request, context):
         """* Pings the endpoint 
         """
@@ -691,11 +679,6 @@ def add_DQLServicer_to_server(servicer, server):
                     request_deserializer=cottontail__pb2.QueryMessage.FromString,
                     response_serializer=cottontail__pb2.QueryResponseMessage.SerializeToString,
             ),
-            'BatchQuery': grpc.unary_stream_rpc_method_handler(
-                    servicer.BatchQuery,
-                    request_deserializer=cottontail__pb2.BatchedQueryMessage.FromString,
-                    response_serializer=cottontail__pb2.QueryResponseMessage.SerializeToString,
-            ),
             'Ping': grpc.unary_unary_rpc_method_handler(
                     servicer.Ping,
                     request_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString,
@@ -747,23 +730,6 @@ def Query(request,
             options, channel_credentials,
             insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
 
-    @staticmethod
-    def BatchQuery(request,
-            target,
-            options=(),
-            channel_credentials=None,
-            call_credentials=None,
-            insecure=False,
-            compression=None,
-            wait_for_ready=None,
-            timeout=None,
-            metadata=None):
-        return grpc.experimental.unary_stream(request, target, '/org.vitrivr.cottontail.grpc.DQL/BatchQuery',
-            cottontail__pb2.BatchedQueryMessage.SerializeToString,
-            cottontail__pb2.QueryResponseMessage.FromString,
-            options, channel_credentials,
-            insecure, call_credentials, compression, wait_for_ready, timeout, metadata)
-
     @staticmethod
     def Ping(request,
             target,

cottontaildb_client/cottontail_pb2_grpc.py

@@ -5,9 +5,9 @@
 from typing import List
 
 from .cottontail_pb2 import SchemaName, CreateSchemaMessage, DropSchemaMessage, EntityName, ColumnDefinition, \
-    EntityDefinition, CreateEntityMessage, Engine, InsertMessage, ColumnName, Scan, From, Type, ListSchemaMessage, \
+    EntityDefinition, CreateEntityMessage, InsertMessage, ColumnName, Scan, From, Type, ListSchemaMessage, \
     ListEntityMessage, EntityDetailsMessage, DropEntityMessage, TruncateEntityMessage, OptimizeEntityMessage, \
-    IndexName, IndexDefinition, IndexType, CreateIndexMessage, DropIndexMessage, RebuildIndexMessage, UpdateMessage, \
+    IndexName, IndexType, CreateIndexMessage, DropIndexMessage, RebuildIndexMessage, UpdateMessage, \
     DeleteMessage, Literal, Vector, FloatVector, BatchInsertMessage, Metadata, QueryMessage, Query
 from .cottontail_pb2_grpc import DDLStub, DMLStub, TXNStub, DQLStub
 
@@ -170,12 +170,10 @@ def create_index(self, schema, entity, index, index_type: IndexType, columns: Li
         """
         schema_name = SchemaName(name=schema)
         entity_name = EntityName(schema=schema_name, name=entity)
-        index_name = IndexName(entity=entity_name, name=index)
         # TODO: map<string,string> params
-        column_names = [ColumnName(entity=entity_name, name=c) for c in columns]
-        index_def = IndexDefinition(name=index_name, type=index_type, columns=column_names)
         response = self._ddl.CreateIndex(
-            CreateIndexMessage(metadata=Metadata(transactionId=self._tid), definition=index_def, rebuild=rebuild))
+            CreateIndexMessage(metadata=Metadata(transactionId=self._tid), entity=entity_name, type=index_type,
+                               indexName=index, columns=columns, rebuild=rebuild))
         return self._parse_query_response(response)
 
     def drop_index(self, schema, entity, index):
@@ -238,7 +236,8 @@ def get_entity_details(self, schema, entity):
         """
         schema_name = SchemaName(name=schema)
         entity_name = EntityName(schema=schema_name, name=entity)
-        response = self._ddl.EntityDetails(EntityDetailsMessage(metadata=Metadata(transactionId=self._tid), entity=entity_name))
+        response = self._ddl.EntityDetails(
+            EntityDetailsMessage(metadata=Metadata(transactionId=self._tid), entity=entity_name))
         entity_data = response.tuples[0]
         data_names = [c.name.name for c in response.columns]
         name_index = data_names.index('dbo')
@@ -344,19 +343,22 @@ def ping(self):
         """Sends a ping message to the endpoint. If method returns without exception endpoint is connected."""
         self._dql.Ping(Empty())
 
-    def query(self, schema, entity, projection, where):
+    def query(self, schema, entity, projection, where, order=None, limit=None, skip=None):
         """
         Queries the specified entity where the provided conditions are met and applies the given projection.
 
         @param schema: the schema containing the queried entity
         @param entity: the entity being queried
         @param projection: the projection to be applied to the result
         @param where: where clause specifying the rows to return
+        @param order: order by clause specifying the order of the result
+        @param limit: maximum number of rows to return
+        @param skip: number of rows to skip
         """
         schema_name = SchemaName(name=schema)
         entity_name = EntityName(schema=schema_name, name=entity)
         from_kwarg = {'from': From(scan=Scan(entity=entity_name))}
-        query = Query(**from_kwarg, projection=projection, where=where)
+        query = Query(**from_kwarg, projection=projection, where=where, order=order, limit=limit, skip=skip)
         responses = self._dql.Query(QueryMessage(metadata=Metadata(transactionId=self._tid), query=query))
         return [r for response in responses for r in self._parse_query_response(response)]
 
@@ -403,8 +405,7 @@ def _insert_helper(self, schema, entity, values):
         return InsertMessage(metadata=Metadata(transactionId=self._tid), **from_kwarg, elements=elements)
 
 
-def column_def(name: str, type_: Type, length: int = None, primary: bool = None, nullable: bool = None,
-               engine: Engine = Engine.MAPDB):
+def column_def(name: str, type_: Type, length: int = None, primary: bool = None, nullable: bool = None):
     """
     Creates a column definition.
 
@@ -413,13 +414,11 @@ def column_def(name: str, type_: Type, length: int = None, primary: bool = None,
     @param length: data length for vector types
     @param primary: if this is a primary column of the entity
     @param nullable: if this column may be null
-    @param engine: storage engine to use (currently only MapDB)
     @return: column definition
     """
     kwargs = {
         'name': ColumnName(name=name),
-        'type': type_,
-        'engine': engine
+        'type': type_
     }
     if length is not None:
         kwargs['length'] = length

cottontaildb_client/cottontaildb_client.py

@@ -1,6 +1,6 @@
 [metadata]
 name = cottontaildb-client
-version = 0.0.5
+version = 0.0.6
 author = Florian Spiess
 author_email = [email protected]
 description = A Cottontail DB gRPC client.

setup.cfg

@@ -4,7 +4,7 @@
 
 from cottontaildb_client import CottontailDBClient, column_def, Type, Literal, float_vector
 from cottontaildb_client.cottontail_pb2 import Where, AtomicBooleanPredicate, ColumnName, AtomicBooleanOperand, \
-    ComparisonOperator, Expressions, Expression, Projection, IndexType
+    ComparisonOperator, Expressions, Expression, Projection, IndexType, EntityName, SchemaName
 
 DB_HOST = 'localhost'
 DB_PORT = 1865
@@ -132,7 +132,7 @@ def test_create_rebuild_drop_index(self):
         self._batch_insert()
         details = self.client.get_entity_details(TEST_SCHEMA, TEST_ENTITY)
         self.assertEqual(len(details['indexes']), 0, 'unexpected number of indexes in entity before index creation')
-        self.client.create_index(TEST_SCHEMA, TEST_ENTITY, TEST_INDEX, IndexType.HASH, [TEST_COLUMN_VALUE])
+        self.client.create_index(TEST_SCHEMA, TEST_ENTITY, TEST_INDEX, IndexType.BTREE, [TEST_COLUMN_VALUE])
         details = self.client.get_entity_details(TEST_SCHEMA, TEST_ENTITY)
         self.assertEqual(len(details['indexes']), 1, 'index was not created')
         self.client.rebuild_index(TEST_SCHEMA, TEST_ENTITY, TEST_INDEX)
@@ -200,8 +200,8 @@ def _insert(self):
         self.client.insert(TEST_SCHEMA, TEST_ENTITY, values)
 
     def _insert_vector(self):
-        list = [0.2, 0.3, 0.5]
-        values = {'id': Literal(stringData='test_0'), 'value': float_vector(*list)}
+        value_list = [0.2, 0.3, 0.5]
+        values = {'id': Literal(stringData='test_0'), 'value': float_vector(*value_list)}
         self.client.insert(TEST_SCHEMA, TEST_VECTOR_ENTITY, values)
 
     def _batch_insert(self):
@@ -236,8 +236,11 @@ def _update_value_with_key(self, key, value):
         self.client.update(TEST_SCHEMA, TEST_ENTITY, where, updates)
 
     def _query_value_with_key(self, key):
-        projection = Projection(op=Projection.ProjectionOperation.SELECT,
-                                elements=[Projection.ProjectionElement(column=ColumnName(name=TEST_COLUMN_VALUE))])
+        schema_name = SchemaName(name=TEST_SCHEMA)
+        entity_name = EntityName(schema=schema_name, name=TEST_ENTITY)
+        expression = Expression(column=ColumnName(entity=entity_name, name=TEST_COLUMN_VALUE))
+        projection_element = Projection.ProjectionElement(expression=expression)
+        projection = Projection(op=Projection.ProjectionOperation.SELECT, elements=[projection_element])
         where = Where(atomic=AtomicBooleanPredicate(
             left=ColumnName(name=TEST_COLUMN_ID),
             right=AtomicBooleanOperand(