add benchmark

.github/workflows/ci-opendb.yml

@@ -15,7 +15,7 @@
 # limitations under the License.
 #
 
-name: CI - FastProto
+name: CI - OpenDB
 on:
   pull_request:
     branches:
@@ -49,4 +49,4 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-maven-
       - name: compile tests
-        run: mvn test
+        run: mvn test -pl !benchmark -am

README.md

@@ -18,4 +18,10 @@ Based on LSM tree-structured database with up to 100,000 data writes per second.
 
     openDB.put(key, value, columnFamilyHandle);
     columnFamilyHandle=openDB.getColumnFamilyHandle("luoluoyuyu").date;
+```
+
+benchmark
+```
+Benchmark                     Mode  Cnt       Score   Error  Units
+OpenDBBenchmark.addKeyValue  thrpt       134061.544          ops/s
 ```

benchmark/pom.xml

@@ -31,6 +31,11 @@
     <artifactId>benchmark</artifactId>
 
     <dependencies>
+        <dependency>
+            <artifactId>opendb-core</artifactId>
+            <groupId>net.openio.opendb</groupId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.openjdk.jmh</groupId>
             <artifactId>jmh-core</artifactId>
@@ -42,5 +47,41 @@
             <version>1.35</version>
         </dependency>
     </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <finalName>benchmarks</finalName>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.openjdk.jmh.Main</mainClass>
+                                </transformer>
+                            </transformers>
+                            <filters>
+                                <filter>
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
 
 </project>

benchmark/src/main/java/net/openio/opendb/OpenDBBenchmark.java

@@ -0,0 +1,88 @@
+/**
+ * Licensed to the OpenIO.Net under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package net.openio.opendb;
+
+
+import net.openio.opendb.db.ColumnFamilyHandle;
+import net.openio.opendb.db.OpenDB;
+import net.openio.opendb.db.OpenDBImp;
+import net.openio.opendb.model.ColumnFamilyDescriptor;
+import net.openio.opendb.model.OperationType;
+import net.openio.opendb.model.Options;
+import net.openio.opendb.model.key.IntKey;
+import net.openio.opendb.model.key.KeyType;
+import net.openio.opendb.model.value.IntValue;
+import net.openio.opendb.model.value.ValueType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Timeout;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+@State(Scope.Benchmark)
+@Warmup(iterations = 0)
+@OutputTimeUnit(TimeUnit.SECONDS)
+@Measurement(iterations = 1,time = 60, timeUnit = TimeUnit.SECONDS)
+@Fork(value = 1)
+public class OpenDBBenchmark {
+
+  static OpenDB openDB;
+  static ColumnFamilyHandle columnFamilyHandle;
+  static int i = 0;
+
+  @Setup
+  public void stepUp(){
+    openDB = OpenDBImp.open(new Options(), "opendb-core/src/test/resources/data/");
+    ColumnFamilyDescriptor columnFamilyDescriptor = new ColumnFamilyDescriptor();
+    columnFamilyDescriptor.setName("luoluoyuyu");
+    columnFamilyDescriptor.setKeyType(KeyType.intKey);
+    columnFamilyDescriptor.setValueType(ValueType.intValue);
+    columnFamilyDescriptor.setBlockSize(1 << 12);
+    openDB.createColumnFamily(columnFamilyDescriptor);
+    columnFamilyHandle = openDB.getColumnFamilyHandle("luoluoyuyu").date;
+    ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
+
+  }
+
+  @Benchmark
+  @Timeout(time = 20, timeUnit = TimeUnit.SECONDS)
+  public void addKeyValue(Blackhole bh) throws Exception {
+    bh.consume(openDB.put(new IntKey(++i), new IntValue(i, OperationType.insert), columnFamilyHandle));
+  }
+
+
+  @Benchmark
+  public void getKeyValue(Blackhole bh) throws Exception {
+    bh.consume(openDB.get(new IntKey(--i), columnFamilyHandle));
+  }
+
+  @TearDown
+  public void closeDatabase() {
+    openDB.close();
+  }
+
+}

opendb-core/src/main/java/net/openio/opendb/db/ColumnFamily.java

@@ -76,6 +76,7 @@ public Value getValue(Key key, Comparator<Key> comparator) {
     for (MemTable memTable : memTables) {
       Value value = memTable.get(key, comparator);
       if (value != null) {
+        readWriteLock.readLock().unlock();
         return value;
       }
     }
@@ -142,10 +143,10 @@ public Levels getCopyLevels() {
 
   public void add(Key key, Value value) {
     KeyValueEntry keyValueEntry = new KeyValueEntry(key, value);
-    readWriteLock.readLock().lock();
     int s = keyType.getByteSize(key) + valueType.getByteSize(value);
     size.addAndGet(s);
     while (true) {
+      readWriteLock.readLock().lock();
       for (MemTable memTable : memTables) {
 
         if (!memTable.isCanWrite()) {
@@ -162,6 +163,7 @@ public void add(Key key, Value value) {
         readWriteLock.readLock().unlock();
         return;
       }
+      readWriteLock.readLock().unlock();
       addMemTable();
     }
 
@@ -171,7 +173,7 @@ public void add(Key key, Value value) {
   public ColumnFamily() {
     memTables = new ArrayList<>(8);
     immMemTable = new ArrayList<>(8);
-    for (int i = 0; i < 128; i++) {
+    for (int i = 0; i < 1; i++) {
       memTables.add(new MemTable(new SkipListRep(), new BloomFilter()));
     }
     levels = new Levels();

opendb-core/src/main/java/net/openio/opendb/db/OpenDB.java

@@ -32,7 +32,7 @@ public interface OpenDB {
 
   Status<Value> update(Key key, Value value, ColumnFamilyHandle columnFamilyHandle);
 
-  Status<Value> delete(Key key, ColumnFamilyHandle columnFamilyHandle);
+  Status<Value> delete(Key key, Value value, ColumnFamilyHandle columnFamilyHandle);
 
   Status<ColumnFamilyHandle> getColumnFamilyHandle(String name);
 

opendb-core/src/main/java/net/openio/opendb/db/OpenDBImp.java

@@ -101,7 +101,7 @@ public static OpenDB open(Options option, String dir) {
       openDBImp.comparator = (a, b) -> {
         int d = a.compareTo(b);
         if (d == 0) {
-          d = a.getSequenceNumber().compareTo(b.getSequenceNumber());
+          d = a.getSequenceNumber().compareTo(b.getSequenceNumber()) >= 0 ? 0 : 1;
         }
         return d;
       };
@@ -118,6 +118,8 @@ private static DataMeta getMetaData(String dir) {
   public Status<Value> get(Key key, ColumnFamilyHandle columnFamilyHandle) {
     SequenceNumber sequenceNumber = writeBatch.getMaxSequenceNumber();
     Snapshot snapshot = snapshotManager.addSnapshot(sequenceNumber);
+    key = key.copy();
+    key.setSequenceNumber(sequenceNumber);
     Status<Value> status = new Status<>(columnFamilyManager.get(columnFamilyHandle.getColumnFamilyId(), key, comparator));
     snapshotManager.removeSnapshot(snapshot);
     return status;
@@ -128,6 +130,7 @@ public Status<Value> put(Key key, Value value, ColumnFamilyHandle columnFamilyHa
     value.setType(OperationType.insert);
     key = key.copy();
     value = value.copy();
+
     writeBatch.addLog(new WalLog(columnFamilyHandle.getColumnFamilyId(), key, value, columnFamilyHandle.keyType,
       columnFamilyHandle.valueType));
     columnFamilyManager.add(key, value, columnFamilyHandle.getColumnFamilyId());
@@ -136,31 +139,21 @@ public Status<Value> put(Key key, Value value, ColumnFamilyHandle columnFamilyHa
 
   @Override
   public Status<Value> update(Key key, Value value, ColumnFamilyHandle columnFamilyHandle) {
-    SequenceNumber sequenceNumber = writeBatch.getMaxSequenceNumber();
-    Snapshot snapshot = snapshotManager.addSnapshot(sequenceNumber);
-    Value value1 = columnFamilyManager.get(columnFamilyHandle.getColumnFamilyId(), key, comparator);
-    snapshotManager.removeSnapshot(snapshot);
-    if (value1 == null) {
-      return Status.fail();
-    }
     key = key.copy();
+    value = value.copy();
     value.setType(OperationType.update);
+    writeBatch.addLog(new WalLog(columnFamilyHandle.getColumnFamilyId(), key, value, columnFamilyHandle.keyType,
+      columnFamilyHandle.valueType));
     columnFamilyManager.add(key, value, columnFamilyHandle.getColumnFamilyId());
-    return new Status<>(value1);
+    return new Status<>();
   }
 
   @Override
-  public Status<Value> delete(Key key, ColumnFamilyHandle columnFamilyHandle) {
-    SequenceNumber sequenceNumber = writeBatch.getMaxSequenceNumber();
-    Snapshot snapshot = snapshotManager.addSnapshot(sequenceNumber);
-    Value value = columnFamilyManager.get(columnFamilyHandle.getColumnFamilyId(), key, comparator);
-    snapshotManager.removeSnapshot(snapshot);
-    if (value == null) {
-      Status.fail();
-    }
+  public Status<Value> delete(Key key, Value value, ColumnFamilyHandle columnFamilyHandle) {
     key = key.copy();
     Value v = value.copy();
-    v.setType(OperationType.delete);
+    writeBatch.addLog(new WalLog(columnFamilyHandle.getColumnFamilyId(), key, value, columnFamilyHandle.keyType,
+      columnFamilyHandle.valueType));
     columnFamilyManager.add(key, value, columnFamilyHandle.getColumnFamilyId());
     return new Status<>(value);
   }

opendb-core/src/main/java/net/openio/opendb/mem/MemTable.java

@@ -77,10 +77,6 @@ public boolean put(KeyValueEntry key, int size) {
 
   public boolean put(List<KeyValueEntry> key) {
 
-    if (increaseCount()) {
-      return false;
-    }
-
     memTableRep.addKeyValue(key);
 
     bloomFilter.add(key);

opendb-core/src/main/java/net/openio/opendb/mem/SkipListRep.java

@@ -34,7 +34,7 @@ public class SkipListRep<K extends Key, V extends Value> implements MemTableRep
 
   private static final int MAX_LEVEL = 32;
 
-  private volatile Node[] head;
+  private final Node[] head;
 
   private static final double P = 0.75;
 
@@ -71,7 +71,7 @@ public void addKeyValue(KeyValueEntry keyValue) {
           i--;
           continue;
         }
-        Key key = ((NodeImp) le[i].next[0]).key;
+        Key key = getNodeImp(le[i].next).key;
         int d = com(key, keyValue.getKey());
         if (d > 0) {
           i--;
@@ -115,19 +115,20 @@ private void add(Node[][] nodes, Node[] node) {
 
   @Override
   public Value getValue(Key key, Comparator<Key> comparator) {
+
     Node[] le = head;
     for (int i = MAX_LEVEL - 1; i >= 0; ) {
       if (le[i].next == null) {
         i--;
         continue;
       }
-      int d = comparator.compare(key, ((NodeImp) le[0]).key);
-      if (d > 0) {
+      int d = comparator.compare(key, getNodeImp(le[i].next).key);
+      if (d < 0) {
         i--;
         continue;
       }
       if (d == 0) {
-        return ((NodeImp) le[i].next[0]).get().getValue();
+        return getNodeImp(le[i].next).getValue();
       }
       le = le[i].next;
     }
@@ -144,6 +145,11 @@ static class Node {
     volatile Node[] next;
   }
 
+  private NodeImp getNodeImp(Node[] node){
+    return (NodeImp) node[0];
+
+  }
+
 
   static final class NodeImp extends Node {
     private Key key;
@@ -157,8 +163,8 @@ Key getKey(Node[] node) {
       return ((NodeImp) node[0]).key;
     }
 
-    Value getValue(Node[] node) {
-      return ((NodeImp) node[0]).values.getValue();
+    Value getValue() {
+      return (this).values.getValue();
     }
 
     private NodeImp(final KeyValueEntry kv) {

opendb-core/src/main/java/net/openio/opendb/model/Status.java

@@ -38,7 +38,7 @@ public Status(K value) {
   }
 
   public Status() {
-
+    success = true;
   }
 
   public static Status fail() {

opendb-core/src/main/java/net/openio/opendb/tool/FileUtils.java

@@ -24,11 +24,6 @@ public static void createDirectoryIfNotExists(String directoryPath) {
     File directory = new File(directoryPath);
     if (!directory.exists()) {
       boolean success = directory.mkdirs();
-      if (success) {
-        System.out.println("目录已创建：" + directoryPath);
-      } else {
-        System.err.println("无法创建目录：" + directoryPath);
-      }
     }
   }
 
@@ -43,13 +38,8 @@ public static void createFileIfNotExists(String filePath) {
 
       try {
         boolean success = file.createNewFile();
-        if (success) {
-          System.out.println("文件已创建：" + filePath);
-        } else {
-          System.err.println("无法创建文件：" + filePath);
-        }
       } catch (Exception e) {
-        System.err.println("创建文件时出现异常：" + e.getMessage());
+        e.printStackTrace();
       }
     }
   }