Frocksdb release guide and helping scripts

HISTORY.md

@@ -1,3 +1,11 @@
+# FRocksdb Change Log
+## 6.11.6-veverica-1.0 (10/22/2020)
+### Features
+* [Flink TTL] compaction filter for background cleanup of state with time-to-live
+* [Flink ListState] Separator-free merge operator
+### Bug FIxes
+* Fix a bug which may cause MultiGet to be slow because it may read more data than requested, but this won't affect correctness. The bug was introduced in 6.10 release.
+
 # Rocksdb Change Log
 ## 6.10.2 (6/5/2020)
 ### Bug fix

Makefile

@@ -6,6 +6,8 @@
 
 #-----------------------------------------------
 
+FROCKSDB_VERSION ?= 1.0
+
 BASH_EXISTS := $(shell which bash)
 SHELL := $(shell which bash)
 PYTHON?=$(shell which python)
@@ -121,6 +123,23 @@ ifeq ($(LITE), 0)
 else
 	OPT += -Os
 endif
+
+ifeq ($(MAKECMDGOALS),frocksdbjavastatic)
+	ifneq ($(DEBUG_LEVEL),2)
+		DEBUG_LEVEL=0
+	endif
+endif
+
+ifeq ($(MAKECMDGOALS),frocksdbjavastaticrelease)
+	DEBUG_LEVEL=0
+endif
+
+ifeq ($(MAKECMDGOALS),frocksdbjavastaticreleasedocker)
+        DEBUG_LEVEL=0
+endif
+
+ifeq ($(MAKECMDGOALS),frocksdbjavastaticpublish)
+	DEBUG_LEVEL=0
 endif
 
 # compile with -O2 if debug level is not 2
@@ -2032,17 +2051,72 @@ rocksdbjavastatic: $(java_static_all_libobjects)
 		strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
 	fi
 	cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
+	jar -uf java/target/$(ROCKSDB_JAR) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
 	cd java/target/apidocs;jar -cf ../$(ROCKSDB_JAVADOCS_JAR) *
 	cd java/src/main/java;jar -cf ../../../target/$(ROCKSDB_SOURCES_JAR) org
 
+	mkdir -p java/target/META-INF
+	cp LICENSE.Apache java/target/META-INF/LICENSE
+	cd java/target;jar -uf $(ROCKSDB_JAR) META-INF/LICENSE
+
 rocksdbjavastaticrelease: rocksdbjavastatic
 	cd java/crossbuild && (vagrant destroy -f || true) && vagrant up linux32 && vagrant halt linux32 && vagrant up linux64 && vagrant halt linux64 && vagrant up linux64-musl && vagrant halt linux64-musl
 	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
-	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
+	jar -uf java/target/$(ROCKSDB_JAR_ALL) HISTORY*.md
+	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib librocksdbjni-win64.dll
 	cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
 
+frocksdbjavastaticreleasedocker: rocksdbjavastaticreleasedocker
+	# update license
+	mkdir -p java/target/META-INF
+	cp LICENSE.Apache java/target/META-INF/LICENSE
+	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) META-INF/LICENSE
+
+	# platform jars
+	$(eval JAR_PREF=rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH))
+	$(eval JAR_DOCS=$(JAR_PREF)-javadoc.jar)
+	$(eval JAR_SOURCES=$(JAR_PREF)-sources.jar)
+	$(eval OSX_JAR=$(JAR_PREF)-osx.jar)
+	$(eval WIN_JAR=$(JAR_PREF)-win64.jar)
+	$(eval LINUX32_JAR=$(JAR_PREF)-linux32.jar)
+	$(eval LINUX64_JAR=$(JAR_PREF)-linux64.jar)
+
+	# update windows jar
+	cd java/target;cp rocksdbjni_classes.jar $(WIN_JAR)
+	cd java;jar -uf target/$(WIN_JAR) HISTORY*.md
+	jar -uf java/target/$(WIN_JAR) HISTORY*.md
+	cd java/target;jar -uf $(WIN_JAR) librocksdbjni-win64.dll
+	cd java/target;jar -uf $(WIN_JAR) META-INF/LICENSE
+
+	# update linux 64 jar with ppc64 lib
+	cd java/target;jar -uf $(LINUX64_JAR) librocksdbjni-linux-ppc64le.so
+
+	cd java/target;jar -uf $(JAR_DOCS) META-INF/LICENSE
+	cd java/target;jar -uf $(JAR_SOURCES) META-INF/LICENSE
+
+	# prepare frocksdb release
+	cd java/target;mkdir -p frocksdb-release
+
+	$(eval FJAR_PREF=frocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-ververica-$(FROCKSDB_VERSION))
+	$(eval FJAR=$(FJAR_PREF).jar)
+	$(eval FJAR_DOCS=$(FJAR_PREF)-javadoc.jar)
+	$(eval FJAR_SOURCES=$(FJAR_PREF)-sources.jar)
+	$(eval OSX_FJAR=$(FJAR_PREF)-osx.jar)
+	$(eval WIN_FJAR=$(FJAR_PREF)-win64.jar)
+	$(eval LINUX32_FJAR=$(FJAR_PREF)-linux32.jar)
+	$(eval LINUX64_FJAR=$(FJAR_PREF)-linux64.jar)
+
+	cd java/target;cp $(ROCKSDB_JAR_ALL) frocksdb-release/$(FJAR)
+	cd java/target;cp $(JAR_DOCS) frocksdb-release/$(FJAR_DOCS)
+	cd java/target;cp $(JAR_SOURCES) frocksdb-release/$(FJAR_SOURCES)
+	cd java/target;cp $(OSX_JAR) frocksdb-release/$(OSX_FJAR)
+	cd java/target;cp $(WIN_JAR) frocksdb-release/$(WIN_FJAR)
+	cd java/target;cp $(LINUX32_JAR) frocksdb-release/$(LINUX32_FJAR)
+	cd java/target;cp $(LINUX64_JAR) frocksdb-release/$(LINUX64_FJAR)
+	cd java;cp rocksjni.pom target/frocksdb-release/$(FJAR_PREF).pom
+
 rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64 rocksdbjavastaticdockerx86musl rocksdbjavastaticdockerx86_64musl
 	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
 	cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib

RELEASE-FROCKSDB.md

@@ -0,0 +1,109 @@
+
+@@ -0,0 +1,104 @@
+## Build in Windows
+
+Use Windows 64 bit machine (e.g. base AWS Windows instance: 4 cores, 16GB RAM, 40GB storage for build).
+
+Install:
+ * git
+ * java 8
+ * maven
+ * Visual Studio Community 15 (2017)
+
+With [chocolatey](https://chocolatey.org/install):
+
+    choco install git.install jdk8 maven visualstudio2017community
+
+Optionally:
+
+    choco install intellijidea-community vscode
+
+Open developer command prompt for vs 2017 and run commands:
+
+    git clone [email protected]:ververica/frocksdb.git
+    cd frocksdb
+    git checkout FRocksDB-6.11.6 # release branch
+    java\crossbuild\build-win.bat
+
+The result native library is `build\java\Release\rocksdbjni-shared.dll`.
+The result windows jar is `build\java\rocksdbjni_classes.jar`.
+
+There is also a how-to in CMakeLists.txt.
+
+## Build in PPC64LE
+
+Use Ubuntu 16.04 (e.g. AWS instance 4 cores, 16GB RAM, 40GB storage for build).
+Install git if not installed. If docker is installed, it might need to be removed.
+
+Setup ppc64le docker machine ([source](https://developer.ibm.com/linuxonpower/2017/06/08/build-test-ppc64le-docker-images-intel/)):
+
+    wget http://ftp.unicamp.br/pub/ppc64el/boot2docker/install.sh && chmod +x ./install.sh && ./install.sh -s
+    docker-machine create -d qemu \
+        --qemu-boot2docker-url=/home/ubuntu/.docker/machine/boot2docker.iso \
+        --qemu-memory 8192 \
+        --qemu-cache-mode none \
+        --qemu-arch ppc64le \
+        vm-ppc64le
+
+Regenerate certs as suggested if it did not work at once.
+
+Prepare docker machine to run rocksdbjni docker image for ppc64le build:
+
+    eval $(docker-machine env vm-ppc64le)
+    git clone [email protected]:ververica/frocksdb.git
+    cd frocksdb
+    git checkout FRocksDB-6.11.6 # release branch
+    docker-machine ssh vm-ppc64le mkdir -p `pwd`
+    docker-machine scp -r . vm-ppc64le:`pwd`
+
+Build frocksdb:
+
+    make rocksdbjavastaticdockerppc64le
+    docker-machine scp vm-ppc64le:`pwd`/java/target/librocksdbjni-linux-ppc64le.so java/target/.
+
+    make rocksdbjavastaticdockerppc64lemusl
+    docker-machine scp vm-ppc64le:`pwd`/java/target/librocksdbjni-linux-ppc64le-musl.so java/target/.     
+
+The result native library is in `java/target/librocksdbjni-linux-ppc64le.so` and `java/target/librocksdbjni-linux-ppc64le-musl.so`.
+
+## Final crossbuild in Mac OSX
+
+Read how to Build cross jar for Mac OSX and linux as described in java/RELEASE.md but do not run it yet.
+
+Run commands:
+
+    make jclean clean
+    mkdir -p java/target
+    cp <path-to-windows-dll>/rocksdbjni-shared.dll java/target/librocksdbjni-win64.dll
+    cp <path-to-windows-jar>/rocksdbjni_classes.jar java/target/rocksdbjni_classes.jar
+    cp <path-to-ppc64le-lib-so>/librocksdbjni-linux-ppc64le.so java/target/librocksdbjni-linux-ppc64le.so
+    FROCKSDB_VERSION=1.0 make frocksdbjavastaticreleasedocker
+
+## Push to maven central
+
+Edit the `frocksdbjni-<release version>.pom` file and replace 
+`<version>-</version>` 
+with 
+`<version><release version></version>` 
+where the `<release version>` is e.g. `6.11.6-veverica-1.0`.
+
+Run:
+```bash
+VERSION=<release version> \
+USER=<sonatype user> \
+PASSWORD=<sonatype password> \
+KEYNAME=<key name> \
+PASSPHRASE=<passphrase> \
+java/publish-frocksdbjni.sh
+```
+
+Go to the staging repositories on Sonatype:
+
+https://oss.sonatype.org/#stagingRepositories
+
+Select the open staging repository and click on "Close".
+
+Test the files in the staging repository 
+which will look something like this `https://oss.sonatype.org/content/repositories/comdata-artisans-1020`.
+
+Press the "Release" button (WARNING: this can not be undone).

java/crossbuild/build-win.bat

@@ -0,0 +1,16 @@
+:: install git, java 8, maven, visual studio community 15 (2017)
+
+set MSBUILD=C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\MSBuild\15.0\Bin\MSBuild.exe
+
+if exist build rd /s /q build
+if exist librocksdbjni-win64.dll del librocksdbjni-win64.dll
+mkdir build && cd build
+
+cmake -G "Visual Studio 15 Win64" -DWITH_JNI=1 ..
+
+"%MSBUILD%" rocksdb.sln /p:Configuration=Release /m
+
+cd ..
+
+copy build\java\Release\rocksdbjni-shared.dll librocksdbjni-win64.dll
+echo Result is in librocksdbjni-win64.dll

java/deploysettings.xml

@@ -0,0 +1,12 @@
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0
+                      http://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <servers>
+    <server>
+      <id>sonatype-nexus-staging</id>
+      <username>${sonatype_user}</username>
+      <password>${sonatype_pw}</password>
+    </server>
+  </servers>
+</settings>

java/publish-frocksdbjni.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+################################################################################
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# fail on errors
+set -e
+
+PREFIX=java/target/frocksdb-release/frocksdbjni-${VERSION}
+
+function deploy() {
+  FILE=$1
+  CLASSIFIER=$2
+  echo "Deploying file=${FILE} with classifier=${CLASSIFIER} to sonatype with prefix=${PREFIX}"
+  sonatype_user=${USER} sonatype_pw=${PASSWORD} mvn gpg:sign-and-deploy-file \
+   --settings java/deploysettings.xml \
+   -Durl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ \
+   -DrepositoryId=sonatype-nexus-staging \
+   -DpomFile=${PREFIX}.pom \
+   -Dfile=$FILE \
+   -Dclassifier=$CLASSIFIER \
+   -Dgpg.keyname=${KEYNAME}  \
+   -Dgpg.passphrase=${PASSPHRASE}
+}
+
+deploy ${PREFIX}-sources.jar sources
+deploy ${PREFIX}-javadoc.jar javadoc
+deploy ${PREFIX}.jar

java/rocksjni.pom

@@ -4,26 +4,21 @@
         xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
     <modelVersion>4.0.0</modelVersion>
-    <name>RocksDB JNI</name>
-    <url>http://rocksdb.org/</url>
-    <groupId>org.rocksdb</groupId>
-    <artifactId>rocksdbjni</artifactId>
+    <name>FRocksDB JNI</name>
+    <url>https://github.com/ververica/frocksdb</url>
+    <groupId>com.ververica</groupId>
+    <artifactId>frocksdbjni</artifactId>
     <!-- Version will be automatically replaced -->
     <version>-</version>
-    <description>RocksDB fat jar that contains .so files for linux32 and linux64 (glibc and musl-libc), jnilib files
-        for Mac OSX, and a .dll for Windows x64.
+    <description>RocksDB fat jar with modifications specific for Apache Flink
+        that contains .so files for linux32 and linux64, jnilib files for Mac OSX, and a .dll for Windows x64.
     </description>
     <licenses>
         <license>
             <name>Apache License 2.0</name>
             <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
             <distribution>repo</distribution>
         </license>
-        <license>
-            <name>GNU General Public License, version 2</name>
-            <url>http://www.gnu.org/licenses/gpl-2.0.html</url>
-            <distribution>repo</distribution>
-        </license>
     </licenses>
     <scm>
         <connection>scm:git:git://github.com/dropwizard/metrics.git</connection>