diff --git a/.gitignore b/.gitignore index e28304f2..9eeed110 100644 --- a/.gitignore +++ b/.gitignore @@ -60,7 +60,6 @@ pink/test/pink_thread_test # Others pink/tags -pink/third *.d.* *.d diff --git a/pink/build.sh b/pink/build.sh index e35612f8..dbe5c138 100755 --- a/pink/build.sh +++ b/pink/build.sh @@ -11,8 +11,8 @@ if test -z $SLASH_PATH; then fi if [[ ! -d $SLASH_PATH ]]; then - mkdir -p $SLASH_PATH - git clone https://github.com/Qihoo360/slash.git $SLASH_PATH + echo "Slash library is not available" + exit 1 fi cd $SLASH_PATH/slash && make diff --git a/pink/third/slash/.gitignore b/pink/third/slash/.gitignore new file mode 100755 index 00000000..08a36b6b --- /dev/null +++ b/pink/third/slash/.gitignore @@ -0,0 +1,47 @@ +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +tags +lib/ +slash/src/build_version.cc +*.d + +# examples +binlog_example +cond_lock_example +conf_example +hash_example +mutex_example + +# tests +base_conf_test +slash_binlog_test +slash_coding_test +slash_env_test +slash_string_test diff --git a/pink/third/slash/.travis.yml b/pink/third/slash/.travis.yml new file mode 100644 index 00000000..14836fbf --- /dev/null +++ b/pink/third/slash/.travis.yml @@ -0,0 +1,19 @@ +sudo: required +dist: trusty +language: cpp + +os: + - linux + +compiler: + - gcc + +language: cpp +notifications: + email: + - chenzongzhi@360.cn + +script: + - make -C slash + - make -C slash example + - make -C slash check diff --git a/pink/third/slash/LICENSE b/pink/third/slash/LICENSE new file mode 100644 index 00000000..8dada3ed --- /dev/null +++ b/pink/third/slash/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pink/third/slash/README.md b/pink/third/slash/README.md new file mode 100644 index 00000000..705a14ca --- /dev/null +++ b/pink/third/slash/README.md @@ -0,0 +1,16 @@ +# Slash: C++ library + +[![Build Status](https://travis-ci.org/Qihoo360/slash.svg?branch=master)](https://travis-ci.org/Qihoo360/slash) + +## Composition + +* File utils; +* configuration utils; +* mutex utils; +* simple log utils; +* string utils; +* simple hash utils; +* coding utils; +* binlog utils; +* unit test utils; +* environment utils; diff --git a/pink/third/slash/slash/Makefile b/pink/third/slash/slash/Makefile new file mode 100644 index 00000000..056c5093 --- /dev/null +++ b/pink/third/slash/slash/Makefile @@ -0,0 +1,188 @@ +CLEAN_FILES = # deliberately empty, so we can append below. +CXX=g++ +LDFLAGS= -lpthread -lrt +CXXFLAGS= -g -std=c++11 -fno-builtin-memcmp -pipe -fPIC +ifeq ($(shell uname -m), x86_64) + CXXFLAGS += -msse -msse4.2 +endif +PROFILING_FLAGS= -pg +ARFLAGS = rs +OPT= + +# Set the default DEBUG_LEVEL to 0 +DEBUG_LEVEL?=0 + +ifeq ($(MAKECMDGOALS),dbg) + DEBUG_LEVEL=2 # compatible with rocksdb +endif + +# compile with -O2 if for release +# if we're compiling for release, compile without debug code (-DNDEBUG) and +# don't treat warnings as errors +ifeq ($(DEBUG_LEVEL),0) +DISABLE_WARNING_AS_ERROR=1 +OPT += -O2 -fno-omit-frame-pointer -DNDEBUG +else +$(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) +OPT += -O0 -D__XDEBUG__ -D_GNU_SOURCE $(PROFILING_FLAGS) +endif + +#----------------------------------------------- + +SRC_DIR=src +VERSION_CC=$(SRC_DIR)/build_version.cc +LIB_SOURCES := $(VERSION_CC) \ + $(filter-out $(VERSION_CC), $(wildcard $(SRC_DIR)/*.cc)) + +AM_DEFAULT_VERBOSITY = 0 + +AM_V_GEN = $(am__v_GEN_$(V)) +am__v_GEN_ = $(am__v_GEN_$(AM_DEFAULT_VERBOSITY)) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_$(V)) +am__v_at_ = $(am__v_at_$(AM_DEFAULT_VERBOSITY)) +am__v_at_0 = @ +am__v_at_1 = + +AM_V_CXX = $(am__v_CXX_$(V)) +am__v_CXX_ = $(am__v_CXX_$(AM_DEFAULT_VERBOSITY)) +am__v_CXX_0 = @echo " CXX " $@; +am__v_CXX_1 = +LD = $(CXX) +AM_V_LD = $(am__v_LD_$(V)) +am__v_LD_ = $(am__v_LD_$(AM_DEFAULT_VERBOSITY)) +am__v_LD_0 = @echo " LD " $@; +am__v_LD_1 = +AM_V_AR = $(am__v_AR_$(V)) +am__v_AR_ = $(am__v_AR_$(AM_DEFAULT_VERBOSITY)) +am__v_AR_0 = @echo " AR " $@; +am__v_AR_1 = + +AM_LINK = $(AM_V_LD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) + +# This (the first rule) must depend on "all". +default: all + +WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare \ + -Wno-unused-parameter -Wno-redundant-decls -Wwrite-strings \ + -Wpointer-arith -Wreorder -Wswitch -Wsign-promo \ + -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers + +ifndef DISABLE_WARNING_AS_ERROR + WARNING_FLAGS += -Werror +endif + +CXXFLAGS += $(WARNING_FLAGS) -I.. $(OPT) + +date := $(shell date +%F) +git_sha := $(shell git rev-parse HEAD 2>/dev/null) +gen_build_version = sed -e s/@@GIT_SHA@@/$(git_sha)/ -e s/@@GIT_DATE_TIME@@/$(date)/ src/build_version.cc.in +# Record the version of the source that we are compiling. +# We keep a record of the git revision in this file. It is then built +# as a regular source file as part of the compilation process. +# One can run "strings executable_filename | grep _build_" to find +# the version of the source that we used to build the executable file. +CLEAN_FILES += src/build_version.cc + +FORCE: +src/build_version.cc: FORCE + $(AM_V_GEN)rm -f $@-t + $(AM_V_at)$(gen_build_version) > $@-t + $(AM_V_at)if test -f $@; then \ + cmp -s $@-t $@ && rm -f $@-t || mv -f $@-t $@; \ + else mv -f $@-t $@; fi + +LIBOBJECTS = $(LIB_SOURCES:.cc=.o) + +# if user didn't config LIBNAME, set the default +ifeq ($(LIBNAME),) +# we should only run slash in production with DEBUG_LEVEL 0 +ifeq ($(DEBUG_LEVEL),0) + LIBNAME=libslash +else + LIBNAME=libslash_debug +endif +endif +LIBOUTPUT = lib +dummy := $(shell mkdir -p $(LIBOUTPUT)) +LIBRARY = $(LIBOUTPUT)/${LIBNAME}.a + +TESTS = slash_string_test slash_binlog_test slash_coding_test base_conf_test \ + slash_env_test + +EXAMPLES = conf_example cond_lock_example binlog_example mutex_example hash_example + +.PHONY: clean dbg static_lib all check example + +all: $(LIBRARY) + +static_lib: $(LIBRARY) + +example: $(LIBRARY) $(EXAMPLES) + +check: $(LIBRARY) $(TESTS) + for t in $(notdir $(TESTS)); do echo "***** Running $$t"; ./$$t || exit 1; done + +dbg: $(LIBRARY) $(EXAMPLES) + +$(LIBRARY): $(LIBOBJECTS) + $(AM_V_AR)rm -f $@ + $(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS) + +%.o : %.cc + $(AM_V_CXX)$(CXX) $(CXXFLAGS) -c $< -o $@ + +%.d: %.cc + @set -e; rm -f $@; $(CXX) -MM $(CXXFLAGS) $< > $@.$$$$; \ + sed 's,\($(notdir $*)\)\.o[ :]*,$(SRC_DIR)/\1.o $@ : ,g' < $@.$$$$ > $@; \ + rm -f $@.$$$$ + +ifneq ($(MAKECMDGOALS),clean) + -include $(LIBOBJECTS:.o=.d) +endif + +clean: + make -C ./examples clean + rm -f $(TESTS) $(EXAMPLES) + rm -f $(LIBRARY) + rm -rf $(CLEAN_FILES) + rm -rf $(LIBOUTPUT) + find . -name "*.[oda]*" -exec rm -f {} \; + find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \; + +# tests + +TEST_MAIN = tests/test_main.o + +slash_string_test: tests/slash_string_test.o $(TEST_MAIN) $(LIBOBJECTS) + $(AM_LINK) + +slash_binlog_test: tests/slash_binlog_test.o $(TEST_MAIN) $(LIBOBJECTS) + $(AM_LINK) + +base_conf_test: tests/base_conf_test.o $(TEST_MAIN) $(LIBOBJECTS) + $(AM_LINK) + +slash_coding_test: tests/slash_coding_test.o $(TEST_MAIN) $(LIBOBJECTS) + $(AM_LINK) + +slash_env_test: tests/slash_env_test.o $(TEST_MAIN) $(LIBOBJECTS) + $(AM_LINK) + +# examples + +conf_example: examples/conf_example.o $(LIBOBJECTS) + $(AM_LINK) + +cond_lock_example: examples/cond_lock_example.o $(LIBOBJECTS) + $(AM_LINK) + +binlog_example: examples/binlog_example.o $(LIBOBJECTS) + $(AM_LINK) + +mutex_example: examples/mutex_example.o $(LIBOBJECTS) + $(AM_LINK) + +hash_example: examples/hash_example.o $(LIBOBJECTS) + $(AM_LINK) diff --git a/pink/third/slash/slash/examples/Makefile b/pink/third/slash/slash/examples/Makefile new file mode 100644 index 00000000..588cd6ec --- /dev/null +++ b/pink/third/slash/slash/examples/Makefile @@ -0,0 +1,34 @@ +CXX=g++ +LDFLAGS= -lpthread -lrt +CXXFLAGS= -O2 -std=c++11 -fno-builtin-memcmp +ifeq ($(shell uname -m), x86_64) + CXXFLAGS += -msse -msse4.2 +endif + +.PHONY: clean libslash + +all: conf_example cond_lock_example binlog_example mutex_example hash_example + +CXXFLAGS+= -I../.. + +conf_example: libslash conf_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../lib/libslash.a $(LDFLAGS) + +cond_lock_example: libslash cond_lock_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../lib/libslash.a $(LDFLAGS) + +binlog_example: libslash binlog_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../lib/libslash.a $(LDFLAGS) + +mutex_example: libslash mutex_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../lib/libslash.a $(LDFLAGS) + +hash_example: libslash hash_example.cc + $(CXX) $(CXXFLAGS) $@.cc -o$@ ../lib/libslash.a $(LDFLAGS) + +clean: + find . -name "*.[oda]*" -exec rm -f {} \; + rm -rf ./conf_example ./cond_lock_example ./binlog_example ./mutex_example ./hash_example + +libslash: + cd .. && $(MAKE) static_lib diff --git a/pink/third/slash/slash/examples/binlog_example.cc b/pink/third/slash/slash/examples/binlog_example.cc new file mode 100644 index 00000000..c44b7029 --- /dev/null +++ b/pink/third/slash/slash/examples/binlog_example.cc @@ -0,0 +1,65 @@ +#include + +#include "slash/include/slash_binlog.h" +#include "slash/include/xdebug.h" + +using namespace slash; + +Binlog *log; + +#define UNUSED(arg) ((void)arg) + +void* Append(void *arg) { + UNUSED(arg); + int i = 0; + while (true) { + std::string item = "item" + std::to_string(i++); + Status s = log->Append(item); + printf ("+ Append (%s) return %s\n", item.c_str(), s.ToString().c_str()); + sleep(1); + } + return NULL; +} + +void* Reader(void *arg) { + UNUSED(arg); + int *id = (int *)arg; + BinlogReader* reader = log->NewBinlogReader(0, 0); + + while (true) { + std::string str; + Status s = reader->ReadRecord(str); + if (s.ok()) { + printf ("- Reader %u: get (%s)\n", *id, str.c_str()); + } else { + printf ("- Reader %lu: %s\n", pthread_self(), s.ToString().c_str()); + } + sleep(1); + } + return NULL; +} + +int main() { + + Status s = Binlog::Open("./binlog", &log); + if (!s.ok()) { + printf ("Open failed %s\n", s.ToString().c_str()); + return -1; + } + + pthread_t pid; + pthread_create(&pid, NULL, &Append, NULL); + + pthread_t cid[3]; + int id[3]; + + for (int i = 0; i < 3; i++) { + id[i] = i; + pthread_create(&cid[i], NULL, &Reader, &id[i]); + } + + void* res; + pthread_join(pid, &res); + + return 0; +} diff --git a/pink/third/slash/slash/examples/cond_lock_example.cc b/pink/third/slash/slash/examples/cond_lock_example.cc new file mode 100644 index 00000000..f3196cb5 --- /dev/null +++ b/pink/third/slash/slash/examples/cond_lock_example.cc @@ -0,0 +1,17 @@ +#include + +#include "slash/include/cond_lock.h" + +#include + +slash::CondLock cl; + +int main() +{ + cl.Lock(); + cl.Unlock(); + uint32_t a = 2000; + cl.Lock(); + cl.TimedWait(a); + return 0; +} diff --git a/pink/third/slash/slash/examples/conf_example.cc b/pink/third/slash/slash/examples/conf_example.cc new file mode 100644 index 00000000..178cddd2 --- /dev/null +++ b/pink/third/slash/slash/examples/conf_example.cc @@ -0,0 +1,23 @@ +#include "slash/include/base_conf.h" +#include "slash/include/xdebug.h" + +using namespace slash; + +int main() +{ + BaseConf b("./conf/pika.conf"); + + if (b.LoadConf() == 0) { + log_info("LoadConf true"); + } else { + log_info("LoodConf error"); + } + + b.SetConfInt("port", 99999); + b.SetConfStr("pidfile", "./anan.pid"); + b.WriteBack(); + b.DumpConf(); + b.WriteSampleConf(); + + return 0; +} diff --git a/pink/third/slash/slash/examples/hash_example.cc b/pink/third/slash/slash/examples/hash_example.cc new file mode 100644 index 00000000..c132641a --- /dev/null +++ b/pink/third/slash/slash/examples/hash_example.cc @@ -0,0 +1,16 @@ +#include +#include + +#include "slash/include/slash_hash.h" + +using namespace slash; +int main() { + std::string input = "grape"; + std::string output1 = sha256(input); + std::string output2 = md5(input); + + std::cout << "sha256('"<< input << "'): " << output1 << std::endl; + std::cout << "md5('"<< input << "'): " << output2<< std::endl; + + return 0; +} diff --git a/pink/third/slash/slash/examples/mutex_example.cc b/pink/third/slash/slash/examples/mutex_example.cc new file mode 100644 index 00000000..02190e52 --- /dev/null +++ b/pink/third/slash/slash/examples/mutex_example.cc @@ -0,0 +1,38 @@ +#include + +#include "slash/include/slash_mutex.h" + +using namespace slash; + +int main() { + Mutex mu; + CondVar cv(&mu); + + mu.Lock(); + // sleep 1 millisecond + cv.TimedWait(1); + printf("sleep 1 millisecond\n"); + + // sleep 1 second + cv.TimedWait(1000); + printf("sleep 1 second\n"); + cv.TimedWait(999); + printf("sleep 999 millisecond\n"); + cv.TimedWait(999); + printf("sleep 999 millisecond\n"); + cv.TimedWait(1); + printf("sleep 1 millisecond\n"); + cv.TimedWait(999); + printf("sleep 999 millisecond\n"); + cv.TimedWait(999); + printf("sleep 999 millisecond\n"); + cv.TimedWait(999); + printf("sleep 999 millisecond\n"); + cv.TimedWait(999); + printf("sleep 999 millisecond\n"); + + mu.Unlock(); + + return 0; +} + diff --git a/pink/third/slash/slash/include/base_conf.h b/pink/third/slash/slash/include/base_conf.h new file mode 100644 index 00000000..39f2fbea --- /dev/null +++ b/pink/third/slash/slash/include/base_conf.h @@ -0,0 +1,94 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SLASH_INCLUDE_BASE_CONF_H_ +#define SLASH_INCLUDE_BASE_CONF_H_ + +#include +#include + +#include +#include + +#include "slash/include/slash_define.h" + +namespace slash { + +class BaseConf; + + +class BaseConf { + public: + struct Rep { + std::string path; + enum ConfType { + kConf = 0, + kComment = 1, + }; + + struct ConfItem { + ConfType type; // 0 means conf, 1 means comment + std::string name; + std::string value; + ConfItem(ConfType t, const std::string &v) : + type(t), + name(""), + value(v) + {} + ConfItem(ConfType t, const std::string &n, const std::string &v) : + type(t), + name(n), + value(v) + {} + }; + + explicit Rep(const std::string &p) + : path(p) { + } + std::vector item; + }; + + explicit BaseConf(const std::string &path); + virtual ~BaseConf(); + + int LoadConf(); + int32_t ReloadConf(); + + // return false if the item dosen't exist + bool GetConfInt(const std::string &name, int* value) const; + bool GetConfInt64(const std::string &name, int64_t* value) const; + + bool GetConfStr(const std::string &name, std::string *value) const; + bool GetConfBool(const std::string &name, bool* value) const; + bool GetConfStrVec(const std::string &name, std::vector *value) const; + + bool SetConfInt(const std::string &name, const int value); + bool SetConfInt64(const std::string &name, const int64_t value); + + bool SetConfStr(const std::string &name, const std::string &value); + bool SetConfBool(const std::string &name, const bool value); + bool SetConfStrVec(const std::string &name, const std::vector &value); + + bool CheckConfExist(const std::string &name) const; + + void DumpConf() const; + bool WriteBack(); + void WriteSampleConf() const; + + void PushConfItem(const Rep::ConfItem& item); + + private: + Rep* rep_; + + /* + * No copy && no assign operator + */ + BaseConf(const BaseConf&); + void operator=(const BaseConf&); +}; + +} // namespace slash + +#endif // SLASH_INCLUDE_BASE_CONF_H_ diff --git a/pink/third/slash/slash/include/cond_lock.h b/pink/third/slash/slash/include/cond_lock.h new file mode 100644 index 00000000..e802759f --- /dev/null +++ b/pink/third/slash/slash/include/cond_lock.h @@ -0,0 +1,49 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +#ifndef INCLUDE_COND_LOCK_H_ +#define INCLUDE_COND_LOCK_H_ +/* + * CondLock is a wrapper for condition variable. + * It contain a mutex in it's class, so we don't need other to protect the + * condition variable. + */ +#include +#include +#include +#include + +#include + +namespace slash { + +class CondLock { + public: + CondLock(); + ~CondLock(); + + void Lock(); + void Unlock(); + + void Wait(); + + /* + * timeout is millisecond + */ + void TimedWait(uint32_t timeout); + void Signal(); + void Broadcast(); + + private: + pthread_mutex_t mutex_; + pthread_cond_t cond_; + + CondLock(const CondLock&) {}; + void operator =(const CondLock&) {}; +}; + +} // namespace slash + +#endif // INCLUDE_COND_LOCK_H_ diff --git a/pink/third/slash/slash/include/env.h b/pink/third/slash/slash/include/env.h new file mode 100644 index 00000000..2800e66b --- /dev/null +++ b/pink/third/slash/slash/include/env.h @@ -0,0 +1,185 @@ +#ifndef SLASH_ENV_H_ +#define SLASH_ENV_H_ + +#include +#include +#include + +#include "slash/include/slash_status.h" + +namespace slash { + +class WritableFile; +class SequentialFile; +class RWFile; +class RandomRWFile; + +/* + * Set the resource limits of a process + */ +int SetMaxFileDescriptorNum(int64_t max_file_descriptor_num); + +/* + * Set size of initial mmap size + */ +void SetMmapBoundSize(size_t size); + +extern const size_t kPageSize; + + +/* + * File Operations + */ +int IsDir(const std::string& path); +int DeleteDir(const std::string& path); +bool DeleteDirIfExist(const std::string& path); +int CreateDir(const std::string& path); +int CreatePath(const std::string& path, mode_t mode = 0755); +uint64_t Du(const std::string& path); + +/* + * Whether the file is exist + * If exist return true, else return false + */ +bool FileExists(const std::string& path); + +Status DeleteFile(const std::string& fname); + +int RenameFile(const std::string& oldname, const std::string& newname); + +class FileLock { + public: + FileLock() { } + virtual ~FileLock() {}; + + int fd_; + std::string name_; + + private: + + // No copying allowed + FileLock(const FileLock&); + void operator=(const FileLock&); +}; + +Status LockFile(const std::string& f, FileLock** l); +Status UnlockFile(FileLock* l); + +int GetChildren(const std::string& dir, std::vector& result); +bool GetDescendant(const std::string& dir, std::vector& result); + + +uint64_t NowMicros(); +void SleepForMicroseconds(int micros); + +Status NewSequentialFile(const std::string& fname, SequentialFile** result); + +Status NewWritableFile(const std::string& fname, WritableFile** result); + +Status NewRWFile(const std::string& fname, RWFile** result); + +Status AppendSequentialFile(const std::string& fname, SequentialFile** result); + +Status AppendWritableFile(const std::string& fname, WritableFile** result, uint64_t write_len = 0); + +Status NewRandomRWFile(const std::string& fname, RandomRWFile** result); + +// A file abstraction for sequential writing. The implementation +// must provide buffering since callers may append small fragments +// at a time to the file. +class WritableFile { + public: + WritableFile() { } + virtual ~WritableFile(); + + virtual Status Append(const Slice& data) = 0; + virtual Status Close() = 0; + virtual Status Flush() = 0; + virtual Status Sync() = 0; + virtual Status Trim(uint64_t offset) = 0; + virtual uint64_t Filesize() = 0; + + private: + // No copying allowed + WritableFile(const WritableFile&); + void operator=(const WritableFile&); +}; + +// A abstract for the sequential readable file +class SequentialFile { + public: + SequentialFile() {}; + virtual ~SequentialFile(); + //virtual Status Read(size_t n, char *&result, char *scratch) = 0; + virtual Status Read(size_t n, Slice* result, char* scratch) = 0; + virtual Status Skip(uint64_t n) = 0; + //virtual Status Close() = 0; + virtual char *ReadLine(char *buf, int n) = 0; +}; + +class RWFile { +public: + RWFile() { } + virtual ~RWFile(); + virtual char* GetData() = 0; + +private: + // No copying allowed + RWFile(const RWFile&); + void operator=(const RWFile&); +}; + +// A file abstraction for random reading and writing. +class RandomRWFile { + public: + RandomRWFile() {} + virtual ~RandomRWFile() {} + + // Write data from Slice data to file starting from offset + // Returns IOError on failure, but does not guarantee + // atomicity of a write. Returns OK status on success. + // + // Safe for concurrent use. + virtual Status Write(uint64_t offset, const Slice& data) = 0; + // Read up to "n" bytes from the file starting at "offset". + // "scratch[0..n-1]" may be written by this routine. Sets "*result" + // to the data that was read (including if fewer than "n" bytes were + // successfully read). May set "*result" to point at data in + // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when + // "*result" is used. If an error was encountered, returns a non-OK + // status. + // + // Safe for concurrent use by multiple threads. + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const = 0; + virtual Status Close() = 0; // closes the file + virtual Status Sync() = 0; // sync data + + /* + * Sync data and/or metadata as well. + * By default, sync only data. + * Override this method for environments where we need to sync + * metadata as well. + */ + virtual Status Fsync() { + return Sync(); + } + + /* + * Pre-allocate space for a file. + */ + virtual Status Allocate(off_t offset, off_t len) { + (void)offset; + (void)len; + return Status::OK(); + } + + private: + // No copying allowed + RandomRWFile(const RandomRWFile&); + void operator=(const RandomRWFile&); +}; + + +} // namespace slash +#endif // SLASH_ENV_H_ diff --git a/pink/third/slash/slash/include/fmacros.h b/pink/third/slash/slash/include/fmacros.h new file mode 100644 index 00000000..dca05159 --- /dev/null +++ b/pink/third/slash/slash/include/fmacros.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _SLASH_FMACRO_H +#define _SLASH_FMACRO_H + +#ifndef _BSD_SOURCE +#define _BSD_SOURCE +#endif + +#if defined(__linux__) +#define _GNU_SOURCE_REDIS +#define _DEFAULT_SOURCE +#endif + +#if defined(_AIX) +#define _ALL_SOURCE +#endif + +#if defined(__linux__) || defined(__OpenBSD__) +#define _XOPEN_SOURCE 700 +/* + * On NetBSD, _XOPEN_SOURCE undefines _NETBSD_SOURCE and + * thus hides inet_aton etc. + */ +#elif !defined(__NetBSD__) +#define _XOPEN_SOURCE +#endif + +#if defined(__sun) +#define _POSIX_C_SOURCE 199506L +#endif + +#ifndef _LARGEFILE_SOURCE +#define _LARGEFILE_SOURCE +#endif + +#define _FILE_OFFSET_BITS 64 + +#endif diff --git a/pink/third/slash/slash/include/lock_mgr.h b/pink/third/slash/slash/include/lock_mgr.h new file mode 100644 index 00000000..83c7f8d4 --- /dev/null +++ b/pink/third/slash/slash/include/lock_mgr.h @@ -0,0 +1,60 @@ +// Copyright (c) 2017-present The blackwidow Authors. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SRC_LOCK_MGR_H_ +#define SRC_LOCK_MGR_H_ + +#include +#include + +#include "slash/include/mutex.h" + +namespace slash { + +namespace lock { +struct LockMap; +struct LockMapStripe; + +class LockMgr { + public: + LockMgr(size_t default_num_stripes, int64_t max_num_locks, + std::shared_ptr factory); + + ~LockMgr(); + + // Attempt to lock key. If OK status is returned, the caller is responsible + // for calling UnLock() on this key. + Status TryLock(const std::string& key); + + // Unlock a key locked by TryLock(). + void UnLock(const std::string& key); + + private: + // Default number of lock map stripes + const size_t default_num_stripes_; + + // Limit on number of keys locked per column family + const int64_t max_num_locks_; + + // Used to allocate mutexes/condvars to use when locking keys + std::shared_ptr mutex_factory_; + + // Map to locked key info + std::shared_ptr lock_map_; + + Status Acquire(LockMapStripe* stripe, const std::string& key); + + Status AcquireLocked(LockMapStripe* stripe, const std::string& key); + + void UnLockKey(const std::string& key, LockMapStripe* stripe); + + // No copying allowed + LockMgr(const LockMgr&); + void operator=(const LockMgr&); +}; + +} // namespace lock +} // namespace slash +#endif // SRC_LOCK_MGR_H_ diff --git a/pink/third/slash/slash/include/mutex.h b/pink/third/slash/slash/include/mutex.h new file mode 100644 index 00000000..aa1950d7 --- /dev/null +++ b/pink/third/slash/slash/include/mutex.h @@ -0,0 +1,89 @@ +// Copyright (c) 2017-present The blackwidow Authors. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SRC_MUTEX_H_ +#define SRC_MUTEX_H_ + +#include + +#include "slash/include/slash_status.h" + +namespace slash { + +namespace lock { + +using Status = slash::Status; + +class Mutex { + public: + virtual ~Mutex() {} + + // Attempt to acquire lock. Return OK on success, or other Status on failure. + // If returned status is OK, BlackWidow will eventually call UnLock(). + virtual Status Lock() = 0; + + // Attempt to acquire lock. If timeout is non-negative, operation may be + // failed after this many microseconds. + // Returns OK on success, + // TimedOut if timed out, + // or other Status on failure. + // If returned status is OK, BlackWidow will eventually call UnLock(). + virtual Status TryLockFor(int64_t timeout_time) = 0; + + // Unlock Mutex that was successfully locked by Lock() or TryLockUntil() + virtual void UnLock() = 0; +}; + +class CondVar { + public: + virtual ~CondVar() {} + + // Block current thread until condition variable is notified by a call to + // Notify() or NotifyAll(). Wait() will be called with mutex locked. + // Returns OK if notified. + // Returns non-OK if BlackWidow should stop waiting and fail the operation. + // May return OK spuriously even if not notified. + virtual Status Wait(std::shared_ptr mutex) = 0; + + // Block current thread until condition variable is notified by a call to + // Notify() or NotifyAll(), or if the timeout is reached. + // Wait() will be called with mutex locked. + // + // If timeout is non-negative, operation should be failed after this many + // microseconds. + // If implementing a custom version of this class, the implementation may + // choose to ignore the timeout. + // + // Returns OK if notified. + // Returns TimedOut if timeout is reached. + // Returns other status if BlackWidow should otherwis stop waiting and + // fail the operation. + // May return OK spuriously even if not notified. + virtual Status WaitFor(std::shared_ptr mutex, + int64_t timeout_time) = 0; + + // If any threads are waiting on *this, unblock at least one of the + // waiting threads. + virtual void Notify() = 0; + + // Unblocks all threads waiting on *this. + virtual void NotifyAll() = 0; +}; + +// Factory class that can allocate mutexes and condition variables. +class MutexFactory { + public: + // Create a Mutex object. + virtual std::shared_ptr AllocateMutex() = 0; + + // Create a CondVar object. + virtual std::shared_ptr AllocateCondVar() = 0; + + virtual ~MutexFactory() {} +}; + +} // namespace lock +} // namespace slash +#endif // SRC_MUTEX_H_ diff --git a/pink/third/slash/slash/include/mutex_impl.h b/pink/third/slash/slash/include/mutex_impl.h new file mode 100644 index 00000000..6a16120d --- /dev/null +++ b/pink/third/slash/slash/include/mutex_impl.h @@ -0,0 +1,23 @@ +// Copyright (c) 2017-present The blackwidow Authors. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SRC_MUTEX_IMPL_H_ +#define SRC_MUTEX_IMPL_H_ + +#include "slash/include/mutex.h" + +#include + +namespace slash { +namespace lock { +// Default implementation of MutexFactory. +class MutexFactoryImpl : public MutexFactory { + public: + std::shared_ptr AllocateMutex() override; + std::shared_ptr AllocateCondVar() override; +}; +} // namespace lock +} // namespace slash +#endif // SRC_MUTEX_IMPL_H_ diff --git a/pink/third/slash/slash/include/posix.h b/pink/third/slash/slash/include/posix.h new file mode 100644 index 00000000..8b36056a --- /dev/null +++ b/pink/third/slash/slash/include/posix.h @@ -0,0 +1,162 @@ +/* $begin csapp.h */ +#ifndef __CSAPP_H__ +#define __CSAPP_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +/* Default file permissions are DEF_MODE & ~DEF_UMASK */ +/* $begin createmasks */ +#define DEF_MODE S_IRUSR|S_IWUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH +#define DEF_UMASK S_IWGRP|S_IWOTH +/* $end createmasks */ + +/* Simplifies calls to bind(), connect(), and accept() */ +/* $begin sockaddrdef */ +typedef struct sockaddr SA; +/* $end sockaddrdef */ + +/* Persistent state for the robust I/O (Rio) package */ +/* $begin rio_t */ +#define RIO_BUFSIZE 8192 +typedef struct { + int rio_fd; /* descriptor for this internal buf */ + int rio_cnt; /* unread bytes in internal buf */ + char *rio_bufptr; /* next unread byte in internal buf */ + char rio_buf[RIO_BUFSIZE]; /* internal buffer */ +} rio_t; +/* $end rio_t */ + +/* External variables */ +extern int h_errno; /* defined by BIND for DNS errors */ +extern char **environ; /* defined by libc */ + +/* Misc constants */ +#define MAXLINE 8192 /* max text line length */ +#define MAXBUF 8192 /* max I/O buffer size */ +#define LISTENQ 1024 /* second argument to listen() */ + +/* Process control wrappers */ +pid_t Fork(void); +void Execve(const char *filename, char *const argv[], char *const envp[]); +pid_t Wait(int *status); +pid_t Waitpid(pid_t pid, int *iptr, int options); +void Kill(pid_t pid, int signum); +unsigned int Sleep(unsigned int secs); +void Pause(void); +unsigned int Alarm(unsigned int seconds); +void Setpgid(pid_t pid, pid_t pgid); +pid_t Getpgrp(); + +/* Signal wrappers */ +typedef void handler_t(int); +handler_t *Signal(int signum, handler_t *handler); +void Sigprocmask(int how, const sigset_t *set, sigset_t *oldset); +void Sigemptyset(sigset_t *set); +void Sigfillset(sigset_t *set); +void Sigaddset(sigset_t *set, int signum); +void Sigdelset(sigset_t *set, int signum); +int Sigismember(const sigset_t *set, int signum); + +/* Unix I/O wrappers */ +int Open(const char *pathname, int flags, mode_t mode); +ssize_t Read(int fd, void *buf, size_t count); +ssize_t Write(int fd, const void *buf, size_t count); +off_t Lseek(int fildes, off_t offset, int whence); +void Close(int fd); +int Select(int n, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, + struct timeval *timeout); +int Dup2(int fd1, int fd2); +void Stat(const char *filename, struct stat *buf); +void Fstat(int fd, struct stat *buf) ; + +/* Memory mapping wrappers */ +void *Mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset); +void Munmap(void *start, size_t length); + +/* Standard I/O wrappers */ +void Fclose(FILE *fp); +FILE *Fdopen(int fd, const char *type); +char *Fgets(char *ptr, int n, FILE *stream); +FILE *Fopen(const char *filename, const char *mode); +void Fputs(const char *ptr, FILE *stream); +size_t Fread(void *ptr, size_t size, size_t nmemb, FILE *stream); +void Fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream); + +/* Dynamic storage allocation wrappers */ +void *Malloc(size_t size); +void *Realloc(void *ptr, size_t size); +void *Calloc(size_t nmemb, size_t size); +void Free(void *ptr); + +/* Sockets interface wrappers */ +int Socket(int domain, int type, int protocol); +void Setsockopt(int s, int level, int optname, const void *optval, int optlen); +void Bind(int sockfd, struct sockaddr *my_addr, int addrlen); +void Listen(int s, int backlog); +int Accept(int s, struct sockaddr *addr, socklen_t *addrlen); +void Connect(int sockfd, struct sockaddr *serv_addr, int addrlen); + +/* DNS wrappers */ +struct hostent *Gethostbyname(const char *name); +struct hostent *Gethostbyaddr(const char *addr, int len, int type); + +/* Pthreads thread control wrappers */ +void Pthread_create(pthread_t *tidp, pthread_attr_t *attrp, + void * (*routine)(void *), void *argp); +void Pthread_join(pthread_t tid, void **thread_return); +void Pthread_cancel(pthread_t tid); +void Pthread_detach(pthread_t tid); +void Pthread_exit(void *retval); +pthread_t Pthread_self(void); +void Pthread_once(pthread_once_t *once_control, void (*init_function)()); + +/* POSIX semaphore wrappers */ +void Sem_init(sem_t *sem, int pshared, unsigned int value); +void P(sem_t *sem); +void V(sem_t *sem); + +/* Rio (Robust I/O) package */ +ssize_t rio_readn(int fd, void *usrbuf, size_t n); +ssize_t rio_writen(int fd, void *usrbuf, size_t n); +void rio_readinitb(rio_t *rp, int fd); +ssize_t rio_readnb(rio_t *rp, void *usrbuf, size_t n); +ssize_t rio_readlineb(rio_t *rp, void *usrbuf, size_t maxlen); + +/* Wrappers for Rio package */ +ssize_t Rio_readn(int fd, void *usrbuf, size_t n); +void Rio_writen(int fd, void *usrbuf, size_t n); +void Rio_readinitb(rio_t *rp, int fd); +ssize_t Rio_readnb(rio_t *rp, void *usrbuf, size_t n); +ssize_t Rio_readlineb(rio_t *rp, void *usrbuf, size_t maxlen); + +/* Client/server helper functions */ +int open_clientfd(char *hostname, int portno); +int open_listenfd(int portno); + +/* Wrappers for client/server helper functions */ +int Open_clientfd(char *hostname, int port); +int Open_listenfd(int port); + +#endif /* __CSAPP_H__ */ +/* $end csapp.h */ diff --git a/pink/third/slash/slash/include/random.h b/pink/third/slash/slash/include/random.h new file mode 100644 index 00000000..08fdcd82 --- /dev/null +++ b/pink/third/slash/slash/include/random.h @@ -0,0 +1,25 @@ +#ifndef SLASH_INCLUDE_RANDOM_H_ +#define SLASH_INCLUDE_RANDOM_H_ + +#include + +namespace slash { + +class Random { + public: + Random() { + srand(time(NULL)); + } + + /* + * return Random number in [1...n] + */ + static uint32_t Uniform(int n) { + return (random() % n) + 1; + } + +}; + +}; // namespace slash + +#endif // SLASH_INCLUDE_RANDOM_H_ diff --git a/pink/third/slash/slash/include/rsync.h b/pink/third/slash/slash/include/rsync.h new file mode 100644 index 00000000..e6c548b9 --- /dev/null +++ b/pink/third/slash/slash/include/rsync.h @@ -0,0 +1,32 @@ +#ifndef SLASH_RSYNC_H_ +#define SLASH_RSYNC_H_ + +#include + +namespace slash { +const std::string kRsyncSecretFile = "slash_rsync.secret"; +const std::string kRsyncConfFile = "slash_rsync.conf"; +const std::string kRsyncLogFile = "slash_rsync.log"; +const std::string kRsyncPidFile = "slash_rsync.pid"; +const std::string kRsyncLockFile = "slash_rsync.lock"; +const std::string kRsyncSubDir = "rsync"; +const std::string kRsyncUser = "rsync_user"; +struct RsyncRemote { + std::string host; + int port; + std::string module; + int kbps; //speed limit + RsyncRemote(const std::string& _host, const int _port, + const std::string& _module, const int _kbps) + : host(_host), port(_port), module(_module), kbps(_kbps) {} +}; + +int StartRsync(const std::string& rsync_path, const std::string& module, const std::string& ip, const int port, const std::string& passwd); +int StopRsync(const std::string& path); +int RsyncSendFile(const std::string& local_file_path, const std::string& remote_file_path, + const std::string& secret_file_path, const RsyncRemote& remote); +int RsyncSendClearTarget(const std::string& local_dir_path, const std::string& remote_dir_path, + const std::string& secret_file_path, const RsyncRemote& remote); + +} +#endif diff --git a/pink/third/slash/slash/include/scope_record_lock.h b/pink/third/slash/slash/include/scope_record_lock.h new file mode 100644 index 00000000..00f9b7e7 --- /dev/null +++ b/pink/third/slash/slash/include/scope_record_lock.h @@ -0,0 +1,66 @@ +// Copyright (c) 2017-present The blackwidow Authors. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SRC_SCOPE_RECORD_LOCK_H_ +#define SRC_SCOPE_RECORD_LOCK_H_ + +#include +#include +#include + +#include "slash/include/lock_mgr.h" + +namespace slash { + +namespace lock { + +class ScopeRecordLock { + public: + ScopeRecordLock(LockMgr* lock_mgr, const Slice& key) : + lock_mgr_(lock_mgr), key_(key) { + lock_mgr_->TryLock(key_.ToString()); + } + ~ScopeRecordLock() { + lock_mgr_->UnLock(key_.ToString()); + } + + private: + LockMgr* const lock_mgr_; + Slice key_; + ScopeRecordLock(const ScopeRecordLock&); + void operator=(const ScopeRecordLock&); +}; + +class MultiScopeRecordLock { + public: + MultiScopeRecordLock(LockMgr* lock_mgr, + const std::vector& keys); + ~MultiScopeRecordLock(); + + private: + LockMgr* const lock_mgr_; + std::vector keys_; + MultiScopeRecordLock(const MultiScopeRecordLock&); + void operator=(const MultiScopeRecordLock&); +}; + +class MultiRecordLock { + public: + explicit MultiRecordLock(LockMgr* lock_mgr) : lock_mgr_(lock_mgr) { + } + ~MultiRecordLock() { + } + void Lock(const std::vector& keys); + void Unlock(const std::vector& keys); + + private: + LockMgr* const lock_mgr_; + MultiRecordLock(const MultiRecordLock&); + void operator=(const MultiRecordLock&); +}; + +} // namespace lock +} // namespace slash +#endif // SRC_SCOPE_RECORD_LOCK_H_ diff --git a/pink/third/slash/slash/include/slash_binlog.h b/pink/third/slash/slash/include/slash_binlog.h new file mode 100644 index 00000000..887db125 --- /dev/null +++ b/pink/third/slash/slash/include/slash_binlog.h @@ -0,0 +1,67 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SLASH_BINLOG_H_ +#define SLASH_BINLOG_H_ + +#include +#include + +#include "slash/include/slash_status.h" +#include "slash/include/xdebug.h" +//#include "slash_mutex.h" + +namespace slash { + +// SyncPoint is a file number and an offset; + +class BinlogReader; + +class Binlog { + public: + static Status Open(const std::string& path, Binlog** logptr); + + Binlog() { } + virtual ~Binlog() { } + + // TODO (aa) + // 1. maybe add Options + + // + // Basic API + // + virtual Status Append(const std::string &item) = 0; + virtual BinlogReader* NewBinlogReader(uint32_t filenum, uint64_t offset) = 0; + + // Set/Get Producer filenum and offset with lock + virtual Status GetProducerStatus(uint32_t* filenum, uint64_t* pro_offset) = 0; + virtual Status SetProducerStatus(uint32_t filenum, uint64_t pro_offset) = 0; + + private: + + // No copying allowed + Binlog(const Binlog&); + void operator=(const Binlog&); +}; + +class BinlogReader { + public: + BinlogReader() { } + virtual ~BinlogReader() { } + + virtual Status ReadRecord(std::string &record) = 0; + //bool ReadRecord(Slice* record, std::string* scratch) = 0; + + private: + + // No copying allowed; + BinlogReader(const BinlogReader&); + void operator=(const BinlogReader&); +}; + +} // namespace slash + + +#endif // SLASH_BINLOG_H_ diff --git a/pink/third/slash/slash/include/slash_coding.h b/pink/third/slash/slash/include/slash_coding.h new file mode 100644 index 00000000..7217fbde --- /dev/null +++ b/pink/third/slash/slash/include/slash_coding.h @@ -0,0 +1,119 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Endian-neutral encoding: +// * Fixed-length numbers are encoded with least-significant byte first +// * In addition we support variable length "varint" encoding +// * Strings are encoded prefixed by their length in varint format + +#ifndef SLASH_CODING_H_ +#define SLASH_CODING_H_ + +#include +#include +#include + +#include "slash/include/slash_slice.h" + +namespace slash { + +// Standard Put... routines append to a string +extern void PutFixed16(std::string* dst, uint16_t value); +extern void PutFixed32(std::string* dst, uint32_t value); +extern void PutFixed64(std::string* dst, uint64_t value); +extern void PutVarint32(std::string* dst, uint32_t value); +extern void PutVarint64(std::string* dst, uint64_t value); +extern void PutLengthPrefixedString(std::string* dst, const std::string& value); + +extern void GetFixed16(std::string* dst, uint16_t* value); +extern void GetFixed32(std::string* dst, uint32_t* value); +extern void GetFixed64(std::string* dst, uint64_t* value); +extern bool GetVarint32(std::string* input, uint32_t* value); +extern bool GetVarint64(std::string* input, uint64_t* value); +extern const char* GetLengthPrefixedSlice(const char* p, const char* limit, + Slice* result); +extern bool GetLengthPrefixedSlice(Slice* input, Slice* result); +extern bool GetLengthPrefixedString(std::string* input, std::string* result); + +// Pointer-based variants of GetVarint... These either store a value +// in *v and return a pointer just past the parsed value, or return +// NULL on error. These routines only look at bytes in the range +// [p..limit-1] +extern const char* GetVarint32Ptr(const char* p,const char* limit, uint32_t* v); +extern const char* GetVarint64Ptr(const char* p,const char* limit, uint64_t* v); + +// Returns the length of the varint32 or varint64 encoding of "v" +extern int VarintLength(uint64_t v); + +// Lower-level versions of Put... that write directly into a character buffer +// REQUIRES: dst has enough space for the value being written +extern void EncodeFixed16(char* dst, uint16_t value); +extern void EncodeFixed32(char* dst, uint32_t value); +extern void EncodeFixed64(char* dst, uint64_t value); + +// Lower-level versions of Put... that write directly into a character buffer +// and return a pointer just past the last byte written. +// REQUIRES: dst has enough space for the value being written +extern char* EncodeVarint32(char* dst, uint32_t value); +extern char* EncodeVarint64(char* dst, uint64_t value); + +// Lower-level versions of Get... that read directly from a character buffer +// without any bounds checking. + +inline uint16_t DecodeFixed16(const char* ptr) { + // Load the raw bytes + uint16_t result; + memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load + return result; +} + +inline uint32_t DecodeFixed32(const char* ptr) { + // Load the raw bytes + uint32_t result; + memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load + return result; +} + +inline uint64_t DecodeFixed64(const char* ptr) { + // Load the raw bytes + uint64_t result; + memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load + return result; +} + +inline void GetFixed16(std::string* dst, uint16_t *value) { + *value = DecodeFixed16(dst->data()); + dst->erase(0, sizeof(uint16_t)); +} + +inline void GetFixed32(std::string* dst, uint32_t *value) { + *value = DecodeFixed32(dst->data()); + dst->erase(0, sizeof(uint32_t)); +} + +inline void GetFixed64(std::string* dst, uint64_t *value) { + *value = DecodeFixed64(dst->data()); + dst->erase(0, sizeof(uint64_t)); +} + +// Internal routine for use by fallback path of GetVarint32Ptr +extern const char* GetVarint32PtrFallback(const char* p, + const char* limit, + uint32_t* value); +inline const char* GetVarint32Ptr(const char* p, + const char* limit, + uint32_t* value) { + if (p < limit) { + uint32_t result = *(reinterpret_cast(p)); + if ((result & 128) == 0) { + *value = result; + return p + 1; + } + } + return GetVarint32PtrFallback(p, limit, value); +} + +} // namespace leveldb + +#endif // SLASH_CODING_H_ diff --git a/pink/third/slash/slash/include/slash_define.h b/pink/third/slash/slash/include/slash_define.h new file mode 100644 index 00000000..28f95263 --- /dev/null +++ b/pink/third/slash/slash/include/slash_define.h @@ -0,0 +1,10 @@ +#ifndef SLASH_DEFINE_H_ +#define SLASH_DEFINE_H_ + +#define SPACE ' ' +#define COLON ':' +#define COMMENT '#' +#define COMMA ',' + + +#endif diff --git a/pink/third/slash/slash/include/slash_hash.h b/pink/third/slash/slash/include/slash_hash.h new file mode 100644 index 00000000..e26f41a0 --- /dev/null +++ b/pink/third/slash/slash/include/slash_hash.h @@ -0,0 +1,88 @@ +/* + * Updated to C++, zedwood.com 2012 + * Based on Olivier Gay's version + * See Modified BSD License below: + * + * FIPS 180-2 SHA-224/256/384/512 implementation + * Issue date: 04/30/2005 + * http://www.ouah.org/ogay/sha2/ + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* MD5 + converted to C++ class by Frank Thilo (thilo@unix-ag.org) + for bzflag (http://www.bzflag.org) + + based on: + + md5.h and md5.c + reference implementation of RFC 1321 + + Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All +rights reserved. + +License to copy and use this software is granted provided that it +is identified as the "RSA Data Security, Inc. MD5 Message-Digest +Algorithm" in all material mentioning or referencing this software +or this function. + +License is also granted to make and use derivative works provided +that such works are identified as "derived from the RSA Data +Security, Inc. MD5 Message-Digest Algorithm" in all material +mentioning or referencing the derived work. + +RSA Data Security, Inc. makes no representations concerning either +the merchantability of this software or the suitability of this +software for any particular purpose. It is provided "as is" +without express or implied warranty of any kind. + +These notices must be retained in any copies of any part of this +documentation and/or software. + +*/ + +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SLASH_HASH_H +#define SLASH_HASH_H +#include +#include +#include + +namespace slash { + +std::string md5(const std::string &str, bool raw = false); +std::string sha256(const std::string &input, bool raw = false); + +} // namespace slash + +#endif // SLASH_HASH_H diff --git a/pink/third/slash/slash/include/slash_mutex.h b/pink/third/slash/slash/include/slash_mutex.h new file mode 100644 index 00000000..eef7f4a3 --- /dev/null +++ b/pink/third/slash/slash/include/slash_mutex.h @@ -0,0 +1,207 @@ +#ifndef SLASH_MUTEXLOCK_H_ +#define SLASH_MUTEXLOCK_H_ + +#include +#include +#include +#include + +namespace slash { + +class CondVar; + +class Mutex { + public: + Mutex(); + ~Mutex(); + + void Lock(); + void Unlock(); + void AssertHeld() { } + + private: + friend class CondVar; + pthread_mutex_t mu_; + + // No copying + Mutex(const Mutex&); + void operator=(const Mutex&); +}; + +// DEPRECATED +class RWLock { + public: + RWLock(pthread_rwlock_t *mu, bool is_rwlock) : + mu_(mu) { + if (is_rwlock) { + res = pthread_rwlock_wrlock(this->mu_); + } else { + res = pthread_rwlock_rdlock(this->mu_); + } + } + ~RWLock() { + if (!res) { + pthread_rwlock_unlock(this->mu_); + } + } + + private: + int res; + pthread_rwlock_t *const mu_; + // No copying allowed + RWLock(const RWLock&); + void operator=(const RWLock&); +}; + +class RWMutex { + public: + RWMutex(); + ~RWMutex(); + + void ReadLock(); + void WriteLock(); + void ReadUnlock(); + void WriteUnlock(); + + private: + pthread_rwlock_t rw_mu_; + + // No copying + RWMutex(const RWMutex&); + void operator=(const RWMutex&); +}; + +class ReadLock { + public: + explicit ReadLock(RWMutex* rw_mu) + : rw_mu_(rw_mu) { + this->rw_mu_->ReadLock(); + } + ~ReadLock() { this->rw_mu_->ReadUnlock(); } + + private: + RWMutex *const rw_mu_; + // No copying + ReadLock(const ReadLock&); + void operator=(const ReadLock&); +}; + +class WriteLock { + public: + WriteLock(RWMutex* rw_mu) + : rw_mu_(rw_mu) { + this->rw_mu_->WriteLock(); + } + ~WriteLock() { this->rw_mu_->WriteUnlock(); } + + private: + RWMutex *const rw_mu_; + // No copying allowed + WriteLock(const WriteLock&); + void operator=(const WriteLock&); +}; + +class CondVar { + public: + explicit CondVar(Mutex* mu); + ~CondVar(); + void Wait(); + /* + * timeout is millisecond + * so if you want to wait for 1 s, you should call + * TimeWait(1000); + * return false if timeout + */ + bool TimedWait(uint32_t timeout); + void Signal(); + void SignalAll(); + + private: + pthread_cond_t cv_; + Mutex* mu_; +}; + +class MutexLock { + public: + explicit MutexLock(Mutex *mu) + : mu_(mu) { + this->mu_->Lock(); + } + ~MutexLock() { this->mu_->Unlock(); } + + private: + Mutex *const mu_; + // No copying allowed + MutexLock(const MutexLock&); + void operator=(const MutexLock&); +}; + +typedef pthread_once_t OnceType; +extern void InitOnce(OnceType* once, void (*initializer)()); + +class RefMutex { + public: + RefMutex(); + ~RefMutex(); + + // Lock and Unlock will increase and decrease refs_, + // should check refs before Unlock + void Lock(); + void Unlock(); + + void Ref(); + void Unref(); + bool IsLastRef() { + return refs_ == 1; + } + + private: + pthread_mutex_t mu_; + int refs_; + + // No copying + RefMutex(const RefMutex&); + void operator=(const RefMutex&); +}; + +class RecordMutex { +public: + RecordMutex() {}; + ~RecordMutex(); + + void MultiLock(const std::vector& keys); + void Lock(const std::string &key); + void MultiUnlock(const std::vector& keys); + void Unlock(const std::string &key); + +private: + + Mutex mutex_; + + std::unordered_map records_; + + // No copying + RecordMutex(const RecordMutex&); + void operator=(const RecordMutex&); +}; + +class RecordLock { + public: + RecordLock(RecordMutex *mu, const std::string &key) + : mu_(mu), key_(key) { + mu_->Lock(key_); + } + ~RecordLock() { mu_->Unlock(key_); } + + private: + RecordMutex *const mu_; + std::string key_; + + // No copying allowed + RecordLock(const RecordLock&); + void operator=(const RecordLock&); +}; + +} + +#endif diff --git a/pink/third/slash/slash/include/slash_slice.h b/pink/third/slash/slash/include/slash_slice.h new file mode 100644 index 00000000..b4111a08 --- /dev/null +++ b/pink/third/slash/slash/include/slash_slice.h @@ -0,0 +1,114 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +// +// Slice is a simple structure containing a pointer into some external +// storage and a size. The user of a Slice must ensure that the slice +// is not used after the corresponding external storage has been +// deallocated. +// +// Multiple threads can invoke const methods on a Slice without +// external synchronization, but if any of the threads may call a +// non-const method, all threads accessing the same Slice must use +// external synchronization. + +#ifndef SLASH_SLICE_H_ +#define SLASH_SLICE_H_ + +#include +#include +#include +#include + +namespace slash { + +class Slice { + public: + // Create an empty slice. + Slice() : data_(""), size_(0) { } + + // Create a slice that refers to d[0,n-1]. + Slice(const char* d, size_t n) : data_(d), size_(n) { } + + // Create a slice that refers to the contents of "s" + Slice(const std::string& s) : data_(s.data()), size_(s.size()) { } + + // Create a slice that refers to s[0,strlen(s)-1] + Slice(const char* s) : data_(s), size_(strlen(s)) { } + + // Return a pointer to the beginning of the referenced data + const char* data() const { return data_; } + + // Return the length (in bytes) of the referenced data + size_t size() const { return size_; } + + // Return true iff the length of the referenced data is zero + bool empty() const { return size_ == 0; } + + // Return the ith byte in the referenced data. + // REQUIRES: n < size() + char operator[](size_t n) const { + assert(n < size()); + return data_[n]; + } + + // Change this slice to refer to an empty array + void clear() { data_ = ""; size_ = 0; } + + // Drop the first "n" bytes from this slice. + void remove_prefix(size_t n) { + assert(n <= size()); + data_ += n; + size_ -= n; + } + + // Return a string that contains the copy of the referenced data. + std::string ToString() const { return std::string(data_, size_); } + + // Three-way comparison. Returns value: + // < 0 iff "*this" < "b", + // == 0 iff "*this" == "b", + // > 0 iff "*this" > "b" + int compare(const Slice& b) const; + + // Return true iff "x" is a prefix of "*this" + bool starts_with(const Slice& x) const { + return ((size_ >= x.size_) && + (memcmp(data_, x.data_, x.size_) == 0)); + } + + private: + const char* data_; + size_t size_; + + // Intentionally copyable +}; + +inline bool operator==(const Slice& x, const Slice& y) { + return ((x.size() == y.size()) && + (memcmp(x.data(), y.data(), x.size()) == 0)); +} + +inline bool operator!=(const Slice& x, const Slice& y) { + return !(x == y); +} + +inline int Slice::compare(const Slice& b) const { + const size_t min_len = (size_ < b.size_) ? size_ : b.size_; + int r = memcmp(data_, b.data_, min_len); + if (r == 0) { + if (size_ < b.size_) r = -1; + else if (size_ > b.size_) r = +1; + } + return r; +} + +} // namespace slash + + +#endif // SLASH_SLICE_H_ diff --git a/pink/third/slash/slash/include/slash_status.h b/pink/third/slash/slash/include/slash_status.h new file mode 100644 index 00000000..05806cf7 --- /dev/null +++ b/pink/third/slash/slash/include/slash_status.h @@ -0,0 +1,148 @@ +#ifndef SLASH_STATUS_H_ +#define SLASH_STATUS_H_ + +#include +#include "slash/include/slash_slice.h" + +namespace slash { + +class Status { + public: + // Create a success status. + Status() : state_(NULL) { } + ~Status() { delete[] state_; } + + // Copy the specified status. + Status(const Status& s); + void operator=(const Status& s); + + // Return a success status. + static Status OK() { return Status(); } + + // Return error status of an appropriate type. + static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotFound, msg, msg2); + } + static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kCorruption, msg, msg2); + } + static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kNotSupported, msg, msg2); + } + static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kInvalidArgument, msg, msg2); + } + static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIOError, msg, msg2); + } + static Status EndFile(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kEndFile, msg, msg2); + } + + static Status Incomplete(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kIncomplete, msg, msg2); + } + + static Status Complete(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kComplete, msg, msg2); + } + + static Status Timeout(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kTimeout, msg, msg2); + } + + static Status AuthFailed(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kAuthFailed, msg, msg2); + } + + static Status Busy(const Slice& msg, const Slice& msg2 = Slice()) { + return Status(kBusy, msg, msg2); + } + + // Returns true if the status indicates success. + bool ok() const { return (state_ == NULL); } + + // Returns true if the status indicates a NotFound error. + bool IsNotFound() const { return code() == kNotFound; } + + // Returns true if the status indicates a Corruption error. + bool IsCorruption() const { return code() == kCorruption; } + + // Returns true if the status indicates a NotSupported error. + bool IsNotSupported() const { return code() == kNotSupported; } + + // Returns true if the status indicates an IOError. + bool IsIOError() const { return code() == kIOError; } + + // Returns true if the status indicates an EOF. + bool IsEndFile() const { return code() == kEndFile; } + + // Returns true if the status is complete. + bool IsComplete() const { return code() == kComplete; } + + // Returns true if the status is Incomplete + bool IsIncomplete() const { return code() == kIncomplete; } + + // Returns true if the status is InvalidArgument + bool IsInvalidArgument() const { return code() == kInvalidArgument; } + + // Returns true if the status is Timeout + bool IsTimeout() const { return code() == kTimeout; } + + // Returns true if the status is AuthFailed + bool IsAuthFailed() const { return code() == kAuthFailed; } + + // Return true if the status is Busy + bool IsBusy() const { return code() == kBusy; } + + // Return a string representation of this status suitable for printing. + // Returns the string "OK" for success. + std::string ToString() const; + + private: + // OK status has a NULL state_. Otherwise, state_ is a new[] array + // of the following form: + // state_[0..3] == length of message + // state_[4] == code + // state_[5..] == message + const char* state_; + + enum Code { + kOk = 0, + kNotFound = 1, + kCorruption = 2, + kNotSupported = 3, + kInvalidArgument = 4, + kIOError = 5, + kEndFile = 6, + kIncomplete = 7, + kComplete = 8, + kTimeout = 9, + kAuthFailed = 10, + kBusy = 11 + }; + + Code code() const { + return (state_ == NULL) ? kOk : static_cast(state_[4]); + } + + Status(Code code, const Slice& msg, const Slice& msg2); + static const char* CopyState(const char* s); +}; + +inline Status::Status(const Status& s) { + state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); +} +inline void Status::operator=(const Status& s) { + // The following condition catches both aliasing (when this == &s), + // and the common case where both s and *this are ok. + if (state_ != s.state_) { + delete[] state_; + state_ = (s.state_ == NULL) ? NULL : CopyState(s.state_); + } +} + +} // namespace slash + + +#endif // SLASH_STATUS_H_ diff --git a/pink/third/slash/slash/include/slash_string.h b/pink/third/slash/slash/include/slash_string.h new file mode 100644 index 00000000..02fbcb21 --- /dev/null +++ b/pink/third/slash/slash/include/slash_string.h @@ -0,0 +1,65 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __SLASH_STRING_H +#define __SLASH_STRING_H + +#include +#include + +namespace slash { + +int stringmatchlen(const char *p, int plen, const char *s, int slen, int nocase); +int stringmatch(const char *p, const char *s, int nocase); +long long memtoll(const char *p, int *err); +int ll2string(char *s, size_t len, long long value); +int string2ll(const char *s, size_t slen, long long *value); +int string2l(const char *s, size_t slen, long *value); +int string2ul(const char *s, size_t slen, unsigned long *value); +int d2string(char *buf, size_t len, double value); +int string2d(const char *buf, size_t len, double *value); +std::vector &StringSplit(const std::string &s, + char delim, std::vector &elems); +std::string StringConcat(const std::vector &elems, char delim); +std::string& StringToLower(std::string& ori); +std::string& StringToUpper(std::string& ori); +std::string IpPortString(const std::string& ip, int port); +std::string ToRead(const std::string& str); +bool ParseIpPortString(const std::string& ip_port, std::string& ip, int &port); +std::string StringTrim(const std::string& ori, const std::string& charlist = " "); + + +} // namespace slash + +#endif // __SLASH_STRING_H diff --git a/pink/third/slash/slash/include/slash_testharness.h b/pink/third/slash/slash/include/slash_testharness.h new file mode 100644 index 00000000..13e5a0d0 --- /dev/null +++ b/pink/third/slash/slash/include/slash_testharness.h @@ -0,0 +1,138 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#ifndef SLASH_TESTHARNESS_H_ +#define SLASH_TESTHARNESS_H_ + +#include +#include +#include +#include "slash/include/slash_status.h" + +namespace slash { +namespace test { + +// Run some of the tests registered by the TEST() macro. If the +// environment variable "SLASH_TESTS" is not set, runs all tests. +// Otherwise, runs only the tests whose name contains the value of +// "SLASH_TESTS" as a substring. E.g., suppose the tests are: +// TEST(Foo, Hello) { ... } +// TEST(Foo, World) { ... } +// SLASH_TESTS=Hello will run the first test +// SLASH_TESTS=o will run both tests +// SLASH_TESTS=Junk will run no tests +// +// Returns 0 if all tests pass. +// Dies or returns a non-zero value if some test fails. +extern int RunAllTests(); + +// Return a randomization seed for this run. Typically returns the +// same number on repeated invocations of this binary, but automated +// runs may be able to vary the seed. +extern int RandomSeed(); + +// An instance of Tester is allocated to hold temporary state during +// the execution of an assertion. +class Tester { + private: + bool ok_; + const char* fname_; + int line_; + std::stringstream ss_; + + public: + Tester(const char* f, int l) + : ok_(true), fname_(f), line_(l) { + } + + ~Tester() { + if (!ok_) { + fprintf(stderr, "%s:%d:%s\n", fname_, line_, ss_.str().c_str()); + exit(1); + } + } + + Tester& Is(bool b, const char* msg) { + if (!b) { + ss_ << " Assertion failure " << msg; + ok_ = false; + } + return *this; + } + + Tester& IsOk(const Status& s) { + if (!s.ok()) { + ss_ << " " << s.ToString(); + ok_ = false; + } + return *this; + } + +#define BINARY_OP(name,op) \ + template \ + Tester& name(const X& x, const Y& y) { \ + if (! (x op y)) { \ + ss_ << " failed: " << x << (" " #op " ") << y; \ + ok_ = false; \ + } \ + return *this; \ + } + + BINARY_OP(IsEq, ==) + BINARY_OP(IsNe, !=) + BINARY_OP(IsGe, >=) + BINARY_OP(IsGt, >) + BINARY_OP(IsLe, <=) + BINARY_OP(IsLt, <) +#undef BINARY_OP + + // Attach the specified value to the error message if an error has occurred + template + Tester& operator<<(const V& value) { + if (!ok_) { + ss_ << " " << value; + } + return *this; + } +}; + +#define ASSERT_TRUE(c) ::slash::test::Tester(__FILE__, __LINE__).Is((c), #c) +#define ASSERT_OK(s) ::slash::test::Tester(__FILE__, __LINE__).IsOk((s)) +#define ASSERT_EQ(a,b) ::slash::test::Tester(__FILE__, __LINE__).IsEq((a),(b)) +#define ASSERT_NE(a,b) ::slash::test::Tester(__FILE__, __LINE__).IsNe((a),(b)) +#define ASSERT_GE(a,b) ::slash::test::Tester(__FILE__, __LINE__).IsGe((a),(b)) +#define ASSERT_GT(a,b) ::slash::test::Tester(__FILE__, __LINE__).IsGt((a),(b)) +#define ASSERT_LE(a,b) ::slash::test::Tester(__FILE__, __LINE__).IsLe((a),(b)) +#define ASSERT_LT(a,b) ::slash::test::Tester(__FILE__, __LINE__).IsLt((a),(b)) + +#define TCONCAT(a,b) TCONCAT1(a,b) +#define TCONCAT1(a,b) a##b + +#define TEST(base,name) \ +class TCONCAT(_Test_,name) : public base { \ + public: \ + void _Run(); \ + static void _RunIt() { \ + TCONCAT(_Test_,name) t; \ + t._Run(); \ + } \ +}; \ +bool TCONCAT(_Test_ignored_,name) = \ + ::slash::test::RegisterTest(#base, #name, &TCONCAT(_Test_,name)::_RunIt); \ +void TCONCAT(_Test_,name)::_Run() + +// Register the specified test. Typically not used directly, but +// invoked via the macro expansion of TEST. +extern bool RegisterTest(const char* base, const char* name, void (*func)()); + + +} // namespace test +} // namespace slash + +#endif // SLASH_TESTHARNESS_H_ diff --git a/pink/third/slash/slash/include/testutil.h b/pink/third/slash/slash/include/testutil.h new file mode 100644 index 00000000..8fc61bad --- /dev/null +++ b/pink/third/slash/slash/include/testutil.h @@ -0,0 +1,15 @@ +#ifndef SLASH_INCLUDE_TESTUTIL_H_ +#define SLASH_INCLUDE_TESTUTIL_H_ + +#include + +namespace slash { + +extern std::string RandomString(const int len); +extern int RandomSeed(); +extern int GetTestDirectory(std::string *result); + +}; // namespace slash + + +#endif // SLASH_INCLUDE_TESTUTIL_H_ diff --git a/pink/third/slash/slash/include/version.h b/pink/third/slash/slash/include/version.h new file mode 100644 index 00000000..b3a408ae --- /dev/null +++ b/pink/third/slash/slash/include/version.h @@ -0,0 +1,14 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +#pragma once + +#define SLASH_MAJOR 1 +#define SLASH_MINOR 0 +#define SLASH_PATCH 1 diff --git a/pink/third/slash/slash/include/xdebug.h b/pink/third/slash/slash/include/xdebug.h new file mode 100644 index 00000000..ceeaa5ab --- /dev/null +++ b/pink/third/slash/slash/include/xdebug.h @@ -0,0 +1,67 @@ +/** + * @file xdebug.h + * @brief debug macros + * @author chenzongzhi + * @version 1.0.0 + * @date 2014-04-25 + */ + +#ifndef __XDEBUG_H_ +#define __XDEBUG_H_ +#include +#include +#include +#include +#include + +#ifdef __XDEBUG__ +#define pint(x) qf_debug("%s = %d", #x, x) +#define psize(x) qf_debug("%s = %zu", #x, x) +#define pstr(x) qf_debug("%s = %s", #x, x) +// 如果A 不对, 那么就输出M +#define qf_check(A, M, ...) if(!(A)) { log_err(M, ##__VA_ARGS__); errno=0; exit(-1);} + +// 用来检测程序是否执行到这里 +#define sentinel(M, ...) { qf_debug(M, ##__VA_ARGS__); errno=0;} + +#define qf_bin_debug(buf, size) \ +{ \ + fwrite(buf, 1, size, stderr); \ +} + +#define _debug_time_def timeval s1, e; +#define _debug_getstart gettimeofday(&s1, NULL) +#define _debug_getend gettimeofday(&e, NULL) +#define _debug_time ((int)(((e.tv_sec - s1.tv_sec) * 1000 + (e.tv_usec - s1.tv_usec) / 1000))) + +#define clean_errno() (errno == 0 ? "None" : strerror(errno)) +#define log_err(M, ...) \ +{ \ + fprintf(stderr, "[ERROR] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__); \ + exit(-1); \ +} +#define log_warn(M, ...) fprintf(stderr, "[WARN] (%s:%d: errno: %s) " M "\n", __FILE__, __LINE__, clean_errno(), ##__VA_ARGS__) +#define log_info(M, ...) fprintf(stderr, "[INFO] (%s:%d) " M "\n", __FILE__, __LINE__, ##__VA_ARGS__) + +#else + +#define pint(x) {} +#define pstr(x) {} +#define qf_bin_debug(buf, size) {} + +#define _debug_time_def {} +#define _debug_getstart {} +#define _debug_getend {} +#define _debug_time 0 + +#define sentinel(M, ...) {} +#define qf_check(A, M, ...) {} +#define log_err(M, ...) {} +#define log_warn(M, ...) {} +#define log_info(M, ...) {} + +#endif + +#endif //__XDEBUG_H_ + +/* vim: set ts=4 sw=4 sts=4 tw=100 */ diff --git a/pink/third/slash/slash/src/base_conf.cc b/pink/third/slash/slash/src/base_conf.cc new file mode 100644 index 00000000..cb794d59 --- /dev/null +++ b/pink/third/slash/slash/src/base_conf.cc @@ -0,0 +1,300 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "slash/include/base_conf.h" + +#include +#include + +#include "slash/include/env.h" +#include "slash/include/xdebug.h" +#include "slash/include/slash_string.h" + +namespace slash { + +static const int kConfItemLen = 1024*1024; + +BaseConf::BaseConf(const std::string &path) + : rep_(new Rep(path)) { +} + +BaseConf::~BaseConf() { + delete rep_; +} + +int BaseConf::LoadConf() { + if (!FileExists(rep_->path)) { + return -1; + } + SequentialFile *sequential_file; + NewSequentialFile(rep_->path, &sequential_file); + + // read conf items + + char line[kConfItemLen]; + char name[kConfItemLen], value[kConfItemLen]; + int line_len = 0; + int name_len = 0, value_len = 0; + int sep_sign = 0; + Rep::ConfType type = Rep::kConf; + + while (sequential_file->ReadLine(line, kConfItemLen) != NULL) { + sep_sign = 0; + name_len = 0; + value_len = 0; + type = Rep::kComment; + line_len = strlen(line); + for (int i = 0; i < line_len; i++) { + if (i == 0 && line[i] == COMMENT) { + type = Rep::kComment; + break; + } + switch (line[i]) { + case SPACE: + case '\r': + case '\n': + break; + case COLON: + if (!sep_sign) { + type = Rep::kConf; + sep_sign = 1; + break; + } + default: + if (sep_sign == 0) { + name[name_len++] = line[i]; + } else { + value[value_len++] = line[i]; + } + } + } + + if (type == Rep::kConf) { + rep_->item.push_back(Rep::ConfItem(Rep::kConf, std::string(name, name_len), std::string(value, value_len))); + } else { + rep_->item.push_back(Rep::ConfItem(Rep::kComment, std::string(line, line_len))); + } + } + + // sequential_file->Close(); + delete sequential_file; + return 0; +} + +int BaseConf::ReloadConf() { + Rep* rep = rep_; + rep_ = new Rep(rep->path); + if (LoadConf() == -1) { + delete rep_; + rep_ = rep; + return -1; + } + delete rep; + return 0; +} + +bool BaseConf::GetConfInt(const std::string &name, int* value) const { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + (*value) = atoi(rep_->item[i].value.c_str()); + return true; + } + } + return false; +} + +bool BaseConf::GetConfInt64(const std::string &name, int64_t* value) const { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + (*value) = strtoll(rep_->item[i].value.c_str(), NULL, 10); + return true; + } + } + return false; +} + +bool BaseConf::GetConfStr(const std::string &name, std::string *val) const { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == 1) { + continue; + } + if (name == rep_->item[i].name) { + (*val) = rep_->item[i].value; + return true; + } + } + return false; +} + +bool BaseConf::GetConfStrVec(const std::string &name, std::vector *value) const { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + std::string val_str = rep_->item[i].value; + std::string::size_type pos; + while (true) { + pos = val_str.find(","); + if (pos == std::string::npos) { + value->push_back(StringTrim(val_str)); + break; + } + value->push_back(StringTrim(val_str.substr(0, pos))); + val_str = val_str.substr(pos+1); + } + return true; + } + } + return false; +} + +bool BaseConf::GetConfBool(const std::string &name, bool* value) const { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + if (rep_->item[i].value == "true" || rep_->item[i].value == "1" || rep_->item[i].value == "yes") { + (*value) = true; + } else if (rep_->item[i].value == "false" || rep_->item[i].value == "0" || rep_->item[i].value == "no") { + (*value) = false; + } + return true; + } + } + return false; +} + +bool BaseConf::SetConfInt(const std::string &name, const int value) { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + rep_->item[i].value = std::to_string(value); + return true; + } + } + return false; +} + +bool BaseConf::SetConfInt64(const std::string &name, const int64_t value) { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + rep_->item[i].value = std::to_string(value); + return true; + } + } + return false; +} + +bool BaseConf::SetConfStr(const std::string &name, const std::string &value) { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + rep_->item[i].value = value; + return true; + } + } + return false; +} + +bool BaseConf::SetConfBool(const std::string &name, const bool value) { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + if (value == true) { + rep_->item[i].value = "true"; + } else { + rep_->item[i].value = "false"; + } + return true; + } + } + return false; +} + +bool BaseConf::SetConfStrVec(const std::string& name, const std::vector &value) { + std::string value_str = StringConcat(value, COMMA); + return SetConfStr(name, value_str); +} + +bool BaseConf::CheckConfExist(const std::string& name) const { + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kComment) { + continue; + } + if (name == rep_->item[i].name) { + return true; + } + } + return false; +} + +void BaseConf::DumpConf() const { + int cnt = 1; + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kConf) { + printf("%2d %s %s\n", cnt++, rep_->item[i].name.c_str(), rep_->item[i].value.c_str()); + } + } +} + +bool BaseConf::WriteBack() { + WritableFile *write_file; + std::string tmp_path = rep_->path + ".tmp"; + Status ret = NewWritableFile(tmp_path, &write_file); + log_info("ret %s", ret.ToString().c_str()); + std::string tmp; + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kConf) { + tmp = rep_->item[i].name + " : " + rep_->item[i].value + "\n"; + write_file->Append(tmp); + } else { + write_file->Append(rep_->item[i].value); + } + } + DeleteFile(rep_->path); + RenameFile(tmp_path, rep_->path); + delete write_file; + return true; +} + +void BaseConf::WriteSampleConf() const { + WritableFile *write_file; + std::string sample_path = rep_->path + ".sample"; + Status ret = NewWritableFile(sample_path, &write_file); + std::string tmp; + for (size_t i = 0; i < rep_->item.size(); i++) { + if (rep_->item[i].type == Rep::kConf) { + tmp = rep_->item[i].name + " :\n"; + write_file->Append(tmp); + } else { + write_file->Append(rep_->item[i].value); + } + } + delete write_file; + return; +} + +void BaseConf::PushConfItem(const Rep::ConfItem& item) { + rep_->item.push_back(item); +} + +} // namespace slash diff --git a/pink/third/slash/slash/src/build_version.cc.in b/pink/third/slash/slash/src/build_version.cc.in new file mode 100644 index 00000000..aa49b7f2 --- /dev/null +++ b/pink/third/slash/slash/src/build_version.cc.in @@ -0,0 +1,4 @@ +#include "slash/include/version.h" +const char* slash_build_git_sha = "slash_build_git_sha:@@GIT_SHA@@"; +const char* slash_build_git_date = "slash_build_git_date:@@GIT_DATE_TIME@@"; +const char* slash_build_compile_date = __DATE__; diff --git a/pink/third/slash/slash/src/build_version.h b/pink/third/slash/slash/src/build_version.h new file mode 100644 index 00000000..8358ec38 --- /dev/null +++ b/pink/third/slash/slash/src/build_version.h @@ -0,0 +1,17 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +// +#pragma once + +// this variable tells us about the git revision +extern const char* slash_build_git_sha; + +// Date on which the code was compiled: +extern const char* slash_build_compile_date; diff --git a/pink/third/slash/slash/src/cond_lock.cc b/pink/third/slash/slash/src/cond_lock.cc new file mode 100644 index 00000000..c0d5e476 --- /dev/null +++ b/pink/third/slash/slash/src/cond_lock.cc @@ -0,0 +1,66 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +#include "slash/include/cond_lock.h" + +#include +#include +#include + +#include "slash/include/xdebug.h" + +namespace slash { + +static void PthreadCall(const char* label, int result) { + if (result != 0) { + fprintf(stderr, "Pthread CondLock error function %s: %s\n", label, strerror(result)); + abort(); + } +} + +CondLock::CondLock() { + PthreadCall("init condlock", pthread_mutex_init(&mutex_, NULL)); +} + +CondLock::~CondLock() { + PthreadCall("destroy condlock", pthread_mutex_unlock(&mutex_)); +} + +void CondLock::Lock() { + PthreadCall("mutex lock", pthread_mutex_lock(&mutex_)); +} + +void CondLock::Unlock() { + PthreadCall("mutex unlock", pthread_mutex_unlock(&mutex_)); +} + +void CondLock::Wait() { + PthreadCall("condlock wait", pthread_cond_wait(&cond_, &mutex_)); +} + +void CondLock::TimedWait(uint32_t timeout) { + /* + * pthread_cond_timedwait api use absolute API + * so we need gettimeofday + timeout + */ + struct timeval now; + gettimeofday(&now, NULL); + struct timespec tsp; + + int64_t usec = now.tv_usec + timeout * 1000LL; + tsp.tv_sec = now.tv_sec + usec / 1000000; + tsp.tv_nsec = (usec % 1000000) * 1000; + + pthread_cond_timedwait(&cond_, &mutex_, &tsp); +} + +void CondLock::Signal() { + PthreadCall("condlock signal", pthread_cond_signal(&cond_)); +} + +void CondLock::Broadcast() { + PthreadCall("condlock broadcast", pthread_cond_broadcast(&cond_)); +} + +} // namespace slash diff --git a/pink/third/slash/slash/src/env.cc b/pink/third/slash/slash/src/env.cc new file mode 100644 index 00000000..177e58cc --- /dev/null +++ b/pink/third/slash/slash/src/env.cc @@ -0,0 +1,808 @@ +#include "slash/include/env.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "slash/include/xdebug.h" + +namespace slash { + +/* + * Set the resource limits of a process + */ + +/* + * 0: success. + * -1: set failed. + * -2: get resource limits failed. + */ +const size_t kPageSize = getpagesize(); + +int SetMaxFileDescriptorNum(int64_t max_file_descriptor_num) { + // Try to Set the number of file descriptor + struct rlimit limit; + if (getrlimit(RLIMIT_NOFILE, &limit) != -1) { + if (limit.rlim_cur < (rlim_t)max_file_descriptor_num) { + // rlim_cur could be set by any user while rlim_max are + // changeable only by root. + limit.rlim_cur = max_file_descriptor_num; + if(limit.rlim_cur > limit.rlim_max) { + limit.rlim_max = max_file_descriptor_num; + } + if (setrlimit(RLIMIT_NOFILE, &limit) != -1) { + return 0; + } else { + return -1; + }; + } else { + return 0; + } + } else { + return -2; + } +} + + +/* + * size of initial mmap size + */ +size_t kMmapBoundSize = 1024 * 1024 * 4; + +void SetMmapBoundSize(size_t size) { + kMmapBoundSize = size; +} + +static Status IOError(const std::string& context, int err_number) { + return Status::IOError(context, strerror(err_number)); +} + +int CreateDir(const std::string& path) { + int res = 0; + + if ((res = mkdir(path.c_str(), 0755)) != 0) { + log_warn("mkdir error is %s", strerror(errno)); + } + return res; +} + +bool FileExists(const std::string& path) { + return access(path.c_str(), F_OK) == 0; +} + +Status DeleteFile(const std::string& fname) { + Status result; + if (unlink(fname.c_str()) != 0) { + result = IOError(fname, errno); + } + return result; +} + +int DoCreatePath(const char *path, mode_t mode) { + struct stat st; + int status = 0; + + if (stat(path, &st) != 0) { + /* Directory does not exist. EEXIST for race + * condition */ + if (mkdir(path, mode) != 0 && errno != EEXIST) + status = -1; + } else if (!S_ISDIR(st.st_mode)) { + errno = ENOTDIR; + status = -1; + } + + return (status); +} + +/** + ** CreatePath - ensure all directories in path exist + ** Algorithm takes the pessimistic view and works top-down to ensure + ** each directory in path exists, rather than optimistically creating + ** the last element and working backwards. + */ +int CreatePath(const std::string &path, mode_t mode) { + char *pp; + char *sp; + int status; + char *copypath = strdup(path.c_str()); + + status = 0; + pp = copypath; + while (status == 0 && (sp = strchr(pp, '/')) != 0) { + if (sp != pp) { + /* Neither root nor double slash in path */ + *sp = '\0'; + status = DoCreatePath(copypath, mode); + *sp = '/'; + } + pp = sp + 1; + } + if (status == 0) + status = DoCreatePath(path.c_str(), mode); + free(copypath); + return (status); +} + +static int LockOrUnlock(int fd, bool lock) { + errno = 0; + struct flock f; + memset(&f, 0, sizeof(f)); + f.l_type = (lock ? F_WRLCK : F_UNLCK); + f.l_whence = SEEK_SET; + f.l_start = 0; + f.l_len = 0; // Lock/unlock entire file + return fcntl(fd, F_SETLK, &f); +} + +Status LockFile(const std::string& fname, FileLock** lock) { + *lock = NULL; + Status result; + int fd = open(fname.c_str(), O_RDWR | O_CREAT, 0644); + if (fd < 0) { + result = IOError(fname, errno); + } else if (LockOrUnlock(fd, true) == -1) { + result = IOError("lock " + fname, errno); + close(fd); + } else { + FileLock* my_lock = new FileLock; + my_lock->fd_ = fd; + my_lock->name_ = fname; + *lock = my_lock; + } + return result; +} + +Status UnlockFile(FileLock* lock) { + Status result; + if (LockOrUnlock(lock->fd_, false) == -1) { + result = IOError("unlock", errno); + } + close(lock->fd_); + delete lock; + return result; +} + +int GetChildren(const std::string& dir, std::vector& result) { + int res = 0; + result.clear(); + DIR* d = opendir(dir.c_str()); + if (d == NULL) { + return errno; + } + struct dirent* entry; + while ((entry = readdir(d)) != NULL) { + if (strcmp(entry->d_name, "..") == 0 || strcmp(entry->d_name, ".") == 0) { + continue; + } + result.push_back(entry->d_name); + } + closedir(d); + return res; +} + +bool GetDescendant(const std::string& dir, std::vector& result) { + DIR* d = opendir(dir.c_str()); + if (d == NULL) { + return false; + } + struct dirent* entry; + std::string fname; + while ((entry = readdir(d)) != NULL) { + if (strcmp(entry->d_name, "..") == 0 || strcmp(entry->d_name, ".") == 0) { + continue; + } + fname = dir + "/" + entry->d_name; + if (0 == IsDir(fname)) { + if (!GetDescendant(fname, result)) { + return false; + } + } else { + result.push_back(fname); + } + } + closedir(d); + return true; +} + +int RenameFile(const std::string& oldname, const std::string& newname) { + return rename(oldname.c_str(), newname.c_str()); +} + +int IsDir(const std::string& path) { + struct stat buf; + int ret = stat(path.c_str(), &buf); + if (0 == ret) { + if (buf.st_mode & S_IFDIR) { + //folder + return 0; + } else { + //file + return 1; + } + } + return -1; +} + +int DeleteDir(const std::string& path) +{ + char chBuf[256]; + DIR * dir = NULL; + struct dirent *ptr; + int ret = 0; + dir = opendir(path.c_str()); + if (NULL == dir) { + return -1; + } + while((ptr = readdir(dir)) != NULL) { + ret = strcmp(ptr->d_name, "."); + if (0 == ret) { + continue; + } + ret = strcmp(ptr->d_name, ".."); + if (0 == ret) { + continue; + } + snprintf(chBuf, 256, "%s/%s", path.c_str(), ptr->d_name); + ret = IsDir(chBuf); + if (0 == ret) { + //is dir + ret = DeleteDir(chBuf); + if (0 != ret) { + return -1; + } + } + else if (1 == ret) { + //is file + ret = remove(chBuf); + if(0 != ret) { + return -1; + } + } + } + (void)closedir(dir); + ret = remove(path.c_str()); + if (0 != ret) { + return -1; + } + return 0; +} + +bool DeleteDirIfExist(const std::string& path) { + if (IsDir(path) == 0 && DeleteDir(path) != 0) { + return false; + } + return true; +} + +uint64_t Du(const std::string& filename) { + struct stat statbuf; + uint64_t sum; + if (lstat(filename.c_str(), &statbuf) != 0) { + return 0; + } + if (S_ISLNK(statbuf.st_mode) && stat(filename.c_str(), &statbuf) != 0) { + return 0; + } + sum = statbuf.st_size; + if (S_ISDIR(statbuf.st_mode)) { + DIR *dir = NULL; + struct dirent *entry; + std::string newfile; + + dir = opendir(filename.c_str()); + if (!dir) { + return sum; + } + while ((entry = readdir(dir))) { + if (strcmp(entry->d_name, "..") == 0 || strcmp(entry->d_name, ".") == 0) { + continue; + } + newfile = filename + "/" + entry->d_name; + sum += Du(newfile); + } + closedir(dir); + } + return sum; +} + +uint64_t NowMicros() { + struct timeval tv; + gettimeofday(&tv, NULL); + return static_cast(tv.tv_sec) * 1000000 + tv.tv_usec; +} + +void SleepForMicroseconds(int micros) { + usleep(micros); +} + +SequentialFile::~SequentialFile() { +} + +class PosixSequentialFile: public SequentialFile { + private: + std::string filename_; + FILE* file_; + + public: + virtual void setUnBuffer() { + setbuf(file_, NULL); + } + + PosixSequentialFile(const std::string& fname, FILE* f) + : filename_(fname), file_(f) { setbuf(file_, NULL); } + + virtual ~PosixSequentialFile() { + if (file_) { + fclose(file_); + } + } + + virtual Status Read(size_t n, Slice* result, char* scratch) override { + Status s; + size_t r = fread_unlocked(scratch, 1, n, file_); + + *result = Slice(scratch, r); + + if (r < n) { + if (feof(file_)) { + s = Status::EndFile(filename_, "end file"); + // We leave status as ok if we hit the end of the file + } else { + // A partial read with an error: return a non-ok status + s = IOError(filename_, errno); + } + } + return s; + } + + virtual Status Skip(uint64_t n) override { + if (fseek(file_, n, SEEK_CUR)) { + return IOError(filename_, errno); + } + return Status::OK(); + } + + virtual char *ReadLine(char* buf, int n) override { + return fgets(buf, n, file_); + } + + virtual Status Close() { + if (fclose(file_) != 0) { + return IOError(filename_, errno); + } + file_ = NULL; + return Status::OK(); + } +}; + +WritableFile::~WritableFile() { +} + +// We preallocate up to an extra megabyte and use memcpy to append new +// data to the file. This is safe since we either properly close the +// file before reading from it, or for log files, the reading code +// knows enough to skip zero suffixes. +class PosixMmapFile : public WritableFile +{ + private: + std::string filename_; + int fd_; + size_t page_size_; + size_t map_size_; // How much extra memory to map at a time + char* base_; // The mapped region + char* limit_; // Limit of the mapped region + char* dst_; // Where to write next (in range [base_,limit_]) + char* last_sync_; // Where have we synced up to + uint64_t file_offset_; // Offset of base_ in file + uint64_t write_len_; // The data that written in the file + + + // Have we done an munmap of unsynced data? + bool pending_sync_; + + // Roundup x to a multiple of y + static size_t Roundup(size_t x, size_t y) { + return ((x + y - 1) / y) * y; + } + + static size_t TrimDown(size_t x, size_t y) { + return (x / y) * y; + } + size_t TruncateToPageBoundary(size_t s) { + s -= (s & (page_size_ - 1)); + assert((s % page_size_) == 0); + return s; + } + + bool UnmapCurrentRegion() { + bool result = true; + if (base_ != NULL) { + if (last_sync_ < limit_) { + // Defer syncing this data until next Sync() call, if any + pending_sync_ = true; + } + if (munmap(base_, limit_ - base_) != 0) { + result = false; + } + file_offset_ += limit_ - base_; + base_ = NULL; + limit_ = NULL; + last_sync_ = NULL; + dst_ = NULL; + + // Increase the amount we map the next time, but capped at 1MB + if (map_size_ < (1<<20)) { + map_size_ *= 2; + } + } + return result; + } + + bool MapNewRegion() { + assert(base_ == NULL); + if (posix_fallocate(fd_, file_offset_, map_size_) != 0) { + log_warn("ftruncate error"); + return false; + } + //log_info("map_size %d fileoffset %llu", map_size_, file_offset_); + void* ptr = mmap(NULL, map_size_, PROT_READ | PROT_WRITE, MAP_SHARED, + fd_, file_offset_); + if (ptr == MAP_FAILED) { + log_warn("mmap failed"); + return false; + } + base_ = reinterpret_cast(ptr); + limit_ = base_ + map_size_; + dst_ = base_ + write_len_; + write_len_ = 0; + last_sync_ = base_; + return true; + } + + public: + PosixMmapFile(const std::string& fname, int fd, size_t page_size, uint64_t write_len = 0) + : filename_(fname), + fd_(fd), + page_size_(page_size), + map_size_(Roundup(kMmapBoundSize, page_size)), + base_(NULL), + limit_(NULL), + dst_(NULL), + last_sync_(NULL), + file_offset_(0), + write_len_(write_len), + pending_sync_(false) { + if (write_len_ != 0) { + while (map_size_ < write_len_) { + map_size_ += (1024 * 1024); + } + } + assert((page_size & (page_size - 1)) == 0); + } + + + ~PosixMmapFile() { + if (fd_ >= 0) { + PosixMmapFile::Close(); + } + } + + virtual Status Append(const Slice& data) { + const char* src = data.data(); + size_t left = data.size(); + while (left > 0) { + assert(base_ <= dst_); + assert(dst_ <= limit_); + size_t avail = limit_ - dst_; + if (avail == 0) { + if (!UnmapCurrentRegion() || !MapNewRegion()) { + return IOError(filename_, errno); + } + } + size_t n = (left <= avail) ? left : avail; + memcpy(dst_, src, n); + dst_ += n; + src += n; + left -= n; + } + return Status::OK(); + } + + virtual Status Close() { + Status s; + size_t unused = limit_ - dst_; + if (!UnmapCurrentRegion()) { + s = IOError(filename_, errno); + } else if (unused > 0) { + // Trim the extra space at the end of the file + if (ftruncate(fd_, file_offset_ - unused) < 0) { + s = IOError(filename_, errno); + } + } + + if (close(fd_) < 0) { + if (s.ok()) { + s = IOError(filename_, errno); + } + } + + fd_ = -1; + base_ = NULL; + limit_ = NULL; + return s; + } + + virtual Status Flush() { + return Status::OK(); + } + + virtual Status Sync() { + Status s; + + if (pending_sync_) { + // Some unmapped data was not synced + pending_sync_ = false; + if (fdatasync(fd_) < 0) { + s = IOError(filename_, errno); + } + } + + if (dst_ > last_sync_) { + // Find the beginnings of the pages that contain the first and last + // bytes to be synced. + size_t p1 = TruncateToPageBoundary(last_sync_ - base_); + size_t p2 = TruncateToPageBoundary(dst_ - base_ - 1); + last_sync_ = dst_; + if (msync(base_ + p1, p2 - p1 + page_size_, MS_SYNC) < 0) { + s = IOError(filename_, errno); + } + } + + return s; + } + + virtual Status Trim(uint64_t target) { + if (!UnmapCurrentRegion()) { + return IOError(filename_, errno); + } + + file_offset_ = target; + + if (!MapNewRegion()) { + return IOError(filename_, errno); + } + return Status::OK(); + } + + virtual uint64_t Filesize() { + return write_len_ + file_offset_ + (dst_ - base_); + } +}; + + +RWFile::~RWFile() { +} + +class MmapRWFile : public RWFile +{ + public: + MmapRWFile(const std::string& fname, int fd, size_t page_size) + : filename_(fname), + fd_(fd), + page_size_(page_size), + map_size_(Roundup(65536, page_size)), + base_(NULL) { + DoMapRegion(); + } + + ~MmapRWFile() { + if (fd_ >= 0) { + munmap(base_, map_size_); + } + } + + bool DoMapRegion() { + if (posix_fallocate(fd_, 0, map_size_) != 0) { + return false; + } + void* ptr = mmap(NULL, map_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0); + if (ptr == MAP_FAILED) { + return false; + } + base_ = reinterpret_cast(ptr); + return true; + } + + char* GetData() { return base_; } + char* base() { return base_; } + + private: + static size_t Roundup(size_t x, size_t y) { + return ((x + y - 1) / y) * y; + } + std::string filename_; + int fd_; + size_t page_size_; + size_t map_size_; + char* base_; +}; + +class PosixRandomRWFile : public RandomRWFile { + private: + const std::string filename_; + int fd_; + bool pending_sync_; + bool pending_fsync_; + //bool fallocate_with_keep_size_; + + public: + PosixRandomRWFile(const std::string& fname, int fd) + : filename_(fname), + fd_(fd), + pending_sync_(false), + pending_fsync_(false) { + //fallocate_with_keep_size_ = options.fallocate_with_keep_size; + } + + ~PosixRandomRWFile() { + if (fd_ >= 0) { + Close(); + } + } + + virtual Status Write(uint64_t offset, const Slice& data) override { + const char* src = data.data(); + size_t left = data.size(); + Status s; + pending_sync_ = true; + pending_fsync_ = true; + + while (left != 0) { + ssize_t done = pwrite(fd_, src, left, offset); + if (done < 0) { + if (errno == EINTR) { + continue; + } + return IOError(filename_, errno); + } + + left -= done; + src += done; + offset += done; + } + + return Status::OK(); + } + + virtual Status Read(uint64_t offset, size_t n, Slice* result, + char* scratch) const override { + Status s; + ssize_t r = -1; + size_t left = n; + char* ptr = scratch; + while (left > 0) { + r = pread(fd_, ptr, left, static_cast(offset)); + if (r <= 0) { + if (errno == EINTR) { + continue; + } + break; + } + ptr += r; + offset += r; + left -= r; + } + *result = Slice(scratch, (r < 0) ? 0 : n - left); + if (r < 0) { + s = IOError(filename_, errno); + } + return s; + } + + virtual Status Close() override { + Status s = Status::OK(); + if (fd_ >= 0 && close(fd_) < 0) { + s = IOError(filename_, errno); + } + fd_ = -1; + return s; + } + + virtual Status Sync() override { + if (pending_sync_ && fdatasync(fd_) < 0) { + return IOError(filename_, errno); + } + pending_sync_ = false; + return Status::OK(); + } + + virtual Status Fsync() override { + if (pending_fsync_ && fsync(fd_) < 0) { + return IOError(filename_, errno); + } + pending_fsync_ = false; + pending_sync_ = false; + return Status::OK(); + } + +// virtual Status Allocate(off_t offset, off_t len) override { +// TEST_KILL_RANDOM(rocksdb_kill_odds); +// int alloc_status = fallocate( +// fd_, fallocate_with_keep_size_ ? FALLOC_FL_KEEP_SIZE : 0, offset, len); +// if (alloc_status == 0) { +// return Status::OK(); +// } else { +// return IOError(filename_, errno); +// } +// } +}; + +Status NewSequentialFile(const std::string& fname, SequentialFile** result) { + FILE* f = fopen(fname.c_str(), "r"); + if (f == NULL) { + *result = NULL; + return IOError(fname, errno); + } else { + *result = new PosixSequentialFile(fname, f); + return Status::OK(); + } +} + +Status NewWritableFile(const std::string& fname, WritableFile** result) { + Status s; + const int fd = open(fname.c_str(), O_CREAT | O_RDWR | O_TRUNC | O_CLOEXEC, 0644); + if (fd < 0) { + *result = NULL; + s = IOError(fname, errno); + } else { + *result = new PosixMmapFile(fname, fd, kPageSize); + } + return s; +} + +Status NewRWFile(const std::string& fname, RWFile** result) { + Status s; + const int fd = open(fname.c_str(), O_CREAT | O_RDWR | O_CLOEXEC, 0644); + if (fd < 0) { + *result = NULL; + s = IOError(fname, errno); + } else { + *result = new MmapRWFile(fname, fd, kPageSize); + } + return s; +} + +Status AppendWritableFile(const std::string& fname, WritableFile** result, uint64_t write_len) { + Status s; + const int fd = open(fname.c_str(), O_RDWR | O_CLOEXEC, 0644); + if (fd < 0) { + *result = NULL; + s = IOError(fname, errno); + } else { + *result = new PosixMmapFile(fname, fd, kPageSize, write_len); + } + return s; +} + +Status NewRandomRWFile(const std::string& fname, RandomRWFile** result) { + Status s; + const int fd = open(fname.c_str(), O_CREAT | O_RDWR, 0644); + if (fd < 0) { + *result = NULL; + s = IOError(fname, errno); + } else { + *result = new PosixRandomRWFile(fname, fd); + } + return s; +} + +} // namespace slash diff --git a/pink/third/slash/slash/src/lock_mgr.cc b/pink/third/slash/slash/src/lock_mgr.cc new file mode 100644 index 00000000..59e6d074 --- /dev/null +++ b/pink/third/slash/slash/src/lock_mgr.cc @@ -0,0 +1,192 @@ +// Copyright (c) 2017-present The blackwidow Authors. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif + +#include "slash/include/lock_mgr.h" + +#include +#include +#include +#include +#include + +#include "slash/include/mutex.h" + +namespace slash { + +namespace lock { + +struct LockMapStripe { + explicit LockMapStripe(std::shared_ptr factory) { + stripe_mutex = factory->AllocateMutex(); + stripe_cv = factory->AllocateCondVar(); + assert(stripe_mutex); + assert(stripe_cv); + } + + // Mutex must be held before modifying keys map + std::shared_ptr stripe_mutex; + + // Condition Variable per stripe for waiting on a lock + std::shared_ptr stripe_cv; + + // Locked keys + std::unordered_set keys; +}; + +// Map of #num_stripes LockMapStripes +struct LockMap { + explicit LockMap(size_t num_stripes, + std::shared_ptr factory) + : num_stripes_(num_stripes) { + lock_map_stripes_.reserve(num_stripes); + for (size_t i = 0; i < num_stripes; i++) { + LockMapStripe* stripe = new LockMapStripe(factory); + lock_map_stripes_.push_back(stripe); + } + } + + ~LockMap() { + for (auto stripe : lock_map_stripes_) { + delete stripe; + } + } + + // Number of sepearate LockMapStripes to create, each with their own Mutex + const size_t num_stripes_; + + // Count of keys that are currently locked. + // (Only maintained if LockMgr::max_num_locks_ is positive.) + std::atomic lock_cnt{0}; + + std::vector lock_map_stripes_; + + size_t GetStripe(const std::string& key) const; +}; + +size_t LockMap::GetStripe(const std::string& key) const { + assert(num_stripes_ > 0); + size_t stripe = std::hash{}(key) % num_stripes_; + return stripe; +} + +LockMgr::LockMgr(size_t default_num_stripes, + int64_t max_num_locks, + std::shared_ptr mutex_factory) + : default_num_stripes_(default_num_stripes), + max_num_locks_(max_num_locks), + mutex_factory_(mutex_factory), + lock_map_(std::shared_ptr( + new LockMap(default_num_stripes, mutex_factory))) {} + +LockMgr::~LockMgr() {} + +Status LockMgr::TryLock(const std::string& key) { +#ifdef LOCKLESS + return Status::OK(); +#else + size_t stripe_num = lock_map_->GetStripe(key); + assert(lock_map_->lock_map_stripes_.size() > stripe_num); + LockMapStripe* stripe = lock_map_->lock_map_stripes_.at(stripe_num); + + return Acquire(stripe, key); +#endif +} + +// Helper function for TryLock(). +Status LockMgr::Acquire(LockMapStripe* stripe, + const std::string& key) { + Status result; + + // we wait indefinitely to acquire the lock + result = stripe->stripe_mutex->Lock(); + + if (!result.ok()) { + // failed to acquire mutex + return result; + } + + // Acquire lock if we are able to + result = AcquireLocked(stripe, key); + + if (!result.ok()) { + // If we weren't able to acquire the lock, we will keep retrying + do { + result = stripe->stripe_cv->Wait(stripe->stripe_mutex); + if (result.ok()) { + result = AcquireLocked(stripe, key); + } + } while (!result.ok()); + } + + stripe->stripe_mutex->UnLock(); + + return result; +} + +// Try to lock this key after we have acquired the mutex. +// REQUIRED: Stripe mutex must be held. +Status LockMgr::AcquireLocked(LockMapStripe* stripe, + const std::string& key) { + Status result; + // Check if this key is already locked + if (stripe->keys.find(key) != stripe->keys.end()) { + // Lock already held + result = Status::Busy("LockTimeout"); + } else { // Lock not held. + // Check lock limit + if (max_num_locks_ > 0 && + lock_map_->lock_cnt.load(std::memory_order_acquire) >= max_num_locks_) { + result = Status::Busy("LockLimit"); + } else { + // acquire lock + stripe->keys.insert(key); + + // Maintain lock count if there is a limit on the number of locks + if (max_num_locks_) { + lock_map_->lock_cnt++; + } + } + } + + return result; +} + +void LockMgr::UnLockKey(const std::string& key, LockMapStripe* stripe) { +#ifdef LOCKLESS +#else + auto stripe_iter = stripe->keys.find(key); + if (stripe_iter != stripe->keys.end()) { + // Found the key locked. unlock it. + stripe->keys.erase(stripe_iter); + if (max_num_locks_ > 0) { + // Maintain lock count if there is a limit on the number of locks. + assert(lock_map_->lock_cnt.load(std::memory_order_relaxed) > 0); + lock_map_->lock_cnt--; + } + } else { + // This key is either not locked or locked by someone else. + } +#endif +} + +void LockMgr::UnLock(const std::string& key) { + // Lock the mutex for the stripe that this key hashes to + size_t stripe_num = lock_map_->GetStripe(key); + assert(lock_map_->lock_map_stripes_.size() > stripe_num); + LockMapStripe* stripe = lock_map_->lock_map_stripes_.at(stripe_num); + + stripe->stripe_mutex->Lock(); + UnLockKey(key, stripe); + stripe->stripe_mutex->UnLock(); + + // Signal waiting threads to retry locking + stripe->stripe_cv->NotifyAll(); +} +} // namespace lock +} // namespace slash diff --git a/pink/third/slash/slash/src/mutex_impl.cc b/pink/third/slash/slash/src/mutex_impl.cc new file mode 100644 index 00000000..585f93fb --- /dev/null +++ b/pink/third/slash/slash/src/mutex_impl.cc @@ -0,0 +1,130 @@ +// Copyright (c) 2017-present The blackwidow Authors. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + + +#include +#include + +#include "slash/include/mutex.h" +#include "slash/include/mutex_impl.h" + +namespace slash { +namespace lock { + +class MutexImpl : public Mutex { + public: + MutexImpl() {} + ~MutexImpl() {} + + Status Lock() override; + + Status TryLockFor(int64_t timeout_time) override; + + void UnLock() override { mutex_.unlock(); } + + friend class CondVarImpl; + + private: + std::mutex mutex_; +}; + +class CondVarImpl : public CondVar { + public: + CondVarImpl() {} + ~CondVarImpl() {} + + Status Wait(std::shared_ptr mutex) override; + + Status WaitFor(std::shared_ptr mutex, + int64_t timeout_time) override; + + void Notify() override { cv_.notify_one(); } + + void NotifyAll() override { cv_.notify_all(); } + + private: + std::condition_variable cv_; +}; + +std::shared_ptr +MutexFactoryImpl::AllocateMutex() { + return std::shared_ptr(new MutexImpl()); +} + +std::shared_ptr +MutexFactoryImpl::AllocateCondVar() { + return std::shared_ptr(new CondVarImpl()); +} + +Status MutexImpl::Lock() { + mutex_.lock(); + return Status::OK(); +} + +Status MutexImpl::TryLockFor(int64_t timeout_time) { + bool locked = true; + + if (timeout_time == 0) { + locked = mutex_.try_lock(); + } else { + // Previously, this code used a std::timed_mutex. However, this was changed + // due to known bugs in gcc versions < 4.9. + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=54562 + // + // Since this mutex isn't held for long and only a single mutex is ever + // held at a time, it is reasonable to ignore the lock timeout_time here + // and only check it when waiting on the condition_variable. + mutex_.lock(); + } + + if (!locked) { + // timeout acquiring mutex + return Status::Timeout("MutexTimeout"); + } + + return Status::OK(); +} + +Status CondVarImpl::Wait( + std::shared_ptr mutex) { + auto mutex_impl = reinterpret_cast(mutex.get()); + + std::unique_lock lock(mutex_impl->mutex_, std::adopt_lock); + cv_.wait(lock); + + // Make sure unique_lock doesn't unlock mutex when it destructs + lock.release(); + + return Status::OK(); +} + +Status CondVarImpl::WaitFor( + std::shared_ptr mutex, int64_t timeout_time) { + Status s; + + auto mutex_impl = reinterpret_cast(mutex.get()); + std::unique_lock lock(mutex_impl->mutex_, std::adopt_lock); + + if (timeout_time < 0) { + // If timeout is negative, do not use a timeout + cv_.wait(lock); + } else { + auto duration = std::chrono::microseconds(timeout_time); + auto cv_status = cv_.wait_for(lock, duration); + + // Check if the wait stopped due to timing out. + if (cv_status == std::cv_status::timeout) { + s = Status::Timeout("MutexTimeout"); + } + } + + // Make sure unique_lock doesn't unlock mutex when it destructs + lock.release(); + + // CV was signaled, or we spuriously woke up (but didn't time out) + return s; +} +} // namespace lock +} // namespace slash diff --git a/pink/third/slash/slash/src/posix.cc b/pink/third/slash/slash/src/posix.cc new file mode 100644 index 00000000..d8052f46 --- /dev/null +++ b/pink/third/slash/slash/src/posix.cc @@ -0,0 +1,834 @@ +#include "slash/include/xdebug.h" +#include "slash/include/posix.h" + +/********************************************* + * Wrappers for Unix process control functions + ********************************************/ + +/* $begin forkwrapper */ +pid_t Fork(void) +{ + pid_t pid; + + if ((pid = fork()) < 0) { + log_err("Fork error: %s\n", strerror(errno)); + } + return pid; +} +/* $end forkwrapper */ + +void Execve(const char *filename, char *const argv[], char *const envp[]) +{ + if (execve(filename, argv, envp) < 0) { + log_err("Execve error: %s\n", strerror(errno)); + } +} + +/* $begin wait */ +pid_t Wait(int *status) +{ + pid_t pid; + + if ((pid = wait(status)) < 0) { + log_err("Wait error: %s\n", strerror(errno)); + } + return pid; +} +/* $end wait */ + +pid_t Waitpid(pid_t pid, int *iptr, int options) +{ + pid_t retpid; + + if ((retpid = waitpid(pid, iptr, options)) < 0) { + log_err("Waitpid error: %s\n", strerror(errno)); + } + return(retpid); +} + +/* $begin kill */ +void Kill(pid_t pid, int signum) +{ + int rc; + + if ((rc = kill(pid, signum)) < 0) { + log_err("Kill error: %s\n", strerror(errno)); + } +} +/* $end kill */ + +void Pause() +{ + (void)pause(); + return; +} + +unsigned int Sleep(unsigned int secs) +{ + return sleep(secs); +} + +unsigned int Alarm(unsigned int seconds) { + return alarm(seconds); +} + +void Setpgid(pid_t pid, pid_t pgid) { + int rc; + + if ((rc = setpgid(pid, pgid)) < 0) { + log_err("Setpgid error: %s\n", strerror(errno)); + } + return; +} + +pid_t Getpgrp(void) { + return getpgrp(); +} + +/************************************ + * Wrappers for Unix signal functions + ***********************************/ + +/* $begin sigaction */ +handler_t *Signal(int signum, handler_t *handler) +{ + struct sigaction action, old_action; + + action.sa_handler = handler; + sigemptyset(&action.sa_mask); /* block sigs of type being handled */ + action.sa_flags = SA_RESTART; /* restart syscalls if possible */ + + if (sigaction(signum, &action, &old_action) < 0) { + log_err("Signal error: %s\n", strerror(errno)); + } + return (old_action.sa_handler); +} +/* $end sigaction */ + +void Sigprocmask(int how, const sigset_t *set, sigset_t *oldset) +{ + if (sigprocmask(how, set, oldset) < 0) { + log_err("Sigprocmask error: %s\n", strerror(errno)); + } + return; +} + +void Sigemptyset(sigset_t *set) +{ + if (sigemptyset(set) < 0) { + log_err("Sigemptyset error: %s\n", strerror(errno)); + } + return; +} + +void Sigfillset(sigset_t *set) +{ + if (sigfillset(set) < 0) { + log_err("Sigfillset error: %s\n", strerror(errno)); + } + return; +} + +void Sigaddset(sigset_t *set, int signum) +{ + if (sigaddset(set, signum) < 0) { + log_err("Sigaddset error: %s\n", strerror(errno)); + } + return; +} + +void Sigdelset(sigset_t *set, int signum) +{ + if (sigdelset(set, signum) < 0) { + log_err("Sigdelset error: %s\n", strerror(errno)); + } + return; +} + +int Sigismember(const sigset_t *set, int signum) +{ + int rc; + if ((rc = sigismember(set, signum)) < 0) { + log_err("Sigismember error: %s\n", strerror(errno)); + } + return rc; +} + + +/******************************** + * Wrappers for Unix I/O routines + ********************************/ + +int Open(const char *pathname, int flags, mode_t mode) +{ + int rc; + + if ((rc = open(pathname, flags, mode)) < 0) { + log_err("Open error: %s\n", strerror(errno)); + } + return rc; +} + +ssize_t Read(int fd, void *buf, size_t count) +{ + ssize_t rc; + + if ((rc = read(fd, buf, count)) < 0) { + log_err("Read error: %s\n", strerror(errno)); + } + return rc; +} + +ssize_t Write(int fd, const void *buf, size_t count) +{ + ssize_t rc; + + if ((rc = write(fd, buf, count)) < 0) { + log_err("Write error: %s\n", strerror(errno)); + } + return rc; +} + +off_t Lseek(int fildes, off_t offset, int whence) +{ + off_t rc; + + if ((rc = lseek(fildes, offset, whence)) < 0) { + log_err("Lseek error: %s\n", strerror(errno)); + } + return rc; +} + +void Close(int fd) +{ + int rc; + + if ((rc = close(fd)) < 0) { + log_err("Close error: %s\n", strerror(errno)); + } +} + +int Select(int n, fd_set *readfds, fd_set *writefds, + fd_set *exceptfds, struct timeval *timeout) +{ + int rc; + + if ((rc = select(n, readfds, writefds, exceptfds, timeout)) < 0) { + log_err("Select error: %s\n", strerror(errno)); + } + return rc; +} + +int Dup2(int fd1, int fd2) +{ + int rc; + + if ((rc = dup2(fd1, fd2)) < 0) { + log_err("Dup2 error: %s\n", strerror(errno)); + } + return rc; +} + +void Stat(const char *filename, struct stat *buf) +{ + if (stat(filename, buf) < 0) { + log_err("Stat error: %s\n", strerror(errno)); + } +} + +void Fstat(int fd, struct stat *buf) +{ + if (fstat(fd, buf) < 0) { + log_err("Fstat error: %s\n", strerror(errno)); + } +} + +/*************************************** + * Wrappers for memory mapping functions + ***************************************/ +void *Mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) +{ + void *ptr; + + if ((ptr = mmap(addr, len, prot, flags, fd, offset)) == ((void *) -1)) { + log_err("mmap error: %s\n", strerror(errno)); + } + return(ptr); +} + +void Munmap(void *start, size_t length) +{ + if (munmap(start, length) < 0) { + log_err("munmap error: %s\n", strerror(errno)); + } +} + +/*************************************************** + * Wrappers for dynamic storage allocation functions + ***************************************************/ + +void *Malloc(size_t size) +{ + void *p; + + if ((p = malloc(size)) == NULL) { + log_err("Malloc error: %s\n", strerror(errno)); + } + return p; +} + +void *Realloc(void *ptr, size_t size) +{ + void *p; + + if ((p = realloc(ptr, size)) == NULL) { + log_err("Realloc error: %s\n", strerror(errno)); + } + return p; +} + +void *Calloc(size_t nmemb, size_t size) +{ + void *p; + + if ((p = calloc(nmemb, size)) == NULL) { + log_err("Calloc error: %s\n", strerror(errno)); + } + return p; +} + +void Free(void *ptr) +{ + free(ptr); +} + +/****************************************** + * Wrappers for the Standard I/O functions. + ******************************************/ +void Fclose(FILE *fp) +{ + if (fclose(fp) != 0) { + log_err("Fclose error: %s\n", strerror(errno)); + } +} + +FILE *Fdopen(int fd, const char *type) +{ + FILE *fp; + + if ((fp = fdopen(fd, type)) == NULL) { + log_err("Fdopen error: %s\n", strerror(errno)); + } + + return fp; +} + +char *Fgets(char *ptr, int n, FILE *stream) +{ + char *rptr; + + if (((rptr = fgets(ptr, n, stream)) == NULL) && ferror(stream)) { + log_err("Fgets error"); + } + + return rptr; +} + +FILE *Fopen(const char *filename, const char *mode) +{ + FILE *fp; + + if ((fp = fopen(filename, mode)) == NULL) { + log_err("Fopen error: %s\n", strerror(errno)); + } + + return fp; +} + +void Fputs(const char *ptr, FILE *stream) +{ + if (fputs(ptr, stream) == EOF) { + log_err("Fputs error: %s\n", strerror(errno)); + } +} + +size_t Fread(void *ptr, size_t size, size_t nmemb, FILE *stream) +{ + size_t n; + + if (((n = fread(ptr, size, nmemb, stream)) < nmemb) && ferror(stream)) { + log_err("Fread error: %s\n", strerror(errno)); + } + return n; +} + +void Fwrite(const void *ptr, size_t size, size_t nmemb, FILE *stream) +{ + if (fwrite(ptr, size, nmemb, stream) < nmemb) { + log_err("Fwrite error: %s\n", strerror(errno)); + } +} + + +/**************************** + * Sockets interface wrappers + ****************************/ + +int Socket(int domain, int type, int protocol) +{ + int rc; + + if ((rc = socket(domain, type, protocol)) < 0) { + log_err("Socket error: %s\n", strerror(errno)); + } + return rc; +} + +void Setsockopt(int s, int level, int optname, const void *optval, int optlen) +{ + int rc; + + if ((rc = setsockopt(s, level, optname, optval, optlen)) < 0) { + log_err("Setsockopt error: %s\n", strerror(errno)); + } +} + +void Bind(int sockfd, struct sockaddr *my_addr, int addrlen) +{ + int rc; + + if ((rc = bind(sockfd, my_addr, addrlen)) < 0) { + log_err("Bind error: %s\n", strerror(errno)); + } +} + +void Listen(int s, int backlog) +{ + int rc; + + if ((rc = listen(s, backlog)) < 0) { + log_err("Listen error: %s\n", strerror(errno)); + } +} + +int Accept(int s, struct sockaddr *addr, socklen_t *addrlen) +{ + int rc; + + if ((rc = accept(s, addr, addrlen)) < 0) { + log_err("Accept error: %s\n", strerror(errno)); + } + return rc; +} + +void Connect(int sockfd, struct sockaddr *serv_addr, int addrlen) +{ + int rc; + + if ((rc = connect(sockfd, serv_addr, addrlen)) < 0) { + log_err("Connect error: %s\n", strerror(errno)); + } +} + +/************************ + * DNS interface wrappers + ***********************/ + +/* $begin gethostbyname */ +struct hostent *Gethostbyname(const char *name) +{ + struct hostent *p; + + if ((p = gethostbyname(name)) == NULL) { + log_err("%s: DNS error %d\n", "Gethostbyname error", h_errno); + } + return p; +} +/* $end gethostbyname */ + +struct hostent *Gethostbyaddr(const char *addr, int len, int type) +{ + struct hostent *p; + + if ((p = gethostbyaddr(addr, len, type)) == NULL) { + log_err("%s: DNS error %d\n", "Gethostbyaddr error", h_errno); + } + return p; +} + +/************************************************ + * Wrappers for Pthreads thread control functions + ************************************************/ + +void Pthread_create(pthread_t *tidp, pthread_attr_t *attrp, + void * (*routine)(void *), void *argp) +{ + int rc; + + if ((rc = pthread_create(tidp, attrp, routine, argp)) != 0) { + log_err("Pthread_create error: %s\n", strerror(rc)); + } +} + +void Pthread_cancel(pthread_t tid) { + int rc; + + if ((rc = pthread_cancel(tid)) != 0) { + log_err("Pthread_cancel error: %s\n", strerror(rc)); + } +} + +void Pthread_join(pthread_t tid, void **thread_return) { + int rc; + + if ((rc = pthread_join(tid, thread_return)) != 0) { + log_err("Pthread_join error: %s\n", strerror(rc)); + } +} + +/* $begin detach */ +void Pthread_detach(pthread_t tid) { + int rc; + + if ((rc = pthread_detach(tid)) != 0) { + log_err("Pthread_detach error: %s\n", strerror(rc)); + } +} +/* $end detach */ + +void Pthread_exit(void *retval) { + pthread_exit(retval); +} + +pthread_t Pthread_self(void) { + return pthread_self(); +} + +void Pthread_once(pthread_once_t *once_control, void (*init_function)()) { + pthread_once(once_control, init_function); +} + +/******************************* + * Wrappers for Posix semaphores + *******************************/ + +void Sem_init(sem_t *sem, int pshared, unsigned int value) +{ + if (sem_init(sem, pshared, value) < 0) { + log_err("Sem_init error: %s\n", strerror(errno)); + } +} + +void P(sem_t *sem) +{ + if (sem_wait(sem) < 0) { + log_err("P error: %s\n", strerror(errno)); + } +} + +void V(sem_t *sem) +{ + if (sem_post(sem) < 0) { + log_err("V error: %s\n", strerror(errno)); + } +} + +/********************************************************************* + * The Rio package - robust I/O functions + **********************************************************************/ +/* + * rio_readn - robustly read n bytes (unbuffered) + */ +/* $begin rio_readn */ +ssize_t rio_readn(int fd, void *usrbuf, size_t n) +{ + size_t nleft = n; + ssize_t nread; + char *bufp = (char *)usrbuf; + + while (nleft > 0) { + if ((nread = read(fd, bufp, nleft)) < 0) { + if (errno == EINTR) /* interrupted by sig handler return */ + nread = 0; /* and call read() again */ + else + return -1; /* errno set by read() */ + } + else if (nread == 0) + break; /* EOF */ + nleft -= nread; + bufp += nread; + } + return (n - nleft); /* return >= 0 */ +} +/* $end rio_readn */ + +/* + * rio_writen - robustly write n bytes (unbuffered) + */ +/* $begin rio_writen */ +ssize_t rio_writen(int fd, void *usrbuf, size_t n) +{ + size_t nleft = n; + ssize_t nwritten; + char *bufp = (char *)usrbuf; + + while (nleft > 0) { + if ((nwritten = write(fd, bufp, nleft)) <= 0) { + if (errno == EINTR) /* interrupted by sig handler return */ + nwritten = 0; /* and call write() again */ + else + return -1; /* errorno set by write() */ + } + nleft -= nwritten; + bufp += nwritten; + } + return n; +} +/* $end rio_writen */ + + +/* + * rio_read - This is a wrapper for the Unix read() function that + * transfers min(n, rio_cnt) bytes from an internal buffer to a user + * buffer, where n is the number of bytes requested by the user and + * rio_cnt is the number of unread bytes in the internal buffer. On + * entry, rio_read() refills the internal buffer via a call to + * read() if the internal buffer is empty. + */ +/* $begin rio_read */ +static ssize_t rio_read(rio_t *rp, char *usrbuf, size_t n) +{ + int cnt; + + while (rp->rio_cnt <= 0) { /* refill if buf is empty */ + rp->rio_cnt = read(rp->rio_fd, rp->rio_buf, + sizeof(rp->rio_buf)); + if (rp->rio_cnt < 0) { + if (errno != EINTR) /* interrupted by sig handler return */ + return -1; + } + else if (rp->rio_cnt == 0) /* EOF */ + return 0; + else + rp->rio_bufptr = rp->rio_buf; /* reset buffer ptr */ + } + + /* Copy min(n, rp->rio_cnt) bytes from internal buf to user buf */ + cnt = n; + if (rp->rio_cnt < (int)n) + cnt = rp->rio_cnt; + memcpy(usrbuf, rp->rio_bufptr, cnt); + rp->rio_bufptr += cnt; + rp->rio_cnt -= cnt; + return cnt; +} +/* $end rio_read */ + +/* + * rio_readinitb - Associate a descriptor with a read buffer and reset buffer + */ +/* $begin rio_readinitb */ +void rio_readinitb(rio_t *rp, int fd) +{ + rp->rio_fd = fd; + rp->rio_cnt = 0; + rp->rio_bufptr = rp->rio_buf; +} +/* $end rio_readinitb */ + +/* + * rio_readnb - Robustly read n bytes (buffered) + */ +/* $begin rio_readnb */ +ssize_t rio_readnb(rio_t *rp, void *usrbuf, size_t n) +{ + size_t nleft = n; + ssize_t nread; + char *bufp = (char *)usrbuf; + + while (nleft > 0) { + if ((nread = rio_read(rp, bufp, nleft)) < 0) { + if (errno == EINTR) /* interrupted by sig handler return */ + nread = 0; /* call read() again */ + else + return -1; /* errno set by read() */ + } + else if (nread == 0) + break; /* EOF */ + nleft -= nread; + bufp += nread; + } + return (n - nleft); /* return >= 0 */ +} +/* $end rio_readnb */ + +/* + * rio_readlineb - robustly read a text line (buffered) + */ +/* $begin rio_readlineb */ +ssize_t rio_readlineb(rio_t *rp, void *usrbuf, size_t maxlen) +{ + size_t n; + int rc; + char c, *bufp = (char *)usrbuf; + + for (n = 1; n < maxlen; n++) { + if ((rc = rio_read(rp, &c, 1)) == 1) { + *bufp++ = c; + if (c == '\n') + break; + } else if (rc == 0) { + if (n == 1) + return 0; /* EOF, no data read */ + else + break; /* EOF, some data was read */ + } else + return -1; /* error */ + } + *bufp = 0; + return n; +} +/* $end rio_readlineb */ + +/********************************** + * Wrappers for robust I/O routines + **********************************/ +ssize_t Rio_readn(int fd, void *ptr, size_t nbytes) +{ + ssize_t n; + + if ((n = rio_readn(fd, ptr, nbytes)) < 0) { + log_err("Rio_readn error: %s\n", strerror(errno)); + } + return n; +} + +void Rio_writen(int fd, void *usrbuf, size_t n) +{ + if (rio_writen(fd, usrbuf, n) != (ssize_t)n) { + log_err("Rio_writen error: %s\n", strerror(errno)); + } +} + +void Rio_readinitb(rio_t *rp, int fd) +{ + rio_readinitb(rp, fd); +} + +ssize_t Rio_readnb(rio_t *rp, void *usrbuf, size_t n) +{ + ssize_t rc; + + if ((rc = rio_readnb(rp, usrbuf, n)) < 0) { + log_err("Rio_readnb error: %s\n", strerror(errno)); + } + return rc; +} + +ssize_t Rio_readlineb(rio_t *rp, void *usrbuf, size_t maxlen) +{ + ssize_t rc; + + if ((rc = rio_readlineb(rp, usrbuf, maxlen)) < 0) { + log_err("Rio_readlineb error: %s\n", strerror(errno)); + } + return rc; +} + +/******************************** + * Client/server helper functions + ********************************/ +/* + * open_clientfd - open connection to server at + * and return a socket descriptor ready for reading and writing. + * Returns -1 and sets errno on Unix error. + * Returns -2 and sets h_errno on DNS (gethostbyname) error. + */ +/* $begin open_clientfd */ +int open_clientfd(char *hostname, int port) +{ + int clientfd; + struct hostent *hp; + struct sockaddr_in serveraddr; + + if ((clientfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) + return -1; /* check errno for cause of error */ + + /* Fill in the server's IP address and port */ + if ((hp = gethostbyname(hostname)) == NULL) + return -2; /* check h_errno for cause of error */ + bzero((char *) &serveraddr, sizeof(serveraddr)); + serveraddr.sin_family = AF_INET; + bcopy((char *)hp->h_addr_list[0], + (char *)&serveraddr.sin_addr.s_addr, hp->h_length); + serveraddr.sin_port = htons(port); + + /* Establish a connection with the server */ + if (connect(clientfd, (SA *) &serveraddr, sizeof(serveraddr)) < 0) + return -1; + return clientfd; +} +/* $end open_clientfd */ + +/* + * open_listenfd - open and return a listening socket on port + * Returns -1 and sets errno on Unix error. + */ +/* $begin open_listenfd */ +int open_listenfd(int port) +{ + int listenfd, optval=1; + struct sockaddr_in serveraddr; + + /* Create a socket descriptor */ + if ((listenfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) + return -1; + + /* Eliminates "Address already in use" error from bind. */ + if (setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, + (const void *)&optval , sizeof(int)) < 0) + return -1; + + /* Listenfd will be an endpoint for all requests to port + on any IP address for this host */ + bzero((char *) &serveraddr, sizeof(serveraddr)); + serveraddr.sin_family = AF_INET; + serveraddr.sin_addr.s_addr = htonl(INADDR_ANY); + serveraddr.sin_port = htons((unsigned short)port); + if (bind(listenfd, (SA *)&serveraddr, sizeof(serveraddr)) < 0) + return -1; + + /* Make it a listening socket ready to accept connection requests */ + if (listen(listenfd, LISTENQ) < 0) + return -1; + return listenfd; +} +/* $end open_listenfd */ + +/****************************************** + * Wrappers for the client/server helper routines + ******************************************/ +int Open_clientfd(char *hostname, int port) +{ + int rc; + + if ((rc = open_clientfd(hostname, port)) < 0) { + if (rc == -1) { + log_err("Open_clientfd Unix error: %s\n", strerror(errno)); + } + else { + log_err("%s: DNS error %d\n", "Open_clientfd DNS error", h_errno); + } + } + return rc; +} + +int Open_listenfd(int port) +{ + int rc; + + if ((rc = open_listenfd(port)) < 0) { + log_err("Open_listenfd error: %s\n", strerror(errno)); + } + return rc; +} diff --git a/pink/third/slash/slash/src/rsync.cc b/pink/third/slash/slash/src/rsync.cc new file mode 100644 index 00000000..0c82ad38 --- /dev/null +++ b/pink/third/slash/slash/src/rsync.cc @@ -0,0 +1,183 @@ +#include +#include +#include + +#include "slash/include/env.h" +#include "slash/include/rsync.h" +#include "slash/include/xdebug.h" + +namespace slash { +// Clean files for rsync info, such as the lock, log, pid, conf file +static bool CleanRsyncInfo(const std::string& path) { + return slash::DeleteDirIfExist(path + kRsyncSubDir); +} + +int StartRsync(const std::string& raw_path, + const std::string& module, + const std::string& ip, + const int port, + const std::string& passwd) { + // Sanity check + if (raw_path.empty() || module.empty() || passwd.empty()) { + return -1; + } + std::string path(raw_path); + if (path.back() != '/') { + path += "/"; + } + std::string rsync_path = path + kRsyncSubDir + "/"; + CreatePath(rsync_path); + + // Generate secret file + std::string secret_file(rsync_path + kRsyncSecretFile); + std::ofstream secret_stream(secret_file.c_str()); + if (!secret_stream) { + log_warn("Open rsync secret file failed!"); + return -1; + } + secret_stream << kRsyncUser << ":" << passwd; + secret_stream.close(); + + // Generate conf file + std::string conf_file(rsync_path + kRsyncConfFile); + std::ofstream conf_stream(conf_file.c_str()); + if (!conf_stream) { + log_warn("Open rsync conf file failed!"); + return -1; + } + + if (geteuid() == 0) { + conf_stream << "uid = root" << std::endl; + conf_stream << "gid = root" << std::endl; + } + conf_stream << "use chroot = no" << std::endl; + conf_stream << "max connections = 10" << std::endl; + conf_stream << "lock file = " << rsync_path + kRsyncLockFile << std::endl; + conf_stream << "log file = " << rsync_path + kRsyncLogFile << std::endl; + conf_stream << "pid file = " << rsync_path + kRsyncPidFile << std::endl; + conf_stream << "list = no" << std::endl; + conf_stream << "strict modes = no" << std::endl; + conf_stream << "auth users = " << kRsyncUser << std::endl; + conf_stream << "secrets file = " << secret_file << std::endl; + conf_stream << "[" << module << "]" << std::endl; + conf_stream << "path = " << path << std::endl; + conf_stream << "read only = no" << std::endl; + conf_stream.close(); + + // Execute rsync command + std::stringstream ss; + ss << "rsync --daemon --config=" << conf_file; + ss << " --address=" << ip; + if (port != 873) { + ss << " --port=" << port; + } + std::string rsync_start_cmd = ss.str(); + int ret = system(rsync_start_cmd.c_str()); + if (ret == 0 || (WIFEXITED(ret) && !WEXITSTATUS(ret))) { + return 0; + } + log_warn("Start rsync deamon failed : %d!", ret); + return ret; +} + +int StopRsync(const std::string& raw_path) { + // Sanity check + if (raw_path.empty()) { + log_warn("empty rsync path!"); + return -1; + } + std::string path(raw_path); + if (path.back() != '/') { + path += "/"; + } + + std::string pid_file(path + kRsyncSubDir + "/" + kRsyncPidFile); + if (!FileExists(pid_file)) { + log_warn("no rsync pid file found"); + return 0; // Rsync deamon is not exist + } + + // Kill Rsync + SequentialFile *sequential_file; + if (!NewSequentialFile(pid_file, &sequential_file).ok()) { + log_warn("no rsync pid file found"); + return 0; + }; + + char line[32]; + if (sequential_file->ReadLine(line, 32) == NULL) { + log_warn("read rsync pid file err"); + delete sequential_file; + return 0; + }; + + delete sequential_file; + + pid_t pid = atoi(line); + + if (pid <= 1) { + log_warn("read rsync pid err"); + return 0; + } + + std::string rsync_stop_cmd = "kill -- -$(ps -o pgid= " + std::to_string(pid) + ")"; + int ret = system(rsync_stop_cmd.c_str()); + if (ret == 0 || (WIFEXITED(ret) && !WEXITSTATUS(ret))) { + log_info("Stop rsync success!"); + } else { + log_warn("Stop rsync deamon failed : %d!", ret); + } + CleanRsyncInfo(path); + return ret; +} + +int RsyncSendFile(const std::string& local_file_path, + const std::string& remote_file_path, + const std::string& secret_file_path, + const RsyncRemote& remote) { + std::stringstream ss; + ss << """rsync -avP --bwlimit=" << remote.kbps + << " --password-file=" << secret_file_path + << " --port=" << remote.port + << " " << local_file_path + << " " << kRsyncUser << "@" << remote.host + << "::" << remote.module << "/" << remote_file_path; + std::string rsync_cmd = ss.str(); + int ret = system(rsync_cmd.c_str()); + if (ret == 0 || (WIFEXITED(ret) && !WEXITSTATUS(ret))) { + return 0; + } + log_warn("Rsync send file failed : %d!", ret); + return ret; +} + +int RsyncSendClearTarget(const std::string& local_dir_path, + const std::string& remote_dir_path, + const std::string& secret_file_path, + const RsyncRemote& remote) { + if (local_dir_path.empty() || remote_dir_path.empty()) { + return -2; + } + std::string local_dir(local_dir_path), remote_dir(remote_dir_path); + if (local_dir_path.back() != '/') { + local_dir.append("/"); + } + if (remote_dir_path.back() != '/') { + remote_dir.append("/"); + } + std::stringstream ss; + ss << "rsync -avP --delete --port=" << remote.port + << " --password-file=" << secret_file_path + << " " << local_dir + << " " << kRsyncUser << "@" << remote.host + << "::" << remote.module << "/" << remote_dir; + std::string rsync_cmd = ss.str(); + int ret = system(rsync_cmd.c_str()); + if (ret == 0 || (WIFEXITED(ret) && !WEXITSTATUS(ret))) { + return 0; + } + log_warn("Rsync send file failed : %d!", ret); + return ret; +} + +} // namespace slash diff --git a/pink/third/slash/slash/src/scope_record_lock.cc b/pink/third/slash/slash/src/scope_record_lock.cc new file mode 100644 index 00000000..e35cb026 --- /dev/null +++ b/pink/third/slash/slash/src/scope_record_lock.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2017-present The blackwidow Authors. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "slash/include/scope_record_lock.h" + +namespace slash { + +namespace lock { + +MultiScopeRecordLock::MultiScopeRecordLock(LockMgr* lock_mgr, + const std::vector& keys) : + lock_mgr_(lock_mgr), + keys_(keys) { + std::string pre_key; + std::sort(keys_.begin(), keys_.end()); + if (!keys_.empty() && + keys_[0].empty()) { + lock_mgr_->TryLock(pre_key); + } + + for (const auto& key : keys_) { + if (pre_key != key) { + lock_mgr_->TryLock(key); + pre_key = key; + } + } +} +MultiScopeRecordLock::~MultiScopeRecordLock() { + std::string pre_key; + if (!keys_.empty() && + keys_[0].empty()) { + lock_mgr_->UnLock(pre_key); + } + + for (const auto& key : keys_) { + if (pre_key != key) { + lock_mgr_->UnLock(key); + pre_key = key; + } + } +} + +void MultiRecordLock::Lock(const std::vector& keys) { + std::vector internal_keys = keys; + std::sort(internal_keys.begin(), internal_keys.end()); + // init to be "" + std::string pre_key; + // consider internal_keys "" "" "a" + if (!internal_keys.empty()) { + lock_mgr_->TryLock(internal_keys.front()); + pre_key = internal_keys.front(); + } + + for (const auto& key : internal_keys) { + if (pre_key != key) { + lock_mgr_->TryLock(key); + pre_key = key; + } + } +} + + +void MultiRecordLock::Unlock(const std::vector& keys) { + std::vector internal_keys = keys; + std::sort(internal_keys.begin(), internal_keys.end()); + std::string pre_key; + if (!internal_keys.empty()) { + lock_mgr_->UnLock(internal_keys.front()); + pre_key = internal_keys.front(); + } + + for (const auto& key : internal_keys) { + if (pre_key != key) { + lock_mgr_->UnLock(key); + pre_key = key; + } + } +} +} // namespace lock +} // namespace slash diff --git a/pink/third/slash/slash/src/slash_binlog_impl.cc b/pink/third/slash/slash/src/slash_binlog_impl.cc new file mode 100644 index 00000000..eca3ece4 --- /dev/null +++ b/pink/third/slash/slash/src/slash_binlog_impl.cc @@ -0,0 +1,578 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include "slash/src/slash_binlog_impl.h" + +#include +#include +#include +#include +#include + +namespace slash { + +std::string NewFileName(const std::string name, const uint32_t current) { + char buf[256]; + snprintf(buf, sizeof(buf), "%s%u", name.c_str(), current); + return std::string(buf); +} + +// Version +Version::Version(RWFile *save) + : pro_offset_(0), + pro_num_(0), + item_num_(0), + save_(save) { + assert(save_ != NULL); +} + +Version::~Version() { + StableSave(); +} + +Status Version::StableSave() { + char *p = save_->GetData(); + memcpy(p, &pro_offset_, sizeof(uint64_t)); + p += 16; + memcpy(p, &item_num_, sizeof(uint32_t)); + p += 4; + memcpy(p, &pro_num_, sizeof(uint32_t)); + return Status::OK(); +} + +Status Version::Init() { + Status s; + if (save_->GetData() != NULL) { + memcpy((char*)(&pro_offset_), save_->GetData(), sizeof(uint64_t)); + memcpy((char*)(&item_num_), save_->GetData() + 16, sizeof(uint32_t)); + memcpy((char*)(&pro_num_), save_->GetData() + 20, sizeof(uint32_t)); + return Status::OK(); + } else { + return Status::Corruption("version init error"); + } +} + +// Binlog +Status Binlog::Open(const std::string& path, Binlog** logptr) { + *logptr = NULL; + + BinlogImpl *impl = new BinlogImpl(path, kBinlogSize); + Status s = impl->Recover(); + if (s.ok()) { + *logptr = impl; + } else { + delete impl; + } + return s; +} + +BinlogImpl::BinlogImpl(const std::string& path, const int file_size) + : exit_all_consume_(false), + path_(path), + file_size_(file_size), + version_(NULL), + queue_(NULL), + versionfile_(NULL) { + if (path_.back() != '/') { + path_.push_back('/'); + } +} + +Status BinlogImpl::Recover() { + CreateDir(path_); + + std::string manifest = path_ + kManifest; + bool exist_flag = false; + if (FileExists(manifest)) { + exist_flag = true; + } + Status s = NewRWFile(manifest, &versionfile_); + if (!s.ok()) { + return s; + } + version_ = new Version(versionfile_); + version_->Init(); + version_->StableSave(); + + pro_num_ = version_->pro_num_; + std::string profile = NewFileName(path_ + kBinlogPrefix, pro_num_); + if (exist_flag) { + s = AppendWritableFile(profile, &queue_, version_->pro_offset_); + if (!s.ok()) { + return s; + } + + // recover memtable + //MemTable *mem = new MemTable(file_size_); + //mem->Ref(); + //mem->RecoverFromFile(profile); + //memtables_[pro_num_] = mem; + } else { + s = NewWritableFile(profile, &queue_); + if (!s.ok()) { + return s; + } + + //MemTable *mem = new MemTable(file_size_); + //mem->Ref(); + //memtables_[pro_num_] = mem; + } + + InitOffset(); + return s; +} + +BinlogImpl::~BinlogImpl() { + delete version_; + delete versionfile_; + delete queue_; +} + +void BinlogImpl::InitOffset() { + assert(queue_ != NULL); + uint64_t filesize = queue_->Filesize(); + block_offset_ = filesize % kBlockSize; +} + +Status BinlogImpl::GetProducerStatus(uint32_t* filenum, uint64_t* offset) { + ReadLock(&version_->rwlock_); + *filenum = version_->pro_num_; + *offset = version_->pro_offset_; + return Status::OK(); +} + +// Note: mutex lock should be held +Status BinlogImpl::Append(const std::string &item) { + Status s; + + // Check to roll log file + uint64_t filesize = queue_->Filesize(); + if (filesize > file_size_) { + delete queue_; + queue_ = NULL; + + pro_num_++; + std::string profile = NewFileName(path_ + kBinlogPrefix, pro_num_); + NewWritableFile(profile, &queue_); + + { + WriteLock(&version_->rwlock_); + version_->pro_offset_ = 0; + version_->pro_num_ = pro_num_; + version_->StableSave(); + //version_->debug(); + } + } + + int pro_offset; + s = Produce(Slice(item.data(), item.size()), &pro_offset); + if (s.ok()) { + WriteLock(&version_->rwlock_); + version_->pro_offset_ = pro_offset; + version_->StableSave(); + } + + return s; +} + +Status BinlogImpl::EmitPhysicalRecord(RecordType t, const char *ptr, size_t n, int *temp_pro_offset) { + Status s; + assert(n <= 0xffffff); + assert(block_offset_ + kHeaderSize + n <= kBlockSize); + + char buf[kHeaderSize]; + + uint64_t now; + struct timeval tv; + gettimeofday(&tv, NULL); + now = tv.tv_sec; + buf[0] = static_cast(n & 0xff); + buf[1] = static_cast((n & 0xff00) >> 8); + buf[2] = static_cast(n >> 16); + buf[3] = static_cast(now & 0xff); + buf[4] = static_cast((now & 0xff00) >> 8); + buf[5] = static_cast((now & 0xff0000) >> 16); + buf[6] = static_cast((now & 0xff000000) >> 24); + buf[7] = static_cast(t); + + s = queue_->Append(Slice(buf, kHeaderSize)); + if (s.ok()) { + s = queue_->Append(Slice(ptr, n)); + if (s.ok()) { + s = queue_->Flush(); + } + } + block_offset_ += static_cast(kHeaderSize + n); + + *temp_pro_offset += kHeaderSize + n; + return s; +} + +Status BinlogImpl::Produce(const Slice &item, int *temp_pro_offset) { + Status s; + const char *ptr = item.data(); + size_t left = item.size(); + bool begin = true; + + *temp_pro_offset = version_->pro_offset_; + do { + const int leftover = static_cast(kBlockSize) - block_offset_; + assert(leftover >= 0); + if (static_cast(leftover) < kHeaderSize) { + if (leftover > 0) { + queue_->Append(Slice("\x00\x00\x00\x00\x00\x00\x00", leftover)); + *temp_pro_offset += leftover; + //version_->StableSave(); + } + block_offset_ = 0; + } + + const size_t avail = kBlockSize - block_offset_ - kHeaderSize; + const size_t fragment_length = (left < avail) ? left : avail; + RecordType type; + const bool end = (left == fragment_length); + if (begin && end) { + type = kFullType; + } else if (begin) { + type = kFirstType; + } else if (end) { + type = kLastType; + } else { + type = kMiddleType; + } + + s = EmitPhysicalRecord(type, ptr, fragment_length, temp_pro_offset); + ptr += fragment_length; + left -= fragment_length; + begin = false; + } while (s.ok() && left > 0); + + return s; +} + +Status BinlogImpl::AppendBlank(WritableFile *file, uint64_t len) { + if (len < kHeaderSize) { + return Status::OK(); + } + + uint64_t pos = 0; + + std::string blank(kBlockSize, ' '); + for (; pos + kBlockSize < len; pos += kBlockSize) { + file->Append(Slice(blank.data(), blank.size())); + } + + // Append a msg which occupy the remain part of the last block + // We simply increase the remain length to kHeaderSize when remain part < kHeaderSize + uint32_t n; + if (len % kBlockSize < kHeaderSize) { + n = 0; + } else { + n = (uint32_t) ((len % kBlockSize) - kHeaderSize); + } + + char buf[kBlockSize]; + uint64_t now; + struct timeval tv; + gettimeofday(&tv, NULL); + now = tv.tv_sec; + buf[0] = static_cast(n & 0xff); + buf[1] = static_cast((n & 0xff00) >> 8); + buf[2] = static_cast(n >> 16); + buf[3] = static_cast(now & 0xff); + buf[4] = static_cast((now & 0xff00) >> 8); + buf[5] = static_cast((now & 0xff0000) >> 16); + buf[6] = static_cast((now & 0xff000000) >> 24); + buf[7] = static_cast(kFullType); + + Status s = file->Append(Slice(buf, kHeaderSize)); + if (s.ok()) { + s = file->Append(Slice(blank.data(), n)); + if (s.ok()) { + s = file->Flush(); + } + } + return s; +} + +Status BinlogImpl::SetProducerStatus(uint32_t pro_num, uint64_t pro_offset) { + MutexLock l(&mutex_); + + // offset smaller than the first header + if (pro_offset < kHeaderSize) { + pro_offset = 0; + } + + delete queue_; + + std::string init_profile = NewFileName(path_ + kBinlogPrefix, 0); + if (FileExists(init_profile)) { + DeleteFile(init_profile); + } + + std::string profile = NewFileName(path_ + kBinlogPrefix, pro_num); + if (FileExists(profile)) { + DeleteFile(profile); + } + + NewWritableFile(profile, &queue_); + BinlogImpl::AppendBlank(queue_, pro_offset); + + pro_num_ = pro_num; + + { + WriteLock(&version_->rwlock_); + version_->pro_num_ = pro_num; + version_->pro_offset_ = pro_offset; + version_->StableSave(); + } + + InitOffset(); + return Status::OK(); +} + +BinlogReader* BinlogImpl::NewBinlogReader(uint32_t filenum, uint64_t offset) { + // Check sync point + uint32_t cur_filenum = 0; + uint64_t cur_offset = 0; + GetProducerStatus(&cur_filenum, &cur_offset); + if (cur_filenum < filenum || (cur_filenum == filenum && cur_offset < offset)) { + return NULL; + } + + std::string confile = NewFileName(path_ + kBinlogPrefix, filenum); + if (!slash::FileExists(confile)) { + // Not found binlog specified by filenum + return NULL; + } + + BinlogReaderImpl* reader = new BinlogReaderImpl(this, path_, filenum, offset); + Status s = reader->Trim(); + if (!s.ok()) { + log_info("Trim offset failed: %s", s.ToString().c_str()); + return NULL; + } + + return reader; +} + +BinlogReaderImpl::BinlogReaderImpl(Binlog* log, const std::string &path, uint32_t filenum, uint64_t offset) + : log_(log), + path_(path), + filenum_(filenum), + offset_(offset), + should_exit_(false), + initial_offset_(0), + last_record_offset_(offset_ % kBlockSize), + end_of_buffer_offset_(kBlockSize), + queue_(NULL), + backing_store_(new char[kBlockSize]) { + std::string confile = NewFileName(path_ + kBinlogPrefix, filenum_); + if (!NewSequentialFile(confile, &queue_).ok()) { + log_info("Reader new sequtialfile failed"); + } +} + +BinlogReaderImpl::~BinlogReaderImpl() { + delete queue_; + delete [] backing_store_; +} + +Status BinlogReaderImpl::Trim() { + Status s; + uint64_t start_block = (offset_ / kBlockSize) * kBlockSize; + s = queue_->Skip(start_block); + if (!s.ok()) { + return s; + } + uint64_t block_offset = offset_ % kBlockSize; + uint64_t ret = 0; + uint64_t res = 0; + + while (true) { + if (res >= block_offset) { + offset_ = start_block + res; + break; + } + ret = GetNext(s); + if (!s.ok()) { + return s; + } + res += ret; + } + last_record_offset_ = offset_ % kBlockSize; + + return Status::OK(); +} + +uint64_t BinlogReaderImpl::GetNext(Status &result) { + uint64_t offset = 0; + Status s; + + while (true) { + buffer_.clear(); + s = queue_->Read(kHeaderSize, &buffer_, backing_store_); + if (!s.ok()) { + result = s; + return 0; + } + + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[0]) & 0xff; + const uint32_t b = static_cast(header[1]) & 0xff; + const uint32_t c = static_cast(header[2]) & 0xff; + const unsigned int type = header[7]; + const uint32_t length = a | (b << 8) | (c << 16); + + if (type == kFullType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + break; + } else if (type == kFirstType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + } else if (type == kMiddleType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + } else if (type == kLastType) { + s = queue_->Read(length, &buffer_, backing_store_); + offset += kHeaderSize + length; + break; + } else { + break; + } + } + result = s; + return offset; +} + +unsigned int BinlogReaderImpl::ReadPhysicalRecord(Slice *result) { + Status s; + uint64_t zero_space = end_of_buffer_offset_ - last_record_offset_; + if (zero_space <= kHeaderSize) { + queue_->Skip(zero_space); + offset_ += zero_space; + last_record_offset_ = 0; + } + buffer_.clear(); + s = queue_->Read(kHeaderSize, &buffer_, backing_store_); + if (s.IsEndFile()) { + return kEof; + } else if (!s.ok()) { + return kBadRecord; + } + + const char* header = buffer_.data(); + const uint32_t a = static_cast(header[0]) & 0xff; + const uint32_t b = static_cast(header[1]) & 0xff; + const uint32_t c = static_cast(header[2]) & 0xff; + const unsigned int type = header[7]; + const uint32_t length = a | (b << 8) | (c << 16); + if (type == kZeroType || length == 0) { + buffer_.clear(); + return kOldRecord; + } + + buffer_.clear(); + //std::cout<<"2 --> offset_: "<Read(length, &buffer_, backing_store_); + *result = Slice(buffer_.data(), buffer_.size()); + last_record_offset_ += kHeaderSize + length; + if (s.ok()) { + offset_ += (kHeaderSize + length); + } + return type; +} + +Status BinlogReaderImpl::Consume(std::string &scratch) { + Status s; + if (last_record_offset_ < initial_offset_) { + return Status::IOError("last_record_offset exceed"); + } + + Slice fragment; + while (true) { + const unsigned int record_type = ReadPhysicalRecord(&fragment); + + switch (record_type) { + case kFullType: + scratch = std::string(fragment.data(), fragment.size()); + s = Status::OK(); + break; + case kFirstType: + scratch.assign(fragment.data(), fragment.size()); + s = Status::NotFound("Middle Status"); + break; + case kMiddleType: + scratch.append(fragment.data(), fragment.size()); + s = Status::NotFound("Middle Status"); + break; + case kLastType: + scratch.append(fragment.data(), fragment.size()); + s = Status::OK(); + break; + case kEof: + return Status::EndFile("Eof"); + case kBadRecord: + return Status::IOError("Data Corruption"); + case kOldRecord: + return Status::EndFile("Eof"); + default: + return Status::IOError("Unknow reason"); + } + // TODO:do handler here + if (s.ok()) { + break; + } + } + //DLOG(INFO) << "Binlog Sender consumer a msg: " << scratch; + return Status::OK(); +} + +// Get a whole message; +// the status will be OK, IOError or Corruption; +Status BinlogReaderImpl::ReadRecord(std::string &scratch) { + scratch.clear(); + Status s; + uint32_t pro_num; + uint64_t pro_offset; + + while (!should_exit_) { + log_->GetProducerStatus(&pro_num, &pro_offset); + if (filenum_ == pro_num && offset_ == pro_offset) { + usleep(10000); + continue; + } + + s = Consume(scratch); + if (s.IsEndFile()) { + std::string confile = NewFileName(path_ + kBinlogPrefix, filenum_ + 1); + + // Roll to next File + if (FileExists(confile)) { + delete queue_; + queue_ = NULL; + NewSequentialFile(confile, &(queue_)); + + filenum_++; + offset_ = 0; + initial_offset_ = 0; + end_of_buffer_offset_ = kBlockSize; + last_record_offset_ = offset_ % kBlockSize; + } else { + usleep(10000); + } + } else { + break; + } + } + + if (should_exit_) { + return Status::Corruption("should exit"); + } + return s; +} + +} // namespace slash diff --git a/pink/third/slash/slash/src/slash_binlog_impl.h b/pink/third/slash/slash/src/slash_binlog_impl.h new file mode 100644 index 00000000..9f1c46f4 --- /dev/null +++ b/pink/third/slash/slash/src/slash_binlog_impl.h @@ -0,0 +1,185 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#ifndef SLASH_BINLOG_IMPL_H_ +#define SLASH_BINLOG_IMPL_H_ + +#include +#include +#include +#include + +#include "slash/include/env.h" +#include "slash/include/slash_binlog.h" +#include "slash/include/slash_status.h" +#include "slash/include/slash_mutex.h" + +namespace slash { + +class Version; +class BinlogReader; + +// SyncPoint is a file number and an offset; + +const std::string kBinlogPrefix = "binlog"; +const std::string kManifest = "manifest"; +const int kBinlogSize = 128; +//const int kBinlogSize = (100 << 20); +const int kBlockSize = (64 << 10); +// Header is Type(1 byte), length (3 bytes), time (4 bytes) +const size_t kHeaderSize = 1 + 3 + 4; + +enum RecordType { + kZeroType = 0, + kFullType = 1, + kFirstType = 2, + kMiddleType = 3, + kLastType = 4, + kEof = 5, + kBadRecord = 6, + kOldRecord = 7 +}; + +class BinlogImpl : public Binlog { + public: + BinlogImpl(const std::string& path, const int file_size = (100 < 20)); + virtual ~BinlogImpl(); + + // + // Basic API + // + virtual Status Append(const std::string &item); + //Status Append(const char* item, int len); + virtual BinlogReader* NewBinlogReader(uint32_t filenum, uint64_t offset); + + virtual Status GetProducerStatus(uint32_t* filenum, uint64_t* offset); + virtual Status SetProducerStatus(uint32_t filenum, uint64_t pro_offset); + + private: + friend class Binlog; + + // + // More specify API, used by Pika + // + Status Recover(); + static Status AppendBlank(WritableFile *file, uint64_t len); + WritableFile *queue() { return queue_; } + uint64_t file_size() { + return file_size_; + } + + void Lock() { mutex_.Lock(); } + void Unlock() { mutex_.Unlock(); } + + void InitOffset(); + Status EmitPhysicalRecord(RecordType t, const char *ptr, size_t n, int *temp_pro_offset); + + // Produce + Status Produce(const Slice &item, int *pro_offset); + + private: + Mutex mutex_; + bool exit_all_consume_; + std::string path_; + uint64_t file_size_; + uint32_t pro_num_; + uint64_t record_num_; + + Version* version_; + WritableFile *queue_; + RWFile *versionfile_; + + int block_offset_; + char* pool_; + + //std::unordered_map memtables_; + + // Not use + //std::string filename; + //int32_t retry_; + //uint32_t consumer_num_; + + // No copying allowed + BinlogImpl(const BinlogImpl&); + void operator=(const BinlogImpl&); +}; + +class Version { + public: + Version(RWFile *save); + ~Version(); + + Status Init(); + // RWLock should be held when access members. + Status StableSave(); + + uint64_t pro_offset_; + uint32_t pro_num_; + uint32_t item_num_; + + RWMutex rwlock_; + + void debug() { + ReadLock(&this->rwlock_); + printf ("Current pro_num %u pro_offset %lu\n", pro_num_, pro_offset_); + } + + private: + + RWFile *save_; + + // Not used + //uint64_t con_offset_; + //uint32_t con_num_; + + // No copying allowed; + Version(const Version&); + void operator=(const Version&); +}; + +class BinlogReaderImpl : public BinlogReader { + public: + BinlogReaderImpl(Binlog* log, const std::string& path, uint32_t filenum, uint64_t offset); + ~BinlogReaderImpl(); + + //bool ReadRecord(Slice* record, std::string* scratch); + virtual Status ReadRecord(std::string &record); + + private: + friend class BinlogImpl; + + //Status Parse(std::string &scratch); + Status Consume(std::string &scratch); + unsigned int ReadPhysicalRecord(Slice *fragment); + + // Tirm offset to first record behind offered offset. + Status Trim(); + // Return next record end offset in a block, store in result if error encounted. + uint64_t GetNext(Status &result); + + Binlog* log_; + std::string path_; + uint32_t filenum_; + uint64_t offset_; + std::atomic should_exit_; + + // not used + uint64_t initial_offset_; + uint64_t last_record_offset_; + uint64_t end_of_buffer_offset_; + + SequentialFile* queue_; + char* const backing_store_; + Slice buffer_; + + // No copying allowed; + BinlogReaderImpl(const BinlogReaderImpl&); + void operator=(const BinlogReaderImpl&); +}; + +} // namespace slash + + +#endif // SLASH_BINLOG_IMPL_H_ diff --git a/pink/third/slash/slash/src/slash_coding.cc b/pink/third/slash/slash/src/slash_coding.cc new file mode 100644 index 00000000..fda2eda6 --- /dev/null +++ b/pink/third/slash/slash/src/slash_coding.cc @@ -0,0 +1,211 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "slash/include/slash_coding.h" +#include "slash/include/slash_slice.h" + +namespace slash { + +void EncodeFixed16(char* buf, uint16_t value) { + memcpy(buf, &value, sizeof(value)); +} + +void EncodeFixed32(char* buf, uint32_t value) { + memcpy(buf, &value, sizeof(value)); +} + +void EncodeFixed64(char* buf, uint64_t value) { + memcpy(buf, &value, sizeof(value)); +} + +void PutFixed16(std::string* dst, uint16_t value) { + char buf[sizeof(value)]; + EncodeFixed16(buf, value); + dst->append(buf, sizeof(buf)); +} + +void PutFixed32(std::string* dst, uint32_t value) { + char buf[sizeof(value)]; + EncodeFixed32(buf, value); + dst->append(buf, sizeof(buf)); +} + +void PutFixed64(std::string* dst, uint64_t value) { + char buf[sizeof(value)]; + EncodeFixed64(buf, value); + dst->append(buf, sizeof(buf)); +} + +char* EncodeVarint32(char* dst, uint32_t v) { + // Operate on characters as unsigneds + unsigned char* ptr = reinterpret_cast(dst); + static const int B = 128; + if (v < (1<<7)) { + *(ptr++) = v; + } else if (v < (1<<14)) { + *(ptr++) = v | B; + *(ptr++) = v>>7; + } else if (v < (1<<21)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = v>>14; + } else if (v < (1<<28)) { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = v>>21; + } else { + *(ptr++) = v | B; + *(ptr++) = (v>>7) | B; + *(ptr++) = (v>>14) | B; + *(ptr++) = (v>>21) | B; + *(ptr++) = v>>28; + } + return reinterpret_cast(ptr); +} + +void PutVarint32(std::string* dst, uint32_t v) { + char buf[5]; + char* ptr = EncodeVarint32(buf, v); + dst->append(buf, ptr - buf); +} + +char* EncodeVarint64(char* dst, uint64_t v) { + static const int B = 128; + unsigned char* ptr = reinterpret_cast(dst); + while (v >= B) { + *(ptr++) = (v & (B-1)) | B; + v >>= 7; + } + *(ptr++) = static_cast(v); + return reinterpret_cast(ptr); +} + +void PutVarint64(std::string* dst, uint64_t v) { + char buf[10]; + char* ptr = EncodeVarint64(buf, v); + dst->append(buf, ptr - buf); +} + +void PutLengthPrefixedString(std::string* dst, const std::string& value) { + PutVarint32(dst, value.size()); + dst->append(value.data(), value.size()); +} + +int VarintLength(uint64_t v) { + int len = 1; + while (v >= 128) { + v >>= 7; + len++; + } + return len; +} + +const char* GetVarint32PtrFallback(const char* p, + const char* limit, + uint32_t* value) { + uint32_t result = 0; + for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { + uint32_t byte = *(reinterpret_cast(p)); + p++; + if (byte & 128) { + // More bytes are present + result |= ((byte & 127) << shift); + } else { + result |= (byte << shift); + *value = result; + return reinterpret_cast(p); + } + } + return NULL; +} + +bool GetVarint32(std::string* input, uint32_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint32Ptr(p, limit, value); + if (q == NULL) { + return false; + } else { + (*input).erase(0, q - p); + return true; + } +} + +bool GetVarint32(Slice* input, uint32_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint32Ptr(p, limit, value); + if (q == NULL) { + return false; + } else { + *input = Slice(q, limit - q); + return true; + } +} + +const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { + uint64_t result = 0; + for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { + uint64_t byte = *(reinterpret_cast(p)); + p++; + if (byte & 128) { + // More bytes are present + result |= ((byte & 127) << shift); + } else { + result |= (byte << shift); + *value = result; + return reinterpret_cast(p); + } + } + return NULL; +} + +bool GetVarint64(Slice* input, uint64_t* value) { + const char* p = input->data(); + const char* limit = p + input->size(); + const char* q = GetVarint64Ptr(p, limit, value); + if (q == NULL) { + return false; + } else { + *input = Slice(q, limit - q); + return true; + } +} + +const char* GetLengthPrefixedSlice(const char* p, const char* limit, + Slice* result) { + uint32_t len; + p = GetVarint32Ptr(p, limit, &len); + if (p == NULL) return NULL; + if (p + len > limit) return NULL; + *result = Slice(p, len); + return p + len; +} + +bool GetLengthPrefixedSlice(Slice* input, Slice* result) { + uint32_t len; + if (GetVarint32(input, &len) && + input->size() >= len) { + *result = Slice(input->data(), len); + input->remove_prefix(len); + return true; + } else { + return false; + } +} + +bool GetLengthPrefixedString(std::string* input, std::string* result) { + uint32_t len; + if (GetVarint32(input, &len) && + input->size() >= len) { + *result = (*input).substr(0, len); + input->erase(0, len); + return true; + } else { + return false; + } +} + +} // namespace leveldb diff --git a/pink/third/slash/slash/src/slash_hash.cc b/pink/third/slash/slash/src/slash_hash.cc new file mode 100644 index 00000000..da5e392f --- /dev/null +++ b/pink/third/slash/slash/src/slash_hash.cc @@ -0,0 +1,638 @@ +/* + * Updated to C++, zedwood.com 2012 + * Based on Olivier Gay's version + * See Modified BSD License below: + * + * FIPS 180-2 SHA-224/256/384/512 implementation + * Issue date: 04/30/2005 + * http://www.ouah.org/ogay/sha2/ + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* MD5 + converted to C++ class by Frank Thilo (thilo@unix-ag.org) + for bzflag (http://www.bzflag.org) + + based on: + + md5.h and md5.c + reference implemantion of RFC 1321 + + Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All + rights reserved. + + License to copy and use this software is granted provided that it + is identified as the "RSA Data Security, Inc. MD5 Message-Digest + Algorithm" in all material mentioning or referencing this software + or this function. + + License is also granted to make and use derivative works provided + that such works are identified as "derived from the RSA Data + Security, Inc. MD5 Message-Digest Algorithm" in all material + mentioning or referencing the derived work. + + RSA Data Security, Inc. makes no representations concerning either + the merchantability of this software or the suitability of this + software for any particular purpose. It is provided "as is" + without express or implied warranty of any kind. + + These notices must be retained in any copies of any part of this + documentation and/or software. +*/ + +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + + +#include +#include +#include +#include "slash/include/slash_hash.h" + +namespace slash { + +class SHA256 { + protected: + typedef unsigned char uint8; + typedef unsigned int uint32; + typedef unsigned long long uint64; + + const static uint32 sha256_k[]; + static const unsigned int SHA224_256_BLOCK_SIZE = (512/8); + public: + void init(); + void update(const unsigned char *message, unsigned int len); + void final(unsigned char *digest); + static const unsigned int DIGEST_SIZE = ( 256 / 8); + + protected: + void transform(const unsigned char *message, unsigned int block_nb); + unsigned int m_tot_len; + unsigned int m_len; + unsigned char m_block[2*SHA224_256_BLOCK_SIZE]; + uint32 m_h[8]; +}; + +#define SHA2_SHFR(x, n) (x >> n) +#define SHA2_ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) +#define SHA2_ROTL(x, n) ((x << n) | (x >> ((sizeof(x) << 3) - n))) +#define SHA2_CH(x, y, z) ((x & y) ^ (~x & z)) +#define SHA2_MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) +#define SHA256_F1(x) (SHA2_ROTR(x, 2) ^ SHA2_ROTR(x, 13) ^ SHA2_ROTR(x, 22)) +#define SHA256_F2(x) (SHA2_ROTR(x, 6) ^ SHA2_ROTR(x, 11) ^ SHA2_ROTR(x, 25)) +#define SHA256_F3(x) (SHA2_ROTR(x, 7) ^ SHA2_ROTR(x, 18) ^ SHA2_SHFR(x, 3)) +#define SHA256_F4(x) (SHA2_ROTR(x, 17) ^ SHA2_ROTR(x, 19) ^ SHA2_SHFR(x, 10)) +#define SHA2_UNPACK32(x, str) \ +{ \ + *((str) + 3) = (uint8) ((x) ); \ + *((str) + 2) = (uint8) ((x) >> 8); \ + *((str) + 1) = (uint8) ((x) >> 16); \ + *((str) + 0) = (uint8) ((x) >> 24); \ +} +#define SHA2_PACK32(str, x) \ +{ \ + *(x) = ((uint32) *((str) + 3) ) \ + | ((uint32) *((str) + 2) << 8) \ + | ((uint32) *((str) + 1) << 16) \ + | ((uint32) *((str) + 0) << 24); \ +} + +// a small class for calculating MD5 hashes of strings or byte arrays +// it is not meant to be fast or secure +// +// usage: 1) feed it blocks of uchars with update() +// 2) finalize() +// 3) get hexdigest() string +// or +// MD5(std::string).hexdigest() +// +// assumes that char is 8 bit and int is 32 bit +class MD5 { + public: + typedef unsigned int size_type; // must be 32bit + + MD5(); + MD5(const std::string& text); + void update(const unsigned char *buf, size_type length); + void update(const char *buf, size_type length); + MD5& finalize(); + std::string hexdigest() const; + std::string rawdigest() const; + friend std::ostream& operator<<(std::ostream&, MD5 md5); + + private: + void init(); + typedef unsigned char uint1; // 8bit + typedef unsigned int uint4; // 32bit + enum {blocksize = 64}; // VC6 won't eat a const static int here + + void transform(const uint1 block[blocksize]); + static void decode(uint4 output[], const uint1 input[], size_type len); + static void encode(uint1 output[], const uint4 input[], size_type len); + + bool finalized; + uint1 buffer[blocksize]; // bytes that didn't fit in last 64 byte chunk + uint4 count[2]; // 64bit counter for number of bits (lo, hi) + uint4 state[4]; // digest so far + uint1 digest[16]; // the result + + // low level logic operations + static inline uint4 F(uint4 x, uint4 y, uint4 z); + static inline uint4 G(uint4 x, uint4 y, uint4 z); + static inline uint4 H(uint4 x, uint4 y, uint4 z); + static inline uint4 I(uint4 x, uint4 y, uint4 z); + static inline uint4 rotate_left(uint4 x, int n); + static inline void FF(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); + static inline void GG(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); + static inline void HH(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); + static inline void II(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac); +}; + +const unsigned int SHA256::sha256_k[64] = {//UL = uint32 + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 +}; + +void SHA256::transform(const unsigned char *message, unsigned int block_nb) { + uint32 w[64]; + uint32 wv[8]; + uint32 t1, t2; + const unsigned char *sub_block; + int i; + int j; + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 6); + for (j = 0; j < 16; j++) { + SHA2_PACK32(&sub_block[j << 2], &w[j]); + } + for (j = 16; j < 64; j++) { + w[j] = SHA256_F4(w[j - 2]) + w[j - 7] + SHA256_F3(w[j - 15]) + w[j - 16]; + } + for (j = 0; j < 8; j++) { + wv[j] = m_h[j]; + } + for (j = 0; j < 64; j++) { + t1 = wv[7] + SHA256_F2(wv[4]) + SHA2_CH(wv[4], wv[5], wv[6]) + + sha256_k[j] + w[j]; + t2 = SHA256_F1(wv[0]) + SHA2_MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + for (j = 0; j < 8; j++) { + m_h[j] += wv[j]; + } + } +} + +void SHA256::init() { + m_h[0] = 0x6a09e667; + m_h[1] = 0xbb67ae85; + m_h[2] = 0x3c6ef372; + m_h[3] = 0xa54ff53a; + m_h[4] = 0x510e527f; + m_h[5] = 0x9b05688c; + m_h[6] = 0x1f83d9ab; + m_h[7] = 0x5be0cd19; + m_len = 0; + m_tot_len = 0; +} + +void SHA256::update(const unsigned char *message, unsigned int len) { + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + tmp_len = SHA224_256_BLOCK_SIZE - m_len; + rem_len = len < tmp_len ? len : tmp_len; + memcpy(&m_block[m_len], message, rem_len); + if (m_len + len < SHA224_256_BLOCK_SIZE) { + m_len += len; + return; + } + new_len = len - rem_len; + block_nb = new_len / SHA224_256_BLOCK_SIZE; + shifted_message = message + rem_len; + transform(m_block, 1); + transform(shifted_message, block_nb); + rem_len = new_len % SHA224_256_BLOCK_SIZE; + memcpy(m_block, &shifted_message[block_nb << 6], rem_len); + m_len = rem_len; + m_tot_len += (block_nb + 1) << 6; +} + +void SHA256::final(unsigned char *digest) { + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + int i; + block_nb = (1 + ((SHA224_256_BLOCK_SIZE - 9) + < (m_len % SHA224_256_BLOCK_SIZE))); + len_b = (m_tot_len + m_len) << 3; + pm_len = block_nb << 6; + memset(m_block + m_len, 0, pm_len - m_len); + m_block[m_len] = 0x80; + SHA2_UNPACK32(len_b, m_block + pm_len - 4); + transform(m_block, block_nb); + for (i = 0 ; i < 8; i++) { + SHA2_UNPACK32(m_h[i], &digest[i << 2]); + } +} + +std::string sha256(const std::string &input, bool raw) { + unsigned char digest[SHA256::DIGEST_SIZE]; + memset(digest,0,SHA256::DIGEST_SIZE); + + SHA256 ctx = SHA256(); + ctx.init(); + ctx.update( (unsigned char*)input.c_str(), input.length()); + ctx.final(digest); + + if (raw) { + std::string res; + for (unsigned int i = 0; i < SHA256::DIGEST_SIZE; ++i) { + res.append(1, digest[i]); + } + return res; + } + char buf[2*SHA256::DIGEST_SIZE+1]; + buf[2*SHA256::DIGEST_SIZE] = 0; + for (size_t i = 0; i < SHA256::DIGEST_SIZE; i++) + sprintf(buf+i*2, "%02x", digest[i]); + return std::string(buf); +} + +// MD5 hash function + +// Constants for MD5Transform routine. +#define S11 7 +#define S12 12 +#define S13 17 +#define S14 22 +#define S21 5 +#define S22 9 +#define S23 14 +#define S24 20 +#define S31 4 +#define S32 11 +#define S33 16 +#define S34 23 +#define S41 6 +#define S42 10 +#define S43 15 +#define S44 21 + +/////////////////////////////////////////////// + +// F, G, H and I are basic MD5 functions. +inline MD5::uint4 MD5::F(uint4 x, uint4 y, uint4 z) { + return (x&y) | (~x&z); +} + +inline MD5::uint4 MD5::G(uint4 x, uint4 y, uint4 z) { + return (x&z) | (y&~z); +} + +inline MD5::uint4 MD5::H(uint4 x, uint4 y, uint4 z) { + return x^y^z; +} + +inline MD5::uint4 MD5::I(uint4 x, uint4 y, uint4 z) { + return y ^ (x | ~z); +} + +// rotate_left rotates x left n bits. +inline MD5::uint4 MD5::rotate_left(uint4 x, int n) { + return (x << n) | (x >> (32-n)); +} + +// FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4. +// Rotation is separate from addition to prevent recomputation. +inline void MD5::FF(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { + a = rotate_left(a+ F(b,c,d) + x + ac, s) + b; +} + +inline void MD5::GG(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { + a = rotate_left(a + G(b,c,d) + x + ac, s) + b; +} + +inline void MD5::HH(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { + a = rotate_left(a + H(b,c,d) + x + ac, s) + b; +} + +inline void MD5::II(uint4 &a, uint4 b, uint4 c, uint4 d, uint4 x, uint4 s, uint4 ac) { + a = rotate_left(a + I(b,c,d) + x + ac, s) + b; +} + +////////////////////////////////////////////// + +// default ctor, just initailize +MD5::MD5() { + init(); +} + +////////////////////////////////////////////// + +// nifty shortcut ctor, compute MD5 for string and finalize it right away +MD5::MD5(const std::string &text) { + init(); + update(text.c_str(), text.length()); + finalize(); +} + +////////////////////////////// + +void MD5::init() { + finalized=false; + + count[0] = 0; + count[1] = 0; + + // load magic initialization constants. + state[0] = 0x67452301; + state[1] = 0xefcdab89; + state[2] = 0x98badcfe; + state[3] = 0x10325476; +} + +////////////////////////////// + +// decodes input (unsigned char) into output (uint4). Assumes len is a multiple of 4. +void MD5::decode(uint4 output[], const uint1 input[], size_type len) { + for (unsigned int i = 0, j = 0; j < len; i++, j += 4) + output[i] = ((uint4)input[j]) | (((uint4)input[j+1]) << 8) | + (((uint4)input[j+2]) << 16) | (((uint4)input[j+3]) << 24); +} + +////////////////////////////// + +// encodes input (uint4) into output (unsigned char). Assumes len is +// a multiple of 4. +void MD5::encode(uint1 output[], const uint4 input[], size_type len) { + for (size_type i = 0, j = 0; j < len; i++, j += 4) { + output[j] = input[i] & 0xff; + output[j+1] = (input[i] >> 8) & 0xff; + output[j+2] = (input[i] >> 16) & 0xff; + output[j+3] = (input[i] >> 24) & 0xff; + } +} + +////////////////////////////// + +// apply MD5 algo on a block +void MD5::transform(const uint1 block[blocksize]) { + uint4 a = state[0], b = state[1], c = state[2], d = state[3], x[16]; + decode (x, block, blocksize); + + /* Round 1 */ + FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */ + FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */ + FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */ + FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */ + FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */ + FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */ + FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */ + FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */ + FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */ + FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */ + FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */ + FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */ + FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */ + FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */ + FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */ + FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */ + + /* Round 2 */ + GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */ + GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */ + GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */ + GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */ + GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */ + GG (d, a, b, c, x[10], S22, 0x2441453); /* 22 */ + GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */ + GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */ + GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */ + GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */ + GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */ + GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */ + GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */ + GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */ + GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */ + GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */ + + /* Round 3 */ + HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */ + HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */ + HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */ + HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */ + HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */ + HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */ + HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */ + HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */ + HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */ + HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */ + HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */ + HH (b, c, d, a, x[ 6], S34, 0x4881d05); /* 44 */ + HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */ + HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */ + HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */ + HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */ + + /* Round 4 */ + II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */ + II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */ + II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */ + II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */ + II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */ + II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */ + II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */ + II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */ + II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */ + II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */ + II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */ + II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */ + II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */ + II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */ + II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */ + II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */ + + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + + // Zeroize sensitive information. + memset(x, 0, sizeof x); +} + +////////////////////////////// + +// MD5 block update operation. Continues an MD5 message-digest +// operation, processing another message block +void MD5::update(const unsigned char input[], size_type length) { + // compute number of bytes mod 64 + size_type index = count[0] / 8 % blocksize; + + // Update number of bits + if ((count[0] += (length << 3)) < (length << 3)) + count[1]++; + count[1] += (length >> 29); + + // number of bytes we need to fill in buffer + size_type firstpart = 64 - index; + + size_type i; + + // transform as many times as possible. + if (length >= firstpart) + { + // fill buffer first, transform + memcpy(&buffer[index], input, firstpart); + transform(buffer); + + // transform chunks of blocksize (64 bytes) + for (i = firstpart; i + blocksize <= length; i += blocksize) + transform(&input[i]); + + index = 0; + } + else + i = 0; + + // buffer remaining input + memcpy(&buffer[index], &input[i], length-i); +} + +////////////////////////////// + +// for convenience provide a verson with signed char +void MD5::update(const char input[], size_type length) { + update((const unsigned char*)input, length); +} + +////////////////////////////// + +// MD5 finalization. Ends an MD5 message-digest operation, writing the +// the message digest and zeroizing the context. +MD5& MD5::finalize() { + static unsigned char padding[64] = { + 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + + if (!finalized) { + // Save number of bits + unsigned char bits[8]; + encode(bits, count, 8); + + // pad out to 56 mod 64. + size_type index = count[0] / 8 % 64; + size_type padLen = (index < 56) ? (56 - index) : (120 - index); + update(padding, padLen); + + // Append length (before padding) + update(bits, 8); + + // Store state in digest + encode(digest, state, 16); + + // Zeroize sensitive information. + memset(buffer, 0, sizeof buffer); + memset(count, 0, sizeof count); + + finalized=true; + } + + return *this; +} + +////////////////////////////// + +// return hex representation of digest as string +std::string MD5::hexdigest() const { + if (!finalized) + return ""; + + char buf[33]; + for (int i=0; i<16; i++) + sprintf(buf+i*2, "%02x", digest[i]); + buf[32]=0; + + return std::string(buf); +} + +std::string MD5::rawdigest() const { + if (!finalized) + return ""; + std::string res; + for (unsigned int i = 0; i < 16; ++i) { + res.append(1, digest[i]); + } + return res; +} + +////////////////////////////// + +std::ostream& operator<<(std::ostream& out, MD5 md5) { + return out << md5.hexdigest(); +} + +////////////////////////////// + +std::string md5(const std::string &str, bool raw) { + MD5 md5 = MD5(str); + + if (raw) { + return md5.rawdigest(); + } + return md5.hexdigest(); +} + +} // namespace slash diff --git a/pink/third/slash/slash/src/slash_mutex.cc b/pink/third/slash/slash/src/slash_mutex.cc new file mode 100644 index 00000000..6e85e947 --- /dev/null +++ b/pink/third/slash/slash/src/slash_mutex.cc @@ -0,0 +1,190 @@ +#include "slash/include/slash_mutex.h" + +#include +#include +#include +#include + +#include + +namespace slash { + +static void PthreadCall(const char* label, int result) { + if (result != 0) { + fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); + abort(); + } +} + +// Return false if timeout +static bool PthreadTimeoutCall(const char* label, int result) { + if (result != 0) { + if (result == ETIMEDOUT) { + return false; + } + fprintf(stderr, "pthread %s: %s\n", label, strerror(result)); + abort(); + } + return true; +} + +Mutex::Mutex() { + PthreadCall("init mutex", pthread_mutex_init(&mu_, NULL)); +} + +Mutex::~Mutex() { + PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); +} + +void Mutex::Lock() { + PthreadCall("lock", pthread_mutex_lock(&mu_)); +} + +void Mutex::Unlock() { + PthreadCall("unlock", pthread_mutex_unlock(&mu_)); +} + +RWMutex::RWMutex() { + PthreadCall("init rw mutex", pthread_rwlock_init(&rw_mu_, NULL)); +} + +RWMutex::~RWMutex() { + PthreadCall("destroy rw mutex", pthread_rwlock_destroy(&rw_mu_)); +} + +void RWMutex::ReadLock() { + PthreadCall("rw readlock", pthread_rwlock_rdlock(&rw_mu_)); +} + +void RWMutex::WriteLock() { + PthreadCall("rw writelock", pthread_rwlock_wrlock(&rw_mu_)); +} + +void RWMutex::WriteUnlock() { + PthreadCall("rw write unlock", pthread_rwlock_unlock(&rw_mu_)); +} + +void RWMutex::ReadUnlock() { + PthreadCall("rw read unlock", pthread_rwlock_unlock(&rw_mu_)); +} + +CondVar::CondVar(Mutex* mu) + : mu_(mu) { + PthreadCall("init cv", pthread_cond_init(&cv_, NULL)); + } + +CondVar::~CondVar() { + PthreadCall("destroy cv", pthread_cond_destroy(&cv_)); +} + +void CondVar::Wait() { + PthreadCall("wait", pthread_cond_wait(&cv_, &mu_->mu_)); +} + +// return false if timeout +bool CondVar::TimedWait(uint32_t timeout) { + /* + * pthread_cond_timedwait api use absolute API + * so we need gettimeofday + timeout + */ + struct timeval now; + gettimeofday(&now, NULL); + struct timespec tsp; + + int64_t usec = now.tv_usec + timeout * 1000LL; + tsp.tv_sec = now.tv_sec + usec / 1000000; + tsp.tv_nsec = (usec % 1000000) * 1000; + + return PthreadTimeoutCall("timewait", + pthread_cond_timedwait(&cv_, &mu_->mu_, &tsp)); +} + +void CondVar::Signal() { + PthreadCall("signal", pthread_cond_signal(&cv_)); +} + +void CondVar::SignalAll() { + PthreadCall("broadcast", pthread_cond_broadcast(&cv_)); +} + +void InitOnce(OnceType* once, void (*initializer)()) { + PthreadCall("once", pthread_once(once, initializer)); +} + +RefMutex::RefMutex() { + refs_ = 0; + PthreadCall("init mutex", pthread_mutex_init(&mu_, nullptr)); +} + +RefMutex::~RefMutex() { + PthreadCall("destroy mutex", pthread_mutex_destroy(&mu_)); +} + +void RefMutex::Ref() { + refs_++; +} +void RefMutex::Unref() { + --refs_; + if (refs_ == 0) { + delete this; + } +} + +void RefMutex::Lock() { + PthreadCall("lock", pthread_mutex_lock(&mu_)); +} + +void RefMutex::Unlock() { + PthreadCall("unlock", pthread_mutex_unlock(&mu_)); +} + +RecordMutex::~RecordMutex() { + mutex_.Lock(); + + std::unordered_map::const_iterator it = records_.begin(); + for (; it != records_.end(); it++) { + delete it->second; + } + mutex_.Unlock(); +} + +void RecordMutex::Lock(const std::string &key) { + mutex_.Lock(); + std::unordered_map::const_iterator it = records_.find(key); + + if (it != records_.end()) { + RefMutex *ref_mutex = it->second; + ref_mutex->Ref(); + mutex_.Unlock(); + + ref_mutex->Lock(); + } else { + RefMutex *ref_mutex = new RefMutex(); + + records_.insert(std::make_pair(key, ref_mutex)); + ref_mutex->Ref(); + mutex_.Unlock(); + + ref_mutex->Lock(); + } +} + +void RecordMutex::Unlock(const std::string &key) { + mutex_.Lock(); + std::unordered_map::const_iterator it = records_.find(key); + + if (it != records_.end()) { + RefMutex *ref_mutex = it->second; + + if (ref_mutex->IsLastRef()) { + records_.erase(it); + } + ref_mutex->Unlock(); + ref_mutex->Unref(); + } + + mutex_.Unlock(); +} + +} + diff --git a/pink/third/slash/slash/src/slash_status.cc b/pink/third/slash/slash/src/slash_status.cc new file mode 100644 index 00000000..0f1d8c8c --- /dev/null +++ b/pink/third/slash/slash/src/slash_status.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include +#include "slash/include/slash_status.h" + +namespace slash { + +const char* Status::CopyState(const char* state) { + uint32_t size; + memcpy(&size, state, sizeof(size)); + char* result = new char[size + 5]; + memcpy(result, state, size + 5); + return result; +} + +Status::Status(Code code, const Slice& msg, const Slice& msg2) { + assert(code != kOk); + const uint32_t len1 = static_cast(msg.size()); + const uint32_t len2 = static_cast(msg2.size()); + const uint32_t size = len1 + (len2 ? (2 + len2) : 0); + char* result = new char[size + 5]; + memcpy(result, &size, sizeof(size)); + result[4] = static_cast(code); + memcpy(result + 5, msg.data(), len1); + if (len2) { + result[5 + len1] = ':'; + result[6 + len1] = ' '; + memcpy(result + 7 + len1, msg2.data(), len2); + } + state_ = result; +} + +std::string Status::ToString() const { + if (state_ == NULL) { + return "OK"; + } else { + char tmp[30]; + const char* type; + switch (code()) { + case kOk: + type = "OK"; + break; + case kNotFound: + type = "NotFound: "; + break; + case kCorruption: + type = "Corruption: "; + break; + case kNotSupported: + type = "Not implemented: "; + break; + case kInvalidArgument: + type = "Invalid argument: "; + break; + case kIOError: + type = "IO error: "; + break; + case kEndFile: + type = "End file: "; + break; + case kIncomplete: + type = "InComplete: "; + break; + case kComplete: + type = "Complete: "; + break; + case kTimeout: + type = "Timeout: "; + break; + case kAuthFailed: + type = "AuthFailed: "; + break; + case kBusy: + type = "Busy:"; + default: + snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", + static_cast(code())); + type = tmp; + break; + } + std::string result(type); + uint32_t length; + memcpy(&length, state_, sizeof(length)); + result.append(state_ + 5, length); + return result; + } +} + + +} // namespace slash diff --git a/pink/third/slash/slash/src/slash_string.cc b/pink/third/slash/slash/src/slash_string.cc new file mode 100644 index 00000000..cd3014ca --- /dev/null +++ b/pink/third/slash/slash/src/slash_string.cc @@ -0,0 +1,667 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +/* + * Copyright (c) 2009-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +#include "slash/include/slash_string.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "slash/include/fmacros.h" + +namespace slash { + +/* Glob-style pattern matching. */ +int stringmatchlen(const char *pattern, int patternLen, + const char *string, int stringLen, int nocase) +{ + while(patternLen) { + switch(pattern[0]) { + case '*': + while (pattern[1] == '*') { + pattern++; + patternLen--; + } + if (patternLen == 1) + return 1; /* match */ + while(stringLen) { + if (stringmatchlen(pattern+1, patternLen-1, + string, stringLen, nocase)) + return 1; /* match */ + string++; + stringLen--; + } + return 0; /* no match */ + break; + case '?': + if (stringLen == 0) + return 0; /* no match */ + string++; + stringLen--; + break; + case '[': + { + int nott; + int match; + + pattern++; + patternLen--; + nott = pattern[0] == '^'; + if (nott) { + pattern++; + patternLen--; + } + match = 0; + while(1) { + if (pattern[0] == '\\') { + pattern++; + patternLen--; + if (pattern[0] == string[0]) + match = 1; + } else if (pattern[0] == ']') { + break; + } else if (patternLen == 0) { + pattern--; + patternLen++; + break; + } else if (pattern[1] == '-' && patternLen >= 3) { + int start = pattern[0]; + int end = pattern[2]; + int c = string[0]; + if (start > end) { + int t = start; + start = end; + end = t; + } + if (nocase) { + start = tolower(start); + end = tolower(end); + c = tolower(c); + } + pattern += 2; + patternLen -= 2; + if (c >= start && c <= end) + match = 1; + } else { + if (!nocase) { + if (pattern[0] == string[0]) + match = 1; + } else { + if (tolower((int)pattern[0]) == tolower((int)string[0])) + match = 1; + } + } + pattern++; + patternLen--; + } + if (nott) + match = !match; + if (!match) + return 0; /* no match */ + string++; + stringLen--; + break; + } + case '\\': + if (patternLen >= 2) { + pattern++; + patternLen--; + } + /* fall through */ + default: + if (!nocase) { + if (pattern[0] != string[0]) + return 0; /* no match */ + } else { + if (tolower((int)pattern[0]) != tolower((int)string[0])) + return 0; /* no match */ + } + string++; + stringLen--; + break; + } + pattern++; + patternLen--; + if (stringLen == 0) { + while(*pattern == '*') { + pattern++; + patternLen--; + } + break; + } + } + if (patternLen == 0 && stringLen == 0) + return 1; + return 0; +} + +int stringmatch(const char *pattern, const char *string, int nocase) { + return stringmatchlen(pattern,strlen(pattern),string,strlen(string),nocase); +} + +/* Convert a string representing an amount of memory into the number of + * bytes, so for instance memtoll("1Gi") will return 1073741824 that is + * (1024*1024*1024). + * + * On parsing error, if *err is not NULL, it's set to 1, otherwise it's + * set to 0 */ +long long memtoll(const char *p, int *err) { + const char *u; + char buf[128]; + long mul; /* unit multiplier */ + long long val; + unsigned int digits; + + if (err) *err = 0; + /* Search the first non digit character. */ + u = p; + if (*u == '-') u++; + while(*u && isdigit(*u)) u++; + if (*u == '\0' || !strcasecmp(u,"b")) { + mul = 1; + } else if (!strcasecmp(u,"k")) { + mul = 1000; + } else if (!strcasecmp(u,"kb")) { + mul = 1024; + } else if (!strcasecmp(u,"m")) { + mul = 1000*1000; + } else if (!strcasecmp(u,"mb")) { + mul = 1024*1024; + } else if (!strcasecmp(u,"g")) { + mul = 1000L*1000*1000; + } else if (!strcasecmp(u,"gb")) { + mul = 1024L*1024*1024; + } else { + if (err) *err = 1; + mul = 1; + } + digits = u-p; + if (digits >= sizeof(buf)) { + if (err) *err = 1; + return LLONG_MAX; + } + memcpy(buf,p,digits); + buf[digits] = '\0'; + val = strtoll(buf,NULL,10); + return val*mul; +} + +/* Return the number of digits of 'v' when converted to string in radix 10. + * See ll2string() for more information. */ +uint32_t digits10(uint64_t v) { + if (v < 10) return 1; + if (v < 100) return 2; + if (v < 1000) return 3; + if (v < 1000000000000UL) { + if (v < 100000000UL) { + if (v < 1000000) { + if (v < 10000) return 4; + return 5 + (v >= 100000); + } + return 7 + (v >= 10000000UL); + } + if (v < 10000000000UL) { + return 9 + (v >= 1000000000UL); + } + return 11 + (v >= 100000000000UL); + } + return 12 + digits10(v / 1000000000000UL); +} + +/* Convert a long long into a string. Returns the number of + * characters needed to represent the number. + * If the buffer is not big enough to store the string, 0 is returned. + * + * Based on the following article (that apparently does not provide a + * novel approach but only publicizes an already used technique): + * + * https://www.facebook.com/notes/facebook-engineering/three-optimization-tips-for-c/10151361643253920 + * + * Modified in order to handle signed integers since the original code was + * designed for unsigned integers. */ +int ll2string(char* dst, size_t dstlen, long long svalue) { + static const char digits[201] = + "0001020304050607080910111213141516171819" + "2021222324252627282930313233343536373839" + "4041424344454647484950515253545556575859" + "6061626364656667686970717273747576777879" + "8081828384858687888990919293949596979899"; + int negative; + unsigned long long value; + + /* The main loop works with 64bit unsigned integers for simplicity, so + * we convert the number here and remember if it is negative. */ + if (svalue < 0) { + if (svalue != LLONG_MIN) { + value = -svalue; + } else { + value = ((unsigned long long) LLONG_MAX)+1; + } + negative = 1; + } else { + value = svalue; + negative = 0; + } + + /* Check length. */ + uint32_t const length = digits10(value)+negative; + if (length >= dstlen) return 0; + + /* Null term. */ + uint32_t next = length; + dst[next] = '\0'; + next--; + while (value >= 100) { + int const i = (value % 100) * 2; + value /= 100; + dst[next] = digits[i + 1]; + dst[next - 1] = digits[i]; + next -= 2; + } + + /* Handle last 1-2 digits. */ + if (value < 10) { + dst[next] = '0' + (uint32_t) value; + } else { + int i = (uint32_t) value * 2; + dst[next] = digits[i + 1]; + dst[next - 1] = digits[i]; + } + + /* Add sign. */ + if (negative) dst[0] = '-'; + return length; +} + +/* Convert a string into a long long. Returns 1 if the string could be parsed + * into a (non-overflowing) long long, 0 otherwise. The value will be set to + * the parsed value when appropriate. */ +int string2ll(const char *s, size_t slen, long long *value) { + const char *p = s; + size_t plen = 0; + int negative = 0; + unsigned long long v; + + if (plen == slen) + return 0; + + /* Special case: first and only digit is 0. */ + if (slen == 1 && p[0] == '0') { + if (value != NULL) *value = 0; + return 1; + } + + if (p[0] == '-') { + negative = 1; + p++; plen++; + + /* Abort on only a negative sign. */ + if (plen == slen) + return 0; + } + + while(plen < slen && p[0] == '0') { + p++; plen++; + } + + if (plen == slen) { + if (value != NULL) *value = 0; + return 1; + } + + /* First digit should be 1-9, otherwise the string should just be 0. */ + if (p[0] >= '1' && p[0] <= '9') { + v = p[0]-'0'; + p++; plen++; + } else if (p[0] == '0' && slen == 1) { + *value = 0; + return 1; + } else { + return 0; + } + + while (plen < slen && p[0] >= '0' && p[0] <= '9') { + if (v > (ULLONG_MAX / 10)) /* Overflow. */ + return 0; + v *= 10; + + if (v > (ULLONG_MAX - (p[0]-'0'))) /* Overflow. */ + return 0; + v += p[0]-'0'; + + p++; plen++; + } + + /* Return if not all bytes were used. */ + if (plen < slen) + return 0; + + if (negative) { + if (v > ((unsigned long long)(-(LLONG_MIN+1))+1)) /* Overflow. */ + return 0; + if (value != NULL) *value = -v; + } else { + if (v > LLONG_MAX) /* Overflow. */ + return 0; + if (value != NULL) *value = v; + } + return 1; +} + +/* Convert a string into a long. Returns 1 if the string could be parsed into a + * (non-overflowing) long, 0 otherwise. The value will be set to the parsed + * value when appropriate. */ +int string2l(const char *s, size_t slen, long *lval) { + long long llval; + + if (!string2ll(s,slen,&llval)) + return 0; + + if (llval < LONG_MIN || llval > LONG_MAX) + return 0; + + *lval = (long)llval; + return 1; +} + +/* Convert a string into a unsigned long. Returns 1 if the string could be parsed into a + * (non-overflowing) unsigned long, 0 otherwise. The value will be set to the parsed + * value when appropriate. */ +int string2ul(const char *s, size_t slen, unsigned long *lval) { + long long llval; + + if (!string2ll(s,slen,&llval)) + return 0; + + if (llval > ULONG_MAX) + return 0; + + *lval = (unsigned long)llval; + return 1; +} + +/* Convert a double to a string representation. Returns the number of bytes + * required. The representation should always be parsable by strtod(3). */ +int d2string(char *buf, size_t len, double value) { + if (std::isnan(value)) { + len = snprintf(buf,len,"nan"); + } else if (std::isinf(value)) { + if (value < 0) + len = snprintf(buf,len,"-inf"); + else + len = snprintf(buf,len,"inf"); + } else if (value == 0) { + /* See: http://en.wikipedia.org/wiki/Signed_zero, "Comparisons". */ + if (1.0/value < 0) + len = snprintf(buf,len,"-0"); + else + len = snprintf(buf,len,"0"); + } else { +#if (DBL_MANT_DIG >= 52) && (LLONG_MAX == 0x7fffffffffffffffLL) + /* Check if the float is in a safe range to be casted into a + * long long. We are assuming that long long is 64 bit here. + * Also we are assuming that there are no implementations around where + * double has precision < 52 bit. + * + * Under this assumptions we test if a double is inside an interval + * where casting to long long is safe. Then using two castings we + * make sure the decimal part is zero. If all this is true we use + * integer printing function that is much faster. */ + double min = -4503599627370495; /* (2^52)-1 */ + double max = 4503599627370496; /* -(2^52) */ + if (value > min && value < max && value == ((double)((long long)value))) + len = ll2string(buf,len,(long long)value); + else +#endif + len = snprintf(buf,len,"%.17g",value); + } + + return len; +} + +int string2d(const char *s, size_t slen, double *dval) { + char *pEnd; + double d = strtod(s, &pEnd); + if (pEnd != s + slen) + return 0; + + if (dval != NULL) *dval = d; + return 1; +} + +/* Generate the Redis "Run ID", a SHA1-sized random number that identifies a + * given execution of Redis, so that if you are talking with an instance + * having run_id == A, and you reconnect and it has run_id == B, you can be + * sure that it is either a different instance or it was restarted. */ +void getRandomHexChars(char *p, unsigned int len) { + FILE *fp = fopen("/dev/urandom","r"); + char charset[] = "0123456789abcdef"; + unsigned int j; + + if (fp == NULL || fread(p,len,1,fp) == 0) { + /* If we can't read from /dev/urandom, do some reasonable effort + * in order to create some entropy, since this function is used to + * generate run_id and cluster instance IDs */ + char *x = p; + unsigned int l = len; + struct timeval tv; + pid_t pid = getpid(); + + /* Use time and PID to fill the initial array. */ + gettimeofday(&tv,NULL); + if (l >= sizeof(tv.tv_usec)) { + memcpy(x,&tv.tv_usec,sizeof(tv.tv_usec)); + l -= sizeof(tv.tv_usec); + x += sizeof(tv.tv_usec); + } + if (l >= sizeof(tv.tv_sec)) { + memcpy(x,&tv.tv_sec,sizeof(tv.tv_sec)); + l -= sizeof(tv.tv_sec); + x += sizeof(tv.tv_sec); + } + if (l >= sizeof(pid)) { + memcpy(x,&pid,sizeof(pid)); + l -= sizeof(pid); + x += sizeof(pid); + } + /* Finally xor it with rand() output, that was already seeded with + * time() at startup. */ + for (j = 0; j < len; j++) + p[j] ^= rand(); + } + /* Turn it into hex digits taking just 4 bits out of 8 for every byte. */ + for (j = 0; j < len; j++) + p[j] = charset[p[j] & 0x0F]; + if (fp) fclose(fp); +} + +std::vector& StringSplit(const std::string &s, + char delim, std::vector &elems) { + elems.clear(); + std::stringstream ss(s); + std::string item; + while (std::getline(ss, item, delim)) { + if (!item.empty()) + elems.push_back(item); + } + return elems; +} + +std::string StringConcat(const std::vector &elems, char delim) { + std::string result; + std::vector::const_iterator it = elems.begin(); + while (it != elems.end()) { + result.append(*it); + result.append(1, delim); + ++it; + } + if (!result.empty()) { + result.resize(result.size() - 1); + } + return result; +} + +std::string& StringToLower(std::string& ori) { + std::transform(ori.begin(), ori.end(), ori.begin(), ::tolower); + return ori; +} + +std::string& StringToUpper(std::string& ori) { + std::transform(ori.begin(), ori.end(), ori.begin(), ::toupper); + return ori; +} + +std::string IpPortString(const std::string& ip, int port) { + if (ip.empty()) { + return std::string(); + } + char buf[10]; + if (ll2string(buf, sizeof(buf), port) <= 0) { + return std::string(); + } + return (ip + ":" + buf); +} + +std::string ToRead(const std::string& str) { + std::string read; + if (str.empty()) { + return read; + } + read.append(1, '"'); + char buf[16]; + std::string::const_iterator iter = str.begin(); + while (iter != str.end()) { + switch (*iter) { + case '\\' : + case '"': + read.append(1, '\\'); + read.append(1, *iter); + break; + case '\n': + read.append("\\n"); + break; + case '\r': + read.append("\\r"); + break; + case '\t': + read.append("\\t"); + break; + case '\a': + read.append("\\a"); + break; + case '\b': + read.append("\\b"); + break; + default: + if (isprint(*iter)) { + read.append(1, *iter); + } else { + snprintf(buf, sizeof(buf), "\\x%02x", static_cast(*iter)); + read.append(buf); + } + break; + } + iter++; + } + read.append(1, '"'); + return read; +} + +bool ParseIpPortString(const std::string& ip_port, std::string& ip, int &port) { + if (ip_port.empty()) { + return false; + } + size_t pos = ip_port.find(':'); + if (pos == std::string::npos) { + return false; + } + ip = ip_port.substr(0, pos); + std::string port_str = ip_port.substr(pos + 1); + long lport = 0; + if (1 != string2l(port_str.data(), port_str.size(), &lport)) { + return false; + } + port = (int)lport; + return true; +} + +// Trim charlist +std::string StringTrim(const std::string& ori, const std::string& charlist) { + if (ori.empty()) + return ori; + + size_t pos = 0; + int rpos = ori.size() - 1; + while (pos < ori.size()) { + bool meet = false; + for (char c : charlist) + if (ori.at(pos) == c) { + meet = true; + break; + } + if (!meet) break; + ++pos; + } + while (rpos >= 0) { + bool meet = false; + for (char c : charlist) + if (ori.at(rpos) == c) { + meet = true; + break; + } + if (!meet) break; + --rpos; + } + return ori.substr(pos, rpos - pos + 1); +} + +} // namespace slash diff --git a/pink/third/slash/slash/src/slash_testharness.cc b/pink/third/slash/slash/src/slash_testharness.cc new file mode 100644 index 00000000..0dd42d7d --- /dev/null +++ b/pink/third/slash/slash/src/slash_testharness.cc @@ -0,0 +1,70 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. +#include "slash/include/slash_testharness.h" + +#include +#include +#include +#include + +#include +#include + +#include "slash/include/env.h" + +namespace slash { +namespace test { + +namespace { +struct Test { + const char* base; + const char* name; + void (*func)(); +}; +std::vector* tests; +} + +bool RegisterTest(const char* base, const char* name, void (*func)()) { + if (tests == NULL) { + tests = new std::vector; + } + Test t; + t.base = base; + t.name = name; + t.func = func; + tests->push_back(t); + return true; +} + +int RunAllTests() { + const char* matcher = getenv("SLASH_TESTS"); + + int num = 0; + if (tests != NULL) { + for (size_t i = 0; i < tests->size(); i++) { + const Test& t = (*tests)[i]; + if (matcher != NULL) { + std::string name = t.base; + name.push_back('.'); + name.append(t.name); + if (strstr(name.c_str(), matcher) == NULL) { + continue; + } + } + fprintf(stderr, "==== Test %s.%s\n", t.base, t.name); + (*t.func)(); + ++num; + } + } + fprintf(stderr, "==== PASSED %d tests\n", num); + return 0; +} + +} // namespace test +} // namespace slash diff --git a/pink/third/slash/slash/src/testutil.cc b/pink/third/slash/slash/src/testutil.cc new file mode 100644 index 00000000..ac44cd6e --- /dev/null +++ b/pink/third/slash/slash/src/testutil.cc @@ -0,0 +1,39 @@ +#include "slash/include/testutil.h" + +#include + +#include "slash/include/env.h" +#include "slash/include/random.h" + +namespace slash { + +std::string RandomString(const int len) { + char buf[len]; + for (int i = 0; i < len; i++) { + buf[i] = Random::Uniform('z' - 'a') + 'a'; + } + return std::string(buf, len); +} + +int GetTestDirectory(std::string *result) { + const char* env = getenv("TEST_TMPDIR"); + if (env && env[0] != '\0') { + *result = env; + } else { + char buf[100]; + snprintf(buf, sizeof(buf), "/tmp/slashtest-%d", int(geteuid())); + *result = buf; + } + return 0; +} + +int RandomSeed() { + const char* env = getenv("TEST_RANDOM_SEED"); + int result = (env != NULL ? atoi(env) : 301); + if (result <= 0) { + result = 301; + } + return result; +} + +} // namespace slash diff --git a/pink/third/slash/slash/tests/base_conf_test.cc b/pink/third/slash/slash/tests/base_conf_test.cc new file mode 100644 index 00000000..42db627c --- /dev/null +++ b/pink/third/slash/slash/tests/base_conf_test.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +#include +#include + +#include "slash/include/env.h" +#include "slash/include/base_conf.h" +#include "slash/include/testutil.h" +#include "slash/include/slash_testharness.h" + + +namespace slash { + +class BaseConfTest { + public: + BaseConfTest() { + GetTestDirectory(&tmpdir_); + DeleteDirIfExist(tmpdir_); + CreateDir(tmpdir_); + test_conf_ = tmpdir_ + "/test.conf"; + } + + Status CreateSampleConf() { + std::vector sample_conf = { + "test_int : 1\n", + "test_str : abkxk\n", + "test_vec : four, five, six\n", + "test_bool : yes\n", + }; + + WritableFile *write_file; + Status ret = NewWritableFile(test_conf_, &write_file); + if (!ret.ok()) + return ret; + for (std::string &item : sample_conf) { + write_file->Append(item); + } + delete write_file; + + return Status::OK(); + } + + protected: + std::string tmpdir_; + std::string test_conf_; +}; + +TEST(BaseConfTest, WriteReadConf) { + ASSERT_OK(CreateSampleConf()); + BaseConf *conf = new BaseConf(test_conf_); + ASSERT_EQ(conf->LoadConf(), 0); + + // Write configuration + ASSERT_TRUE(conf->SetConfInt("test_int", 1345)); + ASSERT_TRUE(conf->SetConfStr("test_str", "kdkbixk")); + ASSERT_TRUE(conf->SetConfStr("test_vec", "one, two, three")); + ASSERT_TRUE(conf->SetConfBool("test_bool", false)); + // Cover test + ASSERT_TRUE(conf->SetConfInt("test_int", 13985)); + ASSERT_TRUE(conf->WriteBack()); + + // Read configuration + int test_int; + std::string test_str; + bool test_bool; + std::vector values; + ASSERT_TRUE(conf->GetConfInt("test_int", &test_int)); + ASSERT_EQ(test_int, 13985); + ASSERT_TRUE(conf->GetConfStr("test_str", &test_str)); + ASSERT_EQ(test_str, "kdkbixk"); + ASSERT_TRUE(conf->GetConfBool("test_bool", &test_bool)); + ASSERT_EQ(test_bool, false); + ASSERT_TRUE(conf->GetConfStrVec("test_vec", &values)); + ASSERT_EQ(values[0], "one"); + ASSERT_EQ(values[1], "two"); + ASSERT_EQ(values[2], "three"); +} + +} // namespace slash diff --git a/pink/third/slash/slash/tests/slash_binlog_test.cc b/pink/third/slash/slash/tests/slash_binlog_test.cc new file mode 100644 index 00000000..b4d9a9c7 --- /dev/null +++ b/pink/third/slash/slash/tests/slash_binlog_test.cc @@ -0,0 +1,94 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. +#include + +#include "slash/include/env.h" +#include "slash/include/testutil.h" +#include "slash/include/slash_testharness.h" +#include "slash/include/slash_binlog.h" +#include "slash/src/slash_binlog_impl.h" + +namespace slash { + +class BinlogTest { + public: + BinlogTest() + : log_(NULL), + reader_(NULL) { + GetTestDirectory(&tmpdir_); + DeleteDirIfExist(tmpdir_); + ASSERT_OK(Binlog::Open(tmpdir_, &log_)); + } + ~BinlogTest() { + delete reader_; + delete log_; + DeleteDirIfExist(tmpdir_); + } + protected: + Binlog *log_; + BinlogReader *reader_; + const std::string test_item_ = "slash_item"; + std::string tmpdir_; +}; + +TEST(BinlogTest, ReadWrite) { + std::string item1 = test_item_ + "1"; + std::string item2 = test_item_ + "2"; + std::string item3 = test_item_ + "3"; + // Write + ASSERT_OK(log_->Append(item1)); + ASSERT_OK(log_->Append(item2)); + ASSERT_OK(log_->Append(item3)); + + // Read + std::string item; + reader_ = log_->NewBinlogReader(0, 0); + ASSERT_TRUE(reader_); + ASSERT_OK(reader_->ReadRecord(item)); + ASSERT_EQ(item, item1); + ASSERT_OK(reader_->ReadRecord(item)); + ASSERT_EQ(item, item2); + ASSERT_OK(reader_->ReadRecord(item)); + ASSERT_EQ(item, item3); +} + +TEST(BinlogTest, OffsetTest) { + std::string second_block_item = "sbi"; + uint64_t offset = 0; + uint32_t filenum = 0; + uint64_t pro_offset = 0; + int i = 0; + + while (true) { + std::string curitem = test_item_ + std::to_string(++i); + log_->Append(curitem); + log_->GetProducerStatus(&filenum, &pro_offset); + offset = filenum * kBinlogSize + pro_offset; + if (offset > kBlockSize) { + log_->Append(second_block_item); + break; + } + } + + std::string str; + reader_ = log_->NewBinlogReader(filenum, pro_offset); + ASSERT_TRUE(reader_); + reader_->ReadRecord(str); + ASSERT_EQ(str, second_block_item); +} + +TEST(BinlogTest, ProducerStatusOp) { + std::cout << "ProducerStatusOp" << std::endl; + uint32_t filenum = 187; + uint64_t pro_offset = 8790; + ASSERT_OK(log_->SetProducerStatus(filenum, pro_offset)); + filenum = 0; + pro_offset = 0; + ASSERT_OK(log_->GetProducerStatus(&filenum, &pro_offset)); + ASSERT_EQ(filenum, 187); + ASSERT_EQ(pro_offset, 8790); +} + +} // namespace slash diff --git a/pink/third/slash/slash/tests/slash_coding_test.cc b/pink/third/slash/slash/tests/slash_coding_test.cc new file mode 100644 index 00000000..5fddf450 --- /dev/null +++ b/pink/third/slash/slash/tests/slash_coding_test.cc @@ -0,0 +1,198 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include + +#include "slash/include/slash_coding.h" +#include "slash/include/slash_testharness.h" + +namespace slash { + +class Coding { }; + +TEST(Coding, Fixed32) { + std::string s; + for (uint32_t v = 0; v < 100000; v++) { + PutFixed32(&s, v); + } + + const char* p = s.data(); + for (uint32_t v = 0; v < 100000; v++) { + uint32_t actual = DecodeFixed32(p); + ASSERT_EQ(v, actual); + p += sizeof(uint32_t); + } +} + +TEST(Coding, Fixed64) { + std::string s; + for (int power = 0; power <= 63; power++) { + uint64_t v = static_cast(1) << power; + PutFixed64(&s, v - 1); + PutFixed64(&s, v + 0); + PutFixed64(&s, v + 1); + } + + const char* p = s.data(); + for (int power = 0; power <= 63; power++) { + uint64_t v = static_cast(1) << power; + uint64_t actual; + actual = DecodeFixed64(p); + ASSERT_EQ(v-1, actual); + p += sizeof(uint64_t); + + actual = DecodeFixed64(p); + ASSERT_EQ(v+0, actual); + p += sizeof(uint64_t); + + actual = DecodeFixed64(p); + ASSERT_EQ(v+1, actual); + p += sizeof(uint64_t); + } +} + +// Test that encoding routines generate little-endian encodings +TEST(Coding, EncodingOutput) { + std::string dst; + PutFixed32(&dst, 0x04030201); + ASSERT_EQ(4, dst.size()); + ASSERT_EQ(0x01, static_cast(dst[0])); + ASSERT_EQ(0x02, static_cast(dst[1])); + ASSERT_EQ(0x03, static_cast(dst[2])); + ASSERT_EQ(0x04, static_cast(dst[3])); + + dst.clear(); + PutFixed64(&dst, 0x0807060504030201ull); + ASSERT_EQ(8, dst.size()); + ASSERT_EQ(0x01, static_cast(dst[0])); + ASSERT_EQ(0x02, static_cast(dst[1])); + ASSERT_EQ(0x03, static_cast(dst[2])); + ASSERT_EQ(0x04, static_cast(dst[3])); + ASSERT_EQ(0x05, static_cast(dst[4])); + ASSERT_EQ(0x06, static_cast(dst[5])); + ASSERT_EQ(0x07, static_cast(dst[6])); + ASSERT_EQ(0x08, static_cast(dst[7])); +} + +TEST(Coding, Varint32) { + std::string s; + for (uint32_t i = 0; i < (32 * 32); i++) { + uint32_t v = (i / 32) << (i % 32); + PutVarint32(&s, v); + } + + const char* p = s.data(); + const char* limit = p + s.size(); + for (uint32_t i = 0; i < (32 * 32); i++) { + uint32_t expected = (i / 32) << (i % 32); + uint32_t actual; + const char* start = p; + p = GetVarint32Ptr(p, limit, &actual); + ASSERT_TRUE(p != NULL); + ASSERT_EQ(expected, actual); + ASSERT_EQ(VarintLength(actual), p - start); + } + ASSERT_EQ(p, s.data() + s.size()); +} + +TEST(Coding, Varint64) { + // Construct the list of values to check + std::vector values; + // Some special values + values.push_back(0); + values.push_back(100); + values.push_back(~static_cast(0)); + values.push_back(~static_cast(0) - 1); + for (uint32_t k = 0; k < 64; k++) { + // Test values near powers of two + const uint64_t power = 1ull << k; + values.push_back(power); + values.push_back(power-1); + values.push_back(power+1); + } + + std::string s; + for (size_t i = 0; i < values.size(); i++) { + PutVarint64(&s, values[i]); + } + + const char* p = s.data(); + const char* limit = p + s.size(); + for (size_t i = 0; i < values.size(); i++) { + ASSERT_TRUE(p < limit); + uint64_t actual; + const char* start = p; + p = GetVarint64Ptr(p, limit, &actual); + ASSERT_TRUE(p != NULL); + ASSERT_EQ(values[i], actual); + ASSERT_EQ(VarintLength(actual), p - start); + } + ASSERT_EQ(p, limit); + +} + +TEST(Coding, Varint32Overflow) { + uint32_t result; + std::string input("\x81\x82\x83\x84\x85\x11"); + ASSERT_TRUE(GetVarint32Ptr(input.data(), input.data() + input.size(), &result) + == NULL); +} + +TEST(Coding, Varint32Truncation) { + uint32_t large_value = (1u << 31) + 100; + std::string s; + PutVarint32(&s, large_value); + uint32_t result; + for (size_t len = 0; len < s.size() - 1; len++) { + ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + len, &result) == NULL); + } + ASSERT_TRUE(GetVarint32Ptr(s.data(), s.data() + s.size(), &result) != NULL); + ASSERT_EQ(large_value, result); +} + +TEST(Coding, Varint64Overflow) { + uint64_t result; + std::string input("\x81\x82\x83\x84\x85\x81\x82\x83\x84\x85\x11"); + ASSERT_TRUE(GetVarint64Ptr(input.data(), input.data() + input.size(), &result) + == NULL); +} + +TEST(Coding, Varint64Truncation) { + uint64_t large_value = (1ull << 63) + 100ull; + std::string s; + PutVarint64(&s, large_value); + uint64_t result; + for (size_t len = 0; len < s.size() - 1; len++) { + ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + len, &result) == NULL); + } + ASSERT_TRUE(GetVarint64Ptr(s.data(), s.data() + s.size(), &result) != NULL); + ASSERT_EQ(large_value, result); +} + +TEST(Coding, Strings) { + std::string s; + PutLengthPrefixedString(&s, ""); + PutLengthPrefixedString(&s, "foo"); + PutLengthPrefixedString(&s, "bar"); + PutLengthPrefixedString(&s, std::string(200, 'x')); + + Slice input(s); + Slice v; + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("foo", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ("bar", v.ToString()); + ASSERT_TRUE(GetLengthPrefixedSlice(&input, &v)); + ASSERT_EQ(std::string(200, 'x'), v.ToString()); + ASSERT_EQ("", input.ToString()); +} + +} // namespace slash diff --git a/pink/third/slash/slash/tests/slash_env_test.cc b/pink/third/slash/slash/tests/slash_env_test.cc new file mode 100644 index 00000000..315eff33 --- /dev/null +++ b/pink/third/slash/slash/tests/slash_env_test.cc @@ -0,0 +1,33 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "slash/include/env.h" +#include "slash/include/testutil.h" +#include "slash/include/slash_testharness.h" + +namespace slash { + +class EnvTest { }; + +TEST(EnvTest, SetMaxFileDescriptorNum) { + ASSERT_EQ(0, SetMaxFileDescriptorNum(10)); + ASSERT_NE(0, SetMaxFileDescriptorNum(2147483647)); +} + +TEST(EnvTest, FileOps) { + std::string tmp_dir; + GetTestDirectory(&tmp_dir); + + ASSERT_TRUE(DeleteDirIfExist(tmp_dir)); + ASSERT_TRUE(!FileExists(tmp_dir)); + ASSERT_EQ(-1, DeleteDir(tmp_dir)); + ASSERT_NE(0, SetMaxFileDescriptorNum(2147483647)); +} + +} // namespace slash diff --git a/pink/third/slash/slash/tests/slash_string_test.cc b/pink/third/slash/slash/tests/slash_string_test.cc new file mode 100644 index 00000000..93810fdb --- /dev/null +++ b/pink/third/slash/slash/tests/slash_string_test.cc @@ -0,0 +1,129 @@ +// Copyright (c) 2015-present, Qihoo, Inc. All rights reserved. +// This source code is licensed under the BSD-style license found in the +// LICENSE file in the root directory of this source tree. An additional grant +// of patent rights can be found in the PATENTS file in the same directory. + +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +#include "slash/include/slash_string.h" +#include "slash/include/slash_testharness.h" + +#include + +namespace slash { + +class StringTest {}; + +TEST(StringTest, StringTrim) { + ASSERT_EQ(StringTrim(" computer "), "computer"); + ASSERT_EQ(StringTrim(" comp uter "), "comp uter"); + ASSERT_EQ(StringTrim(" \n computer \n ", "\n "), "computer"); +} + +TEST(StringTest, ParseIpPort) { + std::string ip; + int port; + ASSERT_TRUE(ParseIpPortString("192.168.1.1:9221", ip, port)); + ASSERT_EQ(ip, "192.168.1.1"); + ASSERT_EQ(port, 9221); +} + +TEST(StringTest, test_string2ll) { + char buf[32]; + long long v; + + /* May not start with +. */ + strcpy(buf,"+1"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 0); + + /* Leading space. */ + strcpy(buf," 1"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 0); + + /* Trailing space. */ + strcpy(buf,"1 "); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 0); + + strcpy(buf,"-1"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, -1); + + strcpy(buf,"0"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, 0); + + strcpy(buf,"1"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, 1); + + strcpy(buf,"99"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, 99); + + strcpy(buf,"-99"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, -99); + + strcpy(buf,"-9223372036854775808"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, LLONG_MIN); + + strcpy(buf,"-9223372036854775809"); /* overflow */ + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 0); + + strcpy(buf,"9223372036854775807"); + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, LLONG_MAX); + + strcpy(buf,"9223372036854775808"); /* overflow */ + ASSERT_EQ(string2ll(buf,strlen(buf),&v), 0); +} + +TEST(StringTest, test_string2l) { + char buf[32]; + long v; + + /* May not start with +. */ + strcpy(buf,"+1"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 0); + + strcpy(buf,"-1"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, -1); + + strcpy(buf,"0"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, 0); + + strcpy(buf,"1"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, 1); + + strcpy(buf,"99"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, 99); + + strcpy(buf,"-99"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, -99); + +#if LONG_MAX != LLONG_MAX + strcpy(buf,"-2147483648"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, LONG_MIN); + + strcpy(buf,"-2147483649"); /* overflow */ + ASSERT_EQ(string2l(buf,strlen(buf),&v), 0); + + strcpy(buf,"2147483647"); + ASSERT_EQ(string2l(buf,strlen(buf),&v), 1); + ASSERT_EQ(v, LONG_MAX); + + strcpy(buf,"2147483648"); /* overflow */ + ASSERT_EQ(string2l(buf,strlen(buf),&v), 0); +#endif +} + +} // namespace slash diff --git a/pink/third/slash/slash/tests/test_main.cc b/pink/third/slash/slash/tests/test_main.cc new file mode 100644 index 00000000..314ba79c --- /dev/null +++ b/pink/third/slash/slash/tests/test_main.cc @@ -0,0 +1,5 @@ +#include "slash/include/slash_testharness.h" + +int main() { + return slash::test::RunAllTests(); +}