[CH-431] not change columns to nulllable in expand operator

.github/workflows/master.yml

@@ -839,6 +839,34 @@ jobs:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ############################################################################################
+##################################### Docker images  #######################################
+############################################################################################
+  DockerServerImages:
+    needs:
+      - BuilderDebRelease
+      - BuilderDebAarch64
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # otherwise we will have no version info
+      - name: Check docker clickhouse/clickhouse-server building
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_server.py --release-type head
+          python3 docker_server.py --release-type head --no-ubuntu \
+            --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:

.github/workflows/pull_request.yml

@@ -894,6 +894,34 @@ jobs:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
           sudo rm -fr "$TEMP_PATH"
 ############################################################################################
+##################################### Docker images  #######################################
+############################################################################################
+  DockerServerImages:
+    needs:
+      - BuilderDebRelease
+      - BuilderDebAarch64
+    runs-on: [self-hosted, style-checker]
+    steps:
+      - name: Clear repository
+        run: |
+          sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0  # otherwise we will have no version info
+      - name: Check docker clickhouse/clickhouse-server building
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 docker_server.py --release-type head --no-push
+          python3 docker_server.py --release-type head --no-push --no-ubuntu \
+            --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper
+      - name: Cleanup
+        if: always()
+        run: |
+          docker kill "$(docker ps -q)" ||:
+          docker rm -f "$(docker ps -a -q)" ||:
+          sudo rm -fr "$TEMP_PATH"
+############################################################################################
 ##################################### BUILD REPORTER #######################################
 ############################################################################################
   BuilderReport:

.github/workflows/release.yml

@@ -14,7 +14,9 @@ jobs:
     steps:
     - name: Deploy packages and assets
       run: |
-        curl '${{ secrets.PACKAGES_RELEASE_URL }}/release/${{ github.ref }}?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true' -d ''
+        GITHUB_TAG="${GITHUB_REF#refs/tags/}"
+        curl --silent --data '' \
+          '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true'
   ############################################################################################
   ##################################### Docker images  #######################################
   ############################################################################################

.gitignore

@@ -166,3 +166,5 @@ tests/queries/0_stateless/*.generated-expect
 /rust/**/target
 # It is autogenerated from *.in
 /rust/**/.cargo/config.toml
+
+utils/local-engine/tests/testConfig.h

CMakeLists.txt

@@ -178,7 +178,7 @@ else ()
     set(NO_WHOLE_ARCHIVE --no-whole-archive)
 endif ()
 
-option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." ON)
+option(ENABLE_CURL_BUILD "Enable curl, azure, sentry build on by default except MacOS." OFF)
 if (OS_DARWIN)
     # Disable the curl, azure, senry build on MacOS
     set (ENABLE_CURL_BUILD OFF)
@@ -422,6 +422,8 @@ endif ()
 
 include(cmake/dbms_glob_sources.cmake)
 
+set (CMAKE_POSITION_INDEPENDENT_CODE ON)
+
 add_library(global-group INTERFACE)
 if (OS_LINUX OR OS_ANDROID)
     include(cmake/linux/default_libs.cmake)
@@ -452,7 +454,6 @@ endif ()
 
 set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
-set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
 if (OS_LINUX AND NOT ARCH_AARCH64)
     # Slightly more efficient code can be generated
     # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
@@ -510,8 +511,10 @@ macro (clickhouse_add_executable target)
     if (${type} STREQUAL EXECUTABLE)
         # disabled for TSAN and gcc since libtsan.a provides overrides too
         if (TARGET clickhouse_new_delete)
-            # operator::new/delete for executables (MemoryTracker stuff)
-            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
+            if (NOT ${target} STREQUAL Git::Git)
+                # operator::new/delete for executables (MemoryTracker stuff)
+                target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES})
+            endif()
         endif()
 
         # In case of static jemalloc, because zone_register() is located in zone.c and

cmake/autogenerated_versions.txt

@@ -5,8 +5,8 @@
 SET(VERSION_REVISION 54470)
 SET(VERSION_MAJOR 23)
 SET(VERSION_MINOR 1)
-SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH 688e488e930c83eefeac4f87c4cc029cc5b231e3)
-SET(VERSION_DESCRIBE v23.1.1.1-testing)
-SET(VERSION_STRING 23.1.1.1)
+SET(VERSION_PATCH 3)
+SET(VERSION_GITHASH 8dfb1700858195fa704221e360fa0798ac6ee9ed)
+SET(VERSION_DESCRIBE v23.1.3.1-stable)
+SET(VERSION_STRING 23.1.3.1)
 # end of autochange

contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in

@@ -139,7 +139,8 @@
 /* #undef JEMALLOC_MUTEX_INIT_CB */
 
 /* Non-empty if the tls_model attribute is supported. */
-#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))
+/*#define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec")))*/
+#define JEMALLOC_TLS_MODEL
 
 /*
  * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables

docs/en/sql-reference/functions/json-functions.md

@@ -200,6 +200,7 @@ Examples:
 ``` sql
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(String, Array(Float64))') = ('hello',[-100,200,300])
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'Tuple(b Array(Float64), a String)') = ([-100,200,300],'hello')
+SELECT JSONExtract('{"a": "hello", "b": "world"}', 'Map(String, String)') = map('a',  'hello', 'b', 'world');
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 'Array(Nullable(Int8))') = [-100, NULL, NULL]
 SELECT JSONExtract('{"a": "hello", "b": [-100, 200.0, 300]}', 'b', 4, 'Nullable(Int64)') = NULL
 SELECT JSONExtract('{"passed": true}', 'passed', 'UInt8') = 1

docs/en/sql-reference/functions/tuple-map-functions.md

@@ -66,6 +66,46 @@ Result:
 
 -   [Map(key, value)](../../sql-reference/data-types/map.md) data type
 
+## mapFromArrays
+
+Merges an [Array](../../sql-reference/data-types/array.md) of keys and an [Array](../../sql-reference/data-types/array.md) of values into a [Map(key, value)](../../sql-reference/data-types/map.md). Notice that the second argument could also be a [Map](../../sql-reference/data-types/map.md), thus it is casted to an Array when executing.
+
+The function is a more convenient alternative to `CAST((key_array, value_array_or_map), 'Map(key_type, value_type)')`. For example, instead of writing `CAST((['aa', 'bb'], [4, 5]), 'Map(String, UInt32)')`, you can write `mapFromArrays(['aa', 'bb'], [4, 5])`.
+
+**Syntax**
+
+```sql
+mapFromArrays(keys, values)
+```  
+
+Alias: `MAP_FROM_ARRAYS(keys, values)`
+
+**Parameters**
+-   `keys` — Given key array to create a map from. The nested type of array must be: [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md), [LowCardinality](../../sql-reference/data-types/lowcardinality.md), [FixedString](../../sql-reference/data-types/fixedstring.md), [UUID](../../sql-reference/data-types/uuid.md), [Date](../../sql-reference/data-types/date.md), [DateTime](../../sql-reference/data-types/datetime.md), [Date32](../../sql-reference/data-types/date32.md), [Enum](../../sql-reference/data-types/enum.md)
+-   `values`  - Given value array or map to create a map from.
+
+**Returned value**
+
+- A map whose keys and values are constructed from the key array and value array/map.
+
+**Example**
+
+Query:
+
+```sql
+select mapFromArrays(['a', 'b', 'c'], [1, 2, 3])
+
+┌─mapFromArrays(['a', 'b', 'c'], [1, 2, 3])─┐
+│ {'a':1,'b':2,'c':3}                       │
+└───────────────────────────────────────────┘
+
+SELECT mapFromArrays([1, 2, 3], map('a', 1, 'b', 2, 'c', 3))
+
+┌─mapFromArrays([1, 2, 3], map('a', 1, 'b', 2, 'c', 3))─┐
+│ {1:('a',1),2:('b',2),3:('c',3)}                       │
+└───────────────────────────────────────────────────────┘
+```
+
 ## mapAdd
 
 Collect all the keys and sum corresponding values.
@@ -429,6 +469,8 @@ Result:
 │ {}                         │
 └────────────────────────────┘
 ```  
+
+
 
 ## mapApply  
 

programs/benchmark/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_BENCHMARK_SOURCES Benchmark.cpp)
+set (CLICKHOUSE_BENCHMARK_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Benchmark.cpp)
 
 set (CLICKHOUSE_BENCHMARK_LINK
     PRIVATE

programs/compressor/CMakeLists.txt

@@ -1,6 +1,6 @@
 # Also in utils
 
-set (CLICKHOUSE_COMPRESSOR_SOURCES Compressor.cpp)
+set (CLICKHOUSE_COMPRESSOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Compressor.cpp)
 
 set (CLICKHOUSE_COMPRESSOR_LINK
     PRIVATE

programs/extract-from-config/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ExtractFromConfig.cpp)
+set (CLICKHOUSE_EXTRACT_FROM_CONFIG_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ExtractFromConfig.cpp)
 
 set (CLICKHOUSE_EXTRACT_FROM_CONFIG_LINK
     PRIVATE

programs/format/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_FORMAT_SOURCES Format.cpp)
+set (CLICKHOUSE_FORMAT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Format.cpp)
 
 set (CLICKHOUSE_FORMAT_LINK
     PRIVATE

programs/git-import/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_GIT_IMPORT_SOURCES git-import.cpp)
+set (CLICKHOUSE_GIT_IMPORT_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/git-import.cpp)
 
 set (CLICKHOUSE_GIT_IMPORT_LINK
     PRIVATE

programs/install/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_INSTALL_SOURCES Install.cpp)
+set (CLICKHOUSE_INSTALL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Install.cpp)
 
 set (CLICKHOUSE_INSTALL_LINK
     PRIVATE

programs/keeper-converter/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES KeeperConverter.cpp)
+set (CLICKHOUSE_KEEPER_CONVERTER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/KeeperConverter.cpp)
 
 set (CLICKHOUSE_KEEPER_CONVERTER_LINK
     PRIVATE

programs/local/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_LOCAL_SOURCES LocalServer.cpp)
+set (CLICKHOUSE_LOCAL_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/LocalServer.cpp)
 
 set (CLICKHOUSE_LOCAL_LINK
     PRIVATE

programs/obfuscator/CMakeLists.txt

@@ -1,4 +1,4 @@
-set (CLICKHOUSE_OBFUSCATOR_SOURCES Obfuscator.cpp)
+set (CLICKHOUSE_OBFUSCATOR_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/Obfuscator.cpp)
 
 set (CLICKHOUSE_OBFUSCATOR_LINK
     PRIVATE

programs/server/CMakeLists.txt

@@ -1,8 +1,8 @@
 include(${ClickHouse_SOURCE_DIR}/cmake/embed_binary.cmake)
 
 set(CLICKHOUSE_SERVER_SOURCES
-    MetricsTransmitter.cpp
-    Server.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/MetricsTransmitter.cpp
+    ${CMAKE_CURRENT_SOURCE_DIR}/Server.cpp
 )
 
 set (LINK_RESOURCE_LIB INTERFACE "-Wl,${WHOLE_ARCHIVE} $<TARGET_FILE:clickhouse_server_configs> -Wl,${NO_WHOLE_ARCHIVE}")
@@ -36,4 +36,6 @@ clickhouse_embed_binaries(
     TARGET clickhouse_server_configs
     RESOURCES config.xml users.xml embedded.xml play.html dashboard.html js/uplot.js
 )
-add_dependencies(clickhouse-server-lib clickhouse_server_configs)
+if(NOT CLICKHOUSE_ONE_SHARED)
+    add_dependencies(clickhouse-server-lib clickhouse_server_configs)
+endif()

src/AggregateFunctions/AggregateFunctionArray.h

@@ -13,7 +13,7 @@ struct Settings;
 
 namespace ErrorCodes
 {
-    extern const int SIZES_OF_ARRAYS_DOESNT_MATCH;
+    extern const int SIZES_OF_ARRAYS_DONT_MATCH;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
@@ -129,7 +129,7 @@ class AggregateFunctionArray final : public IAggregateFunctionHelper<AggregateFu
             const IColumn::Offsets & ith_offsets = ith_column.getOffsets();
 
             if (ith_offsets[row_num] != end || (row_num != 0 && ith_offsets[row_num - 1] != begin))
-                throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
+                throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
         }
 
         for (size_t i = begin; i < end; ++i)

src/AggregateFunctions/AggregateFunctionForEach.h

@@ -19,7 +19,7 @@ namespace ErrorCodes
 {
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int SIZES_OF_ARRAYS_DOESNT_MATCH;
+    extern const int SIZES_OF_ARRAYS_DONT_MATCH;
 }
 
 
@@ -197,7 +197,7 @@ class AggregateFunctionForEach final : public IAggregateFunctionDataHelper<Aggre
             const IColumn::Offsets & ith_offsets = ith_column.getOffsets();
 
             if (ith_offsets[row_num] != end || (row_num != 0 && ith_offsets[row_num - 1] != begin))
-                throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DOESNT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
+                throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Arrays passed to {} aggregate function have different sizes", getName());
         }
 
         AggregateFunctionForEachData & state = ensureAggregateData(place, end - begin, *arena);

src/AggregateFunctions/AggregateFunctionGroupArray.cpp

@@ -121,7 +121,7 @@ AggregateFunctionPtr createAggregateFunctionGroupArraySample(
 
 void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
 {
-    AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
+    AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = true };
 
     factory.registerFunction("groupArray", { createAggregateFunctionGroupArray<false>, properties });
     factory.registerFunction("groupArraySample", { createAggregateFunctionGroupArraySample, properties });

src/Columns/ColumnAggregateFunction.cpp

@@ -525,6 +525,35 @@ void ColumnAggregateFunction::insertDefault()
     pushBackAndCreateState(data, arena, func.get());
 }
 
+void ColumnAggregateFunction::insertRangeSelective(
+    const IColumn & from, const IColumn::Selector & selector, size_t selector_start, size_t length)
+{
+    const ColumnAggregateFunction & from_concrete = static_cast<const ColumnAggregateFunction &>(from);
+    const auto & from_data = from_concrete.data;
+    if (!empty() && src.get() != &from_concrete)
+    {
+        ensureOwnership();
+        Arena & arena = createOrGetArena();
+        Arena * arena_ptr = &arena;
+        data.reserve(size() + length);
+        for (size_t i = 0; i < length; ++i)
+        {
+            pushBackAndCreateState(data, arena, func.get());
+            func->merge(data.back(), from_data[selector[selector_start + i]], arena_ptr);
+        }
+        return;
+    }
+    /// Keep shared ownership of aggregation states.
+    src = from_concrete.getPtr();
+
+    size_t old_size = data.size();
+    data.resize(old_size + length);
+    auto * data_start = data.data();
+    size_t element_size = sizeof(data[0]);
+    for (size_t i = 0; i < length; ++i)
+        memcpy(data_start + old_size + i, &from_concrete.data[selector[selector_start + i]], element_size);
+}
+
 StringRef ColumnAggregateFunction::serializeValueIntoArena(size_t n, Arena & arena, const char *& begin) const
 {
     WriteBufferFromArena out(arena, begin);

src/Columns/ColumnAggregateFunction.h

@@ -184,6 +184,8 @@ class ColumnAggregateFunction final : public COWHelper<IColumn, ColumnAggregateF
 
     void insertRangeFrom(const IColumn & from, size_t start, size_t length) override;
 
+    void insertRangeSelective(const IColumn & src, const IColumn::Selector & selector, size_t selector_start, size_t length) override;
+
     void popBack(size_t n) override;
 
     ColumnPtr filter(const Filter & filter, ssize_t result_size_hint) const override;