diff --git a/turms-server-common/src/main/java/im/turms/server/common/access/admin/web/HttpUtil.java b/turms-server-common/src/main/java/im/turms/server/common/access/admin/web/HttpUtil.java index 8559221956..d43831527e 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/access/admin/web/HttpUtil.java +++ b/turms-server-common/src/main/java/im/turms/server/common/access/admin/web/HttpUtil.java @@ -45,6 +45,11 @@ public static boolean isPreFlightRequest(HttpServerRequest request) { && headers.contains(HttpHeaderNames.ACCESS_CONTROL_REQUEST_METHOD); } + public static boolean isSuccess(HttpResponseStatus status) { + int code = status.code(); + return code >= 200 && code < 300; + } + public static boolean isServerError(HttpResponseStatus status) { return status.code() >= 500; } diff --git a/turms-server-common/src/main/java/im/turms/server/common/access/common/ResponseStatusCode.java b/turms-server-common/src/main/java/im/turms/server/common/access/common/ResponseStatusCode.java index 201917f371..ea02b4c00a 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/access/common/ResponseStatusCode.java +++ b/turms-server-common/src/main/java/im/turms/server/common/access/common/ResponseStatusCode.java @@ -394,7 +394,11 @@ public enum ResponseStatusCode { NOT_FRIEND_TO_QUERY_MESSAGE_ATTACHMENT_INFO_IN_PRIVATE_CONVERSATION(6130, "Only friends can query message attachments in private conversations", 403), NOT_GROUP_MEMBER_TO_QUERY_MESSAGE_ATTACHMENT_INFO_IN_GROUP_CONVERSATION(6131, - "Only group members can query message attachments in group conversations", 403); + "Only group members can query message attachments in group conversations", 403), + + // Search + SEARCHING_USER_IS_DISABLED(7100, "Searching users is disabled", 510), + SEARCHING_GROUP_IS_DISABLED(7200, "Searching groups is disabled", 510); public static final int STATUS_CODE_LENGTH = 4; public static final ResponseStatusCode[] VALUES = values(); diff --git a/turms-server-common/src/main/java/im/turms/server/common/domain/user/po/User.java b/turms-server-common/src/main/java/im/turms/server/common/domain/user/po/User.java index ce712ba892..5f530c77f1 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/domain/user/po/User.java +++ b/turms-server-common/src/main/java/im/turms/server/common/domain/user/po/User.java @@ -19,6 +19,7 @@ import java.util.Date; +import lombok.AllArgsConstructor; import lombok.Data; import im.turms.server.common.access.client.dto.constant.ProfileAccessStrategy; @@ -32,6 +33,7 @@ /** * @author James Chen */ +@AllArgsConstructor @Data @Document(User.COLLECTION_NAME) @Sharded @@ -46,7 +48,7 @@ public final class User extends BaseEntity { private final byte[] password; @Field(Fields.NAME) - private final String name; + private String name; @Field(Fields.INTRO) private final String intro; diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/cluster/service/idgen/ServiceType.java b/turms-server-common/src/main/java/im/turms/server/common/infra/cluster/service/idgen/ServiceType.java index dfe8f69e7c..90533bf117 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/infra/cluster/service/idgen/ServiceType.java +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/cluster/service/idgen/ServiceType.java @@ -35,5 +35,7 @@ public enum ServiceType { USER_LOCATION, USER_PERMISSION_GROUP, - STORAGE_MESSAGE_ATTACHMENT + STORAGE_MESSAGE_ATTACHMENT, + + ELASTICSEARCH_SYNC_LOG } \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/json/JsonUtil.java b/turms-server-common/src/main/java/im/turms/server/common/infra/json/JsonUtil.java index 8a26d92230..6d8fbf2582 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/infra/json/JsonUtil.java +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/json/JsonUtil.java @@ -38,7 +38,6 @@ import im.turms.server.common.infra.lang.StringUtil; import im.turms.server.common.infra.logging.core.logger.Logger; import im.turms.server.common.infra.logging.core.logger.LoggerFactory; -import im.turms.server.common.infra.netty.ReferenceCountUtil; import im.turms.server.common.infra.time.DateUtil; /** @@ -106,12 +105,16 @@ public static Map readStringStringMapValue(InputStream src) { public static ByteBuf write(Object value) { int estimatedSize = estimateSize(value); - ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(estimatedSize); + return write(estimatedSize, value); + } + + public static ByteBuf write(int size, Object value) { + ByteBuf buffer = PooledByteBufAllocator.DEFAULT.directBuffer(size); try (OutputStream bufferOutputStream = new ByteBufOutputStream(buffer)) { MAPPER.writeValue(bufferOutputStream, value); return buffer; } catch (Exception e) { - ReferenceCountUtil.ensureReleased(buffer); + buffer.release(); throw new IllegalArgumentException( "Failed to write the input object as a byte buffer", e); diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/lang/Pair.java b/turms-server-common/src/main/java/im/turms/server/common/infra/lang/Pair.java index 97d84b07c2..b794929a81 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/infra/lang/Pair.java +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/lang/Pair.java @@ -17,15 +17,17 @@ package im.turms.server.common.infra.lang; +import jakarta.annotation.Nullable; + /** * @author James Chen */ public record Pair( - T1 first, - T2 second + @Nullable T1 first, + @Nullable T2 second ) { - public static Pair of(T1 first, T2 second) { + public static Pair of(@Nullable T1 first, @Nullable T2 second) { return new Pair<>(first, second); } diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/ServiceProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/ServiceProperties.java index 84a7415c4e..00324a3a8a 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/ServiceProperties.java +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/ServiceProperties.java @@ -34,6 +34,7 @@ import im.turms.server.common.infra.property.env.service.env.adminapi.AdminApiProperties; import im.turms.server.common.infra.property.env.service.env.clientapi.ClientApiProperties; import im.turms.server.common.infra.property.env.service.env.database.MongoProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.TurmsElasticsearchProperties; import im.turms.server.common.infra.property.env.service.env.push.PushNotificationProperties; import im.turms.server.common.infra.property.env.service.env.redis.TurmsRedisProperties; @@ -54,6 +55,9 @@ public class ServiceProperties { @NestedConfigurationProperty private ClientApiProperties clientApi = new ClientApiProperties(); + @NestedConfigurationProperty + private TurmsElasticsearchProperties elasticsearch = new TurmsElasticsearchProperties(); + @NestedConfigurationProperty private FakeProperties fake = new FakeProperties(); diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchClientProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchClientProperties.java new file mode 100644 index 0000000000..8e3bcf455e --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchClientProperties.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import java.util.Collections; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchClientProperties { + + @Description("Elasticsearch URI") + private String uri = "http://localhost:9200"; + + @Description("Elasticsearch HTTP request headers") + private List requestHeaders = Collections.emptyList(); + + @Description("Elasticsearch username") + private String username = "elastic"; + + @Description("Elasticsearch password") + private String password = ""; + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchGroupUseCaseProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchGroupUseCaseProperties.java new file mode 100644 index 0000000000..a9d0c1a1a8 --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchGroupUseCaseProperties.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * @author James Chen + */ +@AllArgsConstructor +@SuperBuilder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchGroupUseCaseProperties extends ElasticsearchUseCaseProperties { + + private List indexes = + List.of(new ElasticsearchIndexProperties()); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexProperties.java new file mode 100644 index 0000000000..fa369f1ea0 --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexProperties.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import jakarta.validation.constraints.Min; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchIndexProperties { + + @Description("The index will be created for the specified language. " + + "If the language is NONE, this index will be used as the default index for all languages that don't have a specified index for them") + private LanguageCode code = LanguageCode.NONE; + + @Description("The number of shards. -1 means use the default value") + @Min(-1) + private int numberOfShards = -1; + + @Description("The number of replicas. -1 means use the default value") + @Min(-1) + private int numberOfReplicas = -1; + + @NestedConfigurationProperty + private ElasticsearchIndexPropertiesProperties properties = + new ElasticsearchIndexPropertiesProperties(); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexPropertiesFieldProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexPropertiesFieldProperties.java new file mode 100644 index 0000000000..7ee8b7854d --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexPropertiesFieldProperties.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchIndexPropertiesFieldProperties { + + private List textFields = + List.of(new ElasticsearchIndexTextFieldProperties()); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexPropertiesProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexPropertiesProperties.java new file mode 100644 index 0000000000..3fe61f909d --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexPropertiesProperties.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +/** + * @author James Chen + */ +@AllArgsConstructor +@SuperBuilder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchIndexPropertiesProperties { + + @NestedConfigurationProperty + private ElasticsearchIndexPropertiesFieldProperties name = + new ElasticsearchIndexPropertiesFieldProperties(); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexTextFieldProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexTextFieldProperties.java new file mode 100644 index 0000000000..b2775bb73b --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchIndexTextFieldProperties.java @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchIndexTextFieldProperties { + + @Description("The name of the field. A property can have multiple fields in Elasticsearch") + private String fieldName = ""; + + @Description("The name of the analyzer to use for the text field. " + + "If not specified, the default analyzer for the language will be used") + private String analyzer = ""; + + @Description("The name of the search analyzer to use for the text field") + private String searchAnalyzer = ""; + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchLanguageDetectProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchLanguageDetectProperties.java new file mode 100644 index 0000000000..9d2ef9e1d7 --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchLanguageDetectProperties.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import jakarta.validation.constraints.DecimalMax; +import jakarta.validation.constraints.DecimalMin; +import jakarta.validation.constraints.Min; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchLanguageDetectProperties { + + @Description("Whether to enable language detection. " + + "If true, a pipeline for language detection will be created at startup, " + + "and will be used as the default pipeline of new indexes") + private boolean enabled; + + @Description("The confidence score threshold. " + + "Only languages with a confidence score higher than this threshold will be used") + @DecimalMin("0") + @DecimalMax("1") + private float confidenceScoreThreshold = 0.5F; + + @Description("The maximum number of detected languages to use when detecting languages") + @Min(1) + private int maxDetectedLanguages = 3; + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchMongoProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchMongoProperties.java new file mode 100644 index 0000000000..1ad071ff3b --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchMongoProperties.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import im.turms.server.common.infra.property.env.service.env.database.TurmsMongoProperties; +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchMongoProperties extends TurmsMongoProperties { + + @Description("Whether to enable transaction for MongoDB. " + + "If enabled, MongoDB will use transactions when upserting data into both MongoDB and Elasticsearch, " + + "and will roll back data if an error occurs, no matter they are MongoDB or Elasticsearch errors") + private boolean enableTransaction; + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchSyncProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchSyncProperties.java new file mode 100644 index 0000000000..9b6cc1421e --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchSyncProperties.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchSyncProperties { + + @Description("Whether to sync existing data from MongoDB to Elasticsearch. " + + "If true and the current node is the leader, turms will run a full sync at startup if the data has not been synced yet") + private boolean performFullSyncAtStartup = true; + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUseCaseProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUseCaseProperties.java new file mode 100644 index 0000000000..8135dad89a --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUseCaseProperties.java @@ -0,0 +1,55 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@SuperBuilder(toBuilder = true) +@Data +@NoArgsConstructor +public abstract class ElasticsearchUseCaseProperties { + + @Description("Whether to enable this use case") + private boolean enabled = true; + + // Hide the feature currently because the performance on short texts + // of most language detection models is too terrible to use. +// @NestedConfigurationProperty +// private ElasticsearchLanguageDetectProperties languageDetect = +// new ElasticsearchLanguageDetectProperties(); + + @NestedConfigurationProperty + private ElasticsearchClientProperties client = new ElasticsearchClientProperties(); + + @NestedConfigurationProperty + private ElasticsearchSyncProperties sync = new ElasticsearchSyncProperties(); + + @NestedConfigurationProperty + private ElasticsearchMongoProperties mongo = new ElasticsearchMongoProperties(); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUseCasesProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUseCasesProperties.java new file mode 100644 index 0000000000..bb396df5d6 --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUseCasesProperties.java @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchUseCasesProperties { + + @NestedConfigurationProperty + private ElasticsearchUserUseCaseProperties user = new ElasticsearchUserUseCaseProperties(); + + @NestedConfigurationProperty + private ElasticsearchGroupUseCaseProperties group = new ElasticsearchGroupUseCaseProperties(); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUserUseCaseProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUserUseCaseProperties.java new file mode 100644 index 0000000000..b36fce7a33 --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/ElasticsearchUserUseCaseProperties.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.SuperBuilder; + +/** + * @author James Chen + */ +@AllArgsConstructor +@SuperBuilder(toBuilder = true) +@Data +@NoArgsConstructor +public class ElasticsearchUserUseCaseProperties extends ElasticsearchUseCaseProperties { + + private List indexes = + List.of(new ElasticsearchIndexProperties()); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/HttpHeaderProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/HttpHeaderProperties.java new file mode 100644 index 0000000000..fb9b10067b --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/HttpHeaderProperties.java @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class HttpHeaderProperties { + + @Description("The name of the header") + private String name; + + @Description("The value of the header") + private String value; + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/LanguageCode.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/LanguageCode.java new file mode 100644 index 0000000000..79c34acc39 --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/LanguageCode.java @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * @author James Chen + */ +@AllArgsConstructor +public enum LanguageCode { + NONE("", "", ""), + + AF("af", "af", "Afrikaans"), + AM("am", "am", "Amharic"), + AR("ar", "ar", "Arabic"), + AZ("az", "az", "Azerbaijani"), + BE("be", "be", "Belarusian"), + BG("bg", "bg", "Bulgarian"), + BG_LATN("bg-Latn", "bg_latn", "Bulgarian"), + BN("bn", "bn", "Bengali"), + BS("bs", "bs", "Bosnian"), + CA("ca", "ca", "Catalan"), + CEB("ceb", "ceb", "Cebuano"), + CO("co", "co", "Corsican"), + CS("cs", "cs", "Czech"), + CY("cy", "cy", "Welsh"), + DA("da", "da", "Danish"), + DE("de", "de", "German"), + EL("el", "el", "Greek, modern"), + EL_LATN("el-Latn", "el_latn", "Greek, modern"), + EN("en", "en", "English"), + EO("eo", "eo", "Esperanto"), + ES("es", "es", "Spanish, Castilian"), + ET("et", "et", "Estonian"), + EU("eu", "eu", "Basque"), + FA("fa", "fa", "Persian"), + FI("fi", "fi", "Finnish"), + FIL("fil", "fil", "Filipino"), + FR("fr", "fr", "French"), + FY("fy", "fy", "Western Frisian"), + GA("ga", "ga", "Irish"), + GD("gd", "gd", "Gaelic"), + GL("gl", "gl", "Galician"), + GU("gu", "gu", "Gujarati"), + HA("ha", "ha", "Hausa"), + HAW("haw", "haw", "Hawaiian"), + HI("hi", "hi", "Hindi"), + HI_LATN("hi-Latn", "hi_latn", "Hindi"), + HMN("hmn", "hmn", "Hmong"), + HR("hr", "hr", "Croatian"), + HT("ht", "ht", "Haitian"), + HU("hu", "hu", "Hungarian"), + HY("hy", "hy", "Armenian"), + ID("id", "id", "Indonesian"), + IG("ig", "ig", "Igbo"), + IS("is", "is", "Icelandic"), + IT("it", "it", "Italian"), + IW("iw", "iw", "Hebrew"), + JA("ja", "ja", "Japanese"), + JA_LATN("ja-Latn", "ja_latn", "Japanese"), + JV("jv", "jv", "Javanese"), + KA("ka", "ka", "Georgian"), + KK("kk", "kk", "Kazakh"), + KM("km", "km", "Central Khmer"), + KN("kn", "kn", "Kannada"), + KO("ko", "ko", "Korean"), + KU("ku", "ku", "Kurdish"), + KY("ky", "ky", "Kirghiz"), + LA("la", "la", "Latin"), + LB("lb", "lb", "Luxembourgish"), + LO("lo", "lo", "Lao"), + LT("lt", "lt", "Lithuanian"), + LV("lv", "lv", "Latvian"), + MG("mg", "mg", "Malagasy"), + MI("mi", "mi", "Maori"), + MK("mk", "mk", "Macedonian"), + ML("ml", "ml", "Malayalam"), + MN("mn", "mn", "Mongolian"), + MR("mr", "mr", "Marathi"), + MS("ms", "ms", "Malay"), + MT("mt", "mt", "Maltese"), + MY("my", "my", "Burmese"), + NE("ne", "ne", "Nepali"), + NL("nl", "nl", "Dutch, Flemish"), + NO("no", "no", "Norwegian"), + NY("ny", "ny", "Chichewa"), + PA("pa", "pa", "Punjabi"), + PL("pl", "pl", "Polish"), + PS("ps", "ps", "Pashto"), + PT("pt", "pt", "Portuguese"), + RO("ro", "ro", "Romanian"), + RU("ru", "ru", "Russian"), + RU_LATN("ru-Latn", "ru_latn", "Russian"), + SD("sd", "sd", "Sindhi"), + SI("si", "si", "Sinhala"), + SK("sk", "sk", "Slovak"), + SL("sl", "sl", "Slovenian"), + SM("sm", "sm", "Samoan"), + SN("sn", "sn", "Shona"), + SO("so", "so", "Somali"), + SQ("sq", "sq", "Albanian"), + SR("sr", "sr", "Serbian"), + ST("st", "st", "Southern Sotho"), + SU("su", "su", "Sundanese"), + SV("sv", "sv", "Swedish"), + SW("sw", "sw", "Swahili"), + TA("ta", "ta", "Tamil"), + TE("te", "te", "Telugu"), + TG("tg", "tg", "Tajik"), + TH("th", "th", "Thai"), + TR("tr", "tr", "Turkish"), + UK("uk", "uk", "Ukrainian"), + UR("ur", "ur", "Urdu"), + UZ("uz", "uz", "Uzbek"), + VI("vi", "vi", "Vietnamese"), + XH("xh", "xh", "Xhosa"), + YI("yi", "yi", "Yiddish"), + YO("yo", "yo", "Yoruba"), + ZH("zh", "zh", "Chinese"), + ZH_LATN("zh-Latn", "zh_latn", "Chinese"), + ZU("zu", "zu", "Zulu"); + + @Getter + private final String code; + + @Getter + private final String canonicalCode; + + @Getter + private final String language; + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/TurmsElasticsearchProperties.java b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/TurmsElasticsearchProperties.java new file mode 100644 index 0000000000..92d391d830 --- /dev/null +++ b/turms-server-common/src/main/java/im/turms/server/common/infra/property/env/service/env/elasticsearch/TurmsElasticsearchProperties.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.server.common.infra.property.env.service.env.elasticsearch; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +import im.turms.server.common.infra.property.metadata.Description; + +/** + * @author James Chen + */ +@AllArgsConstructor +@Builder(toBuilder = true) +@Data +@NoArgsConstructor +public class TurmsElasticsearchProperties { + + @Description("Whether to enable Elasticsearch") + private boolean enabled; + + @NestedConfigurationProperty + private ElasticsearchUseCasesProperties useCase = new ElasticsearchUseCasesProperties(); + +} \ No newline at end of file diff --git a/turms-server-common/src/main/java/im/turms/server/common/storage/mongo/operation/option/Update.java b/turms-server-common/src/main/java/im/turms/server/common/storage/mongo/operation/option/Update.java index b9d51b88b4..e579b30ba3 100644 --- a/turms-server-common/src/main/java/im/turms/server/common/storage/mongo/operation/option/Update.java +++ b/turms-server-common/src/main/java/im/turms/server/common/storage/mongo/operation/option/Update.java @@ -24,6 +24,7 @@ import jakarta.validation.constraints.NotNull; import org.bson.BsonDocument; +import org.bson.BsonString; import im.turms.server.common.infra.collection.CollectionUtil; import im.turms.server.common.storage.mongo.BsonPool; @@ -53,6 +54,10 @@ public Update set(String field, Object value) { return appendSet(field, value); } + public Update setEnumString(@NotNull String field, @NotNull Enum value) { + return appendSetForEnumString(field, value); + } + public Update setIfNotNull(@NotNull String field, @Nullable Object value) { if (value != null) { if (value instanceof Collection collection) { @@ -106,6 +111,15 @@ private Update appendSet(String key, Object value) { return this; } + private Update appendSetForEnumString(String key, Enum value) { + if (set == null) { + set = new BsonDocument(); + document.append("$set", set); + } + set.put(key, new BsonString(value.name())); + return this; + } + private Update appendSetForEnumStrings(String key, Collection> collection) { if (set == null) { set = new BsonDocument(); diff --git a/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json b/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json index 2490e2a517..a4f1e20fe3 100644 --- a/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json +++ b/turms-server-common/src/test/resources/turms-properties-metadata-with-property-value.json @@ -3294,6 +3294,215 @@ } } }, + "elasticsearch": { + "enabled": { + "deprecated": false, + "description": "Whether to enable Elasticsearch", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean", + "value": false + }, + "useCase": { + "group": { + "client": { + "password": { + "deprecated": false, + "description": "Elasticsearch password", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string", + "value": "" + }, + "requestHeaders": { + "deprecated": false, + "description": "Elasticsearch HTTP request headers", + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.HttpHeaderProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List", + "value": [] + }, + "uri": { + "deprecated": false, + "description": "Elasticsearch URI", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string", + "value": "http://localhost:9200" + }, + "username": { + "deprecated": false, + "description": "Elasticsearch username", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string", + "value": "elastic" + } + }, + "enabled": { + "deprecated": false, + "description": "Whether to enable this use case", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean", + "value": true + }, + "indexes": { + "deprecated": false, + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchIndexProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List", + "value": [ + { + "code": "NONE", + "numberOfReplicas": -1, + "numberOfShards": -1, + "properties": { + "name": { + "textFields": [ + { + "analyzer": "", + "fieldName": "", + "searchAnalyzer": "" + } + ] + } + } + } + ] + }, + "mongo": { + "enableTransaction": { + "deprecated": false, + "description": "Whether to enable transaction for MongoDB. If enabled, MongoDB will use transactions when upserting data into both MongoDB and Elasticsearch, and will roll back data if an error occurs, no matter they are MongoDB or Elasticsearch errors", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean", + "value": false + } + }, + "sync": { + "performFullSyncAtStartup": { + "deprecated": false, + "description": "Whether to sync existing data from MongoDB to Elasticsearch. If true and the current node is the leader, turms will run a full sync at startup if the data has not been synced yet", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean", + "value": true + } + } + }, + "user": { + "client": { + "password": { + "deprecated": false, + "description": "Elasticsearch password", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string", + "value": "" + }, + "requestHeaders": { + "deprecated": false, + "description": "Elasticsearch HTTP request headers", + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.HttpHeaderProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List", + "value": [] + }, + "uri": { + "deprecated": false, + "description": "Elasticsearch URI", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string", + "value": "http://localhost:9200" + }, + "username": { + "deprecated": false, + "description": "Elasticsearch username", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string", + "value": "elastic" + } + }, + "enabled": { + "deprecated": false, + "description": "Whether to enable this use case", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean", + "value": true + }, + "indexes": { + "deprecated": false, + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchIndexProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List", + "value": [ + { + "code": "NONE", + "numberOfReplicas": -1, + "numberOfShards": -1, + "properties": { + "name": { + "textFields": [ + { + "analyzer": "", + "fieldName": "", + "searchAnalyzer": "" + } + ] + } + } + } + ] + }, + "mongo": { + "enableTransaction": { + "deprecated": false, + "description": "Whether to enable transaction for MongoDB. If enabled, MongoDB will use transactions when upserting data into both MongoDB and Elasticsearch, and will roll back data if an error occurs, no matter they are MongoDB or Elasticsearch errors", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean", + "value": false + } + }, + "sync": { + "performFullSyncAtStartup": { + "deprecated": false, + "description": "Whether to sync existing data from MongoDB to Elasticsearch. If true and the current node is the leader, turms will run a full sync at startup if the data has not been synced yet", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean", + "value": true + } + } + } + } + }, "fake": { "clearAllCollectionsBeforeFaking": { "deprecated": false, diff --git a/turms-server-common/src/test/resources/turms-properties-metadata.json b/turms-server-common/src/test/resources/turms-properties-metadata.json index 32ceea0074..d516d78e4a 100644 --- a/turms-server-common/src/test/resources/turms-properties-metadata.json +++ b/turms-server-common/src/test/resources/turms-properties-metadata.json @@ -2885,6 +2885,164 @@ } } }, + "elasticsearch": { + "enabled": { + "deprecated": false, + "description": "Whether to enable Elasticsearch", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean" + }, + "useCase": { + "group": { + "client": { + "password": { + "deprecated": false, + "description": "Elasticsearch password", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string" + }, + "requestHeaders": { + "deprecated": false, + "description": "Elasticsearch HTTP request headers", + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.HttpHeaderProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List" + }, + "uri": { + "deprecated": false, + "description": "Elasticsearch URI", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string" + }, + "username": { + "deprecated": false, + "description": "Elasticsearch username", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string" + } + }, + "enabled": { + "deprecated": false, + "description": "Whether to enable this use case", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean" + }, + "indexes": { + "deprecated": false, + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchIndexProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List" + }, + "mongo": { + "enableTransaction": { + "deprecated": false, + "description": "Whether to enable transaction for MongoDB. If enabled, MongoDB will use transactions when upserting data into both MongoDB and Elasticsearch, and will roll back data if an error occurs, no matter they are MongoDB or Elasticsearch errors", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean" + } + }, + "sync": { + "performFullSyncAtStartup": { + "deprecated": false, + "description": "Whether to sync existing data from MongoDB to Elasticsearch. If true and the current node is the leader, turms will run a full sync at startup if the data has not been synced yet", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean" + } + } + }, + "user": { + "client": { + "password": { + "deprecated": false, + "description": "Elasticsearch password", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string" + }, + "requestHeaders": { + "deprecated": false, + "description": "Elasticsearch HTTP request headers", + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.HttpHeaderProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List" + }, + "uri": { + "deprecated": false, + "description": "Elasticsearch URI", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string" + }, + "username": { + "deprecated": false, + "description": "Elasticsearch username", + "global": false, + "mutable": false, + "sensitive": false, + "type": "string" + } + }, + "enabled": { + "deprecated": false, + "description": "Whether to enable this use case", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean" + }, + "indexes": { + "deprecated": false, + "elementType": "im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchIndexProperties", + "global": false, + "mutable": false, + "sensitive": false, + "type": "java.util.List" + }, + "mongo": { + "enableTransaction": { + "deprecated": false, + "description": "Whether to enable transaction for MongoDB. If enabled, MongoDB will use transactions when upserting data into both MongoDB and Elasticsearch, and will roll back data if an error occurs, no matter they are MongoDB or Elasticsearch errors", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean" + } + }, + "sync": { + "performFullSyncAtStartup": { + "deprecated": false, + "description": "Whether to sync existing data from MongoDB to Elasticsearch. If true and the current node is the leader, turms will run a full sync at startup if the data has not been synced yet", + "global": false, + "mutable": false, + "sensitive": false, + "type": "boolean" + } + } + } + } + }, "fake": { "clearAllCollectionsBeforeFaking": { "deprecated": false, diff --git a/turms-server-common/src/test/resources/turms-properties-only-mutable-metadata.json b/turms-server-common/src/test/resources/turms-properties-only-mutable-metadata.json index 6b23209822..61757b2d23 100644 --- a/turms-server-common/src/test/resources/turms-properties-only-mutable-metadata.json +++ b/turms-server-common/src/test/resources/turms-properties-only-mutable-metadata.json @@ -791,6 +791,20 @@ } } }, + "elasticsearch": { + "useCase": { + "group": { + "client": {}, + "mongo": {}, + "sync": {} + }, + "user": { + "client": {}, + "mongo": {}, + "sync": {} + } + } + }, "fake": {}, "group": { "activateGroupWhenCreated": { diff --git a/turms-service/src/main/java/im/turms/service/domain/common/service/ExpirableEntityService.java b/turms-service/src/main/java/im/turms/service/domain/common/service/ExpirableEntityService.java index 9f32a8a972..302734380c 100644 --- a/turms-service/src/main/java/im/turms/service/domain/common/service/ExpirableEntityService.java +++ b/turms-service/src/main/java/im/turms/service/domain/common/service/ExpirableEntityService.java @@ -41,6 +41,7 @@ public Date getEntityExpirationDate() { return expirableEntityRepository.getEntityExpirationDate(); } + @Nullable protected Date getResponseDateBasedOnStatusForNewRecord( Date now, @Nullable RequestStatus status, diff --git a/turms-service/src/main/java/im/turms/service/domain/conversation/service/ConversationService.java b/turms-service/src/main/java/im/turms/service/domain/conversation/service/ConversationService.java index 9df7943c70..0ce07a6e3d 100644 --- a/turms-service/src/main/java/im/turms/service/domain/conversation/service/ConversationService.java +++ b/turms-service/src/main/java/im/turms/service/domain/conversation/service/ConversationService.java @@ -351,16 +351,8 @@ public Mono deletePrivateConversations( } public Mono deleteGroupConversations( - @NotNull Set groupIds, + @Nullable Set groupIds, @Nullable ClientSession session) { - try { - Validator.notNull(groupIds, "groupIds"); - } catch (ResponseException e) { - return Mono.error(e); - } - if (groupIds.isEmpty()) { - return OperationResultPublisherPool.ACKNOWLEDGED_DELETE_RESULT; - } return groupConversationRepository.deleteByIds(groupIds, session); } diff --git a/turms-service/src/main/java/im/turms/service/domain/group/access/servicerequest/controller/GroupServiceController.java b/turms-service/src/main/java/im/turms/service/domain/group/access/servicerequest/controller/GroupServiceController.java index 581b749de9..f6fc5e1a37 100644 --- a/turms-service/src/main/java/im/turms/service/domain/group/access/servicerequest/controller/GroupServiceController.java +++ b/turms-service/src/main/java/im/turms/service/domain/group/access/servicerequest/controller/GroupServiceController.java @@ -417,7 +417,20 @@ public ClientRequestHandler handleQueryGroupsRequest() { : null; return groupService.authAndQueryGroups(request.getGroupIdsCount() > 0 ? CollectionUtil.newSet(request.getGroupIdsList()) - : Collections.emptySet(), lastUpdatedDate) + : Collections.emptySet(), + request.hasName() + ? request.getName() + : null, + lastUpdatedDate, + request.hasSkip() + ? request.getSkip() + : null, + request.hasLimit() + ? request.getLimit() + : null, + request.getFieldsToHighlightCount() > 0 + ? request.getFieldsToHighlightList() + : null) .map(groups -> { List groupProtos = new ArrayList<>(groups.size()); diff --git a/turms-service/src/main/java/im/turms/service/domain/group/po/Group.java b/turms-service/src/main/java/im/turms/service/domain/group/po/Group.java index c576ed297e..04d3e647f9 100644 --- a/turms-service/src/main/java/im/turms/service/domain/group/po/Group.java +++ b/turms-service/src/main/java/im/turms/service/domain/group/po/Group.java @@ -19,6 +19,7 @@ import java.util.Date; +import lombok.AllArgsConstructor; import lombok.Data; import im.turms.server.common.domain.common.po.BaseEntity; @@ -36,6 +37,7 @@ /** * @author James Chen */ +@AllArgsConstructor @Data @Document(Group.COLLECTION_NAME) @Sharded @@ -62,7 +64,7 @@ public final class Group extends BaseEntity { private final Long ownerId; @Field(Fields.NAME) - private final String name; + private String name; @Field(Fields.INTRO) private final String intro; diff --git a/turms-service/src/main/java/im/turms/service/domain/group/repository/GroupRepository.java b/turms-service/src/main/java/im/turms/service/domain/group/repository/GroupRepository.java index 93d7b71a25..b072af09b5 100644 --- a/turms-service/src/main/java/im/turms/service/domain/group/repository/GroupRepository.java +++ b/turms-service/src/main/java/im/turms/service/domain/group/repository/GroupRepository.java @@ -167,7 +167,7 @@ public Flux findGroups( return mongoClient.findMany(entityClass, filter, options); } - public Flux findNotDeletedGroups(Set ids, @Nullable Date lastUpdatedDate) { + public Flux findNotDeletedGroups(Collection ids, @Nullable Date lastUpdatedDate) { Filter filter = Filter.newBuilder(3) .in(DomainFieldName.ID, ids) .eq(Group.Fields.DELETION_DATE, null) @@ -175,6 +175,12 @@ public Flux findNotDeletedGroups(Set ids, @Nullable Date lastUpdate return mongoClient.findMany(entityClass, filter); } + public Flux findAllNames() { + QueryOptions options = QueryOptions.newBuilder(1) + .include(Group.Fields.NAME); + return mongoClient.findAll(entityClass, options); + } + public Mono findTypeId(Long groupId) { Filter filter = Filter.newBuilder(1) .eq(DomainFieldName.ID, groupId); diff --git a/turms-service/src/main/java/im/turms/service/domain/group/service/GroupService.java b/turms-service/src/main/java/im/turms/service/domain/group/service/GroupService.java index 11c5a4fd56..fda0efab47 100644 --- a/turms-service/src/main/java/im/turms/service/domain/group/service/GroupService.java +++ b/turms-service/src/main/java/im/turms/service/domain/group/service/GroupService.java @@ -19,9 +19,11 @@ import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import jakarta.annotation.Nullable; @@ -81,6 +83,9 @@ import im.turms.service.domain.user.service.UserPermissionGroupService; import im.turms.service.domain.user.service.UserVersionService; import im.turms.service.infra.proto.ProtoModelConvertor; +import im.turms.service.storage.elasticsearch.ElasticsearchManager; +import im.turms.service.storage.elasticsearch.model.Hit; +import im.turms.service.storage.elasticsearch.model.doc.GroupDoc; import im.turms.service.storage.mongo.OperationResultPublisherPool; import static im.turms.server.common.domain.group.constant.GroupConst.DEFAULT_GROUP_TYPE_ID; @@ -98,6 +103,7 @@ public class GroupService { private static final Logger LOGGER = LoggerFactory.getLogger(GroupService.class); private final Node node; + private final ElasticsearchManager elasticsearchManager; private final GroupRepository groupRepository; private final GroupTypeService groupTypeService; private final GroupMemberService groupMemberService; @@ -121,6 +127,7 @@ public class GroupService { */ public GroupService( Node node, + ElasticsearchManager elasticsearchManager, TurmsPropertiesManager propertiesManager, GroupRepository groupRepository, GroupMemberService groupMemberService, @@ -132,6 +139,7 @@ public GroupService( @Lazy MessageService messageService, MetricsService metricsService) { this.node = node; + this.elasticsearchManager = elasticsearchManager; this.groupRepository = groupRepository; this.groupTypeService = groupTypeService; this.groupMemberService = groupMemberService; @@ -208,22 +216,42 @@ public Mono createGroup( now, muteEndDate, isActive); - return groupRepository.inTransaction(session -> groupRepository.insert(group, session) - .then(groupMemberService.addGroupMember(group - .getId(), creatorId, GroupMemberRole.OWNER, null, now, null, session)) - .then(Mono.defer(() -> { - createdGroupsCounter.increment(); - return groupVersionService.upsert(groupId, now) - .onErrorResume(t -> { - LOGGER.error( - "Caught an error while upserting a version for the group ({}) after creating the group", - groupId, - t); - return Mono.empty(); - }); - })) - .thenReturn(group)) + + Boolean putEsDocInTransaction = + elasticsearchManager.isGroupUseCaseEnabled() && StringUtil.isNotBlank(groupName) + ? elasticsearchManager.isTransactionWithMongoEnabledForGroup() + : null; + + Mono addGroup = groupRepository.inTransaction(session -> { + Mono mono = groupRepository.insert(group, session); + if (Boolean.TRUE.equals(putEsDocInTransaction)) { + mono = mono.then(elasticsearchManager.putGroupDoc(groupId, groupName)); + } + return mono + .then(groupMemberService.addGroupMember(group + .getId(), creatorId, GroupMemberRole.OWNER, null, now, null, session)) + .then(Mono.defer(() -> { + createdGroupsCounter.increment(); + return groupVersionService.upsert(groupId, now) + .onErrorResume(t -> { + LOGGER.error( + "Caught an error while upserting a version for the group ({}) after creating the group", + groupId, + t); + return Mono.empty(); + }); + })) + .thenReturn(group); + }) .retryWhen(TRANSACTION_RETRY); + if (Boolean.FALSE.equals(putEsDocInTransaction)) { + addGroup = addGroup.doOnSuccess(ignored -> elasticsearchManager + .putGroupDoc(groupId, groupName) + .subscribe(null, + t -> LOGGER.error("Caught an error while putting the doc of the group: " + + groupId, t))); + } + return addGroup; } /** @@ -302,11 +330,21 @@ public Mono deleteGroupsAndGroupMembers( deleteLogically = deleteGroupLogicallyByDefault; } boolean finalShouldDeleteLogically = deleteLogically; - return groupRepository.inTransaction(session -> { - Mono updateOrDeleteMono = finalShouldDeleteLogically - ? groupRepository.updateGroupsDeletionDate(groupIds, session) - .map(OperationResultConvertor::update2delete) - : groupRepository.deleteByIds(groupIds, session); + Mono delete = groupRepository.inTransaction(session -> { + Mono updateOrDeleteMono; + if (finalShouldDeleteLogically) { + updateOrDeleteMono = groupRepository.updateGroupsDeletionDate(groupIds, session) + .map(OperationResultConvertor::update2delete); + // For logical deletion, we don't need to update the group doc in Elasticsearch. + } else { + updateOrDeleteMono = groupRepository.deleteByIds(groupIds, session); + if (elasticsearchManager.isGroupUseCaseEnabled() + && elasticsearchManager.isTransactionWithMongoEnabledForGroup()) { + updateOrDeleteMono = updateOrDeleteMono.flatMap(result -> (groupIds == null + ? elasticsearchManager.deleteAllGroupDocs() + : elasticsearchManager.deleteGroupDocs(groupIds)).thenReturn(result)); + } + } return updateOrDeleteMono.flatMap(result -> { long count = result.getDeletedCount(); if (count > 0) { @@ -326,6 +364,15 @@ public Mono deleteGroupsAndGroupMembers( }); }) .retryWhen(TRANSACTION_RETRY); + if (elasticsearchManager.isGroupUseCaseEnabled() + && !elasticsearchManager.isTransactionWithMongoEnabledForGroup()) { + delete = delete.doOnSuccess(result -> (groupIds == null + ? elasticsearchManager.deleteAllGroupDocs() + : elasticsearchManager.deleteGroupDocs(groupIds)).subscribe(null, + t -> LOGGER.error("Failed to delete the docs of the groups: " + + groupIds, t))); + } + return delete; } public Flux queryGroups( @@ -621,20 +668,85 @@ public Mono updateGroupsInformation( muteEndDate)) { return OperationResultPublisherPool.ACKNOWLEDGED_UPDATE_RESULT; } - return groupRepository.updateGroups(groupIds, - typeId, - creatorId, - ownerId, - name, - intro, - announcement, - minimumScore, - isActive, - creationDate, - deletionDate, - muteEndDate, - new Date(), - session); + if (!elasticsearchManager.isGroupUseCaseEnabled() || name == null) { + return groupRepository.updateGroups(groupIds, + typeId, + creatorId, + ownerId, + name, + intro, + announcement, + minimumScore, + isActive, + creationDate, + deletionDate, + muteEndDate, + new Date(), + session); + } + if (elasticsearchManager.isTransactionWithMongoEnabledForGroup()) { + if (session == null) { + return groupRepository.inTransaction(clientSession -> groupRepository + .updateGroups(groupIds, + typeId, + creatorId, + ownerId, + name, + intro, + announcement, + minimumScore, + isActive, + creationDate, + deletionDate, + muteEndDate, + new Date(), + clientSession) + .flatMap(updateResult -> (StringUtil.isBlank(name) + ? elasticsearchManager.deleteGroupDocs(groupIds) + : elasticsearchManager.putGroupDocs(groupIds, name)) + .thenReturn(updateResult))); + } else { + return groupRepository + .updateGroups(groupIds, + typeId, + creatorId, + ownerId, + name, + intro, + announcement, + minimumScore, + isActive, + creationDate, + deletionDate, + muteEndDate, + new Date(), + session) + .flatMap(updateResult -> (StringUtil.isBlank(name) + ? elasticsearchManager.deleteGroupDocs(groupIds) + : elasticsearchManager.putGroupDocs(groupIds, name)) + .thenReturn(updateResult)); + } + } + return groupRepository + .updateGroups(groupIds, + typeId, + creatorId, + ownerId, + name, + intro, + announcement, + minimumScore, + isActive, + creationDate, + deletionDate, + muteEndDate, + new Date(), + session) + .doOnSuccess(updateResult -> (StringUtil.isBlank(name) + ? elasticsearchManager.deleteGroupDocs(groupIds) + : elasticsearchManager.putGroupDocs(groupIds, name)).subscribe(null, + t -> LOGGER.error("Failed to update the docs of the groups: " + + groupIds, t))); } public Mono authAndUpdateGroupInformation( @@ -739,8 +851,15 @@ public Mono authAndUpdateGroupInformation( } public Mono> authAndQueryGroups( - @NotNull Set groupIds, - @Nullable Date lastUpdatedDate) { + @Nullable Set groupIds, + @Nullable String name, + @Nullable Date lastUpdatedDate, + @Nullable Integer skip, + @Nullable Integer limit, + @Nullable List fieldsToHighlight) { + if (StringUtil.isNotBlank(name)) { + return search(skip, limit, groupIds, name, fieldsToHighlight); + } try { Validator.notNull(groupIds, "groupIds"); } catch (ResponseException e) { @@ -1071,4 +1190,72 @@ private Mono> queryGroupIdsFromGroupIdsAndMemberIds( .doFinally(signalType -> recyclableList.recycle()); } + private Mono> search( + @Nullable Integer skip, + @Nullable Integer limit, + @Nullable Set groupIds, + @NotNull String name, + @Nullable List fieldsToHighlight) { + if (!elasticsearchManager.isGroupUseCaseEnabled()) { + return Mono + .error(ResponseException.get(ResponseStatusCode.SEARCHING_GROUP_IS_DISABLED)); + } + boolean highlight = CollectionUtil.isNotEmpty(fieldsToHighlight); + return elasticsearchManager + .searchGroupDocs(skip, limit, name, groupIds, highlight, null, null) + .flatMap(response -> { + List> hits = response.hits() + .hits(); + int count = hits.size(); + if (count == 0) { + return PublisherPool.emptyList(); + } + if (highlight) { + Map groupIdToHighlightedName = + CollectionUtil.newMapWithExpectedSize(count); + List ids = new ArrayList<>(count); + for (Hit hit : hits) { + Long id = Long.parseLong(hit.id()); + Collection> highlightValues = hit.highlight() + .values(); + if (!highlightValues.isEmpty()) { + List values = highlightValues.iterator() + .next(); + if (!values.isEmpty()) { + groupIdToHighlightedName.put(id, values.getFirst()); + } + } + ids.add(id); + } + Mono> mono = groupRepository.findNotDeletedGroups(ids, null) + .collect(CollectorUtil.toList(count)); + return mono.map(groups -> { + if (groups.isEmpty()) { + return Collections.emptyList(); + } + Map groupIdToGroup = + CollectionUtil.newMapWithExpectedSize(groups.size()); + for (Group group : groups) { + groupIdToGroup.put(group.getId(), group); + } + for (Map.Entry entry : groupIdToHighlightedName + .entrySet()) { + Group group = groupIdToGroup.get(entry.getKey()); + if (group != null) { + group.setName(entry.getValue()); + } + } + return groups; + }); + } else { + List ids = new ArrayList<>(count); + for (Hit hit : hits) { + ids.add(Long.parseLong(hit.id())); + } + return groupRepository.findNotDeletedGroups(ids, null) + .collect(CollectorUtil.toList(count)); + } + }); + } + } \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/domain/group/service/GroupVersionService.java b/turms-service/src/main/java/im/turms/service/domain/group/service/GroupVersionService.java index cf3728405a..44580471e5 100644 --- a/turms-service/src/main/java/im/turms/service/domain/group/service/GroupVersionService.java +++ b/turms-service/src/main/java/im/turms/service/domain/group/service/GroupVersionService.java @@ -20,7 +20,6 @@ import java.util.Date; import java.util.Set; import jakarta.annotation.Nullable; -import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import com.mongodb.client.result.DeleteResult; @@ -182,13 +181,8 @@ public Mono upsert(@NotNull Long groupId, @NotNull Date timestamp) } public Mono delete( - @NotEmpty Set groupIds, + @Nullable Set groupIds, @Nullable ClientSession session) { - try { - Validator.notEmpty(groupIds, "groupIds"); - } catch (ResponseException e) { - return Mono.error(e); - } return groupVersionRepository.deleteByIds(groupIds, session); } diff --git a/turms-service/src/main/java/im/turms/service/domain/user/access/servicerequest/controller/UserServiceController.java b/turms-service/src/main/java/im/turms/service/domain/user/access/servicerequest/controller/UserServiceController.java index 78f08c3b23..203d012f5c 100644 --- a/turms-service/src/main/java/im/turms/service/domain/user/access/servicerequest/controller/UserServiceController.java +++ b/turms-service/src/main/java/im/turms/service/domain/user/access/servicerequest/controller/UserServiceController.java @@ -156,8 +156,20 @@ public ClientRequestHandler handleQueryUserProfilesRequest() { return userService .authAndQueryUsersProfile(clientRequest.userId(), CollectionUtil.toSet(request.getUserIdsList()), + request.hasName() + ? request.getName() + : null, request.hasLastUpdatedDate() ? new Date(request.getLastUpdatedDate()) + : null, + request.hasSkip() + ? request.getSkip() + : null, + request.hasLimit() + ? request.getLimit() + : null, + request.getFieldsToHighlightCount() > 0 + ? request.getFieldsToHighlightList() : null) .map(users -> { UserInfosWithVersion.Builder userInfosWithVersionBuilder = diff --git a/turms-service/src/main/java/im/turms/service/domain/user/repository/UserRepository.java b/turms-service/src/main/java/im/turms/service/domain/user/repository/UserRepository.java index 5ab6188484..1971f54d3e 100644 --- a/turms-service/src/main/java/im/turms/service/domain/user/repository/UserRepository.java +++ b/turms-service/src/main/java/im/turms/service/domain/user/repository/UserRepository.java @@ -23,6 +23,7 @@ import jakarta.annotation.Nullable; import com.mongodb.client.result.UpdateResult; +import com.mongodb.reactivestreams.client.ClientSession; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import reactor.core.publisher.Flux; @@ -57,7 +58,8 @@ public Mono updateUsers( @Nullable ProfileAccessStrategy profileAccessStrategy, @Nullable Long permissionGroupId, @Nullable Date registrationDate, - @Nullable Boolean isActive) { + @Nullable Boolean isActive, + @Nullable ClientSession session) { Filter filter = Filter.newBuilder(1) .in(DomainFieldName.ID, userIds); Update update = Update.newBuilder(9) @@ -70,7 +72,7 @@ public Mono updateUsers( .setIfNotNull(User.Fields.REGISTRATION_DATE, registrationDate) .setIfNotNull(User.Fields.IS_ACTIVE, isActive) .setIfNotNull(User.Fields.LAST_UPDATED_DATE, new Date()); - return mongoClient.updateMany(entityClass, filter, update); + return mongoClient.updateMany(session, entityClass, filter, update); } public Mono updateUsersDeletionDate(Set userIds) { @@ -133,6 +135,12 @@ public Mono findName(Long userId) { .map(User::getName); } + public Flux findAllNames() { + QueryOptions options = QueryOptions.newBuilder(1) + .include(User.Fields.NAME); + return mongoClient.findAll(entityClass, options); + } + public Mono findProfileAccessIfNotDeleted(Long userId) { Filter filter = Filter.newBuilder(2) .eq(DomainFieldName.ID, userId) diff --git a/turms-service/src/main/java/im/turms/service/domain/user/service/UserPermissionGroupService.java b/turms-service/src/main/java/im/turms/service/domain/user/service/UserPermissionGroupService.java index 20c6ea1471..dc29678510 100644 --- a/turms-service/src/main/java/im/turms/service/domain/user/service/UserPermissionGroupService.java +++ b/turms-service/src/main/java/im/turms/service/domain/user/service/UserPermissionGroupService.java @@ -62,6 +62,7 @@ public class UserPermissionGroupService { private static final Logger LOGGER = LoggerFactory.getLogger(UserPermissionGroupService.class); private final Map idToPermissionGroup = new ConcurrentHashMap<>(16); + private final Node node; private final UserPermissionGroupRepository userPermissionGroupRepository; private final UserService userService; @@ -82,13 +83,16 @@ public UserPermissionGroupService( .onErrorMap(t -> new RuntimeException( "Caught an error while loading all user permission groups", t)) - .then(Mono.defer( - () -> idToPermissionGroup.containsKey(DEFAULT_USER_PERMISSION_GROUP_ID) - ? Mono.empty() - : addDefaultUserPermissionGroup() - .onErrorMap(t -> new RuntimeException( - "Caught an error while adding the default user permission group", - t)))) + .then(Mono.defer(() -> { + UserPermissionGroup userPermissionGroup = + idToPermissionGroup.get(DEFAULT_USER_PERMISSION_GROUP_ID); + if (userPermissionGroup != null) { + return Mono.empty(); + } + return addDefaultUserPermissionGroup().onErrorMap(t -> new RuntimeException( + "Caught an error while adding the default user permission group", + t)); + })) .block(DurationConst.ONE_MINUTE); LOGGER.info( "Loaded all user permission groups and added the default user permission group"); diff --git a/turms-service/src/main/java/im/turms/service/domain/user/service/UserService.java b/turms-service/src/main/java/im/turms/service/domain/user/service/UserService.java index f0b3fcffa4..917e0ba087 100644 --- a/turms-service/src/main/java/im/turms/service/domain/user/service/UserService.java +++ b/turms-service/src/main/java/im/turms/service/domain/user/service/UserService.java @@ -17,10 +17,12 @@ package im.turms.service.domain.user.service; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.List; +import java.util.Map; import java.util.Set; import jakarta.annotation.Nullable; import jakarta.validation.constraints.NotEmpty; @@ -29,6 +31,7 @@ import com.mongodb.client.result.DeleteResult; import com.mongodb.client.result.UpdateResult; +import com.mongodb.reactivestreams.client.ClientSession; import io.micrometer.core.instrument.Counter; import lombok.Getter; import org.springframework.context.annotation.DependsOn; @@ -43,6 +46,7 @@ import im.turms.server.common.domain.user.po.User; import im.turms.server.common.infra.cluster.node.Node; import im.turms.server.common.infra.cluster.service.idgen.ServiceType; +import im.turms.server.common.infra.collection.CollectionUtil; import im.turms.server.common.infra.collection.CollectorUtil; import im.turms.server.common.infra.exception.ResponseException; import im.turms.server.common.infra.exception.ResponseExceptionPublisherPool; @@ -70,6 +74,9 @@ import im.turms.service.domain.user.service.onlineuser.SessionService; import im.turms.service.infra.metrics.MetricNameConst; import im.turms.service.infra.validation.ValidProfileAccess; +import im.turms.service.storage.elasticsearch.ElasticsearchManager; +import im.turms.service.storage.elasticsearch.model.Hit; +import im.turms.service.storage.elasticsearch.model.doc.UserDoc; import im.turms.service.storage.mongo.OperationResultPublisherPool; import static im.turms.server.common.domain.user.constant.UserConst.DEFAULT_USER_PERMISSION_GROUP_ID; @@ -93,6 +100,7 @@ public class UserService { private final MessageService messageService; private final Node node; + private final ElasticsearchManager elasticsearchManager; private final PasswordManager passwordManager; private final UserRepository userRepository; @@ -117,6 +125,7 @@ public class UserService { public UserService( Node node, + ElasticsearchManager elasticsearchManager, TurmsPropertiesManager propertiesManager, PasswordManager passwordManager, UserRepository userRepository, @@ -129,6 +138,7 @@ public UserService( @Lazy MessageService messageService, MetricsService metricsService) { this.node = node; + this.elasticsearchManager = elasticsearchManager; this.passwordManager = passwordManager; this.userRepository = userRepository; @@ -190,10 +200,8 @@ private void updateProperties(TurmsProperties properties) { .getType() .isUserCollectionBasedAuthEnabled(); - boolean localCheckIfTargetActiveAndNotDeleted = - messageProperties.isCheckIfTargetActiveAndNotDeleted(); - checkIfTargetActiveAndNotDeleted = - isUserCollectionBasedAuthEnabled && localCheckIfTargetActiveAndNotDeleted; + checkIfTargetActiveAndNotDeleted = isUserCollectionBasedAuthEnabled + && messageProperties.isCheckIfTargetActiveAndNotDeleted(); } public Mono isAllowedToSendMessageToTarget( @@ -310,20 +318,39 @@ public Mono addUser( now, isActive); Long finalId = id; - return userRepository.inTransaction(session -> userRepository.insert(user, session) - .then(userRelationshipGroupService - .createRelationshipGroup(finalId, 0, "", now, session)) - .then(userVersionService.upsertEmptyUserVersion(user.getId(), date, session) - .onErrorResume(t -> { - LOGGER.error( - "Caught an error while upserting a version for the user ({}) after creating the user", - user.getId(), - t); - return Mono.empty(); - })) - .thenReturn(user)) - .retryWhen(TRANSACTION_RETRY) - .doOnSuccess(ignored -> registeredUsersCounter.increment()); + String finalName = name; + Boolean putEsDocInTransaction = + elasticsearchManager.isUserUseCaseEnabled() && !finalName.isBlank() + ? elasticsearchManager.isTransactionWithMongoEnabledForUser() + : null; + Mono addUser = userRepository.inTransaction(session -> { + Mono mono = userRepository.insert(user, session) + .then(userRelationshipGroupService + .createRelationshipGroup(finalId, 0, "", now, session)); + if (Boolean.TRUE.equals(putEsDocInTransaction)) { + mono = mono.then(elasticsearchManager.putUserDoc(finalId, finalName)); + } + return mono.then(userVersionService.upsertEmptyUserVersion(user.getId(), date, session) + .onErrorResume(t -> { + LOGGER.error( + "Caught an error while upserting a version for the user ({}) after creating the user", + user.getId(), + t); + return Mono.empty(); + })) + .thenReturn(user) + .retryWhen(TRANSACTION_RETRY); + }); + return addUser.doOnSuccess(ignored -> { + registeredUsersCounter.increment(); + if (Boolean.FALSE.equals(putEsDocInTransaction)) { + elasticsearchManager.putUserDoc(finalId, finalName) + .subscribe(null, + t -> LOGGER + .error("Caught an error while creating a doc for the user: " + + finalId, t)); + } + }); } /** @@ -371,10 +398,21 @@ public Mono isAllowToQueryUserProfile( public Mono> authAndQueryUsersProfile( @NotNull Long requesterId, - @NotNull Set userIds, - @Nullable Date lastUpdatedDate) { + @Nullable Set userIds, + @Nullable String name, + @Nullable Date lastUpdatedDate, + @Nullable Integer skip, + @Nullable Integer limit, + @Nullable List fieldsToHighlight) { try { Validator.notNull(requesterId, "requesterId"); + } catch (ResponseException e) { + return Mono.error(e); + } + if (StringUtil.isNotBlank(name)) { + return search(skip, limit, userIds, name, fieldsToHighlight); + } + try { Validator.notNull(userIds, "userIds"); } catch (ResponseException e) { return Mono.error(e); @@ -432,17 +470,20 @@ public Mono deleteUsers( if (deleteLogically) { deleteOrUpdateMono = userRepository.updateUsersDeletionDate(userIds) .map(OperationResultConvertor::update2delete); + // For logical deletion, we don't need to update the user doc in Elasticsearch. } else { + Boolean deleteEsDocInTransaction = elasticsearchManager.isUserUseCaseEnabled() + ? elasticsearchManager.isTransactionWithMongoEnabledForUser() + : null; deleteOrUpdateMono = userRepository .inTransaction(session -> userRepository.deleteByIds(userIds, session) .flatMap(result -> { - long count = result.getDeletedCount(); - if (count > 0) { - deletedUsersCounter.increment(count); - } // TODO: Remove data on Redis - return userRelationshipService - .deleteAllRelationships(userIds, session, false) + return (Boolean.TRUE.equals(deleteEsDocInTransaction) + ? elasticsearchManager.deleteUserDocs(userIds) + : Mono.empty()) + .then(userRelationshipService + .deleteAllRelationships(userIds, session, false)) .then(userRelationshipGroupService .deleteAllRelationshipGroups(userIds, session, @@ -466,7 +507,22 @@ public Mono deleteUsers( t))) .thenReturn(result); })) - .retryWhen(TRANSACTION_RETRY); + .retryWhen(TRANSACTION_RETRY) + .doOnSuccess(result -> { + long count = result.getDeletedCount(); + if (count > 0) { + deletedUsersCounter.increment(count); + } + }); + if (Boolean.FALSE.equals(deleteEsDocInTransaction)) { + deleteOrUpdateMono = deleteOrUpdateMono.doOnSuccess(result -> elasticsearchManager + .deleteUserDocs(userIds) + .subscribe(null, + t -> LOGGER.error( + "Caught an error while deleting the docs of the users: " + + userIds, + t))); + } } return deleteOrUpdateMono.doOnNext(ignored -> sessionService .disconnect(userIds, SessionCloseStatus.USER_IS_DELETED_OR_INACTIVATED) @@ -592,6 +648,74 @@ public Mono updateUsers( byte[] password = rawPassword == null ? null : passwordManager.encodeUserPassword(rawPassword); + + if (name == null || !elasticsearchManager.isUserUseCaseEnabled()) { + return updateUsers(userIds, + name, + intro, + profilePicture, + profileAccessStrategy, + permissionGroupId, + registrationDate, + isActive, + password, + null); + } + if (elasticsearchManager.isTransactionWithMongoEnabledForUser()) { + return userRepository.inTransaction(session -> updateUsers(userIds, + name, + intro, + profilePicture, + profileAccessStrategy, + permissionGroupId, + registrationDate, + isActive, + password, + session).flatMap( + updateResult -> (name.isBlank() + ? elasticsearchManager.deleteUserDocs(userIds) + : elasticsearchManager.putUserDocs(userIds, name)) + .thenReturn(updateResult))); + } + return updateUsers(userIds, + name, + intro, + profilePicture, + profileAccessStrategy, + permissionGroupId, + registrationDate, + isActive, + password, + null).doOnSuccess(ignored -> { + if (name.isBlank()) { + elasticsearchManager.deleteUserDocs(userIds) + .subscribe(null, + t -> LOGGER.error( + "Caught an error while deleting the docs of the users {}", + userIds, + t)); + } else { + elasticsearchManager.putUserDocs(userIds, name) + .subscribe(null, + t -> LOGGER.error( + "Caught an error while updating the docs of the users {}", + userIds, + t)); + } + }); + } + + private Mono updateUsers( + Set userIds, + @Nullable String name, + @Nullable String intro, + @Nullable String profilePicture, + @Nullable ProfileAccessStrategy profileAccessStrategy, + @Nullable Long permissionGroupId, + @Nullable Date registrationDate, + @Nullable Boolean isActive, + byte[] password, + @Nullable ClientSession session) { return userRepository .updateUsers(userIds, password, @@ -601,7 +725,8 @@ public Mono updateUsers( profileAccessStrategy, permissionGroupId, registrationDate, - isActive) + isActive, + session) .flatMap(result -> Boolean.FALSE.equals(isActive) && result.getModifiedCount() > 0 ? sessionService .disconnect(userIds, @@ -617,4 +742,72 @@ public Mono updateUsers( : Mono.just(result)); } + // TODO: support PIT to ensure the result view is consistent within a specified time. + private Mono> search( + @Nullable Integer skip, + @Nullable Integer limit, + @Nullable Set userIds, + @NotNull String name, + @Nullable List fieldsToHighlight) { + if (!elasticsearchManager.isUserUseCaseEnabled()) { + return Mono.error(ResponseException.get(ResponseStatusCode.SEARCHING_USER_IS_DISABLED)); + } + boolean highlight = CollectionUtil.isNotEmpty(fieldsToHighlight); + return elasticsearchManager + .searchUserDocs(skip, limit, name, userIds, highlight, null, null) + .flatMap(response -> { + List> hits = response.hits() + .hits(); + int count = hits.size(); + if (count == 0) { + return PublisherPool.emptyList(); + } + if (highlight) { + Map userIdToHighlightedName = + CollectionUtil.newMapWithExpectedSize(count); + List ids = new ArrayList<>(count); + for (Hit hit : hits) { + Long id = Long.parseLong(hit.id()); + Collection> highlightValues = hit.highlight() + .values(); + if (!highlightValues.isEmpty()) { + List values = highlightValues.iterator() + .next(); + if (!values.isEmpty()) { + userIdToHighlightedName.put(id, values.getFirst()); + } + } + ids.add(id); + } + Mono> mono = userRepository.findNotDeletedUserProfiles(ids, null) + .collect(CollectorUtil.toList(count)); + return mono.map(users -> { + if (users.isEmpty()) { + return Collections.emptyList(); + } + Map userIdToUser = + CollectionUtil.newMapWithExpectedSize(users.size()); + for (User user : users) { + userIdToUser.put(user.getId(), user); + } + for (Map.Entry entry : userIdToHighlightedName + .entrySet()) { + User user = userIdToUser.get(entry.getKey()); + if (user != null) { + user.setName(entry.getValue()); + } + } + return users; + }); + } else { + List ids = new ArrayList<>(count); + for (Hit hit : hits) { + ids.add(Long.parseLong(hit.id())); + } + return userRepository.findNotDeletedUserProfiles(ids, null) + .collect(CollectorUtil.toList(count)); + } + }); + } + } \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/DefaultLanguageSettings.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/DefaultLanguageSettings.java new file mode 100644 index 0000000000..0691e8d619 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/DefaultLanguageSettings.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch; + +import java.util.List; +import java.util.Map; +import jakarta.annotation.Nullable; + +import im.turms.server.common.infra.property.env.service.env.elasticsearch.LanguageCode; +import im.turms.service.storage.elasticsearch.model.IndexSettingsAnalysis; +import im.turms.service.storage.elasticsearch.model.Property; + +/** + * @author James Chen + */ +public class DefaultLanguageSettings { + + // Use the analyzer name as the field name + // so that it is convenient and consistent to introduce + // new fields with different analyzers in the future. + public static final String ANALYZER_KUROMOJI = "turms_kuromoji_analyzer"; + public static final String ANALYZER_NGRAM = "turms_ngram_analyzer"; + public static final String TOKENIZER_KUROMOJI = "turms_kuromoji_tokenizer"; + public static final String TOKENIZER_NGRAM = "turms_ngram_tokenizer"; + + public static final IndexTextFieldSetting DEFAULT = new IndexTextFieldSetting( + Map.of("standard", new Property(Property.Type.TEXT, "standard", null, null)), + null); + + private static final IndexTextFieldSetting JA = new IndexTextFieldSetting( + Map.of("kuromoji", + new Property(Property.Type.TEXT, ANALYZER_KUROMOJI, null, null), + "ngram", + new Property(Property.Type.TEXT, ANALYZER_NGRAM, null, null)), + new IndexSettingsAnalysis( + Map.of( + // Used to improve precision. + ANALYZER_KUROMOJI, + Map.of("type", + "custom", + "char_filter", + List.of("icu_normalizer"), + "tokenizer", + TOKENIZER_KUROMOJI, + "filter", + List.of("kuromoji_baseform", + "kuromoji_part_of_speech", + "cjk_width", + "ja_stop", + "kuromoji_stemmer", + "lowercase")), + // Used to improve recall. + ANALYZER_NGRAM, + Map.of("type", + "custom", + "char_filter", + List.of("icu_normalizer"), + "tokenizer", + TOKENIZER_NGRAM, + "filter", + List.of("lowercase"))), + null, + null, + null, + Map.of(TOKENIZER_KUROMOJI, + Map.of("mode", "search", "type", "kuromoji_tokenizer"), + TOKENIZER_NGRAM, + Map.of("type", + "ngram", + // Use the same values to avoid disk usage explosion. + "min_gram", + 2, + "max_gram", + 2, + "token_char", + List.of("letter", "digit"))))); + private static final IndexTextFieldSetting ZH = new IndexTextFieldSetting( + Map.of("ik", new Property(Property.Type.TEXT, "ik_max_word", "ik_smart", null)), + null); + + private static final Map LANGUAGE_CODE_TO_SETTING = + Map.of(LanguageCode.NONE, DEFAULT, LanguageCode.JA, JA, LanguageCode.ZH, ZH); + + private DefaultLanguageSettings() { + } + + @Nullable + public static IndexTextFieldSetting getSetting(LanguageCode code) { + return LANGUAGE_CODE_TO_SETTING.get(code); + } + +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ElasticsearchClient.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ElasticsearchClient.java new file mode 100644 index 0000000000..51d1d53aa5 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ElasticsearchClient.java @@ -0,0 +1,271 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.function.Function; +import java.util.function.Supplier; +import jakarta.annotation.Nullable; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectReader; +import io.netty.buffer.ByteBuf; +import io.netty.handler.codec.http.HttpMethod; +import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; +import reactor.netty.http.client.HttpClientResponse; + +import im.turms.server.common.access.admin.web.HttpUtil; +import im.turms.server.common.infra.io.InputOutputException; +import im.turms.server.common.infra.json.JsonCodecPool; +import im.turms.server.common.infra.json.JsonUtil; +import im.turms.service.storage.elasticsearch.model.BulkRequest; +import im.turms.service.storage.elasticsearch.model.BulkResponse; +import im.turms.service.storage.elasticsearch.model.ClosePointInTimeRequest; +import im.turms.service.storage.elasticsearch.model.CreateIndexRequest; +import im.turms.service.storage.elasticsearch.model.DeleteByQueryRequest; +import im.turms.service.storage.elasticsearch.model.DeleteByQueryResponse; +import im.turms.service.storage.elasticsearch.model.DeleteResponse; +import im.turms.service.storage.elasticsearch.model.ErrorResponse; +import im.turms.service.storage.elasticsearch.model.HealthResponse; +import im.turms.service.storage.elasticsearch.model.SearchRequest; +import im.turms.service.storage.elasticsearch.model.SearchResponse; +import im.turms.service.storage.elasticsearch.model.UpdateByQueryRequest; +import im.turms.service.storage.elasticsearch.model.UpdateByQueryResponse; + +/** + * @author James Chen + * @implNote Don't use the Elasticsearch official implementation because: + *

+ * 1. Its jar (elasticsearch-java:8.13.0 is 11.2 MB) is unnecessarily large for our use + * cases. + *

+ * 2. Our implementation is more concise and efficient. + */ +public class ElasticsearchClient { + + private static final Mono EMPTY_RESPONSE = Mono.error(new RuntimeException("Empty response")); + public static final ObjectReader READER = JsonCodecPool.MAPPER.reader() + .withoutFeatures(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); + private static final ObjectReader READER_HEALTH_RESPONSE = READER.forType(HealthResponse.class); + private static final ObjectReader READER_DELETE_RESPONSE = READER.forType(DeleteResponse.class); + private static final ObjectReader READER_DELETE_BY_QUERY_RESPONSE = + READER.forType(DeleteByQueryResponse.class); + private static final ObjectReader READER_UPDATE_BY_QUERY_RESPONSE = + READER.forType(UpdateByQueryResponse.class); + private static final ObjectReader READER_ERROR_RESPONSE = READER.forType(ErrorResponse.class); + + private final HttpClient httpClient; + + public ElasticsearchClient(HttpClient httpClient) { + this.httpClient = httpClient; + } + + public Mono healthcheck() { + HttpClient.ResponseReceiver send = httpClient.request(HttpMethod.GET) + .uri("/_cluster/health"); + return parseResponse(send, inputStream -> { + try { + return READER_HEALTH_RESPONSE.readValue(inputStream); + } catch (IOException e) { + throw new InputOutputException("Failed to parse the health response", e); + } + }); + } + + public Mono putIndex(String index, CreateIndexRequest request) { + HttpClient.ResponseReceiver send = httpClient.put() + .uri("/" + + index) + .send(Mono.fromCallable(() -> JsonUtil.write(512, request))); + return ignoreResponseBody(send); + } + + public Mono putDoc(String index, String id, Supplier payloadSupplier) { + HttpClient.ResponseReceiver send = httpClient.put() + .uri("/" + + index + + "/_doc/" + + id) + .send(Mono.fromCallable(payloadSupplier::get)); + return ignoreResponseBody(send); + } + + public Mono deleteDoc(String index, String id) { + HttpClient.ResponseReceiver send = httpClient.delete() + .uri("/" + + index + + "/_doc/" + + id); + return parseResponse(send, inputStream -> { + try { + return READER_DELETE_RESPONSE.readValue(inputStream); + } catch (IOException e) { + throw new InputOutputException("Failed to parse the result", e); + } + }); + } + + public Mono deleteByQuery(String index, DeleteByQueryRequest request) { + HttpClient.ResponseReceiver send = httpClient.post() + .uri("/" + + index + + "/_delete_by_query") + .send(Mono.fromCallable(() -> JsonUtil.write(request))); + return parseResponse(send, inputStream -> { + try { + return READER_DELETE_BY_QUERY_RESPONSE.readValue(inputStream); + } catch (IOException e) { + throw new InputOutputException("Failed to parse the result", e); + } + }); + } + + public Mono updateByQuery(String index, UpdateByQueryRequest request) { + HttpClient.ResponseReceiver send = httpClient.post() + .uri("/" + + index + + "/_update_by_query") + .send(Mono.fromCallable(() -> JsonUtil.write(request))); + return parseResponse(send, inputStream -> { + try { + return READER_UPDATE_BY_QUERY_RESPONSE.readValue(inputStream); + } catch (IOException e) { + throw new InputOutputException("Failed to parse the result", e); + } + }); + } + + public Mono> search( + String index, + SearchRequest request, + ObjectReader reader) { + HttpClient.ResponseReceiver send = httpClient.request(HttpMethod.GET) + .uri("/" + + index + + "/_search") + .send(Mono.fromCallable(() -> JsonUtil.write(request))); + return parseResponse(send, inputStream -> { + try { + return reader.readValue(inputStream); + } catch (IOException e) { + throw new InputOutputException("Failed to parse the result", e); + } + }); + } + + public Mono bulk(BulkRequest request) { + HttpClient.ResponseReceiver send = httpClient.post() + .uri("/_bulk") + .send(Mono.fromCallable(() -> JsonUtil.write(request))); + return parseResponse(send, inputStream -> { + try { + return READER.readValue(inputStream, BulkResponse.class); + } catch (IOException e) { + throw new InputOutputException("Failed to parse the result", e); + } + }); + } + + public Mono deletePit(String scrollId) { + HttpClient.ResponseReceiver send = httpClient.delete() + .uri("/_pit") + .send(Mono.fromCallable(() -> { + ClosePointInTimeRequest request = new ClosePointInTimeRequest(scrollId); + return JsonUtil.write(request); + })); + return ignoreResponseBody(send); + } + + private Mono ignoreResponseBody(HttpClient.ResponseReceiver responseReceiver) { + return responseReceiver.responseSingle((response, byteBufMono) -> { + if (HttpUtil.isSuccess(response.status())) { + return Mono.empty(); + } + return byteBufMono.asInputStream() + .switchIfEmpty(emptyResponse()) + .flatMap(inputStream -> handleResponse(null, response, inputStream)); + }); + } + + private Mono parseResponse( + HttpClient.ResponseReceiver responseReceiver, + Function responseTransformer) { + return responseReceiver.responseSingle((response, byteBufMono) -> byteBufMono + .asInputStream() + .switchIfEmpty(emptyResponse()) + .flatMap( + inputStream -> handleResponse(responseTransformer, response, inputStream))); + } + + private Mono handleResponse( + @Nullable Function responseTransformer, + HttpClientResponse response, + InputStream inputStream) { + try { + if (inputStream.available() == 0) { + return Mono.error(new RuntimeException( + "HTTP response error status: " + + response.status())); + } + } catch (IOException e) { + return Mono.error(new RuntimeException("Failed to read HTTP response payload", e)); + } + if (!HttpUtil.isSuccess(response.status())) { + inputStream.mark(0); + try { + ErrorResponse errorResponse = READER_ERROR_RESPONSE.readValue(inputStream); + return Mono.error(new ErrorResponseException(errorResponse)); + } catch (Exception e) { + try { + inputStream.reset(); + return Mono.error(new RuntimeException( + "HTTP response error status: " + + response.status() + + ". Response payload: " + + convertStreamToString(inputStream))); + } catch (IOException ex) { + return Mono.error(new RuntimeException( + "HTTP response error status: " + + response.status())); + } + } + } + if (responseTransformer == null) { + return Mono.empty(); + } + T responsePayload = responseTransformer.apply(inputStream); + return Mono.just(responsePayload); + } + + private Mono emptyResponse() { + return EMPTY_RESPONSE; + } + + @Nullable + private static String convertStreamToString(InputStream inputStream) { + try { + return new String(inputStream.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException e) { + return null; + } + } + +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ElasticsearchManager.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ElasticsearchManager.java new file mode 100644 index 0000000000..76f9c1bc87 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ElasticsearchManager.java @@ -0,0 +1,907 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import jakarta.annotation.Nullable; +import jakarta.validation.constraints.NotNull; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectReader; +import io.netty.handler.codec.http.HttpHeaderNames; +import lombok.Getter; +import org.eclipse.collections.impl.set.mutable.UnifiedSet; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.netty.http.client.HttpClient; + +import im.turms.server.common.domain.user.po.User; +import im.turms.server.common.infra.cluster.node.Node; +import im.turms.server.common.infra.cluster.service.idgen.ServiceType; +import im.turms.server.common.infra.codec.Base64Util; +import im.turms.server.common.infra.collection.CollectionUtil; +import im.turms.server.common.infra.collection.CollectorUtil; +import im.turms.server.common.infra.json.JsonUtil; +import im.turms.server.common.infra.lang.StringUtil; +import im.turms.server.common.infra.logging.core.logger.Logger; +import im.turms.server.common.infra.logging.core.logger.LoggerFactory; +import im.turms.server.common.infra.property.TurmsPropertiesManager; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchClientProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchGroupUseCaseProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchIndexProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchIndexPropertiesFieldProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchIndexTextFieldProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchUseCasesProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.ElasticsearchUserUseCaseProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.HttpHeaderProperties; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.LanguageCode; +import im.turms.server.common.infra.property.env.service.env.elasticsearch.TurmsElasticsearchProperties; +import im.turms.server.common.infra.time.DurationConst; +import im.turms.server.common.storage.mongo.DomainFieldName; +import im.turms.server.common.storage.mongo.TurmsMongoClient; +import im.turms.server.common.storage.mongo.exception.DuplicateKeyException; +import im.turms.server.common.storage.mongo.operation.option.Filter; +import im.turms.server.common.storage.mongo.operation.option.Update; +import im.turms.service.domain.group.po.Group; +import im.turms.service.domain.group.repository.GroupRepository; +import im.turms.service.domain.user.repository.UserRepository; +import im.turms.service.storage.elasticsearch.model.BulkRequest; +import im.turms.service.storage.elasticsearch.model.CreateIndexRequest; +import im.turms.service.storage.elasticsearch.model.DeleteByQueryRequest; +import im.turms.service.storage.elasticsearch.model.DeleteByQueryResponse; +import im.turms.service.storage.elasticsearch.model.DeleteResponse; +import im.turms.service.storage.elasticsearch.model.DynamicMapping; +import im.turms.service.storage.elasticsearch.model.ErrorResponse; +import im.turms.service.storage.elasticsearch.model.FieldCollapse; +import im.turms.service.storage.elasticsearch.model.HealthStatus; +import im.turms.service.storage.elasticsearch.model.Highlight; +import im.turms.service.storage.elasticsearch.model.IndexSettings; +import im.turms.service.storage.elasticsearch.model.IndexSettingsAnalysis; +import im.turms.service.storage.elasticsearch.model.OperationType; +import im.turms.service.storage.elasticsearch.model.PointInTimeReference; +import im.turms.service.storage.elasticsearch.model.Property; +import im.turms.service.storage.elasticsearch.model.Script; +import im.turms.service.storage.elasticsearch.model.SearchRequest; +import im.turms.service.storage.elasticsearch.model.SearchResponse; +import im.turms.service.storage.elasticsearch.model.TypeMapping; +import im.turms.service.storage.elasticsearch.model.UpdateByQueryRequest; +import im.turms.service.storage.elasticsearch.model.UpdateByQueryResponse; +import im.turms.service.storage.elasticsearch.model.doc.BaseDoc; +import im.turms.service.storage.elasticsearch.model.doc.GroupDoc; +import im.turms.service.storage.elasticsearch.model.doc.UserDoc; +import im.turms.service.storage.elasticsearch.mongo.SyncLog; +import im.turms.service.storage.elasticsearch.mongo.SyncStatus; + +/** + * @author James Chen + */ +@Component +public class ElasticsearchManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(ElasticsearchManager.class); + + private static final String USER_INDEX = "turms_user"; + private static final String GROUP_INDEX = "turms_group"; + + private static final int SYNC_BATCH_SIZE = 1000; + + public static final String PRE_TAG = "\u0002"; + public static final String POST_TAG = "\u0003"; + + private static final Highlight HIGHLIGHT_NAME = new Highlight( + Map.of(BaseDoc.Fields.NAME + + "*", Collections.emptyMap()), + List.of(PRE_TAG), + List.of(POST_TAG), + "plain", + // Use 0 so that Elasticsearch won't trim our tags. + 0); + + private static final ObjectReader READER_USER_DOC = + ElasticsearchClient.READER.forType(new TypeReference>() { + }); + private static final ObjectReader READER_GROUP_DOC = + ElasticsearchClient.READER.forType(new TypeReference>() { + }); + public static final FieldCollapse COLLAPSE_ID = new FieldCollapse(BaseDoc.Fields.ID); + + private final ElasticsearchClient elasticsearchClientForUserDocs; + private final ElasticsearchClient elasticsearchClientForGroupDocs; + private final boolean shareClient; + + @Getter + private final boolean isUserUseCaseEnabled; + @Getter + private final boolean isGroupUseCaseEnabled; + @Getter + private final boolean isTransactionWithMongoEnabledForUser; + @Getter + private final boolean isTransactionWithMongoEnabledForGroup; + private List allIndexesForUserDocs = Collections.emptyList(); + private List allIndexesForGroupDocs = Collections.emptyList(); + private boolean onlyOneIndexForUserDocs; + private boolean onlyOneIndexForGroupDocs; + + private final Node node; + private final UserRepository userRepository; + private final GroupRepository groupRepository; + + public ElasticsearchManager( + Node node, + TurmsPropertiesManager propertiesManager, + UserRepository userRepository, + GroupRepository groupRepository) { + this.node = node; + this.userRepository = userRepository; + this.groupRepository = groupRepository; + + TurmsElasticsearchProperties properties = propertiesManager.getLocalProperties() + .getService() + .getElasticsearch(); + + ElasticsearchUseCasesProperties useCasesProperties = properties.getUseCase(); + ElasticsearchUserUseCaseProperties userUseCaseProperties = useCasesProperties.getUser(); + ElasticsearchGroupUseCaseProperties groupUseCaseProperties = useCasesProperties.getGroup(); + boolean enabled = properties.isEnabled(); + isUserUseCaseEnabled = enabled && userUseCaseProperties.isEnabled(); + isGroupUseCaseEnabled = enabled && groupUseCaseProperties.isEnabled(); + isTransactionWithMongoEnabledForUser = isUserUseCaseEnabled + && userUseCaseProperties.getMongo() + .isEnableTransaction(); + isTransactionWithMongoEnabledForGroup = isGroupUseCaseEnabled + && groupUseCaseProperties.getMongo() + .isEnableTransaction(); + + if (!isUserUseCaseEnabled && !isGroupUseCaseEnabled) { + elasticsearchClientForUserDocs = null; + elasticsearchClientForGroupDocs = null; + shareClient = false; + return; + } + + ElasticsearchClientProperties elasticsearchClientForGroupProperties = + groupUseCaseProperties.getClient(); + if (isUserUseCaseEnabled) { + ElasticsearchClientProperties elasticsearchClientForUserProperties = + userUseCaseProperties.getClient(); + elasticsearchClientForUserDocs = + initElasticsearchClient(elasticsearchClientForUserProperties); + if (isGroupUseCaseEnabled) { + if (elasticsearchClientForUserProperties + .equals(elasticsearchClientForGroupProperties)) { + elasticsearchClientForGroupDocs = elasticsearchClientForUserDocs; + shareClient = true; + } else { + elasticsearchClientForGroupDocs = + initElasticsearchClient(elasticsearchClientForGroupProperties); + shareClient = false; + } + } else { + elasticsearchClientForGroupDocs = null; + shareClient = false; + } + } else if (isGroupUseCaseEnabled) { + elasticsearchClientForUserDocs = null; + elasticsearchClientForGroupDocs = + initElasticsearchClient(elasticsearchClientForGroupProperties); + shareClient = false; + } else { + elasticsearchClientForUserDocs = null; + elasticsearchClientForGroupDocs = null; + shareClient = false; + } + + init(userUseCaseProperties, groupUseCaseProperties).blockLast(DurationConst.FIVE_MINUTES); + } + + private ElasticsearchClient initElasticsearchClient(ElasticsearchClientProperties properties) { + String uri = properties.getUri(); + List requestHeaders = properties.getRequestHeaders(); + HttpClient httpClient = HttpClient.create() + .baseUrl(uri) + .headers(entries -> entries.add(HttpHeaderNames.CONTENT_TYPE, "application/json")); + String username = properties.getUsername(); + String password = properties.getPassword(); + boolean hasUsername = StringUtil.isNotBlank(username); + boolean hasPassword = StringUtil.isNotBlank(password); + boolean hasRequestHeaders = CollectionUtil.isNotEmpty(requestHeaders); + if (hasUsername || hasPassword || hasRequestHeaders) { + httpClient = httpClient.headers(headers -> { + if (hasUsername || hasPassword) { + headers.add("Authorization", + "Basic " + + Base64Util.encodeToString((hasUsername + ? username + : "") + + ":" + + (hasPassword + ? password + : ""))); + } + if (hasRequestHeaders) { + for (HttpHeaderProperties requestHeader : requestHeaders) { + headers.add(requestHeader.getName(), requestHeader.getValue()); + } + } + }); + } + return new ElasticsearchClient(httpClient); + } + + private Flux init( + ElasticsearchUserUseCaseProperties userUseCaseProperties, + ElasticsearchGroupUseCaseProperties groupUseCaseProperties) { + List> jobs = List.of(ensureHealthy(), + createIndexes(userUseCaseProperties, groupUseCaseProperties), + fullSyncIfEnabled(userUseCaseProperties, groupUseCaseProperties)); + return Flux.concat(jobs); + } + + private Mono fullSyncIfEnabled( + ElasticsearchUserUseCaseProperties userUseCaseProperties, + ElasticsearchGroupUseCaseProperties groupUseCaseProperties) { + boolean isUserDocsFullSyncEnabled = isUserUseCaseEnabled + && userUseCaseProperties.getSync() + .isPerformFullSyncAtStartup(); + boolean isGroupDocsFullSyncEnabled = isGroupUseCaseEnabled + && groupUseCaseProperties.getSync() + .isPerformFullSyncAtStartup(); + if (!isUserDocsFullSyncEnabled && !isGroupDocsFullSyncEnabled) { + return Mono.empty(); + } + if (!node.isLocalNodeLeader()) { + LOGGER.info("Skip the full sync because this node is not the cluster leader"); + return Mono.empty(); + } + if (shareClient) { + return TurmsMongoClient.of(userUseCaseProperties.getMongo(), "elasticsearch") + .flatMap(mongoClient -> { + mongoClient.registerEntitiesByClasses(SyncLog.class); + return mongoClient.findMany(SyncLog.class, + Filter.newBuilder(1) + .inIfNotNullForEnumStrings(SyncLog.Fields.status, + List.of(SyncStatus.IN_PROGRESS, + SyncStatus.COMPLETED))) + .collect(CollectorUtil.toChunkedList()) + .flatMap(syncLogs -> performFullSyncs(mongoClient, + syncLogs, + isUserDocsFullSyncEnabled, + isGroupDocsFullSyncEnabled)); + }); + } + + Mono performFullSyncForUserDocs = Mono.defer(() -> TurmsMongoClient + .of(userUseCaseProperties.getMongo(), "elasticsearch-for-user-docs") + .flatMap(mongoClient -> { + mongoClient.registerEntitiesByClasses(SyncLog.class); + return mongoClient.findMany(SyncLog.class, + Filter.newBuilder(1) + .inIfNotNullForEnumStrings(SyncLog.Fields.status, + List.of(SyncStatus.IN_PROGRESS, SyncStatus.COMPLETED))) + .collect(CollectorUtil.toChunkedList()) + .flatMap(syncLogs -> performFullSyncs(mongoClient, + syncLogs, + true, + false)); + })); + Mono performFullSyncForGroupDocs = Mono.defer(() -> TurmsMongoClient + .of(groupUseCaseProperties.getMongo(), "elasticsearch-for-group-docs") + .flatMap(mongoClient -> { + mongoClient.registerEntitiesByClasses(SyncLog.class); + return mongoClient.findMany(SyncLog.class, + Filter.newBuilder(1) + .inIfNotNullForEnumStrings(SyncLog.Fields.status, + List.of(SyncStatus.IN_PROGRESS, SyncStatus.COMPLETED))) + .collect(CollectorUtil.toChunkedList()) + .flatMap(syncLogs -> performFullSyncs(mongoClient, + syncLogs, + false, + true)); + })); + if (isUserDocsFullSyncEnabled && isGroupDocsFullSyncEnabled) { + return performFullSyncForUserDocs.then(performFullSyncForGroupDocs); + } else if (isUserDocsFullSyncEnabled) { + return performFullSyncForUserDocs; + } else { + return performFullSyncForGroupDocs; + } + } + + @NotNull + private Mono performFullSyncs( + TurmsMongoClient mongoClient, + List syncLogs, + boolean isUserFullSyncEnabled, + boolean isGroupFullSyncEnabled) { + Map> syncedOrSyncingCollectionToIndexes; + if (syncLogs.isEmpty()) { + syncedOrSyncingCollectionToIndexes = Collections.emptyMap(); + } else { + syncedOrSyncingCollectionToIndexes = + CollectionUtil.newMapWithExpectedSize(syncLogs.size()); + for (SyncLog syncLog : syncLogs) { + syncedOrSyncingCollectionToIndexes + .computeIfAbsent(syncLog.getMongoCollection(), k -> new UnifiedSet<>(8)) + .add(syncLog.getEsIndex()); + } + } + List> syncJobs = new ArrayList<>(2); + Date now = new Date(); + if (isUserFullSyncEnabled) { + syncJobs.add(performFullSyncForUserDocs(mongoClient, + syncedOrSyncingCollectionToIndexes, + now)); + } + if (isGroupFullSyncEnabled) { + syncJobs.add(performFullSyncForGroupDocs(mongoClient, + syncedOrSyncingCollectionToIndexes, + now)); + } + return Flux.concat(syncJobs) + .then(); + } + + private Mono performFullSyncForUserDocs( + TurmsMongoClient mongoClient, + Map> syncedOrSyncingCollectionToIndexes, + Date now) { + List indexesForSync = new ArrayList<>(allIndexesForUserDocs); + indexesForSync.removeAll(syncedOrSyncingCollectionToIndexes + .getOrDefault(User.COLLECTION_NAME, Collections.emptySet())); + if (indexesForSync.isEmpty()) { + // TODO: check if the initiator node is still alive because it may have crashed + // and leaved the dirty syncing status. + LOGGER.info("Skip syncing user docs because they are already synced or syncing"); + return Mono.empty(); + } + LOGGER.info("Start syncing user docs to the indexes: " + + indexesForSync); + // TODO: fetch as batches + return userRepository.findAllNames() + .collect(CollectorUtil.toChunkedList()) + .flatMap(users -> { + List> syncs = new ArrayList<>(indexesForSync.size()); + for (String index : indexesForSync) { + syncs.add(performFullSync(mongoClient, + now, + User.COLLECTION_NAME, + index, + () -> fullSync(users, batch -> { + List operations = new ArrayList<>(batch.size() << 1); + for (User user : batch) { + Long id = user.getId(); + operations.add(Map.of(OperationType.INDEX, + Map.of("_index", index, "_id", id))); + operations.add(new UserDoc(id, user.getName())); + } + return elasticsearchClientForUserDocs + .bulk(new BulkRequest(operations)) + .flatMap(bulkResponse -> { + if (bulkResponse.errors()) { + return Mono.error(new RuntimeException( + "Failed to index user docs: " + + bulkResponse)); + } + LOGGER.info("Indexed user docs: {}", bulkResponse); + return Mono.empty(); + }) + .then(); + }))); + } + return Flux.concat(syncs) + .then(); + }); + } + + private Mono performFullSyncForGroupDocs( + TurmsMongoClient mongoClient, + Map> syncedOrSyncingCollectionToIndexes, + Date now) { + List indexesForSync = new ArrayList<>(allIndexesForGroupDocs); + indexesForSync.removeAll(syncedOrSyncingCollectionToIndexes + .getOrDefault(Group.COLLECTION_NAME, Collections.emptySet())); + if (indexesForSync.isEmpty()) { + LOGGER.info("Skip syncing group docs because they are already synced or syncing"); + return Mono.empty(); + } + LOGGER.info("Start syncing group docs to the indexes: " + + indexesForSync); + return groupRepository.findAllNames() + .collect(CollectorUtil.toChunkedList()) + .flatMap(groups -> { + List> syncs = new ArrayList<>(indexesForSync.size()); + for (String index : indexesForSync) { + syncs.add(performFullSync(mongoClient, + now, + Group.COLLECTION_NAME, + index, + () -> fullSync(groups, batch -> { + List operations = new ArrayList<>(batch.size() << 1); + for (Group group : batch) { + Long id = group.getId(); + operations.add(Map.of(OperationType.INDEX, + Map.of("_index", index, "_id", id))); + operations.add(new GroupDoc(id, group.getName())); + } + return elasticsearchClientForGroupDocs + .bulk(new BulkRequest(operations)) + .flatMap(bulkResponse -> { + if (bulkResponse.errors()) { + return Mono.error(new RuntimeException( + "Failed to index group docs: " + + bulkResponse)); + } + LOGGER.info("Indexed group docs: {}", bulkResponse); + return Mono.empty(); + }) + .then(); + }))); + } + return Flux.concat(syncs) + .then(); + }); + } + + private Mono performFullSync( + TurmsMongoClient mongoClient, + Date creationDate, + String mongoCollection, + String esIndex, + Supplier> sync) { + return insertSyncLog(mongoClient, creationDate, mongoCollection, esIndex) + .onErrorMap(t -> new RuntimeException("Failed to perform user full sync", t)) + .flatMap(logId -> sync.get() + .onErrorResume(t -> mongoClient + .updateOne(SyncLog.class, + Filter.newBuilder(1) + .eq(DomainFieldName.ID, logId), + Update.newBuilder(2) + .setEnumString(SyncLog.Fields.status, + SyncStatus.FAILED) + .set(SyncLog.Fields.lastUpdateDate, new Date())) + .materialize() + .flatMap(signal -> { + Throwable updateSyncLogThrowable = signal.getThrowable(); + if (updateSyncLogThrowable != null) { + t.addSuppressed(new RuntimeException( + "Failed to update the sync log: " + + logId, + updateSyncLogThrowable)); + } + return Mono.error(new RuntimeException( + "Failed to perform user full sync", + t)); + })) + .then(Mono.defer(() -> mongoClient + .updateOne(SyncLog.class, + Filter.newBuilder(1) + .eq(DomainFieldName.ID, logId), + Update.newBuilder(2) + .setEnumString(SyncLog.Fields.status, + SyncStatus.COMPLETED) + .set(SyncLog.Fields.lastUpdateDate, new Date())) + .onErrorMap(t -> new RuntimeException( + "Failed to update the sync log: " + + logId, + t)) + .then()))); + } + + private Mono fullSync(List records, Function, Mono> batchHandler) { + if (records.isEmpty()) { + return Mono.empty(); + } + int size = records.size(); + int startIndex = 0; + int count = size / SYNC_BATCH_SIZE + (size % SYNC_BATCH_SIZE == 0 + ? 0 + : 1); + List> inserts = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + List batch = + records.subList(startIndex, Math.min(size, startIndex + SYNC_BATCH_SIZE)); + inserts.add(batchHandler.apply(batch)); + startIndex += SYNC_BATCH_SIZE; + } + return Flux.concat(inserts) + .then(); + } + + private Mono insertSyncLog( + TurmsMongoClient mongoClient, + Date creationDate, + String mongoCollection, + String esIndex) { + long logId = node.nextIncreasingId(ServiceType.ELASTICSEARCH_SYNC_LOG); + return mongoClient + .insert(new SyncLog( + logId, + node.getLocalMemberId(), + mongoCollection, + esIndex, + SyncStatus.IN_PROGRESS, + creationDate, + creationDate, + 0)) + .thenReturn(logId) + .onErrorResume(DuplicateKeyException.class, + e -> insertSyncLog(mongoClient, creationDate, mongoCollection, esIndex)); + } + + private Mono ensureHealthy() { + if (shareClient) { + return elasticsearchClientForUserDocs.healthcheck() + .flatMap(healthResponse -> { + HealthStatus status = healthResponse.status(); + if (HealthStatus.GREEN != status && HealthStatus.YELLOW != status) { + return Mono.error(new IllegalStateException( + "Health check failed. Expected: GREEN or YELLOW. Actual: " + + status)); + } + return Mono.empty(); + }); + } + Mono healthcheckForUserDocs = isUserUseCaseEnabled + ? elasticsearchClientForUserDocs.healthcheck() + .flatMap(healthResponse -> { + HealthStatus status = healthResponse.status(); + if (HealthStatus.GREEN != status && HealthStatus.YELLOW != status) { + return Mono.error(new IllegalStateException( + "Health check failed for user docs. Expected: GREEN or YELLOW. Actual: " + + status)); + } + return Mono.empty(); + }) + : Mono.empty(); + Mono healthcheckForGroupDocs = isGroupUseCaseEnabled + ? elasticsearchClientForGroupDocs.healthcheck() + .flatMap(healthResponse -> { + HealthStatus status = healthResponse.status(); + if (HealthStatus.GREEN != status && HealthStatus.YELLOW != status) { + return Mono.error(new IllegalStateException( + "Health check failed for group docs. Expected: GREEN or YELLOW. Actual: " + + status)); + } + return Mono.empty(); + }) + : Mono.empty(); + return Mono.when(healthcheckForUserDocs, healthcheckForGroupDocs); + } + + private Mono createIndexes( + ElasticsearchUserUseCaseProperties userUseCaseProperties, + ElasticsearchGroupUseCaseProperties groupUseCaseProperties) { + return Mono.when(isUserUseCaseEnabled + ? createIndexes(elasticsearchClientForUserDocs, + userUseCaseProperties.getIndexes(), + USER_INDEX).doOnSuccess(indexes -> { + allIndexesForUserDocs = indexes; + onlyOneIndexForUserDocs = indexes.size() == 1; + }) + : Mono.empty(), + isGroupUseCaseEnabled + ? createIndexes(elasticsearchClientForGroupDocs, + groupUseCaseProperties.getIndexes(), + GROUP_INDEX).doOnSuccess(indexes -> { + allIndexesForGroupDocs = indexes; + onlyOneIndexForGroupDocs = indexes.size() == 1; + }) + : Mono.empty()); + } + + private Mono> createIndexes( + ElasticsearchClient elasticsearchClient, + List indexPropertiesList, + String indexPrefix) { + int indexCount = indexPropertiesList.size(); + List> createIndexes = new ArrayList<>(indexCount); + Set codes = CollectionUtil.newSetWithExpectedSize(indexCount); + for (ElasticsearchIndexProperties indexProperties : indexPropertiesList) { + LanguageCode code = indexProperties.getCode(); + if (!codes.add(code)) { + return Mono.error(new IllegalArgumentException( + "Duplicate index properties for the language code: " + + code)); + } + } + for (ElasticsearchIndexProperties indexProperties : indexPropertiesList) { + createIndexes.add(createIndex(elasticsearchClient, indexPrefix, indexProperties)); + } + return Flux.merge(createIndexes) + .collect(CollectorUtil.toList(indexCount)); + } + + private static Mono createIndex( + ElasticsearchClient elasticsearchClient, + String indexPrefix, + ElasticsearchIndexProperties indexProperties) { + LanguageCode code = indexProperties.getCode(); + int tempNumberOfShards = indexProperties.getNumberOfShards(); + int tempNumberOfReplicas = indexProperties.getNumberOfReplicas(); + Integer numberOfShards = tempNumberOfShards >= 0 + ? tempNumberOfShards + : null; + Integer numberOfReplicas = tempNumberOfReplicas >= 0 + ? tempNumberOfReplicas + : null; + ElasticsearchIndexPropertiesFieldProperties nameFieldProperties = + indexProperties.getProperties() + .getName(); + List nameTextFieldsPropertiesList = + nameFieldProperties.getTextFields(); + + IndexSettingsAnalysis indexSettingsAnalysis = null; + + String index = indexPrefix; + Map fieldToProperty = + CollectionUtil.newMapWithExpectedSize(nameTextFieldsPropertiesList.size()); + + for (ElasticsearchIndexTextFieldProperties textFieldProperties : nameTextFieldsPropertiesList) { + String fieldName = textFieldProperties.getFieldName(); + String analyzer = textFieldProperties.getAnalyzer(); + String searchAnalyzer = textFieldProperties.getSearchAnalyzer(); + if (code != LanguageCode.NONE) { + index = indexPrefix + + "-" + + code.getCanonicalCode(); + } + if (StringUtil.isBlank(analyzer)) { + if (StringUtil.isNotBlank(searchAnalyzer)) { + throw new IllegalArgumentException( + "The search analyzer must be blank if the analyzer is blank. " + + "The language code: " + + code); + } + IndexTextFieldSetting indexTextFieldSetting = + DefaultLanguageSettings.getSetting(code); + if (indexTextFieldSetting == null) { + indexTextFieldSetting = DefaultLanguageSettings.DEFAULT; + } + IndexSettingsAnalysis analysis = indexTextFieldSetting.analysis(); + if (analysis != null) { + indexSettingsAnalysis = indexSettingsAnalysis == null + ? analysis + : indexSettingsAnalysis.merge(analysis); + } + fieldToProperty.putAll(indexTextFieldSetting.fieldToProperty()); + } else { + Property property = + new Property(Property.Type.TEXT, analyzer, searchAnalyzer, null); + fieldToProperty.put(fieldName, property); + } + } + + CreateIndexRequest request = new CreateIndexRequest( + new TypeMapping( + DynamicMapping.STRICT, + Map.of(BaseDoc.Fields.ID, + new Property(Property.Type.KEYWORD, null, null, null), + BaseDoc.Fields.NAME, + new Property(Property.Type.KEYWORD, null, null, fieldToProperty))), + new IndexSettings( + new IndexSettings( + null, + numberOfShards, + numberOfReplicas, + null, + indexSettingsAnalysis), + null, + null, + null, + null)); + String finalIndex = index; + return elasticsearchClient.putIndex(index, request) + .doOnSuccess(unused -> LOGGER.info("Created an index: " + + finalIndex)) + .onErrorResume(ErrorResponseException.class, e -> { + ErrorResponse errorResponse = e.getErrorResponse(); + if ("resource_already_exists_exception".equals(errorResponse.error() + .type())) { + LOGGER.info("The index already exists: \"" + + finalIndex + + "\""); + return Mono.empty(); + } + return Mono.error(e); + }) + .thenReturn(finalIndex); + } + + public Mono putUserDoc(Long userId, String name) { + if (onlyOneIndexForUserDocs) { + return elasticsearchClientForUserDocs.putDoc(allIndexesForUserDocs.getFirst(), + userId.toString(), + () -> JsonUtil.write(64, new UserDoc(userId, name))); + } + List indexes = allIndexesForUserDocs; + List operations = new ArrayList<>(indexes.size() << 1); + UserDoc userDoc = new UserDoc(userId, name); + for (String index : indexes) { + operations.add(Map.of(OperationType.INDEX, Map.of("_index", index, "_id", userId))); + operations.add(userDoc); + } + BulkRequest request = new BulkRequest(operations); + return elasticsearchClientForUserDocs.bulk(request) + .flatMap(bulkResponse -> bulkResponse.errors() + ? Mono.error(new IllegalStateException( + "Bulk response contains errors: " + + bulkResponse)) + : Mono.empty()); + } + + public Mono putUserDocs(Collection userIds, String name) { + return elasticsearchClientForUserDocs.updateByQuery(USER_INDEX + + "*", + new UpdateByQueryRequest( + Map.of("ids", Map.of("values", userIds)), + new Script( + "ctx._source." + + BaseDoc.Fields.NAME + + "=" + + JsonUtil.writeAsString(name)))); + } + + public Mono deleteUserDoc(Long userId) { + return elasticsearchClientForUserDocs.deleteDoc(USER_INDEX + + "*", userId.toString()); + } + + public Mono deleteUserDocs(Collection userIds) { + return elasticsearchClientForUserDocs.deleteByQuery(USER_INDEX + + "*", new DeleteByQueryRequest(Map.of("terms", Map.of("_ids", userIds)))); + } + + public Mono> searchUserDocs( + @Nullable Integer from, + @Nullable Integer size, + String name, + @Nullable Collection ids, + boolean highlight, + @Nullable String scrollId, + @Nullable String keepAlive) { + return search(READER_USER_DOC, + elasticsearchClientForUserDocs, + USER_INDEX + + "*", + from, + size, + name, + ids, + highlight, + scrollId, + keepAlive); + } + + public Mono putGroupDoc(Long groupId, String name) { + if (onlyOneIndexForGroupDocs) { + return elasticsearchClientForGroupDocs.putDoc(allIndexesForGroupDocs.getFirst(), + groupId.toString(), + () -> JsonUtil.write(64, new GroupDoc(groupId, name))); + } + List indexes = allIndexesForGroupDocs; + List operations = new ArrayList<>(indexes.size() << 1); + GroupDoc groupDoc = new GroupDoc(groupId, name); + for (String index : indexes) { + operations.add(Map.of(OperationType.INDEX, Map.of("_index", index, "_id", groupId))); + operations.add(groupDoc); + } + BulkRequest request = new BulkRequest(operations); + return elasticsearchClientForGroupDocs.bulk(request) + .flatMap(bulkResponse -> bulkResponse.errors() + ? Mono.error(new IllegalStateException( + "Bulk response contains errors: " + + bulkResponse)) + : Mono.empty()); + } + + public Mono putGroupDocs(Collection groupIds, String name) { + return elasticsearchClientForGroupDocs.updateByQuery(GROUP_INDEX + + "*", + new UpdateByQueryRequest( + Map.of("ids", Map.of("values", groupIds)), + new Script( + "ctx._source." + + BaseDoc.Fields.NAME + + "=" + + JsonUtil.writeAsString(name)))); + } + + public Mono deleteGroupDocs(Collection groupIds) { + return elasticsearchClientForGroupDocs.deleteByQuery(GROUP_INDEX + + "*", new DeleteByQueryRequest(Map.of("terms", Map.of("_ids", groupIds)))); + } + + public Mono deleteAllGroupDocs() { + return elasticsearchClientForGroupDocs.deleteByQuery(GROUP_INDEX + + "*", new DeleteByQueryRequest(Map.of("match_all", Collections.emptyMap()))); + } + + public Mono> searchGroupDocs( + @Nullable Integer from, + @Nullable Integer size, + String name, + @Nullable Collection ids, + boolean highlight, + @Nullable String scrollId, + @Nullable String keepAlive) { + return search(READER_GROUP_DOC, + elasticsearchClientForGroupDocs, + GROUP_INDEX + + "*", + from, + size, + name, + ids, + highlight, + scrollId, + keepAlive); + } + + private Mono> search( + ObjectReader docReader, + ElasticsearchClient elasticsearchClient, + String index, + @Nullable Integer from, + @Nullable Integer size, + String name, + @Nullable Collection ids, + boolean highlight, + @Nullable String scrollId, + @Nullable String keepAlive) { + Map multiMatch = Map.of("multi_match", + Map.of("fields", + List.of(BaseDoc.Fields.NAME + + "*", + BaseDoc.Fields.NAME + + "*.standard^0.5", + BaseDoc.Fields.NAME + + "*.ngram^0.25"), + "query", + name)); + Map query = CollectionUtil.isEmpty(ids) + ? multiMatch + : Map.of("bool", + Map.of("must", List.of(Map.of("ids", Map.of("values", ids)), multiMatch))); + + SearchRequest request = new SearchRequest( + from, + size, + COLLAPSE_ID, + highlight + ? HIGHLIGHT_NAME + : null, + scrollId == null + ? null + : new PointInTimeReference(scrollId, keepAlive), + query); + return elasticsearchClient.search(index, request, docReader); + } + + public Mono deletePitForUserDocs(String scrollId) { + return elasticsearchClientForUserDocs.deletePit(scrollId); + } +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ErrorResponseException.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ErrorResponseException.java new file mode 100644 index 0000000000..754578e487 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/ErrorResponseException.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +import im.turms.service.storage.elasticsearch.model.ErrorResponse; + +/** + * @author James Chen + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class ErrorResponseException extends RuntimeException { + + private final ErrorResponse errorResponse; + + public ErrorResponseException(ErrorResponse errorResponse) { + this.errorResponse = errorResponse; + } +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/IndexTextFieldSetting.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/IndexTextFieldSetting.java new file mode 100644 index 0000000000..fa911610e0 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/IndexTextFieldSetting.java @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch; + +import java.util.Map; +import jakarta.annotation.Nullable; + +import im.turms.service.storage.elasticsearch.model.IndexSettingsAnalysis; +import im.turms.service.storage.elasticsearch.model.Property; + +/** + * @author James Chen + */ +public record IndexTextFieldSetting( + Map fieldToProperty, + @Nullable IndexSettingsAnalysis analysis +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkRequest.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkRequest.java new file mode 100644 index 0000000000..fdb5d7aee5 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkRequest.java @@ -0,0 +1,54 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.io.IOException; +import java.util.List; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +/** + * @author James Chen + */ +@JsonSerialize(using = BulkRequest.BulkRequestSerializer.class) +public record BulkRequest( + List operations +) { + + public static class BulkRequestSerializer extends JsonSerializer { + @Override + public void serialize(BulkRequest value, JsonGenerator gen, SerializerProvider serializers) + throws IOException { + boolean isFirstOperation = true; + for (Object operation : value.operations) { + if (isFirstOperation) { + isFirstOperation = false; + } else { + gen.writeRawValue("\n"); + } + gen.writeObject(operation); + } + // The bulk request must be terminated by a newline. + gen.writeRawValue("\n"); + } + } + +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkResponse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkResponse.java new file mode 100644 index 0000000000..759b92c5f2 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkResponse.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record BulkResponse( + @JsonProperty("errors") boolean errors, + @JsonProperty("items") List items +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkResponseItem.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkResponseItem.java new file mode 100644 index 0000000000..40fee69c64 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/BulkResponseItem.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record BulkResponseItem( + @JsonProperty("_id") @Nullable String id, + @JsonProperty("_index") String index, + @JsonProperty("status") int status, + @JsonProperty("error") @Nullable ErrorCause error, + @JsonProperty("result") @Nullable String result, + @JsonProperty("_seq_no") @Nullable Long seqNo, + @JsonProperty("_shards") @Nullable ShardStatistics shards, + @JsonProperty("_version") @Nullable Long version +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ClosePointInTimeRequest.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ClosePointInTimeRequest.java new file mode 100644 index 0000000000..b1128f78e3 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ClosePointInTimeRequest.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record ClosePointInTimeRequest( + @JsonProperty("id") String id +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/CreateIndexRequest.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/CreateIndexRequest.java new file mode 100644 index 0000000000..0009c4f936 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/CreateIndexRequest.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record CreateIndexRequest( + @JsonProperty("mappings") @Nullable TypeMapping mappings, + @JsonProperty("settings") @Nullable IndexSettings settings +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteByQueryRequest.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteByQueryRequest.java new file mode 100644 index 0000000000..a02c696ab7 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteByQueryRequest.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record DeleteByQueryRequest( + @JsonProperty("query") Map query +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteByQueryResponse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteByQueryResponse.java new file mode 100644 index 0000000000..4c463ad473 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteByQueryResponse.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record DeleteByQueryResponse( + @JsonProperty("deleted") @Nullable Long deleted +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteResponse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteResponse.java new file mode 100644 index 0000000000..753b1a17db --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DeleteResponse.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record DeleteResponse( + @JsonProperty("result") Result result +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DynamicMapping.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DynamicMapping.java new file mode 100644 index 0000000000..458a1414b6 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/DynamicMapping.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public enum DynamicMapping { + @JsonProperty("strict") + STRICT, + + @JsonProperty("runtime") + RUNTIME, + + @JsonProperty("true") + TRUE, + + @JsonProperty("false") + FALSE, + +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ErrorCause.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ErrorCause.java new file mode 100644 index 0000000000..bbb96d747c --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ErrorCause.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.List; +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record ErrorCause( + @JsonProperty("type") @Nullable String type, + @JsonProperty("reason") @Nullable String reason, + @JsonProperty("stack_trace") @Nullable String stackTrace, + @JsonProperty("caused_by") @Nullable ErrorCause causedBy, + @JsonProperty("root_cause") List rootCause, + @JsonProperty("suppressed") List suppressed +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ErrorResponse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ErrorResponse.java new file mode 100644 index 0000000000..a3813b1568 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ErrorResponse.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record ErrorResponse( + @JsonProperty("error") ErrorCause error, + @JsonProperty("status") int status +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/FieldCollapse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/FieldCollapse.java new file mode 100644 index 0000000000..c31ece2073 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/FieldCollapse.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record FieldCollapse( + @JsonProperty("field") String field +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HealthResponse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HealthResponse.java new file mode 100644 index 0000000000..d1b5fd24a3 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HealthResponse.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record HealthResponse( + @JsonProperty("cluster_name") String clusterName, + @JsonProperty("status") HealthStatus status +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HealthStatus.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HealthStatus.java new file mode 100644 index 0000000000..24f30a3f72 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HealthStatus.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public enum HealthStatus { + @JsonProperty("green") + GREEN, + + @JsonProperty("yellow") + YELLOW, + + @JsonProperty("red") + RED +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Highlight.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Highlight.java new file mode 100644 index 0000000000..d00d684713 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Highlight.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.List; +import java.util.Map; +import jakarta.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record Highlight( + @JsonProperty("fields") Map> fields, + @JsonProperty("pre_tags") @Nullable List preTags, + @JsonProperty("post_tags") @Nullable List postTags, + @JsonProperty("type") @Nullable String type, + @JsonProperty("number_of_fragments") @Nullable Integer numberOfFragments +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Hit.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Hit.java new file mode 100644 index 0000000000..df7fec53df --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Hit.java @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.List; +import java.util.Map; +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record Hit( + @JsonProperty("_index") String index, + @JsonProperty("_id") String id, + @JsonProperty("_score") @Nullable Double score, + @JsonProperty("highlight") Map> highlight, + @JsonProperty("_source") @Nullable T source +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HitsMetadata.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HitsMetadata.java new file mode 100644 index 0000000000..5a30955c26 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/HitsMetadata.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record HitsMetadata( + @JsonProperty("hits") List> hits +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/IndexSettings.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/IndexSettings.java new file mode 100644 index 0000000000..b4c83a3dd7 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/IndexSettings.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record IndexSettings( + @JsonProperty("index") @Nullable IndexSettings index, + @JsonProperty("number_of_shards") @Nullable Integer numberOfShards, + @JsonProperty("number_of_replicas") @Nullable Integer numberOfReplicas, + @JsonProperty("default_pipeline") @Nullable String defaultPipeline, + @JsonProperty("analysis") @Nullable IndexSettingsAnalysis analysis +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/IndexSettingsAnalysis.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/IndexSettingsAnalysis.java new file mode 100644 index 0000000000..a5cd0325a7 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/IndexSettingsAnalysis.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.Map; +import jakarta.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import im.turms.server.common.infra.collection.CollectionUtil; + +/** + * @author James Chen + */ +public record IndexSettingsAnalysis( + @JsonProperty("analyzer") @Nullable Map> analyzer, + @JsonProperty("char_filter") @Nullable Map> charFilter, + @JsonProperty("filter") @Nullable Map> filter, + @JsonProperty("normalizer") @Nullable Map> normalizer, + @JsonProperty("tokenizer") @Nullable Map> tokenizer +) { + + public IndexSettingsAnalysis merge(IndexSettingsAnalysis analysis) { + return new IndexSettingsAnalysis( + analyzer == null + ? analysis.analyzer + : analysis.analyzer == null + ? null + : CollectionUtil.merge(analyzer, analysis.analyzer), + charFilter == null + ? analysis.charFilter + : analysis.charFilter == null + ? null + : CollectionUtil.merge(charFilter, analysis.charFilter), + filter == null + ? analysis.filter + : analysis.filter == null + ? null + : CollectionUtil.merge(filter, analysis.filter), + normalizer == null + ? analysis.normalizer + : analysis.normalizer == null + ? null + : CollectionUtil.merge(normalizer, analysis.normalizer), + tokenizer == null + ? analysis.tokenizer + : analysis.tokenizer == null + ? null + : CollectionUtil.merge(tokenizer, analysis.tokenizer)); + } + +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/OperationType.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/OperationType.java new file mode 100644 index 0000000000..a76149b9cd --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/OperationType.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public enum OperationType { + @JsonProperty("index") + INDEX, + + @JsonProperty("create") + CREATE, + + @JsonProperty("update") + UPDATE, + + @JsonProperty("delete") + DELETE, +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/PointInTimeReference.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/PointInTimeReference.java new file mode 100644 index 0000000000..9f1f3508e7 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/PointInTimeReference.java @@ -0,0 +1,29 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +/** + * @author James Chen + */ +public record PointInTimeReference( + String id, + @Nullable String keepAlive +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Property.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Property.java new file mode 100644 index 0000000000..b143b76fb8 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Property.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.Map; +import jakarta.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record Property( + @JsonProperty("type") Type type, + @JsonProperty("analyzer") @Nullable String analyzer, + @JsonProperty("search_analyzer") @Nullable String searchAnalyzer, + @JsonProperty("fields") @Nullable Map fields +) { + + public enum Type { + @JsonProperty("keyword") + KEYWORD, + + @JsonProperty("text") + TEXT + } + +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Result.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Result.java new file mode 100644 index 0000000000..f0ec2e329b --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Result.java @@ -0,0 +1,40 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public enum Result { + @JsonProperty("created") + CREATED, + + @JsonProperty("updated") + UPDATED, + + @JsonProperty("deleted") + DELETED, + + @JsonProperty("not_found") + NOT_FOUND, + + @JsonProperty("noop") + NO_OP +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Script.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Script.java new file mode 100644 index 0000000000..ed10f5d410 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/Script.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record Script( + @JsonProperty("source") String source +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/SearchRequest.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/SearchRequest.java new file mode 100644 index 0000000000..5f4b408f1d --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/SearchRequest.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.Map; +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record SearchRequest( + @JsonProperty("from") @Nullable Integer from, + @JsonProperty("size") @Nullable Integer size, + + @JsonProperty("collapse") @Nullable FieldCollapse collapse, + @JsonProperty("highlight") @Nullable Highlight highlight, + @JsonProperty("pit") @Nullable PointInTimeReference pit, + @JsonProperty("query") @Nullable Map query +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/SearchResponse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/SearchResponse.java new file mode 100644 index 0000000000..14addc3ac1 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/SearchResponse.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record SearchResponse( + @JsonProperty("hits") HitsMetadata hits, + @JsonProperty("_scroll_id") @Nullable String scrollId +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ShardFailure.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ShardFailure.java new file mode 100644 index 0000000000..2b41fa6c3f --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ShardFailure.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record ShardFailure( + @JsonProperty("index") @Nullable String index, + @JsonProperty("node") @Nullable String node, + @JsonProperty("reason") ErrorCause reason, + @JsonProperty("shard") int shard, + @JsonProperty("status") @Nullable String status +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ShardStatistics.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ShardStatistics.java new file mode 100644 index 0000000000..0a33c1c63a --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/ShardStatistics.java @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.List; +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record ShardStatistics( + @JsonProperty("failed") Long failed, + @JsonProperty("successful") Long successful, + @JsonProperty("total") Long total, + @JsonProperty("failures") List failures, + @JsonProperty("skipped") @Nullable Long skipped +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/TypeMapping.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/TypeMapping.java new file mode 100644 index 0000000000..9d429e0a56 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/TypeMapping.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.Map; +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record TypeMapping( + @JsonProperty("dynamic") @Nullable DynamicMapping dynamic, + @JsonProperty("properties") @Nullable Map properties +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/UpdateByQueryRequest.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/UpdateByQueryRequest.java new file mode 100644 index 0000000000..b5a361f00f --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/UpdateByQueryRequest.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record UpdateByQueryRequest( + @JsonProperty("query") Map query, + @JsonProperty("script") Script script +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/UpdateByQueryResponse.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/UpdateByQueryResponse.java new file mode 100644 index 0000000000..8c40389ec1 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/UpdateByQueryResponse.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model; + +import javax.annotation.Nullable; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author James Chen + */ +public record UpdateByQueryResponse( + @JsonProperty("updated") @Nullable Long updated +) { +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/BaseDoc.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/BaseDoc.java new file mode 100644 index 0000000000..bf59f5653d --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/BaseDoc.java @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model.doc; + +import lombok.Data; + +/** + * @author James Chen + */ +@Data +public abstract class BaseDoc { + + private final Long id; + + public static class Fields { + public static final String ID = "id"; + public static final String NAME = "name"; + + private Fields() { + } + } +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/GroupDoc.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/GroupDoc.java new file mode 100644 index 0000000000..58e8bb5da7 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/GroupDoc.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model.doc; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author James Chen + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class GroupDoc extends BaseDoc { + @JsonProperty(Fields.NAME) + private final String name; + + @JsonCreator + public GroupDoc(Long id, String name) { + super(id); + this.name = name; + } +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/UserDoc.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/UserDoc.java new file mode 100644 index 0000000000..f6fdded168 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/model/doc/UserDoc.java @@ -0,0 +1,39 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.model.doc; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * @author James Chen + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class UserDoc extends BaseDoc { + @JsonProperty(Fields.NAME) + private final String name; + + @JsonCreator + public UserDoc(Long id, String name) { + super(id); + this.name = name; + } +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/mongo/SyncLog.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/mongo/SyncLog.java new file mode 100644 index 0000000000..42b45d241c --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/mongo/SyncLog.java @@ -0,0 +1,62 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.mongo; + +import java.util.Date; + +import lombok.Data; +import lombok.experimental.FieldNameConstants; + +import im.turms.server.common.domain.common.po.BaseEntity; +import im.turms.server.common.storage.mongo.entity.annotation.Document; +import im.turms.server.common.storage.mongo.entity.annotation.Id; +import im.turms.server.common.storage.mongo.entity.annotation.Indexed; + +/** + * @author James Chen + */ +@Data +@Document(SyncLog.COLLECTION_NAME) +@FieldNameConstants +public class SyncLog extends BaseEntity { + + public static final String COLLECTION_NAME = "syncLog"; + + @Id + private final Long id; + + private final String nodeId; + + private final String mongoCollection; + + private final String esIndex; + + private final SyncStatus status; + + @Indexed + private final Date creationDate; + + private final Date lastUpdateDate; + + /** + * Our sync logic may change over time. This field records the version of the logic, so that we + * can support smooth migration. + */ + private final int version; + +} \ No newline at end of file diff --git a/turms-service/src/main/java/im/turms/service/storage/elasticsearch/mongo/SyncStatus.java b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/mongo/SyncStatus.java new file mode 100644 index 0000000000..91ad07afa1 --- /dev/null +++ b/turms-service/src/main/java/im/turms/service/storage/elasticsearch/mongo/SyncStatus.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2019 The Turms Project + * https://github.com/turms-im/turms + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package im.turms.service.storage.elasticsearch.mongo; + +/** + * @author James Chen + */ +public enum SyncStatus { + IN_PROGRESS, + COMPLETED, + FAILED +} \ No newline at end of file