Merge branch 'master' into merge-master

.github/workflows/e2e-operation.yml

@@ -51,7 +51,7 @@ jobs:
 
   e2e-operation:
     if: ${{ needs.global-environment.outputs.GLOBAL_JOB_ENABLED == 'true' }}
-    name: E2E - Operation
+    name: E2E - ${{ matrix.operation }} on ${{ matrix.image.version }}
     needs: [ detect-changed-files, global-environment ]
     runs-on: ubuntu-latest
     timeout-minutes: 40

.github/workflows/e2e-sql.yml

@@ -110,7 +110,7 @@ jobs:
         database: [ MySQL, PostgreSQL ]
         # Fix me #25051
         #scenario: [ dbtbl_with_readwrite_splitting, dbtbl_with_readwrite_splitting_and_encrypt, sharding_and_encrypt, encrypt_and_readwrite_splitting, encrypt_shadow, readwrite_splitting_and_shadow, sharding_and_shadow, sharding_encrypt_shadow, mask_encrypt, mask_sharding, mask_encrypt_sharding ]
-        scenario: [ empty_rules, rdl_empty_rules, passthrough, tbl, encrypt, readwrite_splitting, shadow, mask, dbtbl_with_readwrite_splitting_and_encrypt, sharding_and_encrypt, encrypt_and_readwrite_splitting, encrypt_shadow, readwrite_splitting_and_shadow, sharding_and_shadow, sharding_encrypt_shadow, mask_encrypt, mask_sharding, mask_encrypt_sharding ]
+        scenario: [ empty_rules, rdl_empty_rules, passthrough, db, tbl, encrypt, readwrite_splitting, shadow, mask, dbtbl_with_readwrite_splitting_and_encrypt, sharding_and_encrypt, encrypt_and_readwrite_splitting, encrypt_shadow, readwrite_splitting_and_shadow, sharding_and_shadow, sharding_encrypt_shadow, mask_encrypt, mask_sharding, mask_encrypt_sharding ]
         additional-options: [ '' ]
         include:
           - adapter: proxy

.github/workflows/nightly-build.yml

@@ -71,7 +71,7 @@ jobs:
           remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
 
   build-proxy-image:
-    if: ${{ needs.global-environment.outputs.GLOBAL_IS_NIGHTLY_JOB_EXECUTABLE == 'true' }}
+    if: github.repository == 'apache/shardingsphere'
     name: Build Proxy Image
     needs: global-environment
     runs-on: ${{ needs.global-environment.outputs.GLOBAL_RUNS_ON }}

.github/workflows/nightly-check.yml

@@ -23,7 +23,7 @@ on:
   workflow_dispatch:
 
 env:
-  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dspotless.apply.skip=true
+  MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
 
 jobs:
   global-environment:
@@ -45,7 +45,7 @@ jobs:
           restore-keys: |
             ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-
       - name: Run SpotBugs
-        run: ./mvnw clean install spotbugs:check -T1C
+        run: ./mvnw clean install spotbugs:check -Pcheck -T1C
 
   check-and-report-sonarcloud:
     if: github.repository == 'apache/shardingsphere'
@@ -82,7 +82,7 @@ jobs:
     name: Checkstyle - SonarCloud
     needs: global-environment
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    timeout-minutes: 100
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-java@v3
@@ -101,7 +101,8 @@ jobs:
           restore-keys: |
             ${{ needs.global-environment.outputs.GLOBAL_CACHE_PREFIX }}-maven-third-party-
       - name: Aggregate checkstyle report
-        run:
+        run: |
+          export MAVEN_OPTS='-Xmx4g -Xms512m'
           ./mvnw clean verify checkstyle:checkstyle-aggregate
       - name: Upload checkstyle report to SonarCloud
         env:
@@ -192,4 +193,3 @@ jobs:
           stale-issue-message: >
             There hasn't been any activity on this issue recently, and in order to prioritize active issues, it will be
             marked as stale.
-

.github/workflows/required-reusable.yml

@@ -43,6 +43,7 @@ jobs:
             echo ${line} >> $GITHUB_OUTPUT
           done
           echo "GLOBAL_REPOSITORY=${{ env.GLOBAL_REPOSITORY }}"
+          echo "DISABLE_CI=${{ env.DISABLE_CI }}"
           GLOBAL_CACHE_PREFIX=$(echo '${{ env.GLOBAL_REPOSITORY }}' | sed 's/\//_/g')
           echo "GLOBAL_CACHE_PREFIX=${GLOBAL_CACHE_PREFIX}" >> $GITHUB_OUTPUT
       - name: Check If Job is Executable
@@ -56,13 +57,9 @@ jobs:
           if [ ${{ toJSON(github.repository) }} = ${{ steps.import-variables.outputs.GLOBAL_REPOSITORY }} ]; then
             echo "GLOBAL_IS_NIGHTLY_JOB_EXECUTABLE=true" >> $GITHUB_OUTPUT
             echo "GLOBAL_JOB_ENABLED=false" >> $GITHUB_OUTPUT
-            echo '${{ env.JSON_LABELS }}' | while read line
-            do
-              if [ ${line}="type: ${{ steps.import-variables.outputs.RUNNABLE_CI_LABEL }}" ] || [ -z "${{ steps.import-variables.outputs.RUNNABLE_CI_LABEL }}" ]; then
-                echo "GLOBAL_JOB_ENABLED=true" >> $GITHUB_OUTPUT
-                break
-              fi
-            done
+            if [ -z "${{ steps.import-variables.outputs.DISABLE_CI }}"] || [ "${{ steps.import-variables.outputs.DISABLE_CI }}"=="false" ] ; then
+              echo "GLOBAL_JOB_ENABLED=true" >> $GITHUB_OUTPUT
+            fi
           fi
     outputs:
       GLOBAL_RUNS_ON: ${{ steps.import-variables.outputs.GLOBAL_RUNS_ON }}

.github/workflows/resources/filter/filters.yml

@@ -22,9 +22,9 @@ pipeline:
   - '.github/workflows/e2e-operation.yml'
   - 'infra/**/src/main/**'
   - 'mode/**/src/main/**'
-  - 'features/encrypt/src/main/**'
-  - 'features/features/readwrite-splitting/src/main/**'
-  - 'features/sharding/src/main/**'
+  - 'features/encrypt/**/src/main/**'
+  - 'features/readwrite-splitting/**/src/main/**'
+  - 'features/sharding/**/src/main/**'
   - 'proxy/**/src/main/**'
   - 'jdbc/core/src/main/**'
   - '**/*-distsql*/**/src/main/**'

.github/workflows/resources/linkcheck/markdown-link-check.json

@@ -17,6 +17,9 @@
     },
     {
       "pattern": "^https://starchart.cc/"
+    },
+    {
+      "pattern": "^https://www.apiseven.com"
     }
   ], "replacementPatterns": [
   {

.github/workflows/resources/variables/env-variables.properties

@@ -1,3 +1,3 @@
 GLOBAL_REPOSITORY=apache/shardingsphere
 GLOBAL_RUNS_ON=ubuntu-latest
-RUNNABLE_CI_LABEL=
+DISABLE_CI=false

.gitignore

@@ -39,7 +39,6 @@ logs/
 *.diff
 *.patch
 *.tmp
-.hugo_build.lock
 
 # system ignore
 .DS_Store
@@ -51,3 +50,8 @@ gen/
 
 # profiler ignore
 .profiler/
+
+# hugo ignore
+public/
+.hugo_build.lock
+*.html-e

CONTRIBUTING.md

@@ -2,11 +2,11 @@ You can report a bug, submit a new feature enhancement recommendation, or commit
 
 ## Reporting Bug
 
- - Before report a bug, please search from google to confirm you cannot find any hint on it.
- - Look [issues List](https://github.com/apache/shardingsphere/issues) to confirm this issue is not a duplicated one.
+ - Before reporting a bug, please search from google to confirm that you cannot find any hint on it.
+ - Look at the [issues List](https://github.com/apache/shardingsphere/issues) to confirm this issue is not a duplicated one.
  - [Create](https://github.com/apache/shardingsphere/issues/new) a new issue.
  - Define a clear and descriptive title for the issue.
- - If bug reported, please provide information below:
+ - If a bug is reported, please provide information below:
        - Details for reproduce bug step by step. Include SQL, configuration, expected results, actual results and tracing log.
        - ShardingSphere and your OS version.
        - Source code that reproduce the bug on GitHub, can be linked here.
@@ -18,7 +18,7 @@ You can report a bug, submit a new feature enhancement recommendation, or commit
        - Explain why this enhancement is general feature for most developers.
        - List similar features which already available in other product if possible. Both open source and commercial software are available.
  - Assign label after issue created. Label should be bug, enhancement, discussion and so on.
- - Please pay attention on the issue and provide more information during discuss.
+ - Please pay attention to the issue and provide more information during discuss.
  - Please close issue when it is resolved. If you don't close it, we will close it after 3 days。
  - If this issue has new information, please reopen it again. Please note, issue can reopen which only closed by yourself.
 

README.md

@@ -26,7 +26,7 @@
 
 <hr>
 
-Apache ShardingSphere is a distributed SQL transaction & query engine for data sharding, scaling, encryption, and more - on any database. Our community's guiding development concept is Database Plus for creating a complete ecosystem that allows you to transform any database into a distributed database system. 
+Apache ShardingSphere is a distributed SQL transaction & query engine that allows for data sharding, scaling, encryption, and more - on any database. Our community's guiding development concept is Database Plus for creating a complete ecosystem that allows you to transform any database into a distributed database system. 
 
 It focuses on repurposing existing databases, by placing a standardized upper layer above existing and fragmented databases, rather than creating a new database. 
 
@@ -133,7 +133,7 @@ Keep an eye on the [milestones page](https://github.com/apache/shardingsphere/mi
 <hr>
 
 Apache ShardingSphere includes 2 independent products: JDBC & Proxy.
-They all provide functions of data scale-out, distributed transaction and distributed governance, applicable in a variety of situations such as Java isomorphism, heterogeneous language and Cloud-Native.
+They all provide functions of data scale-out, distributed transaction and distributed governance, applicable in a variety of situations such as Java-based isomorphism, heterogeneous language and Cloud-Native.
 
 ### ShardingSphere-JDBC
 
@@ -154,7 +154,7 @@ With the client end connecting directly to the database, it provides services in
 [![Download](https://img.shields.io/badge/release-download-orange.svg)](https://www.apache.org/dyn/closer.lua/shardingsphere/5.3.2/apache-shardingsphere-5.3.2-shardingsphere-proxy-bin.tar.gz)
 [![Docker Pulls](https://img.shields.io/docker/pulls/apache/shardingsphere-proxy.svg)](https://store.docker.com/community/images/apache/shardingsphere-proxy)
 
-A transparent database proxy, providing a database server that encapsulates database binary protocol to support heterogeneous languages. 
+A transparent database proxy, providing a database server that encapsulates the database binary protocol to support heterogeneous languages. 
 Friendlier to DBAs, the MySQL and PostgreSQL version now provided can use any kind of terminal.
 
 :link: For more details, follow this [link to the official website](https://shardingsphere.apache.org/document/current/en/overview/#shardingsphere-proxy).
@@ -164,9 +164,9 @@ Friendlier to DBAs, the MySQL and PostgreSQL version now provided can use any ki
 <hr>
 
 ShardingSphere-JDBC adopts a decentralized architecture, applicable to high-performance light-weight OLTP applications developed with Java. 
-ShardingSphere-Proxy provides static entry and all languages support, suitable for OLAP application and sharding databases management and operation.
+ShardingSphere-Proxy provides static entry and all languages support, suitable for an OLAP application and sharding databases management and operation.
 
-Through the mixed use of ShardingSphere-JDBC & ShardingSphere-Proxy together with a unified sharding strategy by the same registry center, the ShardingSphere ecosystem can build an application system suitable to all kinds of scenarios.
+Through the combination of ShardingSphere-JDBC & ShardingSphere-Proxy together with a unified sharding strategy by the same registry center, the ShardingSphere ecosystem can build an application system suitable to all kinds of scenarios.
 
 :link: More details can be found following this [link to the official website](https://shardingsphere.apache.org/document/current/en/overview/#hybrid-architecture).
 

agent/core/src/main/java/org/apache/shardingsphere/agent/core/yaml/AgentYamlConstructor.java

@@ -18,6 +18,7 @@
 package org.apache.shardingsphere.agent.core.yaml;
 
 import org.apache.shardingsphere.agent.core.util.AgentPreconditions;
+import org.yaml.snakeyaml.LoaderOptions;
 import org.yaml.snakeyaml.constructor.Constructor;
 
 /**
@@ -27,8 +28,8 @@ public final class AgentYamlConstructor extends Constructor {
 
     private final Class<?> rootClass;
 
-    public AgentYamlConstructor(final Class<?> rootClass) {
-        super(rootClass);
+    public AgentYamlConstructor(final Class<?> rootClass, final LoaderOptions loadingConfig) {
+        super(rootClass, loadingConfig);
         this.rootClass = rootClass;
     }
 

agent/core/src/main/java/org/apache/shardingsphere/agent/core/yaml/AgentYamlEngine.java

@@ -21,6 +21,7 @@
 import lombok.NoArgsConstructor;
 import org.apache.shardingsphere.agent.core.advisor.config.yaml.entity.YamlAdvisorsConfiguration;
 import org.apache.shardingsphere.agent.core.plugin.config.yaml.entity.YamlAgentConfiguration;
+import org.yaml.snakeyaml.LoaderOptions;
 import org.yaml.snakeyaml.Yaml;
 
 import java.io.InputStream;
@@ -38,7 +39,7 @@ public final class AgentYamlEngine {
      * @return YAML agent configuration
      */
     public static YamlAgentConfiguration unmarshalYamlAgentConfiguration(final InputStream inputStream) {
-        return new Yaml(new AgentYamlConstructor(YamlAgentConfiguration.class)).loadAs(inputStream, YamlAgentConfiguration.class);
+        return new Yaml(new AgentYamlConstructor(YamlAgentConfiguration.class, createLoaderOptions())).loadAs(inputStream, YamlAgentConfiguration.class);
     }
 
     /**
@@ -48,6 +49,12 @@ public static YamlAgentConfiguration unmarshalYamlAgentConfiguration(final Input
      * @return YAML advisors configuration
      */
     public static YamlAdvisorsConfiguration unmarshalYamlAdvisorsConfiguration(final InputStream inputStream) {
-        return new Yaml(new AgentYamlConstructor(YamlAdvisorsConfiguration.class)).loadAs(inputStream, YamlAdvisorsConfiguration.class);
+        return new Yaml(new AgentYamlConstructor(YamlAdvisorsConfiguration.class, createLoaderOptions())).loadAs(inputStream, YamlAdvisorsConfiguration.class);
+    }
+
+    private static LoaderOptions createLoaderOptions() {
+        LoaderOptions result = new LoaderOptions();
+        result.setCodePointLimit(Integer.MAX_VALUE);
+        return result;
     }
 }

agent/plugins/core/src/main/java/org/apache/shardingsphere/agent/plugin/core/util/SQLStatementUtils.java

@@ -20,11 +20,11 @@
 import lombok.AccessLevel;
 import lombok.NoArgsConstructor;
 import org.apache.shardingsphere.agent.plugin.core.enums.SQLStatementType;
-import org.apache.shardingsphere.distsql.parser.statement.DistSQLStatement;
-import org.apache.shardingsphere.distsql.parser.statement.ral.RALStatement;
-import org.apache.shardingsphere.distsql.parser.statement.rdl.RDLStatement;
-import org.apache.shardingsphere.distsql.parser.statement.rql.RQLStatement;
-import org.apache.shardingsphere.distsql.parser.statement.rul.RULStatement;
+import org.apache.shardingsphere.distsql.statement.DistSQLStatement;
+import org.apache.shardingsphere.distsql.statement.ral.RALStatement;
+import org.apache.shardingsphere.distsql.statement.rdl.RDLStatement;
+import org.apache.shardingsphere.distsql.statement.rql.RQLStatement;
+import org.apache.shardingsphere.distsql.statement.rul.RULStatement;
 import org.apache.shardingsphere.sql.parser.sql.common.statement.SQLStatement;
 import org.apache.shardingsphere.sql.parser.sql.common.statement.dal.DALStatement;
 import org.apache.shardingsphere.sql.parser.sql.common.statement.dcl.DCLStatement;

agent/plugins/metrics/core/src/main/java/org/apache/shardingsphere/agent/plugin/metrics/core/exporter/impl/proxy/ProxyMetaDataInfoExporter.java

@@ -53,6 +53,6 @@ public Optional<GaugeMetricFamilyMetricsCollector> export(final String pluginTyp
     }
 
     private int getStorageUnitCount(final MetaDataContexts metaDataContexts) {
-        return metaDataContexts.getMetaData().getDatabases().values().stream().map(each -> each.getResourceMetaData().getStorageUnitMetaData().getStorageUnits().size()).reduce(0, Integer::sum);
+        return metaDataContexts.getMetaData().getDatabases().values().stream().map(each -> each.getResourceMetaData().getStorageUnits().size()).reduce(0, Integer::sum);
     }
 }

agent/plugins/metrics/core/src/test/java/org/apache/shardingsphere/agent/plugin/metrics/core/advice/SQLParseCountAdviceTest.java

@@ -22,9 +22,9 @@
 import org.apache.shardingsphere.agent.plugin.metrics.core.config.MetricConfiguration;
 import org.apache.shardingsphere.agent.plugin.metrics.core.fixture.collector.MetricsCollectorFixture;
 import org.apache.shardingsphere.agent.plugin.metrics.core.fixture.TargetAdviceObjectFixture;
-import org.apache.shardingsphere.distsql.parser.statement.rdl.create.RegisterStorageUnitStatement;
-import org.apache.shardingsphere.distsql.parser.statement.rql.show.ShowStorageUnitsStatement;
-import org.apache.shardingsphere.distsql.parser.statement.rul.sql.FormatStatement;
+import org.apache.shardingsphere.distsql.statement.rdl.create.RegisterStorageUnitStatement;
+import org.apache.shardingsphere.distsql.statement.rql.show.ShowStorageUnitsStatement;
+import org.apache.shardingsphere.distsql.statement.rul.sql.FormatStatement;
 import org.apache.shardingsphere.migration.distsql.statement.ShowMigrationListStatement;
 import org.apache.shardingsphere.sql.parser.sql.common.segment.generic.DatabaseSegment;
 import org.apache.shardingsphere.sql.parser.sql.common.statement.SQLStatement;

agent/plugins/metrics/core/src/test/java/org/apache/shardingsphere/agent/plugin/metrics/core/exporter/impl/proxy/ProxyMetaDataInfoExporterTest.java

@@ -75,7 +75,7 @@ void assertExportWithContextManager() {
 
     private ContextManager mockContextManager() {
         ShardingSphereDatabase database = mock(ShardingSphereDatabase.class, RETURNS_DEEP_STUBS);
-        when(database.getResourceMetaData().getStorageUnitMetaData().getStorageUnits()).thenReturn(Collections.singletonMap("ds_0", mock(StorageUnit.class)));
+        when(database.getResourceMetaData().getStorageUnits()).thenReturn(Collections.singletonMap("ds_0", mock(StorageUnit.class)));
         when(database.getProtocolType()).thenReturn(TypedSPILoader.getService(DatabaseType.class, "FIXTURE"));
         ShardingSphereMetaData metaData = mock(ShardingSphereMetaData.class);
         when(metaData.getDatabases()).thenReturn(Collections.singletonMap("sharding_db", database));

agent/plugins/tracing/core/src/main/java/org/apache/shardingsphere/agent/plugin/tracing/core/advice/TracingJDBCExecutorCallbackAdvice.java

@@ -41,8 +41,8 @@ public abstract class TracingJDBCExecutorCallbackAdvice<T> implements InstanceMe
     public final void beforeMethod(final TargetAdviceObject target, final Method method, final Object[] args, final String pluginType) {
         JDBCExecutionUnit executionUnit = (JDBCExecutionUnit) args[0];
         ResourceMetaData resourceMetaData = AgentReflectionUtils.getFieldValue(target, "resourceMetaData");
-        ConnectionProperties connectionProps = resourceMetaData.getConnectionProperties(executionUnit.getExecutionUnit().getDataSourceName());
-        DatabaseType storageType = resourceMetaData.getStorageType(executionUnit.getExecutionUnit().getDataSourceName());
+        ConnectionProperties connectionProps = resourceMetaData.getStorageUnits().get(executionUnit.getExecutionUnit().getDataSourceName()).getConnectionProperties();
+        DatabaseType storageType = resourceMetaData.getStorageUnits().get(executionUnit.getExecutionUnit().getDataSourceName()).getStorageType();
         recordExecuteInfo(RootSpanContext.get(), target, executionUnit, (boolean) args[1], connectionProps, storageType);
     }
 

agent/plugins/tracing/type/opentelemetry/pom.xml

@@ -27,8 +27,9 @@
     <name>${project.artifactId}</name>
 
     <properties>
-        <opentelemetry-bom.version>1.22.0</opentelemetry-bom.version>
-        <opentelemetry-autoconfigure.version>1.22.0-alpha</opentelemetry-autoconfigure.version>
+        <opentelemetry-bom.version>1.31.0</opentelemetry-bom.version>
+        <okhttp.version>4.12.0</okhttp.version>
+        <kotlin-stdlib.version>1.9.10</kotlin-stdlib.version>
     </properties>
 
     <dependencyManagement>
@@ -40,6 +41,26 @@
                 <type>pom</type>
                 <scope>import</scope>
             </dependency>
+            <dependency>
+                <groupId>com.squareup.okhttp3</groupId>
+                <artifactId>okhttp</artifactId>
+                <version>${okhttp.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-stdlib</artifactId>
+                <version>${kotlin-stdlib.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-stdlib-common</artifactId>
+                <version>${kotlin-stdlib.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>org.jetbrains.kotlin</groupId>
+                <artifactId>kotlin-stdlib-jdk8</artifactId>
+                <version>${kotlin-stdlib.version}</version>
+            </dependency>
         </dependencies>
     </dependencyManagement>
 
@@ -67,7 +88,6 @@
         <dependency>
             <groupId>io.opentelemetry</groupId>
             <artifactId>opentelemetry-sdk-extension-autoconfigure</artifactId>
-            <version>${opentelemetry-autoconfigure.version}</version>
         </dependency>
         <dependency>
             <groupId>io.opentelemetry</groupId>