Do not reload table schema when update catalog cache (#2667) (#2677)

pingcap · Apr 27, 2023 · 0947e43 · 0947e43
1 parent 1d7009e
commit 0947e43
Show file tree

Hide file tree

Showing 7 changed files with 83 additions and 29 deletions.
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -60,8 +60,11 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+#    - name: Autobuild
+#      uses: github/codeql-action/autobuild@v1
+    - name: build
+      run: |
+          mvn clean package -Dmaven.test.skip=true -B
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl

diff --git a/core/src/main/scala/com/pingcap/tispark/TiConfigConst.scala b/core/src/main/scala/com/pingcap/tispark/TiConfigConst.scala
@@ -97,4 +97,7 @@ object TiConfigConst {
   val GC_MAX_WAIT_TIME: String = "spark.tispark.gc_max_wait_time"
   val DEFAULT_GC_MAX_WAIT_TIME: Long = 24 * 60 * 60
   val DEFAULT_GC_SAFE_POINT_TTL: Int = 5 * 60
+  // cache load
+  val LOAD_TABLES: String = "spark.tispark.load_tables"
+  val DEFAULT_LOAD_TABLES: Boolean = true
 }
diff --git a/core/src/main/scala/com/pingcap/tispark/utils/TiUtil.scala b/core/src/main/scala/com/pingcap/tispark/utils/TiUtil.scala
@@ -255,6 +255,9 @@ object TiUtil {
     if (conf.contains(TiConfigConst.ENABLE_GRPC_FORWARD)) {
       tiConf.setEnableGrpcForward(conf.get(TiConfigConst.ENABLE_GRPC_FORWARD).toBoolean)
     }
+
+    tiConf.setLoadTables(
+      conf.get(TiConfigConst.LOAD_TABLES, TiConfigConst.DEFAULT_LOAD_TABLES.toString).toBoolean)
     tiConf
   }
 

diff --git a/docs/userguide_3.0.md b/docs/userguide_3.0.md
diff --git a/tikv-client/src/main/java/com/pingcap/tikv/ClientSession.java b/tikv-client/src/main/java/com/pingcap/tikv/ClientSession.java
@@ -62,7 +62,11 @@ public Catalog getCatalog() {
       synchronized (this) {
         if (catalog == null) {
           catalog =
-              new Catalog(this::createSnapshot, getConf().isShowRowId(), getConf().getDBPrefix());
+              new Catalog(
+                  this::createSnapshot,
+                  getConf().isShowRowId(),
+                  getConf().getDBPrefix(),
+                  getConf().getLoadTables());
         }
         res = catalog;
       }
@@ -117,9 +121,11 @@ public synchronized Catalog getOrCreateSnapShotCatalog(TiTimestamp ts) {
     if (snapshotCatalog == null) {
       snapshotCatalog =
           new Catalog(
-              this::createSnapshotWithSnapshotTimestamp, conf.isShowRowId(), conf.getDBPrefix());
+              this::createSnapshotWithSnapshotTimestamp,
+              conf.isShowRowId(),
+              conf.getDBPrefix(),
+              conf.getLoadTables());
     }
-    snapshotCatalog.reloadCache(true);
     return snapshotCatalog;
   }
 

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/TiConfiguration.java b/tikv-client/src/main/java/com/pingcap/tikv/TiConfiguration.java
@@ -88,6 +88,11 @@ public Optional<Boolean> getNewCollationEnable() {
   private int tikvRegionSplitSizeInMB = DEF_TIKV_REGION_SPLIT_SIZE_IN_MB;
   private static final int DEF_PARTITION_PER_SPLIT = 10;
   private int partitionPerSplit = DEF_PARTITION_PER_SPLIT;
+  private boolean loadTables = true;
+
+  public boolean getLoadTables() {
+    return loadTables;
+  }
   // ----------     same with client-java  ------------
 
   public String getPdAddrsString() {

diff --git a/tikv-client/src/main/java/com/pingcap/tikv/catalog/Catalog.java b/tikv-client/src/main/java/com/pingcap/tikv/catalog/Catalog.java
@@ -41,13 +41,16 @@ public class Catalog implements AutoCloseable {
   private final Supplier<Snapshot> snapshotProvider;
   private CatalogCache metaCache;
   private static final AtomicLong lastUpdateTime = new AtomicLong(0);
+  private final boolean loadTables;
 
-  public Catalog(Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix) {
+  public Catalog(
+      Supplier<Snapshot> snapshotProvider, boolean showRowId, String dbPrefix, boolean loadTables) {
     this.snapshotProvider = Objects.requireNonNull(snapshotProvider, "Snapshot Provider is null");
     this.showRowId = showRowId;
     this.dbPrefix = dbPrefix;
-    metaCache = new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, false);
-    reloadCache(true);
+    this.loadTables = loadTables;
+    metaCache =
+        new CatalogCache(new CatalogTransaction(snapshotProvider.get()), dbPrefix, loadTables);
   }
 
   @Override
@@ -67,18 +70,14 @@ public void reloadCache(boolean loadTables) {
     }
   }
 
-  private void reloadCache() {
-    reloadCache(false);
-  }
-
   public List<TiDBInfo> listDatabases() {
-    reloadCache();
+    reloadCache(false);
     return metaCache.listDatabases();
   }
 
   public List<TiTableInfo> listTables(TiDBInfo database) {
     Objects.requireNonNull(database, "database is null");
-    reloadCache(true);
+    reloadCache(loadTables);
     if (showRowId) {
       return metaCache
           .listTables(database)
@@ -133,7 +132,7 @@ public TiTableInfo getTableFromCache(TiDBInfo database, String tableName) {
 
   public TiDBInfo getDatabase(String dbName) {
     Objects.requireNonNull(dbName, "dbName is null");
-    reloadCache();
+    reloadCache(false);
     return metaCache.getDatabase(dbName);
   }
 
@@ -148,7 +147,7 @@ public TiTableInfo getTable(String dbName, String tableName) {
   public TiTableInfo getTable(TiDBInfo database, String tableName) {
     Objects.requireNonNull(database, "database is null");
     Objects.requireNonNull(tableName, "tableName is null");
-    reloadCache(true);
+    reloadCache(loadTables);
     TiTableInfo table = metaCache.getTable(database, tableName);
     if (showRowId && table != null) {
       return table.copyTableWithRowId();