Merge pull request #51 from hellojavaer/1.0.x

1.0.x

Author: hellojavaer

ddal-ddr/src/main/java/org/hellojavaer/ddal/ddr/datasource/manager/rw/DefaultReadWriteDataSourceManager.java

@@ -76,7 +76,7 @@ public class DefaultReadWriteDataSourceManager implements ReadWriteDataSourceMan
     private boolean                                                initialized                                = false;
 
     // cache
-    private Map<String, Set<String>>                               physicalTables                             = null;
+    private volatile Map<String, Set<String>>                      physicalTables                             = null;
 
     private DefaultReadWriteDataSourceManager() {
         this.metaDataChecker = new DefaultMetaDataChecker("mysql");
@@ -565,7 +565,15 @@ private void check(Connection conn, String scName) {
             return;
         }
         Map<String, Set<String>> physicalTables = getPhysicalTables();
-        metaDataChecker.check(conn, scName, physicalTables.get(scName));
+        if (physicalTables != null) {
+            Set<String> tbNames = physicalTables.get(scName);
+            if (tbNames != null && !tbNames.isEmpty()) {
+                metaDataChecker.check(conn, scName, tbNames);
+                if (stdLogger.isInfoEnabled()) {
+                    stdLogger.info("MetaDataCheck - sc:{}, tb:{} meta data checking is passed", scName, tbNames);
+                }
+            }
+        }
     }
 
     private void initWriteOnlyDataSource(List<WriteOnlyDataSourceBinding> bindings) {

ddal-ddr/src/main/java/org/hellojavaer/ddal/ddr/shard/simple/SimpleShardRouter.java

@@ -15,8 +15,10 @@
  */
 package org.hellojavaer.ddal.ddr.shard.simple;
 
-import org.hellojavaer.ddal.ddr.expression.range.RangeExpressionParser;
+import com.googlecode.concurrentlinkedhashmap.ConcurrentLinkedHashMap;
+import com.googlecode.concurrentlinkedhashmap.Weighers;
 import org.hellojavaer.ddal.ddr.expression.range.RangeExpressionItemVisitor;
+import org.hellojavaer.ddal.ddr.expression.range.RangeExpressionParser;
 import org.hellojavaer.ddal.ddr.shard.*;
 import org.hellojavaer.ddal.ddr.shard.exception.*;
 import org.hellojavaer.ddal.ddr.utils.DDRStringUtils;
@@ -31,11 +33,21 @@
  */
 public class SimpleShardRouter implements ShardRouter {
 
-    private Logger                                               logger            = LoggerFactory.getLogger(getClass());
-    private List<SimpleShardRouteRuleBinding>                    routeRuleBindings = null;
-    private Map<String, InnerSimpleShardRouteRuleBindingWrapper> cache             = Collections.EMPTY_MAP;
-    private Map<String, List<ShardRouteInfo>>                    routeInfoMap      = new HashMap<>();
-    private Map<String, Set<String>>                             routedTables      = new HashMap<>();
+    private Logger                                               logger                        = LoggerFactory.getLogger(getClass());
+
+    // schema+table级路由配置
+    private List<SimpleShardRouteRuleBinding>                    routeRuleBindings             = null;
+    private Map<String, InnerSimpleShardRouteRuleBindingWrapper> routeInfoCache                = Collections.EMPTY_MAP;
+    private Map<String, List<ShardRouteInfo>>                    routeInfosCache               = new HashMap<>();
+    private Map<String, Set<String>>                             routedTables                  = new HashMap<>();
+
+    // schema级路由配置
+    private Map<String, SimpleShardRouteRuleBinding>             schemaBindingMap              = new HashMap<>();
+    private SimpleShardRouteRuleBinding                          defaultSchemaBinding          = null;
+    private Map<String, List<ShardRouteInfo>>                    routeInfosCacheForSchemaLevel = new ConcurrentLinkedHashMap.Builder<String, List<ShardRouteInfo>>()//
+                                                                                               .maximumWeightedCapacity(10000)//
+                                                                                               .weigher(Weighers.singleton())//
+                                                                                               .build();
 
     private SimpleShardRouter() {
     }
@@ -63,47 +75,62 @@ public void setRouteRuleBindings(List<SimpleShardRouteRuleBinding> bindings) {
                 if (scName == null) {
                     throw new IllegalArgumentException("'scName' can't be empty");
                 }
-                if (tbName == null) {
-                    throw new IllegalArgumentException("'tbName' can't be empty");
-                }
-                final SimpleShardRouteRuleBinding b0 = new SimpleShardRouteRuleBinding();
-                b0.setScName(scName);
-                b0.setTbName(tbName);
-                b0.setSdKey(sdKey);
-                b0.setRule(binding.getRule());
-                StringBuilder sb = new StringBuilder();
-                sb.append(scName).append('.').append(tbName);
-                putToCache(cache, sb.toString(), b0, true);
-                putToCache(cache, tbName, b0, false);
-
-                final Set<ShardRouteInfo> routeInfos = new LinkedHashSet<>();
-                if (sdValues != null) {
-                    new RangeExpressionParser(sdValues).visit(new RangeExpressionItemVisitor() {
+                if (tbName != null) {
+                    final SimpleShardRouteRuleBinding b0 = new SimpleShardRouteRuleBinding();
+                    b0.setScName(scName);
+                    b0.setTbName(tbName);
+                    b0.setSdKey(sdKey);
+                    b0.setRule(binding.getRule());
+                    StringBuilder sb = new StringBuilder();
+                    sb.append(scName).append('.').append(tbName);
+                    putToCache(cache, sb.toString(), b0, true);
+                    putToCache(cache, tbName, b0, false);
 
-                        @Override
-                        public void visit(Object val) {
-                            ShardRouteInfo routeInfo = getRouteInfo(b0, scName, tbName, val);
-                            routeInfos.add(routeInfo);
+                    // 构建 逻辑sc+逻辑tb下包含的所有物理表信息
+                    final Set<ShardRouteInfo> routeInfos = new LinkedHashSet<>();
+                    if (binding.getRule() == null) {// 如果路由规则为空,则使用原表名
+                        ShardRouteInfo routeInfo = new ShardRouteInfo();
+                        routeInfo.setScName(scName);
+                        routeInfo.setTbName(tbName);
+                        routeInfos.add(routeInfo);
+                    } else {
+                        if (sdValues != null) {
+                            new RangeExpressionParser(sdValues).visit(new RangeExpressionItemVisitor() {
+
+                                @Override
+                                public void visit(Object val) {
+                                    ShardRouteInfo routeInfo = getRouteInfo(b0, scName, tbName, val);
+                                    routeInfos.add(routeInfo);
+                                }
+                            });
                         }
-                    });
-                }
-                String key = buildQueryKey(scName, tbName);
-                if (routeInfoMap.containsKey(key)) {
-                    throw new IllegalArgumentException("Duplicate route config for table '" + key + "'");
-                } else {
-                    Set<String> tables = routedTables.get(scName);
-                    if (tables == null) {
-                        tables = new LinkedHashSet<>();
-                        routedTables.put(scName, tables);
                     }
-                    tables.add(tbName);
-                    routeInfoMap.put(key, new ArrayList(routeInfos));
+                    String key = buildQueryKey(scName, tbName);
+                    if (routeInfoMap.containsKey(key)) {
+                        throw new IllegalArgumentException("Duplicate route config for table '" + key + "'");
+                    } else {
+                        Set<String> tables = routedTables.get(scName);
+                        if (tables == null) {
+                            tables = new LinkedHashSet<>();
+                            routedTables.put(scName, tables);
+                        }
+                        tables.add(tbName);
+                        routeInfoMap.put(key, new ArrayList(routeInfos));
+                    }
+                } else {// 配置schema级别路由规则
+                    if (defaultSchemaBinding == null) {// 使用列表中的第一个作为默认配置
+                        defaultSchemaBinding = binding;
+                    }
+                    if (schemaBindingMap.put(scName, binding) != null) {
+                        throw new DuplicateRouteRuleBindingException("Duplicate route rule binding for scName:"
+                                                                     + scName);
+                    }
                 }
             }
         }
         this.routeRuleBindings = bindings;
-        this.cache = cache;
-        this.routeInfoMap = routeInfoMap;
+        this.routeInfoCache = cache;
+        this.routeInfosCache = routeInfoMap;
         this.routedTables = routedTables;
     }
 
@@ -164,7 +191,7 @@ private InnerSimpleShardRouteRuleBindingWrapper getBinding(String scName, String
         if (scName != null) {
             queryKey = new StringBuilder().append(scName).append('.').append(tbName).toString();
         }
-        InnerSimpleShardRouteRuleBindingWrapper ruleBindingWrapper = cache.get(queryKey);
+        InnerSimpleShardRouteRuleBindingWrapper ruleBindingWrapper = routeInfoCache.get(queryKey);
         if (ruleBindingWrapper == null) {
             return null;
         } else if (ruleBindingWrapper.getConflictSchemas().size() > 1) {
@@ -179,23 +206,47 @@ private InnerSimpleShardRouteRuleBindingWrapper getBinding(String scName, String
     public ShardRouteRule getRouteRule(String scName, String tbName) {
         scName = DDRStringUtils.toLowerCase(scName);
         tbName = DDRStringUtils.toLowerCase(tbName);
+        if (tbName == null) {
+            throw new IllegalArgumentException("tbName can't be null");
+        }
         InnerSimpleShardRouteRuleBindingWrapper bindingWrapper = getBinding(scName, tbName);
-        if (bindingWrapper == null || bindingWrapper.getRuleBinding() == null) {
-            return null;
-        } else {
-            return bindingWrapper.getRuleBinding().getRule();
+        if (bindingWrapper != null) {// 1.从schema+table级中获取
+            if (bindingWrapper.getRuleBinding() != null) {
+                return bindingWrapper.getRuleBinding().getRule();
+            } else {
+                return null;
+            }
+        } else {// 2.从schema级别获取
+            SimpleShardRouteRuleBinding binding = getRouteRuleBindingBySchemaLevel(scName);
+            if (binding != null) {
+                return binding.getRule();
+            } else {
+                return null;
+            }
         }
     }
 
     @Override
     public ShardRouteConfig getRouteConfig(String scName, String tbName) {
         scName = DDRStringUtils.toLowerCase(scName);
         tbName = DDRStringUtils.toLowerCase(tbName);
+        if (tbName == null) {
+            throw new IllegalArgumentException("tbName can't be null");
+        }
         InnerSimpleShardRouteRuleBindingWrapper bindingWrapper = getBinding(scName, tbName);
-        if (bindingWrapper == null) {
-            return null;
-        } else {
+        if (bindingWrapper != null) {// 1.从schema+table级中获取
             return bindingWrapper.getRouteConfig();
+        } else {// 2.从schema级别获取
+            SimpleShardRouteRuleBinding binding = getRouteRuleBindingBySchemaLevel(scName);
+            if (binding == null) {
+                return null;
+            } else {
+                ShardRouteConfig config = new ShardRouteConfig();
+                config.setScName(binding.getScName());
+                config.setTbName(tbName);// note here
+                config.setSdKey(binding.getSdKey());
+                return config;
+            }
         }
     }
 
@@ -205,20 +256,86 @@ public ShardRouteInfo getRouteInfo(String scName, String tbName, Object sdValue)
                                                                                     ShardRouteException {
         scName = DDRStringUtils.toLowerCase(scName);
         tbName = DDRStringUtils.toLowerCase(tbName);
+        if (tbName == null) {
+            throw new IllegalArgumentException("tbName can't be null");
+        }
+        SimpleShardRouteRuleBinding binding = null;
         InnerSimpleShardRouteRuleBindingWrapper bindingWrapper = getBinding(scName, tbName);
-        SimpleShardRouteRuleBinding binding = bindingWrapper.getRuleBinding();
+        if (bindingWrapper != null) {// 1.从schema+table级中获取
+            binding = bindingWrapper.getRuleBinding();
+        } else {// 2.从schema级获取
+            binding = getRouteRuleBindingBySchemaLevel(scName);
+        }
         if (binding == null) {
             return null;
-        } else {// 必须使用 binding 中的 scName,因为sql中的scName可能为空
-            ShardRouteInfo info = getRouteInfo(binding, binding.getScName(), binding.getTbName(), sdValue);
+        } else {// 必须使用binding中的scName,因为sql中的scName可能为空,binding.getTbName()可能为null必须使用tbName
+            ShardRouteInfo info = getRouteInfo(binding, binding.getScName(), tbName, sdValue);
             return info;
         }
     }
 
+    /**
+     * 
+     * @param scName can be null
+     * @param tbName can't be null
+     * @return if sdValues is null or scName-tbName don't hit config return null
+     * @throws ShardValueNotFoundException
+     * @throws ShardRouteException
+     */
     @Override
     public List<ShardRouteInfo> getRouteInfos(String scName, String tbName) throws ShardValueNotFoundException,
                                                                            ShardRouteException {
-        return routeInfoMap.get(buildQueryKey(scName, tbName));
+        scName = DDRStringUtils.toLowerCase(scName);
+        tbName = DDRStringUtils.toLowerCase(tbName);
+        if (tbName == null) {
+            throw new IllegalArgumentException("tbName can't be null");
+        }
+        List<ShardRouteInfo> result = routeInfosCache.get(buildQueryKey(scName, tbName));
+        if (result != null) {
+            return result;
+        } else {// 查询schema配置级别
+            SimpleShardRouteRuleBinding binding = getRouteRuleBindingBySchemaLevel(scName);
+            if (binding == null) {
+                return null;
+            } else {
+                String key = buildQueryKey(binding.getScName(), tbName);
+                String sdValues = binding.getSdValues();
+                if (sdValues != null) {
+                    List<ShardRouteInfo> list = routeInfosCacheForSchemaLevel.get(key);
+                    if (list == null) {
+                        final List<ShardRouteInfo> routeInfos = new ArrayList<>();
+                        final String scName0 = scName;
+                        final String tbName0 = tbName;
+                        new RangeExpressionParser(sdValues).visit(new RangeExpressionItemVisitor() {
+
+                            @Override
+                            public void visit(Object val) {
+                                ShardRouteInfo routeInfo = getRouteInfo(scName0, tbName0, val);
+                                routeInfos.add(routeInfo);
+                            }
+                        });
+                        routeInfosCacheForSchemaLevel.put(key, routeInfos);
+                        list = routeInfos;
+                    }
+                    return list;
+                } else {
+                    return null;
+                }
+            }
+        }
+    }
+
+    /**
+     * 
+     * @param scName can be null
+     * @return
+     */
+    private SimpleShardRouteRuleBinding getRouteRuleBindingBySchemaLevel(String scName) {
+        if (scName != null) {
+            return schemaBindingMap.get(scName);
+        } else {
+            return defaultSchemaBinding;
+        }
     }
 
     @Override

ddal-example/ddal-example-example0/src/main/resources/datasource.xml

@@ -31,7 +31,7 @@
             <list>
                 <bean class="org.hellojavaer.ddal.ddr.shard.simple.SimpleShardRouteRuleBinding">
                     <property name="scName" value="base"></property>
-                    <property name="tbName" value="user"></property>
+                    <!--<property name="tbName" value="user"></property>-->
                     <property name="rule" ref="idRule"></property>
                     <property name="sdKey" value="id"></property>
                     <property name="sdValues" value="[0..7]"></property>

ddal-example/ddal-example-example0/src/main/resources/log4j.xml

@@ -15,7 +15,7 @@
 
     <logger name="org.hellojavaer.ddr.ds" additivity="false">
         <!-- if the value of 'value' is 'trace', target datasource will be printed out -->
-        <priority value="error"/>
+        <priority value="info"/>
         <appender-ref ref="console"/>
     </logger>
 

ddal-example/ddal-example-example1/src/main/resources/datasource.xml

@@ -27,7 +27,7 @@
             <list>
                 <bean class="org.hellojavaer.ddal.ddr.shard.simple.SimpleShardRouteRuleBinding">
                     <property name="scName" value="acl"></property>
-                    <property name="tbName" value="role"></property>
+                    <!--<property name="tbName" value="role"></property>-->
                 </bean>
             </list>
         </property>
@@ -45,6 +45,7 @@
     </bean>
     <bean id="dataSourceManager"
           class="org.hellojavaer.ddal.ddr.datasource.manager.rw.DefaultReadWriteDataSourceManager">
+        <property name="shardRouter" ref="shardRouter"/>
         <property name="readOnlyDataSourceMonitorServer">
             <bean class="org.hellojavaer.ddal.ddr.datasource.manager.rw.monitor.mbean.MBeanReadOnlyDataSourceMonitorServer"></bean>
         </property>