Use local datasource for References service

whois-api/src/main/java/net/ripe/db/whois/api/rest/ReferencesService.java

@@ -18,9 +18,9 @@
 import net.ripe.db.whois.api.rest.marshal.StreamingHelper;
 import net.ripe.db.whois.common.Message;
 import net.ripe.db.whois.common.Messages;
+import net.ripe.db.whois.common.dao.ReferencesDao;
 import net.ripe.db.whois.common.dao.RpslObjectDao;
 import net.ripe.db.whois.common.dao.RpslObjectInfo;
-import net.ripe.db.whois.common.dao.RpslObjectUpdateDao;
 import net.ripe.db.whois.common.rpsl.AttributeTemplate;
 import net.ripe.db.whois.common.rpsl.AttributeType;
 import net.ripe.db.whois.common.rpsl.ObjectTemplate;
@@ -42,6 +42,7 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.dao.DataAccessException;
 import org.springframework.dao.EmptyResultDataAccessException;
@@ -94,7 +95,7 @@ public class ReferencesService {
     private static final Logger LOGGER = LoggerFactory.getLogger(ReferencesService.class);
 
     private final RpslObjectDao rpslObjectDao;
-    private final RpslObjectUpdateDao rpslObjectUpdateDao;
+    private final ReferencesDao referencesDao;
     private final SourceContext sourceContext;
     private final InternalUpdatePerformer updatePerformer;
     private final SsoTranslator ssoTranslator;
@@ -104,17 +105,16 @@ public class ReferencesService {
     private final String dummyRole;
     @Autowired
     public ReferencesService(
-            final RpslObjectDao rpslObjectDao,
-            final RpslObjectUpdateDao rpslObjectUpdateDao,
+            @Qualifier("jdbcRpslObjectSlaveDao") final RpslObjectDao rpslObjectDao,
+            final ReferencesDao referencesDao,
             final SourceContext sourceContext,
             final InternalUpdatePerformer updatePerformer,
             final SsoTranslator ssoTranslator,
             final LoggerContext loggerContext,
             final WhoisObjectMapper whoisObjectMapper,
             final @Value("#{${whois.dummy}}") Map<String, String> dummyMap) {
-
         this.rpslObjectDao = rpslObjectDao;
-        this.rpslObjectUpdateDao = rpslObjectUpdateDao;
+        this.referencesDao = referencesDao;
         this.sourceContext = sourceContext;
         this.updatePerformer = updatePerformer;
         this.ssoTranslator = ssoTranslator;
@@ -572,7 +572,7 @@ private void validateReferences(final RpslObject primaryObject, final Map<RpslOb
 
         for (final Map.Entry<RpslObjectInfo, RpslObject> entry : references.entrySet()) {
             final RpslObject reference = entry.getValue();
-            for (final RpslObjectInfo referenceToReference : rpslObjectUpdateDao.getReferences(reference)) {
+            for (final RpslObjectInfo referenceToReference : referencesDao.getReferences(reference)) {
                 if (!referenceMatches(referenceToReference, primaryObject) && !references.keySet().contains(referenceToReference)) {
                     throw new IllegalArgumentException("Referencing object " + reference.getKey()  + " itself is referenced by " + referenceToReference.getKey());
                 }
@@ -718,7 +718,7 @@ private RpslObject lookupObjectByKey(final String primaryKey, final ObjectType o
     private Map<RpslObjectInfo, RpslObject> findReferences(final RpslObject rpslObject) {
         final Map<RpslObjectInfo, RpslObject> references = Maps.newHashMap();
         try {
-            for (final RpslObjectInfo rpslObjectInfo : rpslObjectUpdateDao.getReferences(rpslObject)) {
+            for (final RpslObjectInfo rpslObjectInfo : referencesDao.getReferences(rpslObject)) {
                 references.put(rpslObjectInfo, rpslObjectDao.getById(rpslObjectInfo.getObjectId()));
             }
         } catch (EmptyResultDataAccessException e) {

whois-commons/src/main/java/net/ripe/db/whois/common/dao/ReferencesDao.java

@@ -0,0 +1,23 @@
+package net.ripe.db.whois.common.dao;
+
+import net.ripe.db.whois.common.domain.CIString;
+import net.ripe.db.whois.common.rpsl.AttributeType;
+import net.ripe.db.whois.common.rpsl.RpslAttribute;
+import net.ripe.db.whois.common.rpsl.RpslObject;
+
+import javax.annotation.CheckForNull;
+import java.util.Map;
+import java.util.Set;
+
+public interface ReferencesDao {
+
+    boolean isReferenced(RpslObject object);
+
+    Set<RpslObjectInfo> getReferences(RpslObject object);
+
+    Map<RpslAttribute, Set<CIString>> getInvalidReferences(RpslObject object);
+
+    @CheckForNull
+    RpslObjectInfo getAttributeReference(AttributeType attributeType, CIString keyValue);
+
+}

whois-commons/src/main/java/net/ripe/db/whois/common/dao/RpslObjectDao.java

@@ -6,9 +6,9 @@
 import net.ripe.db.whois.common.rpsl.AttributeType;
 import net.ripe.db.whois.common.rpsl.ObjectType;
 import net.ripe.db.whois.common.rpsl.RpslObject;
-import java.time.LocalDateTime;
 
 import javax.annotation.Nullable;
+import java.time.LocalDateTime;
 import java.util.Collection;
 import java.util.List;
 import java.util.Set;

whois-commons/src/main/java/net/ripe/db/whois/common/dao/RpslObjectUpdateDao.java

@@ -1,15 +1,8 @@
 package net.ripe.db.whois.common.dao;
 
-import net.ripe.db.whois.common.domain.CIString;
-import net.ripe.db.whois.common.rpsl.AttributeType;
 import net.ripe.db.whois.common.rpsl.ObjectType;
-import net.ripe.db.whois.common.rpsl.RpslAttribute;
 import net.ripe.db.whois.common.rpsl.RpslObject;
 
-import javax.annotation.CheckForNull;
-import java.util.Map;
-import java.util.Set;
-
 public interface RpslObjectUpdateDao {
 
     RpslObjectUpdateInfo lookupObject(ObjectType type, String pkey);
@@ -21,13 +14,4 @@ public interface RpslObjectUpdateDao {
     RpslObjectUpdateInfo updateObject(int objectId, RpslObject object);
 
     RpslObjectUpdateInfo createObject(RpslObject object);
-
-    boolean isReferenced(RpslObject object);
-
-    Set<RpslObjectInfo> getReferences(RpslObject object);
-
-    Map<RpslAttribute, Set<CIString>> getInvalidReferences(RpslObject object);
-
-    @CheckForNull
-    RpslObjectInfo getAttributeReference(AttributeType attributeType, CIString keyValue);
 }

whois-commons/src/main/java/net/ripe/db/whois/common/dao/jdbc/JdbcReferencesDao.java

@@ -0,0 +1,175 @@
+package net.ripe.db.whois.common.dao.jdbc;
+
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import net.ripe.db.whois.common.aspects.RetryFor;
+import net.ripe.db.whois.common.collect.CollectionHelper;
+import net.ripe.db.whois.common.dao.ReferencesDao;
+import net.ripe.db.whois.common.dao.RpslObjectInfo;
+import net.ripe.db.whois.common.dao.jdbc.index.IndexStrategies;
+import net.ripe.db.whois.common.dao.jdbc.index.IndexStrategy;
+import net.ripe.db.whois.common.domain.CIString;
+import net.ripe.db.whois.common.rpsl.AttributeType;
+import net.ripe.db.whois.common.rpsl.ObjectTemplate;
+import net.ripe.db.whois.common.rpsl.ObjectType;
+import net.ripe.db.whois.common.rpsl.RpslAttribute;
+import net.ripe.db.whois.common.rpsl.RpslObject;
+import net.ripe.db.whois.common.source.SourceContext;
+import org.apache.commons.lang.Validate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.dao.RecoverableDataAccessException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Repository;
+
+import javax.annotation.CheckForNull;
+import javax.sql.DataSource;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static net.ripe.db.whois.common.domain.CIString.ciString;
+
+@Repository
+@RetryFor(RecoverableDataAccessException.class)
+// TODO: [ES] sourceaware only seems to use the correct source when extending JdbcRpslObjectDao?
+public class JdbcReferencesDao extends JdbcRpslObjectDao implements ReferencesDao {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(JdbcReferencesDao.class);
+
+    final SourceContext sourceContext;      // TODO: [ES] debug only
+    private final JdbcTemplate jdbcTemplate;
+
+    // TODO: [ES] if we use a hardcoded @Qualifier("whoisSlaveDataSource") final DataSource dataSource
+    // then whois-update will use the possibly out-of-date slave source when validating references.
+    @Autowired
+    public JdbcReferencesDao(@Qualifier("sourceAwareDataSource") final DataSource dataSource, final SourceContext sourceContext) {
+        super(dataSource, sourceContext);
+        this.jdbcTemplate = new JdbcTemplate(dataSource);
+        this.sourceContext = sourceContext;
+    }
+
+    @Override
+    public boolean isReferenced(final RpslObject object) {
+        LOGGER.warn("*** isReferenced: currentSource = {} ***", sourceContext.getCurrentSource());
+
+        for (final RpslAttribute attribute : object.findAttributes(ObjectTemplate.getTemplate(object.getType()).getKeyAttributes())) {
+            for (final IndexStrategy indexStrategy : IndexStrategies.getReferencing(object.getType())) {
+                for (final CIString value : attribute.getReferenceValues()) {
+                    for (final RpslObjectInfo result : indexStrategy.findInIndex(jdbcTemplate, value)) {
+                        if (object.getKey().equals(ciString(result.getKey())) && result.getObjectType().equals(object.getType())) {
+                            continue;
+                        }
+
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
+
+    @Override
+    public Set<RpslObjectInfo> getReferences(final RpslObject object) {
+        LOGGER.warn("*** getReferences: currentSource = {} ***", sourceContext.getCurrentSource());
+
+        final Set<RpslObjectInfo> references = Sets.newHashSet();
+        final List<IndexStrategy> indexStrategies = IndexStrategies.getReferencing(object.getType());
+
+        // for route(6), individually check each key
+        for (final RpslAttribute keyAttr : object.findAttributes(ObjectTemplate.getTemplate(object.getType()).getKeyAttributes())) {
+            for (final IndexStrategy indexStrategy : indexStrategies) {
+                for (final CIString value : keyAttr.getReferenceValues()) {
+                    final List<RpslObjectInfo> results = indexStrategy.findInIndex(jdbcTemplate, value);
+                    for (final RpslObjectInfo result : results) {
+                        if (object.getKey().equals(result.getKey()) && result.getObjectType().equals(object.getType())) {
+                            continue;
+                        }
+
+                        references.add(result);
+                    }
+                }
+            }
+        }
+
+        return references;
+    }
+
+    @Override
+    public Map<RpslAttribute, Set<CIString>> getInvalidReferences(final RpslObject object) {
+        LOGGER.warn("*** getInvalidReferences: currentSource = {} ***", sourceContext.getCurrentSource());
+
+        final Map<RpslAttribute, Set<CIString>> invalidReferenceMap = Maps.newHashMap();
+
+        for (final RpslAttribute attribute : object.getAttributes()) {
+            final Set<CIString> invalidReferenceValues = getInvalidReferences(object, attribute);
+            if (!invalidReferenceValues.isEmpty()) {
+                invalidReferenceMap.put(attribute, invalidReferenceValues);
+            }
+        }
+
+        return invalidReferenceMap;
+    }
+
+    private Set<CIString> getInvalidReferences(final RpslObject object, final RpslAttribute attribute) {
+        AttributeType attributeType = attribute.getType();
+        if (attributeType != null && attributeType.getReferences().isEmpty()) {
+            return Collections.emptySet();
+        }
+
+        final Set<CIString> invalidReferences = Sets.newLinkedHashSet();
+        for (final CIString value : attribute.getReferenceValues()) {
+            if (isInvalidReference(object, attributeType, value)) {
+                invalidReferences.add(value);
+            }
+        }
+
+        return invalidReferences;
+    }
+
+    private boolean isInvalidReference(final RpslObject object, final AttributeType attributeType, final CIString referenceValue) {
+        final Set<ObjectType> references = attributeType.getReferences(referenceValue);
+        if (references.isEmpty()) {
+            return false;
+        }
+
+        for (final ObjectType reference : references) {
+            if (reference.equals(object.getType()) && object.getKey().equals(referenceValue)) {
+                return false;
+            }
+
+            if (getAttributeReference(reference, referenceValue) != null) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    @CheckForNull
+    public RpslObjectInfo getAttributeReference(final AttributeType attributeType, final CIString value) {
+        LOGGER.warn("*** getAttributeReference: currentSource = {} ***", sourceContext.getCurrentSource());
+
+        final CIString referenceValue = new RpslAttribute(attributeType, value.toString()).getReferenceValue();
+        for (final ObjectType objectType : attributeType.getReferences()) {
+            final RpslObjectInfo result = getAttributeReference(objectType, referenceValue);
+            if (result != null) {
+                return result;
+            }
+        }
+
+        return null;
+    }
+
+    private RpslObjectInfo getAttributeReference(final ObjectType objectType, final CIString keyValue) {
+        final ObjectTemplate referenceTemplate = ObjectTemplate.getTemplate(objectType);
+        final Set<AttributeType> referenceKeyAttributes = referenceTemplate.getKeyAttributes();
+        Validate.isTrue(referenceKeyAttributes.size() == 1, "We can never have a reference to a composed key");
+        final IndexStrategy indexStrategy = IndexStrategies.get(referenceKeyAttributes.iterator().next());
+        final List<RpslObjectInfo> result = indexStrategy.findInIndex(jdbcTemplate, keyValue);
+        return CollectionHelper.uniqueResult(result);
+    }
+}

whois-commons/src/main/java/net/ripe/db/whois/common/dao/jdbc/JdbcRpslObjectDao.java

@@ -346,4 +346,5 @@ public Collection<RpslObjectInfo> relatedTo(final RpslObject identifiable, final
 
         return result;
     }
+
 }