Skip to content

Commit

Permalink
Adding spring integration for messages and redis for clustering (#27)
Browse files Browse the repository at this point in the history
* Spring Integration
* Caffeine Cache
* Redis Profile for clustering
  • Loading branch information
cjmalloy authored Apr 2, 2024
1 parent ae30dec commit 9b3b282
Show file tree
Hide file tree
Showing 126 changed files with 3,347 additions and 2,453 deletions.
152 changes: 72 additions & 80 deletions README.md

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,12 @@ services:
POSTGRES_USER: jasper
POSTGRES_PASSWORD: jasper
POSTGRES_DB: jasper
redis:
image: redis
ports:
- 6379:6379
redis-monitor:
image: redis
entrypoint: sh -c 'sleep 10 && redis-cli -h redis monitor'
depends_on:
- redis
27 changes: 21 additions & 6 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
<rome.version>2.1.0</rome.version>
<apache-httpclient.version>4.5.14</apache-httpclient.version>
<commons-io.version>2.15.1</commons-io.version>
<varv.version>0.10.4</varv.version>
<jsoup.version>1.17.2</jsoup.version>
<scim2-client.version>2.3.8</scim2-client.version>
<openai-gpt.version>0.18.2</openai-gpt.version>
Expand Down Expand Up @@ -79,6 +78,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-cache</artifactId>
</dependency>
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
</dependency>
<dependency>
<groupId>org.liquibase</groupId>
<artifactId>liquibase-core</artifactId>
Expand Down Expand Up @@ -134,11 +137,6 @@
<artifactId>commons-io</artifactId>
<version>${commons-io.version}</version>
</dependency>
<dependency>
<groupId>io.vavr</groupId>
<artifactId>vavr</artifactId>
<version>${varv.version}</version>
</dependency>
<dependency>
<groupId>org.openapitools</groupId>
<artifactId>jackson-databind-nullable</artifactId>
Expand Down Expand Up @@ -329,6 +327,18 @@
<groupId>org.springframework.security</groupId>
<artifactId>spring-security-data</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-integration</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.integration</groupId>
<artifactId>spring-integration-redis</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-redis</artifactId>
</dependency>
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
Expand All @@ -353,6 +363,11 @@
<artifactId>spring-security-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.integration</groupId>
<artifactId>spring-integration-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/jasper/JasperApplication.java
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
import jasper.config.Props;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration;
import org.springframework.boot.autoconfigure.liquibase.LiquibaseProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cloud.kubernetes.fabric8.discovery.KubernetesCatalogWatchAutoConfiguration;
import org.springframework.cloud.kubernetes.fabric8.discovery.KubernetesDiscoveryClientAutoConfiguration;

@SpringBootApplication(exclude = { KubernetesDiscoveryClientAutoConfiguration.class, KubernetesCatalogWatchAutoConfiguration.class })
@SpringBootApplication(exclude = { KubernetesDiscoveryClientAutoConfiguration.class, KubernetesCatalogWatchAutoConfiguration.class, RedisAutoConfiguration.class })
@EnableConfigurationProperties({ LiquibaseProperties.class, Props.class })
@EnableCaching
public class JasperApplication {
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/jasper/client/JasperClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import java.util.List;
import java.util.Map;

@FeignClient(value = "jasper", url = "https://jasperkms.info")
@FeignClient(value = "jasper", url = "https://jasperkm.info")
public interface JasperClient {

@RequestLine("GET /pub/api/v1/repl/ref")
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/jasper/component/AccessToken.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,17 @@ public String getAdminToken() {

private URI baseUri() {
try {
return new URI(auth.getClient().getAuthentication().getJwt().getTokenEndpoint());
return new URI(auth.security().getTokenEndpoint());
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}

private String getSecret() {
return props.getSecurity().getClient(props.getLocalOrigin()).getAuthentication().getJwt().getSecret();
return auth.security().getSecret();
}

private String getClientId() {
return props.getSecurity().getClient(props.getLocalOrigin()).getAuthentication().getJwt().getClientId();
return auth.security().getClientId();
}
}
209 changes: 209 additions & 0 deletions src/main/java/jasper/component/ConfigCache.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
package jasper.component;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
import jasper.component.dto.ComponentDtoMapper;
import jasper.config.Config.SecurityConfig;
import jasper.config.Config.ServerConfig;
import jasper.config.Props;
import jasper.domain.Plugin;
import jasper.domain.Template;
import jasper.repository.PluginRepository;
import jasper.repository.RefRepository;
import jasper.repository.TemplateRepository;
import jasper.repository.UserRepository;
import jasper.repository.filter.RefFilter;
import jasper.service.dto.PluginDto;
import jasper.service.dto.TemplateDto;
import jasper.service.dto.UserDto;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Component;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.PostConstruct;
import java.util.List;
import java.util.Optional;

@Component
public class ConfigCache {
private static final Logger logger = LoggerFactory.getLogger(ConfigCache.class);

@Autowired
Props props;

@Autowired
RefRepository refRepository;

@Autowired
UserRepository userRepository;

@Autowired
PluginRepository pluginRepository;

@Autowired
TemplateRepository templateRepository;

@Autowired
IngestTemplate ingest;

@Autowired
ObjectMapper objectMapper;

@Autowired
ComponentDtoMapper dtoMapper;

@PostConstruct
public void init() {
if (templateRepository.findByTemplateAndOrigin("_config/server", "").isEmpty()) {
ingest.push(config());
}
}

@CacheEvict(value = "config-cache", allEntries = true)
public void clearConfigCache() {
logger.info("Cleared config cache.");
}

@CacheEvict(value = "user-cache", allEntries = true)
public void clearUserCache() {
logger.info("Cleared user cache.");
}

@CacheEvict(value = {
"plugin-cache",
"plugin-config-cache",
"plugin-metadata-cache",
"all-plugins-cache",
},
allEntries = true)
public void clearPluginCache() {
logger.info("Cleared plugin cache.");
}

@CacheEvict(value = {
"template-cache",
"template-config-cache",
"template-cache-wrapped",
"template-schemas-cache",
"all-templates-cache",
},
allEntries = true)
public void clearTemplateCache() {
logger.info("Cleared template cache.");
}

@Cacheable("user-cache")
@Transactional(readOnly = true)
public UserDto getUser(String tag) {
return userRepository.findOneByQualifiedTag(tag)
.map(dtoMapper::domainToDto)
.orElse(null);
}

@Cacheable(value = "config-cache", key = "#tag + #origin + '@' + #url")
@Transactional(readOnly = true)
public <T> T getConfig(String url, String origin, String tag, Class<T> toValueType) {
return refRepository.findOneByUrlAndOrigin(url, origin)
.map(r -> r.getPlugin(tag, toValueType))
.orElse(objectMapper.convertValue(objectMapper.createObjectNode(), toValueType));
}

@Cacheable(value = "config-cache", key = "#tag + #origin")
@Transactional(readOnly = true)
public <T> List<T> getAllConfigs(String origin, String tag, Class<T> toValueType) {
return refRepository.findAll(
RefFilter.builder()
.origin(origin)
.query(tag).build().spec()).stream()
.map(r -> r.getPlugin(tag, toValueType))
.toList();
}

@Cacheable(value = "plugin-config-cache", key = "#tag + #origin")
@Transactional(readOnly = true)
public <T> T getPluginConfig(String tag, String origin, Class<T> toValueType) {
return pluginRepository.findByTagAndOrigin(tag, origin)
.map(r -> r.getConfig(toValueType))
.orElse(objectMapper.convertValue(objectMapper.createObjectNode(), toValueType));
}

@Cacheable(value = "plugin-cache", key = "#tag + #origin")
@Transactional(readOnly = true)
public Optional<Plugin> getPlugin(String tag, String origin) {
return pluginRepository.findByTagAndOrigin(tag, origin);
}

@Cacheable("plugin-metadata-cache")
@Transactional(readOnly = true)
public List<String> getMetadataPlugins(String origin) {
return pluginRepository.findAllByGenerateMetadataByOrigin(origin);
}

@Cacheable("all-plugins-cache")
@Transactional(readOnly = true)
public List<PluginDto> getAllPlugins(String origin) {
return pluginRepository.findAllByOrigin(origin)
.stream()
.map(dtoMapper::domainToDto)
.toList();
}

@Cacheable(value = "template-config-cache", key = "#template + #origin")
@Transactional(readOnly = true)
public <T> T getTemplateConfig(String template, String origin, Class<T> toValueType) {
return templateRepository.findByTemplateAndOrigin(template, origin)
.map(r -> r.getConfig(toValueType))
.orElse(objectMapper.convertValue(objectMapper.createObjectNode(), toValueType));
}

@Cacheable(value = "template-cache", key = "#template + #origin")
@Transactional(readOnly = true)
public Optional<Template> getTemplate(String template, String origin) {
return templateRepository.findByTemplateAndOrigin(template, origin);
}

@Cacheable(value = "template-cache", key = "'_config/server'")
@Transactional(readOnly = true)
public ServerConfig root() {
return getTemplateConfig("_config/server", "", ServerConfig.class);
}

@Cacheable(value = "template-cache-wrapped", key = "'_config/security' + #origin")
@Transactional(readOnly = true)
public SecurityConfig security(String origin) {
// TODO: crawl origin hierarchy until found
return getTemplateConfig("_config/security", origin, SecurityConfig.class)
.wrap(props);
}

@Cacheable("template-schemas-cache")
@Transactional(readOnly = true)
public List<TemplateDto> getSchemas(String tag, String origin) {
return templateRepository.findAllForTagAndOriginWithSchema(tag, origin)
.stream()
.map(dtoMapper::domainToDto)
.toList();
}

@Cacheable("all-templates-cache")
@Transactional(readOnly = true)
public List<TemplateDto> getAllTemplates(String origin) {
return templateRepository.findAllByOrigin(origin)
.stream()
.map(dtoMapper::domainToDto)
.toList();
}

private Template config() {
var config = objectMapper.convertValue(objectMapper.createObjectNode(), ServerConfig.class);
var template = new Template();
template.setTag("_config/server");
template.setName("Server Config");
template.setConfig(objectMapper.convertValue(config, ObjectNode.class));
return template;
}
}
4 changes: 0 additions & 4 deletions src/main/java/jasper/component/Ingest.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
import jasper.errors.DuplicateModifiedDateException;
import jasper.errors.ModifiedException;
import jasper.errors.NotFoundException;
import jasper.repository.PluginRepository;
import jasper.repository.RefRepository;
import org.hibernate.exception.ConstraintViolationException;
import org.slf4j.Logger;
Expand Down Expand Up @@ -38,9 +37,6 @@ public class Ingest {
@Autowired
RefRepository refRepository;

@Autowired
PluginRepository pluginRepository;

@Autowired
EntityManager em;

Expand Down
6 changes: 6 additions & 0 deletions src/main/java/jasper/component/IngestExt.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ public class IngestExt {
@Autowired
Validate validate;

@Autowired
Messages messages;

@Autowired
PlatformTransactionManager transactionManager;

Expand All @@ -57,12 +60,14 @@ public void create(Ext ext, boolean force) {
}
validate.ext(ext, force);
ensureCreateUniqueModified(ext);
messages.updateExt(ext);
}

@Timed(value = "jasper.ext", histogram = true)
public void update(Ext ext, boolean force) {
validate.ext(ext, force);
ensureUpdateUniqueModified(ext);
messages.updateExt(ext);
}

@Timed(value = "jasper.ext", histogram = true)
Expand All @@ -78,6 +83,7 @@ public void push(Ext ext) {
} else {
delete(deletorTag(ext.getTag()) + ext.getOrigin());
}
messages.updateExt(ext);
}

@Timed(value = "jasper.ext", histogram = true)
Expand Down
Loading

0 comments on commit 9b3b282

Please sign in to comment.