Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PLT-191: optimizations for methods in metastore #2455

Closed
wants to merge 2 commits into from
Closed

PLT-191: optimizations for methods in metastore #2455

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7fb6b2b
PLT-191: caching support in metastore
n5nk Dec 4, 2023
72f1e1e
atlassearch refresh
n5nk Dec 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .github/workflows/maven.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ on:
- development
- master
- lineageondemand
- atlassearch

jobs:
build:
Expand All @@ -51,6 +52,16 @@ jobs:
run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
id: get_branch

- name: Create Maven Settings
uses: s4u/[email protected]
with:
servers: |
[{
"id": "github",
"username": "atlan-ci",
"password": "${{ secrets.my_pat }}"
}]

- name: Build with Maven
run: |
branch_name=${{ steps.get_branch.outputs.branch }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/trivy-docker-scan.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ on:
- development
- master
- lineageondemand
- atlassearch


jobs:
Expand Down
4 changes: 2 additions & 2 deletions build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ unzip -o keycloak-15.0.2.1.zip -d ~/.m2/repository/org
echo "Maven Building"

if [ "$1" == "build_without_dashboard" ]; then
mvn -pl '!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge,!dashboardv2,!dashboardv3' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipOverlay -DskipEnunciate=true package -Pdist
mvn -pl '!test-tools,!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge,!dashboardv2,!dashboardv3' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipOverlay -DskipEnunciate=true package -Pdist
else
mvn -pl '!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipEnunciate=true package -Pdist
mvn -pl '!test-tools,!addons/hdfs-model,!addons/hive-bridge,!addons/hive-bridge-shim,!addons/falcon-bridge-shim,!addons/falcon-bridge,!addons/sqoop-bridge,!addons/sqoop-bridge-shim,!addons/hbase-bridge,!addons/hbase-bridge-shim,!addons/hbase-testing-util,!addons/kafka-bridge,!addons/impala-hook-api,!addons/impala-bridge-shim,!addons/impala-bridge' -Dmaven.test.skip -DskipTests -Drat.skip=true -DskipEnunciate=true package -Pdist
fi

echo "[DEBUG listing distro/target"
Expand Down
46 changes: 45 additions & 1 deletion common/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,51 @@
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-all</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-common</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-resolver</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-codec</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-buffer</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-epoll</artifactId>
<version>${netty4.version}</version>
</dependency>
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-transport-native-unix-common</artifactId>
<version>${netty4.version}</version>
</dependency>
</dependencies>

<build>
Expand Down
2 changes: 2 additions & 0 deletions common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public abstract class AbstractRedisService implements RedisService {
private static final String ATLAS_REDIS_LOCK_WATCHDOG_TIMEOUT_MS = "atlas.redis.lock.watchdog_timeout.ms";
private static final int DEFAULT_REDIS_WAIT_TIME_MS = 15_000;
private static final int DEFAULT_REDIS_LOCK_WATCHDOG_TIMEOUT_MS = 600_000;
private static final String ATLAS_METASTORE_SERVICE = "atlas-metastore-service";

RedissonClient redisClient;
Map<String, RLock> keyLockMap;
Expand Down Expand Up @@ -99,6 +100,7 @@ Config getLocalConfig() throws AtlasException {
Config getProdConfig() throws AtlasException {
Config config = initAtlasConfig();
config.useSentinelServers()
.setClientName(ATLAS_METASTORE_SERVICE)
.setReadMode(ReadMode.MASTER_SLAVE)
.setCheckSentinelsList(false)
.setMasterName(atlasConfig.getString(ATLAS_REDIS_MASTER_NAME))
Expand Down
4 changes: 4 additions & 0 deletions graphdb/janus/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,10 @@
<groupId>org.codehaus.woodstox</groupId>
<artifactId>woodstox-core-asl</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- CVE Overrides for Lucene -->
Expand Down
13 changes: 12 additions & 1 deletion pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -717,7 +717,7 @@
<httpcomponents-httpcore.version>4.4.13</httpcomponents-httpcore.version>
<jackson.databind.version>2.11.3</jackson.databind.version>
<jackson.version>2.11.3</jackson.version>
<janusgraph.version>0.6.0</janusgraph.version>
<janusgraph.version>0.6.01</janusgraph.version>
<janusgraph.cassandra.version>0.5.3</janusgraph.cassandra.version>
<javax-inject.version>1</javax-inject.version>
<javax.servlet.version>3.1.0</javax.servlet.version>
Expand Down Expand Up @@ -778,6 +778,7 @@
<zookeeper.version>3.4.6</zookeeper.version>
<redis.client.version>3.20.1</redis.client.version>
<micrometer.version>1.11.1</micrometer.version>
<netty4.version>4.1.61.Final</netty4.version>
</properties>

<modules>
Expand Down Expand Up @@ -835,6 +836,16 @@
<enabled>false</enabled>
</snapshots>
</repository>
<repository>
<id>github</id>
<url>https://maven.pkg.github.com/atlanhq/atlan-janusgraph</url>
<snapshots>
<enabled>true</enabled>
</snapshots>
<releases>
<enabled>true</enabled>
</releases>
</repository>
<repository>
<id>hortonworks.repo</id>
<url>https://repo.hortonworks.com/content/repositories/releases</url>
Expand Down
1 change: 1 addition & 0 deletions repository/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,7 @@
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-testtools</artifactId>
<version>${project.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
Expand Down
1 change: 1 addition & 0 deletions webapp/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -552,6 +552,7 @@
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-testtools</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>

<dependency>
Expand Down
Loading