Skip to content

Commit

Permalink
Merge pull request #373 from dromara/dev
Browse files Browse the repository at this point in the history
[ISSUE #372] add heap memory usage in alarm message
  • Loading branch information
yanhom1314 committed Dec 17, 2023
2 parents 9cab076 + 9588da4 commit e1121ff
Show file tree
Hide file tree
Showing 10 changed files with 405 additions and 52 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import com.baidu.cloud.starlight.api.rpc.StarlightClient;
import com.baidu.cloud.starlight.api.rpc.threadpool.ThreadPoolFactory;
import com.baidu.cloud.starlight.core.rpc.SingleStarlightClient;
import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.apache.commons.collections4.CollectionUtils;
Expand Down Expand Up @@ -61,8 +60,7 @@ protected String getTpPrefix() {
protected void initialize() {
super.initialize();

List<StarlightClient> starlightClients = Lists.newArrayList();
starlightClients.addAll(JVMTI.getInstances(StarlightClient.class));
List<StarlightClient> starlightClients = JVMTI.getInstances(StarlightClient.class);
if (CollectionUtils.isEmpty(starlightClients)) {
log.warn("Cannot find beans of type StarlightClient.");
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ protected void initialize() {
if (MapUtils.isNotEmpty(executorMap)) {
executorMap.forEach((k, v) -> {
ThreadPoolExecutor proxy = getProxy((ThreadPoolExecutor) v);
executorMap.replace(k, proxy);
dataStore.put(EXECUTOR_SERVICE_COMPONENT_KEY, k, proxy);
putAndFinalize(genTpName(k), (ExecutorService) v, proxy);
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@

import com.google.common.collect.Lists;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.dromara.dynamictp.common.em.CollectorTypeEnum;
import org.dromara.dynamictp.common.entity.DtpExecutorProps;
import org.dromara.dynamictp.common.entity.NotifyPlatform;
Expand All @@ -33,7 +32,6 @@
* @author yanhom
* @since 1.0.0
**/
@Slf4j
@Data
public class DtpProperties {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -98,14 +98,14 @@ protected void doRefresh(DtpProperties properties) {
publishEvent(properties);
}

protected boolean needRefresh(Set<String> keys) {
if (CollectionUtils.isEmpty(keys)) {
protected boolean needRefresh(Set<String> changedKeys) {
if (CollectionUtils.isEmpty(changedKeys)) {
return false;
}
keys = keys.stream()
changedKeys = changedKeys.stream()
.filter(str -> str.startsWith(MAIN_PROPERTIES_PREFIX))
.collect(Collectors.toSet());
return CollectionUtils.isNotEmpty(keys);
return CollectionUtils.isNotEmpty(changedKeys);
}

private void publishEvent(DtpProperties dtpProperties) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.dromara.dynamictp.core.system;

import lombok.extern.slf4j.Slf4j;

import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean;
import java.util.concurrent.TimeUnit;

/**
* CpuMetricsCaptor related
*
* @author yanhom
* @since 1.1.6
*/
@Slf4j
public class CpuMetricsCaptor implements Runnable {

private double currProcessCpuUsage = -1;

private long prevProcessCpuTime = 0;

private long prevUpTime = 0;

public double getProcessCpuUsage() {
return currProcessCpuUsage;
}

@Override
public void run() {
try {
OperatingSystemMXBean osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
int cpuCores = osBean.getAvailableProcessors();

long newProcessCpuTime = OperatingSystemBeanManager.getProcessCpuTime();
RuntimeMXBean runtimeBean = ManagementFactory.getPlatformMXBean(RuntimeMXBean.class);
long newUpTime = runtimeBean.getUptime();
long elapsedCpu = TimeUnit.NANOSECONDS.toMillis(newProcessCpuTime - prevProcessCpuTime);
long elapsedTime = newUpTime - prevUpTime;
double processCpuUsage = (double) elapsedCpu / elapsedTime / cpuCores;
prevProcessCpuTime = newProcessCpuTime;
prevUpTime = newUpTime;
currProcessCpuUsage = processCpuUsage;
} catch (Throwable e) {
log.error("Get system metrics error.", e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.dromara.dynamictp.core.system;

import lombok.extern.slf4j.Slf4j;
import lombok.val;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;

import java.lang.management.ManagementFactory;
import java.lang.management.MemoryPoolMXBean;
import java.lang.management.MemoryUsage;
import java.util.function.ToLongFunction;

/**
* MemoryMetricsCaptor related
*
* @author yanhom
* @since 1.1.6
**/
@Slf4j
public class MemoryMetricsCaptor implements Runnable {

private double max = -1;

private double used = -1;

public double getLongLivedMemoryUsage() {
if (max == -1 || used == -1) {
return -1;
}
return used / max;
}

@Override
public void run() {
try {
val memoryPoolBeans = ManagementFactory.getPlatformMXBeans(MemoryPoolMXBean.class);
if (CollectionUtils.isEmpty(memoryPoolBeans)) {
return;
}
for (MemoryPoolMXBean memoryPoolBean : memoryPoolBeans) {
String name = memoryPoolBean.getName();
boolean isLongLivedPool = isLongLivedPool(name);
if (isLongLivedPool) {
used = getUsageValue(memoryPoolBean, MemoryUsage::getUsed);
max = getUsageValue(memoryPoolBean, MemoryUsage::getMax);
break;
}
}
} catch (Exception e) {
log.warn("MemoryMetricsCaptor run failed.", e);
}
}

private double getUsageValue(MemoryPoolMXBean memoryPoolMXBean, ToLongFunction<MemoryUsage> getter) {
MemoryUsage usage = getUsage(memoryPoolMXBean);
if (usage == null) {
return -1;
}
return getter.applyAsLong(usage);
}

private MemoryUsage getUsage(MemoryPoolMXBean memoryPoolMXBean) {
try {
return memoryPoolMXBean.getUsage();
} catch (InternalError e) {
return null;
}
}

private boolean isLongLivedPool(String name) {
return StringUtils.isNotBlank(name) && (name.endsWith("Old Gen") ||
name.endsWith("Tenured Gen") ||
"ZHeap".equals(name) ||
"Shenandoah".equals(name) ||
name.endsWith("balanced-old") ||
name.contains("tenured") ||
"JavaHeap".equals(name)
);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@

import java.lang.management.ManagementFactory;
import java.lang.management.OperatingSystemMXBean;
import java.lang.management.RuntimeMXBean;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;

Expand All @@ -35,62 +34,37 @@
@Slf4j
public class SystemMetricManager {

private static final SystemMetricPoller METRIC_POLLER;
private static final CpuMetricsCaptor CPU_METRICS_CAPTOR;

private static final MemoryMetricsCaptor MEMORY_METRICS_CAPTOR;

private static final ScheduledExecutorService EXECUTOR = ThreadPoolCreator.newScheduledThreadPool("dtp-system-metric", 1);

static {
METRIC_POLLER = new SystemMetricPoller();
EXECUTOR.scheduleAtFixedRate(METRIC_POLLER, 0, 2, TimeUnit.SECONDS);
CPU_METRICS_CAPTOR = new CpuMetricsCaptor();
MEMORY_METRICS_CAPTOR = new MemoryMetricsCaptor();
EXECUTOR.scheduleAtFixedRate(CPU_METRICS_CAPTOR, 0, 2, TimeUnit.SECONDS);
EXECUTOR.scheduleAtFixedRate(MEMORY_METRICS_CAPTOR, 0, 2, TimeUnit.SECONDS);
}

public static String getSystemMetric() {
OperatingSystemMXBean osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
double systemAvgLoad = osBean.getSystemLoadAverage();
double systemCpuUsage = OperatingSystemBeanManager.getSystemCpuUsage();
int cpuCores = osBean.getAvailableProcessors();
return String.format("SystemMetric{sAvgLoad=%.2f, sCpuUsage=%.2f, pCpuUsage=%.2f, cpuCores=%d}",
systemAvgLoad, systemCpuUsage, getProcessCpuUsage(), cpuCores);
return String.format("SystemMetric{sAvgLoad=%.2f, sCpuUsage=%.2f, pCpuUsage=%.2f, cpuCores=%d, oldMemUsage=%.2f}",
systemAvgLoad, systemCpuUsage, getProcessCpuUsage(), cpuCores, getLongLivedMemoryUsage());
}

public static double getProcessCpuUsage() {
return METRIC_POLLER.getProcessCpuUsage();
return CPU_METRICS_CAPTOR.getProcessCpuUsage();
}

public static void stop() {
EXECUTOR.shutdown();
public static double getLongLivedMemoryUsage() {
return MEMORY_METRICS_CAPTOR.getLongLivedMemoryUsage();
}

private static class SystemMetricPoller implements Runnable {

private double currProcessCpuUsage = -1;

private long prevProcessCpuTime = 0;

private long prevUpTime = 0;

public double getProcessCpuUsage() {
return currProcessCpuUsage;
}

@Override
public void run() {
try {
OperatingSystemMXBean osBean = ManagementFactory.getPlatformMXBean(OperatingSystemMXBean.class);
int cpuCores = osBean.getAvailableProcessors();

long newProcessCpuTime = OperatingSystemBeanManager.getProcessCpuTime();
RuntimeMXBean runtimeBean = ManagementFactory.getPlatformMXBean(RuntimeMXBean.class);
long newUpTime = runtimeBean.getUptime();
long elapsedCpu = TimeUnit.NANOSECONDS.toMillis(newProcessCpuTime - prevProcessCpuTime);
long elapsedTime = newUpTime - prevUpTime;
double processCpuUsage = (double) elapsedCpu / elapsedTime / cpuCores;
prevProcessCpuTime = newProcessCpuTime;
prevUpTime = newUpTime;
currProcessCpuUsage = processCpuUsage;
} catch (Throwable e) {
log.error("Get system metrics error.", e);
}
}
public static void stop() {
EXECUTOR.shutdown();
}
}
2 changes: 1 addition & 1 deletion dependencies/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
<url>https://github.com/yanhom1314/dynamic-tp</url>

<properties>
<revision>1.1.5</revision>
<revision>1.1.6</revision>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

<lombok.version>1.18.24</lombok.version>
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
<url>https://github.com/yanhom1314/dynamic-tp</url>

<properties>
<revision>1.1.5</revision>
<revision>1.1.6</revision>

<maven.compiler.source>8</maven.compiler.source>
<maven.compiler.target>8</maven.compiler.target>
Expand Down
Loading

0 comments on commit e1121ff

Please sign in to comment.