Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

OAK-11478 Node store statistics: support the tree store #2071

Merged
merged 3 commits into from
Mar 24, 2025
Merged

OAK-11478 Node store statistics: support the tree store #2071

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9397c54
OAK-11478 Node store statistics: support the tree store
thomasmueller Feb 12, 2025
9e314e7
OAK-11478 Node store statistics: support the tree store
thomasmueller Mar 20, 2025
6d58ab6
OAK-11478 Node store statistics: support the tree store
thomasmueller Mar 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/Profiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ public class Profiler implements Runnable {
"sun," +
"com.sun.," +
"com.mongodb.," +
"org.apache.jackrabbit.oak.commons.Profiler.," +
"org.bson.,"
).split(",");
private final String[] ignorePackages = (
Expand Down Expand Up @@ -380,15 +381,19 @@ private void tick() {

private void processList(List<Object[]> list) {
for (Object[] dump : list) {
if (startsWithAny(dump[0].toString(), ignoreThreads)) {
String el = dump[0].toString();
if (el.startsWith("app//")) {
el = el.substring("app//".length());
}
if (startsWithAny(el, ignoreThreads)) {
continue;
}
StringBuilder buff = new StringBuilder();
// simple recursive calls are ignored
String last = null;
boolean packageCounts = false;
for (int j = 0, i = 0; i < dump.length && j < depth; i++) {
String el = dump[i].toString();
el = dump[i].toString();
if (el.startsWith("app//")) {
el = el.substring("app//".length());
}
Expand Down
64 changes: 31 additions & 33 deletions ...java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/analysis/StatsBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,54 +34,52 @@
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.modules.TopLargestBinaries;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.stream.NodeData;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.stream.NodeDataReader;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.stream.NodeLineReader;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.stream.NodeStreamReader;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.stream.NodeStreamReaderCompressed;
import org.apache.jackrabbit.oak.index.indexer.document.flatfile.analysis.stream.NodeTreeStoreReader;

/**
* Builder for commonly used statistics for flat file stores.
*/
public class StatsBuilder {

private static final boolean ONLY_READ = false;

/**
* Read a flat file store and build statistics.
*
* @param args the file name
*/
public static void main(String... args) throws Exception {
String fileName = null;
String nodeNameFilter = null;
boolean stream = false;
boolean compressedStream = false;
boolean profiler = false;
for(int i = 0; i<args.length; i++) {
String a = args[i];
if (a.equals("--fileName")) {
fileName = args[++i];
} else if (a.equals("--nodeNameFilter")) {
nodeNameFilter = args[++i];
} else if (a.endsWith("--stream")) {
stream = true;
} else if (a.equals("--compressedStream")) {
compressedStream = true;
} else if (a.equals("--profiler")) {
profiler = true;
}
}
if (fileName == null) {
System.out.println("Command line arguments:");
System.out.println(" --fileName <file name> (flat file store file name; mandatory)");
System.out.println(" --nodeNameFilter <filter> (node name filter for binaries; optional)");
System.out.println(" --stream (use a stream file; optional)");
System.out.println(" --compressedStream (use a compressed stream file; optional)");
System.out.println(" --profiler (enable the build-in profiler; optional)");
return;
}
buildStats(fileName, nodeNameFilter, profiler);
}

/**
* Read a flat file store and build statistics.
*
* @param fileName the file name
* @param nodeNameFilter the node names to filter
* @param profiler also run the profiler
*/
public static void buildStats(String fileName, String nodeNameFilter, boolean profiler) throws IOException {
System.out.println("Processing " + fileName);
ListCollector collectors = new ListCollector();
collectors.add(new NodeCount(1000, 1));
PropertyStats ps = new PropertyStats(false, 1);
collectors.add(ps);
collectors.add(new NodeTypeCount());
if (nodeNameFilter != null) {
if (nodeNameFilter != null && !nodeNameFilter.isEmpty()) {
collectors.add(new NodeNameFilter(nodeNameFilter, new BinarySize(false, 1)));
collectors.add(new NodeNameFilter(nodeNameFilter, new BinarySize(true, 1)));
collectors.add(new NodeNameFilter(nodeNameFilter, new BinarySizeHistogram(1)));
Expand All @@ -94,18 +92,16 @@ public static void main(String... args) throws Exception {
collectors.add(new DistinctBinarySizeHistogram(1));
collectors.add(new DistinctBinarySize(16, 16));

Profiler prof = new Profiler().startCollecting();
NodeDataReader reader;
if (compressedStream) {
reader = NodeStreamReaderCompressed.open(fileName);
} else if (stream) {
reader = NodeStreamReader.open(fileName);
} else {
reader = NodeLineReader.open(fileName);
Profiler prof = null;
if (profiler) {
prof = new Profiler().startCollecting();
}
NodeDataReader reader = NodeTreeStoreReader.open(fileName);
collect(reader, collectors);

System.out.println(prof.getTop(10));
if (profiler) {
System.out.println(prof.getTop(10));
}
System.out.println();
System.out.println("Results");
System.out.println();
Expand All @@ -123,11 +119,13 @@ private static void collect(NodeDataReader reader, StatsCollector collector) thr
if (node == null) {
break;
}
if (++lineCount % 1000000 == 0) {
System.out.println(lineCount + " lines; " + reader.getProgressPercent() + "%");
}
if (ONLY_READ) {
continue;
if (++lineCount % 1_000_000 == 0) {
String msg = lineCount + " entries";
int progressPercent = reader.getProgressPercent();
if (progressPercent != 0) {
msg += "; " + progressPercent + "%";
}
System.out.println(msg);
}
if (last != null) {
while (last != null && last.getPathElements().size() >= node.getPathElements().size()) {
Expand Down
8 changes: 4 additions & 4 deletions ...e/jackrabbit/oak/index/indexer/document/flatfile/analysis/modules/DistinctBinarySize.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public void add(NodeData node) {
referenceCount += list.size();
for(BinaryId id : list) {
referenceSize += id.getLength();
if (largeBinariesCountMax > 0 && id.getLength() >= largeBinarySizeThreshold) {
if (largeBinariesCountMax > 0 && id.getLength() > largeBinarySizeThreshold) {
largeBinaries.add(id);
truncateLargeBinariesSet();
} else {
Expand All @@ -132,15 +132,15 @@ private void truncateLargeBinariesSet() {
}
long[] lengths = new long[largeBinaries.size()];
int i = 0;
for(BinaryId id : largeBinaries) {
for (BinaryId id : largeBinaries) {
lengths[i++] = id.getLength();
}
Arrays.sort(lengths);
// the new threshold is the median of all the lengths
largeBinarySizeThreshold = lengths[largeBinariesCountMax];
for(Iterator<BinaryId> it = largeBinaries.iterator(); it.hasNext();) {
for (Iterator<BinaryId> it = largeBinaries.iterator(); it.hasNext();) {
BinaryId id = it.next();
if (id.getLength() < largeBinarySizeThreshold) {
if (id.getLength() <= largeBinarySizeThreshold) {
addToBloomFilter(id);
it.remove();
}
Expand Down
103 changes: 0 additions & 103 deletions ...e/jackrabbit/oak/index/indexer/document/flatfile/analysis/stream/NodeStreamConverter.java
View file
Open in desktop

This file was deleted.

Loading
Loading