-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement MatchingStats for SQL mode
- Loading branch information
Showing
30 changed files
with
1,002 additions
and
311 deletions.
There are no files selected for viewing
114 changes: 114 additions & 0 deletions
114
backend/src/main/java/com/bakdata/conquery/mode/cluster/WorkerMatchingStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| package com.bakdata.conquery.mode.cluster; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
|
|
||
| import com.bakdata.conquery.models.common.daterange.CDateRange; | ||
| import com.bakdata.conquery.models.datasets.Column; | ||
| import com.bakdata.conquery.models.datasets.Table; | ||
| import com.bakdata.conquery.models.datasets.concepts.MatchingStats; | ||
| import com.bakdata.conquery.models.events.Bucket; | ||
| import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; | ||
| import com.fasterxml.jackson.annotation.JsonIgnore; | ||
| import it.unimi.dsi.fastutil.ints.IntOpenHashSet; | ||
| import it.unimi.dsi.fastutil.ints.IntSet; | ||
| import lombok.AllArgsConstructor; | ||
| import lombok.Data; | ||
| import lombok.NoArgsConstructor; | ||
| import lombok.Setter; | ||
|
|
||
| @Setter | ||
| public class WorkerMatchingStats implements MatchingStats { | ||
|
|
||
| private Map<WorkerId, Entry> entries = new HashMap<>(); | ||
|
|
||
| @JsonIgnore | ||
| private transient CDateRange span; | ||
|
|
||
| @JsonIgnore | ||
| private transient long numberOfEvents = -1L; | ||
|
|
||
| @JsonIgnore | ||
| private transient long numberOfEntities = -1L; | ||
|
|
||
| @Override | ||
| public long countEvents() { | ||
| if (numberOfEvents == -1L) { | ||
| synchronized (this) { | ||
| if (numberOfEvents == -1L) { | ||
| numberOfEvents = entries.values().stream().mapToLong(Entry::getNumberOfEvents).sum(); | ||
| } | ||
| } | ||
| } | ||
| return numberOfEvents; | ||
| } | ||
|
|
||
|
|
||
| @Override | ||
| public long countEntities() { | ||
| if (numberOfEntities == -1L) { | ||
| synchronized (this) { | ||
| if (numberOfEntities == -1L) { | ||
| numberOfEntities = entries.values().stream().mapToLong(Entry::getNumberOfEntities).sum(); | ||
| } | ||
| } | ||
| } | ||
| return numberOfEntities; | ||
| } | ||
|
|
||
| @Override | ||
| public CDateRange spanEvents() { | ||
| if (span == null) { | ||
| synchronized (this) { | ||
| if (span == null) { | ||
| span = entries.values().stream().map(Entry::getSpan).reduce(CDateRange.all(), CDateRange::spanClosed); | ||
| } | ||
| } | ||
| } | ||
| return span; | ||
|
|
||
| } | ||
|
|
||
| public void putEntry(WorkerId source, Entry entry) { | ||
| synchronized (this) { | ||
| entries.put(source, entry); | ||
| span = null; | ||
| numberOfEntities = -1L; | ||
| numberOfEvents = -1L; | ||
| } | ||
| } | ||
|
|
||
| @Data | ||
| @NoArgsConstructor | ||
| @AllArgsConstructor | ||
| public static class Entry { | ||
| private long numberOfEvents; | ||
|
|
||
| @JsonIgnore | ||
| private final IntSet foundEntities = new IntOpenHashSet(); | ||
| private long numberOfEntities; | ||
| private CDateRange span; | ||
|
|
||
|
|
||
| public void addEvent(Table table, Bucket bucket, int event, int entityForEvent) { | ||
| numberOfEvents++; | ||
| if (foundEntities.add(entityForEvent)) { | ||
| numberOfEntities++; | ||
| } | ||
|
|
||
| for (Column c : table.getColumns()) { | ||
| if (!c.getType().isDateCompatible()) { | ||
| continue; | ||
| } | ||
|
|
||
| if (!bucket.has(event, c)) { | ||
| continue; | ||
| } | ||
|
|
||
| final CDateRange time = bucket.getAsDateRange(event, c); | ||
| span = time.spanClosed(span); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
backend/src/main/java/com/bakdata/conquery/mode/local/SqlMatchingStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| package com.bakdata.conquery.mode.local; | ||
|
|
||
| import com.bakdata.conquery.models.common.daterange.CDateRange; | ||
| import com.bakdata.conquery.models.datasets.concepts.MatchingStats; | ||
| import lombok.Value; | ||
|
|
||
| @Value | ||
| public class SqlMatchingStats implements MatchingStats { | ||
|
|
||
| long numberOfEvents; | ||
| long numberOfEntities; | ||
| CDateRange span; | ||
|
|
||
| @Override | ||
| public long countEvents() { | ||
| return numberOfEvents; | ||
| } | ||
|
|
||
| @Override | ||
| public long countEntities() { | ||
| return numberOfEntities; | ||
| } | ||
|
|
||
| @Override | ||
| public CDateRange spanEvents() { | ||
| return span; | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
103 changes: 6 additions & 97 deletions
103
backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/MatchingStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,107 +1,16 @@ | ||
| package com.bakdata.conquery.models.datasets.concepts; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import javax.annotation.Nullable; | ||
|
|
||
| import com.bakdata.conquery.models.common.daterange.CDateRange; | ||
| import com.bakdata.conquery.models.datasets.Column; | ||
| import com.bakdata.conquery.models.datasets.Table; | ||
| import com.bakdata.conquery.models.events.Bucket; | ||
| import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; | ||
| import com.fasterxml.jackson.annotation.JsonIgnore; | ||
| import it.unimi.dsi.fastutil.ints.IntOpenHashSet; | ||
| import it.unimi.dsi.fastutil.ints.IntSet; | ||
| import lombok.*; | ||
|
|
||
| @Getter | ||
| @Setter | ||
| public class MatchingStats { | ||
| public interface MatchingStats { | ||
|
|
||
| private Map<WorkerId, Entry> entries = new HashMap<>(); | ||
| @JsonIgnore | ||
| private transient CDateRange span; | ||
| long countEvents(); | ||
|
|
||
| @JsonIgnore | ||
| private transient long numberOfEvents = -1L; | ||
| long countEntities(); | ||
|
|
||
| @JsonIgnore | ||
| private transient long numberOfEntities = -1L; | ||
|
|
||
| public long countEvents() { | ||
| if (numberOfEvents == -1L) { | ||
| synchronized (this) { | ||
| if (numberOfEvents == -1L) { | ||
| numberOfEvents = entries.values().stream().mapToLong(Entry::getNumberOfEvents).sum(); | ||
| } | ||
| } | ||
| } | ||
| return numberOfEvents; | ||
| } | ||
|
|
||
|
|
||
| public long countEntities() { | ||
| if (numberOfEntities == -1L) { | ||
| synchronized (this) { | ||
| if (numberOfEntities == -1L) { | ||
| numberOfEntities = entries.values().stream().mapToLong(Entry::getNumberOfEntities).sum(); | ||
| } | ||
| } | ||
| } | ||
| return numberOfEntities; | ||
| } | ||
|
|
||
| public CDateRange spanEvents() { | ||
| if (span == null) { | ||
| synchronized (this) { | ||
| if (span == null) { | ||
| span = entries.values().stream().map(Entry::getSpan).reduce(CDateRange.all(), CDateRange::spanClosed); | ||
| } | ||
| } | ||
| } | ||
| return span; | ||
|
|
||
| } | ||
|
|
||
| public void putEntry(WorkerId source, Entry entry) { | ||
| synchronized (this) { | ||
| entries.put(source, entry); | ||
| span = null; | ||
| numberOfEntities = -1L; | ||
| numberOfEvents = -1L; | ||
| } | ||
| } | ||
|
|
||
| @Data | ||
| @NoArgsConstructor | ||
| @AllArgsConstructor | ||
| public static class Entry { | ||
| private long numberOfEvents; | ||
|
|
||
| @JsonIgnore | ||
| private final IntSet foundEntities = new IntOpenHashSet(); | ||
| private long numberOfEntities; | ||
| private CDateRange span; | ||
|
|
||
|
|
||
| public void addEvent(Table table, Bucket bucket, int event, int entityForEvent) { | ||
| numberOfEvents++; | ||
| if (foundEntities.add(entityForEvent)) { | ||
| numberOfEntities++; | ||
| } | ||
|
|
||
| for (Column c : table.getColumns()) { | ||
| if (!c.getType().isDateCompatible()) { | ||
| continue; | ||
| } | ||
|
|
||
| if (!bucket.has(event, c)) { | ||
| continue; | ||
| } | ||
|
|
||
| final CDateRange time = bucket.getAsDateRange(event, c); | ||
| span = time.spanClosed(span); | ||
| } | ||
| } | ||
| } | ||
| @Nullable | ||
| CDateRange spanEvents(); | ||
|
|
||
| } |
Oops, something went wrong.