Merge pull request #3639 from ingef/reintegrate-main

Reintegrate Main

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -107,35 +107,41 @@ public class QueryProcessor {
 	private Validator validator;
 
 
-	public Stream<ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
+	public Stream<? extends ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
 		final Stream<ManagedExecution> allQueries = storage.getAllExecutions();
 
 		return getQueriesFiltered(dataset.getId(), RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders);
 	}
 
-	public Stream<ExecutionStatus> getQueriesFiltered(DatasetId datasetId, UriBuilder uriBuilder, Subject subject, Stream<ManagedExecution> allQueries, boolean allProviders) {
+	public Stream<? extends ExecutionStatus> getQueriesFiltered(DatasetId datasetId, UriBuilder uriBuilder, Subject subject, Stream<ManagedExecution> allQueries, boolean allProviders) {
 
 		return allQueries
-						 // The following only checks the dataset, under which the query was submitted, but a query can target more that
-						 // one dataset.
-						 .filter(q -> q.getDataset().equals(datasetId))
-						 // to exclude subtypes from somewhere else
-						 .filter(QueryProcessor::canFrontendRender)
-						 .filter(Predicate.not(ManagedExecution::isSystem))
-						 .filter(q -> {
-							 ExecutionState state = q.getState();
-									 return state == ExecutionState.NEW || state == ExecutionState.DONE;
-								 }
-						 )
-						 .filter(q -> subject.isPermitted(q, Ability.READ))
-						 .map(mq -> {
-							 final OverviewExecutionStatus status = mq.buildStatusOverview(subject);
-
-							 if (mq.isReadyToDownload()) {
-								 status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders));
-							 }
-							 return status;
-						 });
+				// The following only checks the dataset, under which the query was submitted, but a query can target more that
+				// one dataset.
+				.filter(q -> q.getDataset().equals(datasetId))
+				// to exclude subtypes from somewhere else
+				.filter(QueryProcessor::canFrontendRender)
+				.filter(Predicate.not(ManagedExecution::isSystem))
+				.filter(q -> {
+							ExecutionState state = q.getState();
+							return state == ExecutionState.NEW || state == ExecutionState.DONE;
+						})
+				.filter(q -> subject.isPermitted(q, Ability.READ))
+				.map(mq -> {
+					try {
+						final OverviewExecutionStatus status = mq.buildStatusOverview(subject);
+
+						if (mq.isReadyToDownload()) {
+							status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders));
+						}
+						return status;
+					}
+					catch (Exception e) {
+						log.error("FAILED building status for {}", mq, e);
+					}
+					return null;
+				})
+				.filter(Objects::nonNull);
 	}
 
 	/**

backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java

@@ -200,7 +200,7 @@ private static Map<ColumnId, Integer> calculateColumnPositions(
 				for (Column column : table.getConnector().resolve().getResolvedTable().getColumns()) {
 
 					// ValidityDates are handled separately in column=0
-					if (validityDates.stream().anyMatch(vd -> vd.containsColumn(column))) {
+					if (validityDates.stream().anyMatch(vd -> vd.containsColumn(column.getId()))) {
 						continue;
 					}
 

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java

@@ -148,10 +148,11 @@ public String defaultLabel(Locale locale) {
 		builder.append(" ");
 
 		for (ConceptElementId<?> id : elements) {
-			ConceptElement<?> conceptElement = id.resolve();
-			if (conceptElement.equals(getConcept())) {
+			if (id.equals(getConceptId())) {
 				continue;
 			}
+
+			ConceptElement<?> conceptElement = id.resolve();
 			builder.append(conceptElement.getLabel()).append("+");
 		}
 
@@ -274,9 +275,7 @@ public RequiredEntities collectRequiredEntities(QueryExecutionContext context) {
 		final Set<ConnectorId> connectors = getTables().stream().map(CQTable::getConnector).collect(Collectors.toSet());
 
 		return new RequiredEntities(context.getBucketManager()
-										   .getEntitiesWithConcepts(getElements().stream()
-																				 .<ConceptElement<?>>map(ConceptElementId::resolve)
-																				 .toList(),
+										   .getEntitiesWithConcepts(getElements(),
 																	connectors, context.getDateRestriction()));
 	}
 

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/DateFormat.java

@@ -58,7 +58,11 @@ public void readDates(String value, DateReader dateReader, CDateSet out) {
 	DATE_SET {
 		@Override
 		public void readDates(String value, DateReader dateReader, CDateSet out) {
-			out.addAll(dateReader.parseToCDateSet(value));
+			CDateSet parsed = dateReader.parseToCDateSet(value);
+			if (parsed == null ) {
+				return;
+			}
+			out.addAll(parsed);
 		}
 	},
 	ALL {

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java

@@ -15,6 +15,7 @@
 import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 import com.bakdata.conquery.util.DateReader;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 
 @Slf4j
 public class EntityResolverUtil {
@@ -41,7 +42,7 @@ public static CDateSet[] readDates(String[][] values, List<String> format, DateR
 		 but can also don't contribute to any date aggregation.
 		 */
 		if (dateFormats.stream().allMatch(Objects::isNull)) {
-			// Initialize empty
+			// Initialize empty, so all lines appear als resolved
 			for (int row = 0; row < values.length; row++) {
 				out[row] = CDateSet.createEmpty();
 			}
@@ -59,10 +60,19 @@ public static CDateSet[] readDates(String[][] values, List<String> format, DateR
 					if (dateFormat == null) {
 						continue;
 					}
-					dateFormat.readDates(values[row][col], dateReader, dates);
+					String value = values[row][col];
+
+					if (StringUtils.isBlank(value)) {
+						log.trace("Found blank/null value in {}/{} (row/col)", row,col);
+						continue;
+					}
+
+					dateFormat.readDates(value, dateReader, dates);
 				}
 
 				if (dates.isEmpty()) {
+					// Don't set an empty dateset here, because this flags the line as: unresolvedDate
+					// TODO It might be better to set an empty dateset nonetheless, because it seems to be intentionally empty, as we had no problem while parsing a value
 					continue;
 				}
 
@@ -73,7 +83,9 @@ public static CDateSet[] readDates(String[][] values, List<String> format, DateR
 				out[row].addAll(dates);
 			}
 			catch (Exception e) {
-				log.warn("Failed to parse Date from {}", row, e);
+				// If a value is not parsable, it is included in the exceptions cause message  (see DateReader)
+				log.trace("Failed to parse Date in row {}", row, e);
+				// This catch causes `out[row]` to remain `null` which later flags this line as: unresolvedDate
 			}
 		}
 
@@ -142,6 +154,7 @@ public static String tryResolveId(String[] row, List<Function<String[], External
 	 */
 	public static Map<String, String>[] readExtras(String[][] values, List<String> format) {
 		final String[] names = values[0];
+		@SuppressWarnings("unchecked")
 		final Map<String, String>[] extrasByRow = new Map[values.length];
 
 

backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java

@@ -12,13 +12,12 @@
 
 import c10n.C10N;
 import com.bakdata.conquery.internationalization.ExcelSheetNameC10n;
-import com.bakdata.conquery.models.common.CDate;
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.config.ExcelConfig;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.i18n.I18n;
 import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper;
-import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.models.query.SingleTableResult;
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
@@ -54,18 +53,20 @@ public class ExcelRenderer {
 	private final ExcelConfig config;
 	private final PrintSettings settings;
 	private final ImmutableMap<String, CellStyle> styles;
+
 	public ExcelRenderer(ExcelConfig config, PrintSettings settings) {
 		workbook = new SXSSFWorkbook();
 		this.config = config;
 		styles = config.generateStyles(workbook, settings);
 		this.settings = settings;
 	}
 
-	public <E extends ManagedExecution & SingleTableResult> void renderToStream(List<ResultInfo> idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings)
+	public <E extends ManagedExecution & SingleTableResult> void renderToStream(
+			List<ResultInfo> idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings, MetaStorage storage)
 			throws IOException {
 		final List<ResultInfo> resultInfosExec = exec.getResultInfos();
 
-		setMetaData(exec);
+		setMetaData(exec, storage);
 
 		final SXSSFSheet sheet = workbook.createSheet(C10N.get(ExcelSheetNameC10n.class, I18n.LOCALE.get()).result());
 		try {
@@ -91,12 +92,21 @@ public <E extends ManagedExecution & SingleTableResult> void renderToStream(List
 	/**
 	 * Include meta data in the xlsx such as the title, owner/author, tag and the name of this instance.
 	 */
-	private <E extends ManagedExecution & SingleTableResult> void setMetaData(E exec) {
+	private <E extends ManagedExecution & SingleTableResult> void setMetaData(E exec, MetaStorage metaStorage) {
 		final POIXMLProperties.CoreProperties coreProperties = workbook.getXSSFWorkbook().getProperties().getCoreProperties();
 		coreProperties.setTitle(exec.getLabelWithoutAutoLabelSuffix());
 
-		final UserId owner = exec.getOwner();
-		coreProperties.setCreator(owner != null ? owner.resolve().getLabel() : config.getApplicationName());
+		String creator = config.getApplicationName();
+
+		if (exec.getOwner() != null) {
+			final User user = metaStorage.get(exec.getOwner());
+
+			if (user != null) {
+				creator = user.getLabel();
+			}
+		}
+
+		coreProperties.setCreator(creator);
 		coreProperties.setKeywords(String.join(" ", exec.getTags()));
 		final POIXMLProperties.ExtendedProperties extendedProperties = workbook.getXSSFWorkbook().getProperties().getExtendedProperties();
 		extendedProperties.setApplication(config.getApplicationName());
@@ -180,7 +190,8 @@ private int writeBody(
 		// Row 0 is the Header the data starts at 1
 		final AtomicInteger currentRow = new AtomicInteger(1);
 
-		final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.createPrinter(printerFactory, settings), settings)).toArray(TypeWriter[]::new);
+		final TypeWriter[] writers =
+				infos.stream().map(info -> writer(info.getType(), info.createPrinter(printerFactory, settings), settings)).toArray(TypeWriter[]::new);
 		final PrintIdMapper idMapper = settings.getIdMapper();
 
 		final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, sheet, writers, idMapper)).sum();
@@ -215,10 +226,32 @@ private void postProcessTable(SXSSFSheet sheet, XSSFTable table, int writtenLine
 		sheet.createFreezePane(size, 1);
 	}
 
+	private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings) {
+		if (type instanceof ResultType.ListT<?>) {
+			//Excel cannot handle LIST types so we just toString them.
+			return (value, cell, styles) -> writeStringCell(cell, value, printer);
+		}
+
+		return switch (((ResultType.Primitive) type)) {
+			case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer);
+			case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles);
+			case MONEY -> (value, cell, styles) -> writeMoneyCell(value, cell, printer, settings, styles);
+			case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles);
+			case DATE -> (value, cell, styles) -> writeDateCell(value, cell, printer, styles);
+			default -> (value, cell, styles) -> writeStringCell(cell, value, printer);
+		};
+	}
+
 	/**
 	 * Writes the result lines for each entity.
 	 */
-	private int writeRowsForEntity(List<ResultInfo> infos, EntityResult internalRow, final AtomicInteger currentRow, SXSSFSheet sheet, TypeWriter[] writers, PrintIdMapper idMapper) {
+	private int writeRowsForEntity(
+			List<ResultInfo> infos,
+			EntityResult internalRow,
+			final AtomicInteger currentRow,
+			SXSSFSheet sheet,
+			TypeWriter[] writers,
+			PrintIdMapper idMapper) {
 
 		final String[] ids = idMapper.map(internalRow).getExternalId();
 
@@ -286,22 +319,6 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) {
 		}
 	}
 
-	private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings) {
-		if (type instanceof ResultType.ListT<?>) {
-			//Excel cannot handle LIST types so we just toString them.
-			return (value, cell, styles) -> writeStringCell(cell, value, printer);
-		}
-
-		return switch (((ResultType.Primitive) type)) {
-			case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer);
-			case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles);
-			case MONEY -> (value, cell, styles) -> writeMoneyCell(value, cell, printer, settings, styles);
-			case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles);
-			case DATE -> (value, cell, styles) -> writeDateCell(value, cell, printer, styles);
-			default -> (value, cell, styles) -> writeStringCell(cell, value, printer);
-		};
-	}
-
 	// Type specific cell writers
 	private static void writeStringCell(Cell cell, Object value, Printer printer) {
 		cell.setCellValue((String) printer.apply(value));

backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java

@@ -10,6 +10,7 @@
 import jakarta.ws.rs.core.StreamingOutput;
 
 import com.bakdata.conquery.io.result.ResultUtil;
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.Subject;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.ExcelConfig;
@@ -33,6 +34,8 @@ public class ResultExcelProcessor {
 
 	// Media type according to https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
 	public static final MediaType MEDIA_TYPE = new MediaType("application", "vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+
+	private final MetaStorage metaStorage;
 	private final DatasetRegistry datasetRegistry;
 	private final ConqueryConfig conqueryConfig;
 
@@ -57,7 +60,7 @@ public <E extends ManagedExecution & SingleTableResult> Response createResult(Su
 		final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings);
 
 		final StreamingOutput out = output -> {
-			excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(), exec, output, limit, settings);
+			excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(), exec, output, limit, settings, metaStorage);
 			log.trace("FINISHED downloading {}", exec.getId());
 		};
 

backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java

@@ -1,15 +1,13 @@
 package com.bakdata.conquery.mode.cluster;
 
-import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.collectExtraData;
-import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.readDates;
-import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.tryResolveId;
-import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.verifyOnlySingles;
+import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.*;
 
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
+import jakarta.validation.constraints.NotEmpty;
 
 import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver;
 import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil;
@@ -19,7 +17,6 @@
 import com.bakdata.conquery.models.identifiable.mapping.ExternalId;
 import com.bakdata.conquery.util.DateReader;
 import com.bakdata.conquery.util.io.IdColumnUtil;
-import jakarta.validation.constraints.NotEmpty;
 
 public class ClusterEntityResolver implements EntityResolver {
 
@@ -58,18 +55,18 @@ public ResolveStatistic resolveEntities(
 
 			final String[] row = values[rowNum];
 
-			if (rowDates[rowNum] == null) {
-				unresolvedDate.add(row);
-				continue;
-			}
-
+			// Try to resolve the id first, because it has higher priority for the uploader than the dates
 			String resolvedId = tryResolveId(row, readers, mapping);
-
 			if (resolvedId == null) {
 				unresolvedId.add(row);
 				continue;
 			}
 
+			if (rowDates[rowNum] == null) {
+				unresolvedDate.add(row);
+				continue;
+			}
+
 			// read the dates from the row
 			resolved.put(resolvedId, rowDates[rowNum]);