Merge branch 'release' into fix/check-connectorId

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -107,35 +107,41 @@ public class QueryProcessor {
 	private Validator validator;
 
 
-	public Stream<ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
+	public Stream<? extends ExecutionStatus> getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) {
 		final Stream<ManagedExecution> allQueries = storage.getAllExecutions();
 
 		return getQueriesFiltered(dataset.getId(), RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders);
 	}
 
-	public Stream<ExecutionStatus> getQueriesFiltered(DatasetId datasetId, UriBuilder uriBuilder, Subject subject, Stream<ManagedExecution> allQueries, boolean allProviders) {
+	public Stream<? extends ExecutionStatus> getQueriesFiltered(DatasetId datasetId, UriBuilder uriBuilder, Subject subject, Stream<ManagedExecution> allQueries, boolean allProviders) {
 
 		return allQueries
-						 // The following only checks the dataset, under which the query was submitted, but a query can target more that
-						 // one dataset.
-						 .filter(q -> q.getDataset().equals(datasetId))
-						 // to exclude subtypes from somewhere else
-						 .filter(QueryProcessor::canFrontendRender)
-						 .filter(Predicate.not(ManagedExecution::isSystem))
-						 .filter(q -> {
-							 ExecutionState state = q.getState();
-									 return state == ExecutionState.NEW || state == ExecutionState.DONE;
-								 }
-						 )
-						 .filter(q -> subject.isPermitted(q, Ability.READ))
-						 .map(mq -> {
-							 final OverviewExecutionStatus status = mq.buildStatusOverview(subject);
-
-							 if (mq.isReadyToDownload()) {
-								 status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders));
-							 }
-							 return status;
-						 });
+				// The following only checks the dataset, under which the query was submitted, but a query can target more that
+				// one dataset.
+				.filter(q -> q.getDataset().equals(datasetId))
+				// to exclude subtypes from somewhere else
+				.filter(QueryProcessor::canFrontendRender)
+				.filter(Predicate.not(ManagedExecution::isSystem))
+				.filter(q -> {
+							ExecutionState state = q.getState();
+							return state == ExecutionState.NEW || state == ExecutionState.DONE;
+						})
+				.filter(q -> subject.isPermitted(q, Ability.READ))
+				.map(mq -> {
+					try {
+						final OverviewExecutionStatus status = mq.buildStatusOverview(subject);
+
+						if (mq.isReadyToDownload()) {
+							status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders));
+						}
+						return status;
+					}
+					catch (Exception e) {
+						log.error("FAILED building status for {}", mq, e);
+					}
+					return null;
+				})
+				.filter(Objects::nonNull);
 	}
 
 	/**

backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java

@@ -200,7 +200,7 @@ private static Map<ColumnId, Integer> calculateColumnPositions(
 				for (Column column : table.getConnector().resolve().getResolvedTable().getColumns()) {
 
 					// ValidityDates are handled separately in column=0
-					if (validityDates.stream().anyMatch(vd -> vd.containsColumn(column))) {
+					if (validityDates.stream().anyMatch(vd -> vd.containsColumn(column.getId()))) {
 						continue;
 					}
 

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java

@@ -148,10 +148,11 @@ public String defaultLabel(Locale locale) {
 		builder.append(" ");
 
 		for (ConceptElementId<?> id : elements) {
-			ConceptElement<?> conceptElement = id.resolve();
-			if (conceptElement.equals(getConcept())) {
+			if (id.equals(getConceptId())) {
 				continue;
 			}
+
+			ConceptElement<?> conceptElement = id.resolve();
 			builder.append(conceptElement.getLabel()).append("+");
 		}
 
@@ -274,9 +275,7 @@ public RequiredEntities collectRequiredEntities(QueryExecutionContext context) {
 		final Set<ConnectorId> connectors = getTables().stream().map(CQTable::getConnector).collect(Collectors.toSet());
 
 		return new RequiredEntities(context.getBucketManager()
-										   .getEntitiesWithConcepts(getElements().stream()
-																				 .<ConceptElement<?>>map(ConceptElementId::resolve)
-																				 .toList(),
+										   .getEntitiesWithConcepts(getElements(),
 																	connectors, context.getDateRestriction()));
 	}
 

backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java

@@ -12,13 +12,12 @@
 
 import c10n.C10N;
 import com.bakdata.conquery.internationalization.ExcelSheetNameC10n;
-import com.bakdata.conquery.models.common.CDate;
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.config.ExcelConfig;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.i18n.I18n;
 import com.bakdata.conquery.models.identifiable.mapping.PrintIdMapper;
-import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
 import com.bakdata.conquery.models.query.PrintSettings;
 import com.bakdata.conquery.models.query.SingleTableResult;
 import com.bakdata.conquery.models.query.resultinfo.ResultInfo;
@@ -54,18 +53,20 @@ public class ExcelRenderer {
 	private final ExcelConfig config;
 	private final PrintSettings settings;
 	private final ImmutableMap<String, CellStyle> styles;
+
 	public ExcelRenderer(ExcelConfig config, PrintSettings settings) {
 		workbook = new SXSSFWorkbook();
 		this.config = config;
 		styles = config.generateStyles(workbook, settings);
 		this.settings = settings;
 	}
 
-	public <E extends ManagedExecution & SingleTableResult> void renderToStream(List<ResultInfo> idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings)
+	public <E extends ManagedExecution & SingleTableResult> void renderToStream(
+			List<ResultInfo> idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings, MetaStorage storage)
 			throws IOException {
 		final List<ResultInfo> resultInfosExec = exec.getResultInfos();
 
-		setMetaData(exec);
+		setMetaData(exec, storage);
 
 		final SXSSFSheet sheet = workbook.createSheet(C10N.get(ExcelSheetNameC10n.class, I18n.LOCALE.get()).result());
 		try {
@@ -91,12 +92,21 @@ public <E extends ManagedExecution & SingleTableResult> void renderToStream(List
 	/**
 	 * Include meta data in the xlsx such as the title, owner/author, tag and the name of this instance.
 	 */
-	private <E extends ManagedExecution & SingleTableResult> void setMetaData(E exec) {
+	private <E extends ManagedExecution & SingleTableResult> void setMetaData(E exec, MetaStorage metaStorage) {
 		final POIXMLProperties.CoreProperties coreProperties = workbook.getXSSFWorkbook().getProperties().getCoreProperties();
 		coreProperties.setTitle(exec.getLabelWithoutAutoLabelSuffix());
 
-		final UserId owner = exec.getOwner();
-		coreProperties.setCreator(owner != null ? owner.resolve().getLabel() : config.getApplicationName());
+		String creator = config.getApplicationName();
+
+		if (exec.getOwner() != null) {
+			final User user = metaStorage.get(exec.getOwner());
+
+			if (user != null) {
+				creator = user.getLabel();
+			}
+		}
+
+		coreProperties.setCreator(creator);
 		coreProperties.setKeywords(String.join(" ", exec.getTags()));
 		final POIXMLProperties.ExtendedProperties extendedProperties = workbook.getXSSFWorkbook().getProperties().getExtendedProperties();
 		extendedProperties.setApplication(config.getApplicationName());
@@ -180,7 +190,8 @@ private int writeBody(
 		// Row 0 is the Header the data starts at 1
 		final AtomicInteger currentRow = new AtomicInteger(1);
 
-		final TypeWriter[] writers = infos.stream().map(info -> writer(info.getType(), info.createPrinter(printerFactory, settings), settings)).toArray(TypeWriter[]::new);
+		final TypeWriter[] writers =
+				infos.stream().map(info -> writer(info.getType(), info.createPrinter(printerFactory, settings), settings)).toArray(TypeWriter[]::new);
 		final PrintIdMapper idMapper = settings.getIdMapper();
 
 		final int writtenLines = resultLines.mapToInt(l -> writeRowsForEntity(infos, l, currentRow, sheet, writers, idMapper)).sum();
@@ -215,10 +226,32 @@ private void postProcessTable(SXSSFSheet sheet, XSSFTable table, int writtenLine
 		sheet.createFreezePane(size, 1);
 	}
 
+	private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings) {
+		if (type instanceof ResultType.ListT<?>) {
+			//Excel cannot handle LIST types so we just toString them.
+			return (value, cell, styles) -> writeStringCell(cell, value, printer);
+		}
+
+		return switch (((ResultType.Primitive) type)) {
+			case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer);
+			case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles);
+			case MONEY -> (value, cell, styles) -> writeMoneyCell(value, cell, printer, settings, styles);
+			case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles);
+			case DATE -> (value, cell, styles) -> writeDateCell(value, cell, printer, styles);
+			default -> (value, cell, styles) -> writeStringCell(cell, value, printer);
+		};
+	}
+
 	/**
 	 * Writes the result lines for each entity.
 	 */
-	private int writeRowsForEntity(List<ResultInfo> infos, EntityResult internalRow, final AtomicInteger currentRow, SXSSFSheet sheet, TypeWriter[] writers, PrintIdMapper idMapper) {
+	private int writeRowsForEntity(
+			List<ResultInfo> infos,
+			EntityResult internalRow,
+			final AtomicInteger currentRow,
+			SXSSFSheet sheet,
+			TypeWriter[] writers,
+			PrintIdMapper idMapper) {
 
 		final String[] ids = idMapper.map(internalRow).getExternalId();
 
@@ -286,22 +319,6 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) {
 		}
 	}
 
-	private static TypeWriter writer(ResultType type, Printer printer, PrintSettings settings) {
-		if (type instanceof ResultType.ListT<?>) {
-			//Excel cannot handle LIST types so we just toString them.
-			return (value, cell, styles) -> writeStringCell(cell, value, printer);
-		}
-
-		return switch (((ResultType.Primitive) type)) {
-			case BOOLEAN -> (value, cell, styles) -> writeBooleanCell(value, cell, printer);
-			case INTEGER -> (value, cell, styles) -> writeIntegerCell(value, cell, printer, styles);
-			case MONEY -> (value, cell, styles) -> writeMoneyCell(value, cell, printer, settings, styles);
-			case NUMERIC -> (value, cell, styles) -> writeNumericCell(value, cell, printer, styles);
-			case DATE -> (value, cell, styles) -> writeDateCell(value, cell, printer, styles);
-			default -> (value, cell, styles) -> writeStringCell(cell, value, printer);
-		};
-	}
-
 	// Type specific cell writers
 	private static void writeStringCell(Cell cell, Object value, Printer printer) {
 		cell.setCellValue((String) printer.apply(value));

backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java

@@ -10,6 +10,7 @@
 import jakarta.ws.rs.core.StreamingOutput;
 
 import com.bakdata.conquery.io.result.ResultUtil;
+import com.bakdata.conquery.io.storage.MetaStorage;
 import com.bakdata.conquery.models.auth.entities.Subject;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.ExcelConfig;
@@ -33,6 +34,8 @@ public class ResultExcelProcessor {
 
 	// Media type according to https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
 	public static final MediaType MEDIA_TYPE = new MediaType("application", "vnd.openxmlformats-officedocument.spreadsheetml.sheet");
+
+	private final MetaStorage metaStorage;
 	private final DatasetRegistry datasetRegistry;
 	private final ConqueryConfig conqueryConfig;
 
@@ -57,7 +60,7 @@ public <E extends ManagedExecution & SingleTableResult> Response createResult(Su
 		final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings);
 
 		final StreamingOutput out = output -> {
-			excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(), exec, output, limit, settings);
+			excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(), exec, output, limit, settings, metaStorage);
 			log.trace("FINISHED downloading {}", exec.getId());
 		};
 

backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java

@@ -63,7 +63,7 @@ private static void handleImport(Namespace namespace, InputStream inputStream, b
 
 			readAndDistributeImport(((DistributedNamespace) namespace), table, header, parser, datasetRegistry);
 
-			clearDependentConcepts(namespace.getStorage().getAllConcepts(), table);
+			clearDependentConcepts(namespace.getStorage().getAllConcepts(), table.getId());
 		}
 	}
 
@@ -150,10 +150,10 @@ private static void readAndDistributeImport(DistributedNamespace namespace, Tabl
 
 	}
 
-	private static void clearDependentConcepts(Stream<Concept<?>> allConcepts, Table table) {
+	private static void clearDependentConcepts(Stream<Concept<?>> allConcepts, TableId table) {
 		allConcepts.map(Concept::getConnectors)
 				   .flatMap(List::stream)
-				   .filter(con -> con.getResolvedTableId().equals(table.getId()))
+				   .filter(con -> con.getResolvedTableId().equals(table))
 				   .map(Connector::getConcept)
 				   .forEach(Concept::clearMatchingStats);
 	}
@@ -183,7 +183,7 @@ public void deleteImport(Import imp) {
 		final DatasetId id = imp.getTable().getDataset();
 		final DistributedNamespace namespace = datasetRegistry.get(id);
 
-		clearDependentConcepts(namespace.getStorage().getAllConcepts(), imp.getTable().resolve());
+		clearDependentConcepts(namespace.getStorage().getAllConcepts(), imp.getTable());
 
 		namespace.getStorage().removeImport(imp.getId());
 		namespace.getWorkerHandler().sendToAll(new RemoveImportJob(imp.getId()));

backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java

@@ -3,7 +3,6 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 import jakarta.validation.Valid;
@@ -180,7 +179,6 @@ public List<Select> getSelects() {
 		return getInfoCardSelects().stream()
 								   .map(InfoCardSelect::select)
 								   .map(SelectId::<Select>resolve)
-								   .filter(Objects::nonNull)
 								   .collect(Collectors.toList());
 	}
 
@@ -191,7 +189,6 @@ public List<FilterId> resolveSearchFilters() {
 
 		return searchFilters.stream()
 							.map(FilterId::resolve)
-							.filter(Objects::nonNull)
 							.map(Filter::getId)
 							.toList();
 	}