diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/config/Services.java b/projects/saturn/src/main/java/io/fairspace/saturn/config/Services.java index f392857ce..4ae86d379 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/config/Services.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/config/Services.java @@ -21,7 +21,10 @@ import io.fairspace.saturn.services.metadata.MetadataPermissions; import io.fairspace.saturn.services.metadata.MetadataService; import io.fairspace.saturn.services.metadata.validation.*; +import io.fairspace.saturn.services.search.FileSearchService; +import io.fairspace.saturn.services.search.JdbcFileSearchService; import io.fairspace.saturn.services.search.SearchService; +import io.fairspace.saturn.services.search.SparqlFileSearchService; import io.fairspace.saturn.services.users.UserService; import io.fairspace.saturn.services.views.*; import io.fairspace.saturn.services.workspaces.WorkspaceService; @@ -31,6 +34,7 @@ import io.fairspace.saturn.webdav.blobstore.LocalBlobStore; import static io.fairspace.saturn.config.ConfigLoader.CONFIG; +import static io.fairspace.saturn.services.views.ViewStoreClientFactory.protectedResources; import static io.fairspace.saturn.vocabulary.Vocabularies.VOCABULARY; @Log4j2 @@ -48,6 +52,7 @@ public class Services { private final ViewService viewService; private final QueryService queryService; private final SearchService searchService; + private final FileSearchService fileSearchService; private final BlobStore blobStore; private final DavFactory davFactory; private final HttpServlet davServlet; @@ -115,6 +120,18 @@ public Services( ? new SparqlQueryService(config.search, viewsConfig, filteredDataset) : new JdbcQueryService( config.search, viewsConfig, viewStoreClientFactory, transactions, davFactory.root); + + // File search should be done using JDBC for performance reasons. However, if the view store is not available, + // or collections and files view is not configured, we fall back to using SPARQL queries on the RDF database + // directly. + boolean useSparqlFileSearchService = viewStoreClientFactory == null + || viewsConfig.views.stream().noneMatch(view -> protectedResources.containsAll(view.types)); + + fileSearchService = useSparqlFileSearchService + ? new SparqlFileSearchService(filteredDataset) + : new JdbcFileSearchService( + config.search, viewsConfig, viewStoreClientFactory, transactions, davFactory.root); + viewService = new ViewService(config, viewsConfig, filteredDataset, viewStoreClientFactory, metadataPermissions); diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/config/SparkFilterFactory.java b/projects/saturn/src/main/java/io/fairspace/saturn/config/SparkFilterFactory.java index 4385f9558..a8cd0b75c 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/config/SparkFilterFactory.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/config/SparkFilterFactory.java @@ -18,7 +18,7 @@ public static Filter createSparkFilter(String apiPathPrefix, Services svc, Confi new WorkspaceApp(apiPathPrefix + "/workspaces", svc.getWorkspaceService()), new MetadataApp(apiPathPrefix + "/metadata", svc.getMetadataService()), new ViewApp(apiPathPrefix + "/views", svc.getViewService(), svc.getQueryService()), - new SearchApp(apiPathPrefix + "/search", svc.getSearchService(), svc.getQueryService()), + new SearchApp(apiPathPrefix + "/search", svc.getSearchService(), svc.getFileSearchService()), new VocabularyApp(apiPathPrefix + "/vocabulary"), new UserApp(apiPathPrefix + "/users", svc.getUserService()), new FeaturesApp(apiPathPrefix + "/features", config.features), diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/search/FileSearchService.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/FileSearchService.java new file mode 100644 index 000000000..818cf2ad1 --- /dev/null +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/FileSearchService.java @@ -0,0 +1,7 @@ +package io.fairspace.saturn.services.search; + +import java.util.List; + +public interface FileSearchService { + List searchFiles(FileSearchRequest request); +} diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/search/JdbcFileSearchService.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/JdbcFileSearchService.java new file mode 100644 index 000000000..f625d82b5 --- /dev/null +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/JdbcFileSearchService.java @@ -0,0 +1,49 @@ +package io.fairspace.saturn.services.search; + +import java.util.List; +import java.util.stream.Collectors; + +import io.milton.resource.CollectionResource; +import lombok.SneakyThrows; +import lombok.extern.log4j.Log4j2; + +import io.fairspace.saturn.config.Config; +import io.fairspace.saturn.config.ViewsConfig; +import io.fairspace.saturn.rdf.transactions.Transactions; +import io.fairspace.saturn.services.views.ViewStoreClientFactory; +import io.fairspace.saturn.services.views.ViewStoreReader; + +import static io.fairspace.saturn.webdav.PathUtils.getCollectionNameByUri; + +@Log4j2 +public class JdbcFileSearchService implements FileSearchService { + private final Transactions transactions; + private final CollectionResource rootSubject; + private final Config.Search searchConfig; + private final ViewsConfig viewsConfig; + private final ViewStoreClientFactory viewStoreClientFactory; + + public JdbcFileSearchService( + Config.Search searchConfig, + ViewsConfig viewsConfig, + ViewStoreClientFactory viewStoreClientFactory, + Transactions transactions, + CollectionResource rootSubject) { + this.searchConfig = searchConfig; + this.viewStoreClientFactory = viewStoreClientFactory; + this.transactions = transactions; + this.rootSubject = rootSubject; + this.viewsConfig = viewsConfig; + } + + @SneakyThrows + public List searchFiles(FileSearchRequest request) { + var collectionsForUser = transactions.calculateRead(m -> rootSubject.getChildren().stream() + .map(collection -> getCollectionNameByUri(rootSubject.getUniqueId(), collection.getUniqueId())) + .collect(Collectors.toList())); + + try (var viewStoreReader = new ViewStoreReader(searchConfig, viewsConfig, viewStoreClientFactory)) { + return viewStoreReader.searchFiles(request, collectionsForUser); + } + } +} diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/search/SearchApp.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/SearchApp.java index 3b3051c45..1b7030d9d 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/services/search/SearchApp.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/SearchApp.java @@ -1,19 +1,18 @@ package io.fairspace.saturn.services.search; import io.fairspace.saturn.services.BaseApp; -import io.fairspace.saturn.services.views.QueryService; import static org.eclipse.jetty.http.MimeTypes.Type.APPLICATION_JSON; import static spark.Spark.post; public class SearchApp extends BaseApp { private final SearchService searchService; - private final QueryService queryService; + private final FileSearchService fileSearchService; - public SearchApp(String basePath, SearchService searchService, QueryService queryService) { + public SearchApp(String basePath, SearchService searchService, FileSearchService fileSearchService) { super(basePath); this.searchService = searchService; - this.queryService = queryService; + this.fileSearchService = fileSearchService; } @Override @@ -21,7 +20,7 @@ protected void initApp() { post("/files", (req, res) -> { res.type(APPLICATION_JSON.asString()); var request = mapper.readValue(req.body(), FileSearchRequest.class); - var searchResult = queryService.searchFiles(request); + var searchResult = fileSearchService.searchFiles(request); SearchResultsDTO resultDto = SearchResultsDTO.builder() .results(searchResult) diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/search/SparqlFileSearchService.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/SparqlFileSearchService.java new file mode 100644 index 000000000..a38ada61e --- /dev/null +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/search/SparqlFileSearchService.java @@ -0,0 +1,54 @@ +package io.fairspace.saturn.services.search; + +import java.util.List; + +import lombok.extern.log4j.Log4j2; +import org.apache.jena.query.Dataset; +import org.apache.jena.query.Query; +import org.apache.jena.query.QueryFactory; +import org.apache.jena.query.QuerySolutionMap; + +import io.fairspace.saturn.rdf.SparqlUtils; +import io.fairspace.saturn.vocabulary.FS; + +import static io.fairspace.saturn.util.ValidationUtils.validateIRI; + +import static org.apache.jena.rdf.model.ResourceFactory.createStringLiteral; + +@Log4j2 +public class SparqlFileSearchService implements FileSearchService { + private final Dataset ds; + + public SparqlFileSearchService(Dataset ds) { + this.ds = ds; + } + + public List searchFiles(FileSearchRequest request) { + var query = getSearchForFilesQuery(request.getParentIRI()); + var binding = new QuerySolutionMap(); + binding.add("regexQuery", createStringLiteral(SparqlUtils.getQueryRegex(request.getQuery()))); + return SparqlUtils.getByQuery(query, binding, ds); + } + + private Query getSearchForFilesQuery(String parentIRI) { + var builder = new StringBuilder("PREFIX fs: <") + .append(FS.NS) + .append(">\nPREFIX rdfs: \n\n") + .append("SELECT ?id ?label ?comment ?type\n") + .append("WHERE {\n"); + + if (parentIRI != null && !parentIRI.trim().isEmpty()) { + validateIRI(parentIRI); + builder.append("?id fs:belongsTo* <").append(parentIRI).append("> .\n"); + } + + builder.append("?id rdfs:label ?label ; a ?type .\n") + .append("FILTER (?type in (fs:File, fs:Directory, fs:Collection))\n") + .append("OPTIONAL { ?id rdfs:comment ?comment }\n") + .append("FILTER NOT EXISTS { ?id fs:dateDeleted ?anydate }\n") + .append("FILTER (regex(?label, ?regexQuery, \"i\") || regex(?comment, ?regexQuery, \"i\"))\n") + .append("}\nLIMIT 10000"); + + return QueryFactory.create(builder.toString()); + } +} diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/JdbcQueryService.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/JdbcQueryService.java index a45234373..bfc2629a1 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/JdbcQueryService.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/JdbcQueryService.java @@ -1,14 +1,8 @@ package io.fairspace.saturn.services.views; -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; import java.sql.SQLException; import java.sql.SQLTimeoutException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import io.milton.resource.CollectionResource; @@ -19,8 +13,8 @@ import io.fairspace.saturn.config.ViewsConfig; import io.fairspace.saturn.rdf.transactions.Transactions; import io.fairspace.saturn.rdf.transactions.TxnIndexDatasetGraph; -import io.fairspace.saturn.services.search.FileSearchRequest; -import io.fairspace.saturn.services.search.SearchResultDTO; + +import static io.fairspace.saturn.webdav.PathUtils.getCollectionNameByUri; import static java.lang.Integer.min; @@ -52,12 +46,6 @@ public JdbcQueryService( this.viewsConfig = viewsConfig; } - public String getCollectionName(String uri) { - var rootLocation = rootSubject.getUniqueId() + "/"; - var location = uri.substring(rootLocation.length()); - return URLDecoder.decode(location.split("/")[0], StandardCharsets.UTF_8); - } - ViewStoreReader getViewStoreReader() throws SQLException { return new ViewStoreReader(searchConfig, viewsConfig, viewStoreClientFactory); } @@ -71,14 +59,14 @@ protected void applyCollectionsFilterIfRequired(String view, List fi return; } var collections = transactions.calculateRead(m -> rootSubject.getChildren().stream() - .map(collection -> (Object) getCollectionName(collection.getUniqueId())) + .map(collection -> (Object) getCollectionNameByUri(rootSubject.getUniqueId(), collection.getUniqueId())) .collect(Collectors.toList())); if (filters.stream().anyMatch(filter -> filter.getField().equalsIgnoreCase("Resource_collection"))) { // Update existing filters in place filters.stream() .filter(filter -> filter.getField().equalsIgnoreCase("Resource_collection")) .forEach(filter -> filter.setValues(filter.values.stream() - .map(value -> getCollectionName(value.toString())) + .map(value -> getCollectionNameByUri(rootSubject.getUniqueId(), value.toString())) .filter(collections::contains) .collect(Collectors.toList()))); return; @@ -131,15 +119,4 @@ public CountDTO count(CountRequest request) { return new CountDTO(0, true); } } - - @SneakyThrows - public List searchFiles(FileSearchRequest request) { - var collectionsForUser = transactions.calculateRead(m -> rootSubject.getChildren().stream() - .map(collection -> getCollectionName(collection.getUniqueId())) - .collect(Collectors.toList())); - - try (var viewStoreReader = getViewStoreReader()) { - return viewStoreReader.searchFiles(request, collectionsForUser); - } - } } diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/QueryService.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/QueryService.java index f8eba9025..eb4952140 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/QueryService.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/QueryService.java @@ -1,10 +1,5 @@ package io.fairspace.saturn.services.views; -import java.util.List; - -import io.fairspace.saturn.services.search.FileSearchRequest; -import io.fairspace.saturn.services.search.SearchResultDTO; - /** * High-level interface for fetching metadata view pages and counts. * Implemented using Sparql queries on the RDF database directly @@ -22,6 +17,4 @@ public interface QueryService { ViewPageDTO retrieveViewPage(ViewRequest request); CountDTO count(CountRequest request); - - List searchFiles(FileSearchRequest request); } diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/SparqlQueryService.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/SparqlQueryService.java index 28c43c85a..dd669e243 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/SparqlQueryService.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/SparqlQueryService.java @@ -16,7 +16,6 @@ import org.apache.jena.query.QueryCancelledException; import org.apache.jena.query.QueryExecutionFactory; import org.apache.jena.query.QueryFactory; -import org.apache.jena.query.QuerySolutionMap; import org.apache.jena.rdf.model.RDFNode; import org.apache.jena.rdf.model.Resource; import org.apache.jena.rdf.model.Statement; @@ -38,9 +37,6 @@ import io.fairspace.saturn.config.ViewsConfig; import io.fairspace.saturn.config.ViewsConfig.ColumnType; import io.fairspace.saturn.config.ViewsConfig.View; -import io.fairspace.saturn.rdf.SparqlUtils; -import io.fairspace.saturn.services.search.FileSearchRequest; -import io.fairspace.saturn.services.search.SearchResultDTO; import io.fairspace.saturn.vocabulary.FS; import static io.fairspace.saturn.rdf.ModelUtils.getResourceProperties; @@ -54,7 +50,6 @@ import static java.util.stream.Collectors.toSet; import static org.apache.jena.graph.NodeFactory.createURI; import static org.apache.jena.rdf.model.ResourceFactory.createProperty; -import static org.apache.jena.rdf.model.ResourceFactory.createStringLiteral; import static org.apache.jena.sparql.expr.NodeValue.makeBoolean; import static org.apache.jena.sparql.expr.NodeValue.makeDate; import static org.apache.jena.sparql.expr.NodeValue.makeDecimal; @@ -158,13 +153,6 @@ private Map> fetch(Resource resource, String viewName) { return result; } - public List searchFiles(FileSearchRequest request) { - var query = getSearchForFilesQuery(request.getParentIRI()); - var binding = new QuerySolutionMap(); - binding.add("regexQuery", createStringLiteral(SparqlUtils.getQueryRegex(request.getQuery()))); - return SparqlUtils.getByQuery(query, binding, ds); - } - private Set getValues(Resource resource, View.Column column) { return new TreeSet<>(resource.listProperties(createProperty(column.source)) .mapWith(Statement::getObject) @@ -384,28 +372,6 @@ private View.Column getColumn(String name) { }); } - private Query getSearchForFilesQuery(String parentIRI) { - var builder = new StringBuilder("PREFIX fs: <") - .append(FS.NS) - .append(">\nPREFIX rdfs: \n\n") - .append("SELECT ?id ?label ?comment ?type\n") - .append("WHERE {\n"); - - if (parentIRI != null && !parentIRI.trim().isEmpty()) { - validateIRI(parentIRI); - builder.append("?id fs:belongsTo* <").append(parentIRI).append("> .\n"); - } - - builder.append("?id rdfs:label ?label ; a ?type .\n") - .append("FILTER (?type in (fs:File, fs:Directory, fs:Collection))\n") - .append("OPTIONAL { ?id rdfs:comment ?comment }\n") - .append("FILTER NOT EXISTS { ?id fs:dateDeleted ?anydate }\n") - .append("FILTER (regex(?label, ?regexQuery, \"i\") || regex(?comment, ?regexQuery, \"i\"))\n") - .append("}\nLIMIT 10000"); - - return QueryFactory.create(builder.toString()); - } - private static Calendar convertDateValue(String value) { var calendar = Calendar.getInstance(); calendar.setTimeInMillis(Instant.parse(value).toEpochMilli()); diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewService.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewService.java index 22599e91c..ef156dad1 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewService.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewService.java @@ -243,7 +243,7 @@ private FacetDTO getFacetInfo(View view, View.Column column) { } case Number, Date -> { if (viewStoreClientFactory != null) { - var range = getColumnRange(view, column); + var range = getViewStoreColumnRange(view, column); if (range != null) { min = range.getStart(); max = range.getEnd(); @@ -278,7 +278,7 @@ private Object convertLiteralValue(Object value) { } @SneakyThrows - private Range getColumnRange(View view, View.Column column) { + private Range getViewStoreColumnRange(View view, View.Column column) { if (!EnumSet.of(ColumnType.Date, ColumnType.Number).contains(column.type)) { return null; } diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/webdav/PathUtils.java b/projects/saturn/src/main/java/io/fairspace/saturn/webdav/PathUtils.java index 962f40ff0..eabb62b7e 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/webdav/PathUtils.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/webdav/PathUtils.java @@ -1,5 +1,8 @@ package io.fairspace.saturn.webdav; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; + import io.milton.http.exceptions.BadRequestException; import static org.apache.commons.lang3.StringUtils.strip; @@ -37,4 +40,10 @@ public static void validateCollectionName(String name) throws BadRequestExceptio throw new BadRequestException("The collection name contains an illegal character (\\)"); } } + + public static String getCollectionNameByUri(String rootSubjectUri, String uri) { + var rootLocation = rootSubjectUri + "/"; + var location = uri.substring(rootLocation.length()); + return URLDecoder.decode(location.split("/")[0], StandardCharsets.UTF_8); + } } diff --git a/projects/saturn/src/test/java/io/fairspace/saturn/services/search/JdbcFileSearchServiceTest.java b/projects/saturn/src/test/java/io/fairspace/saturn/services/search/JdbcFileSearchServiceTest.java new file mode 100644 index 000000000..deb632897 --- /dev/null +++ b/projects/saturn/src/test/java/io/fairspace/saturn/services/search/JdbcFileSearchServiceTest.java @@ -0,0 +1,239 @@ +package io.fairspace.saturn.services.search; + +import java.io.IOException; +import java.sql.SQLException; + +import io.milton.http.ResourceFactory; +import io.milton.http.exceptions.BadRequestException; +import io.milton.http.exceptions.ConflictException; +import io.milton.http.exceptions.NotAuthorizedException; +import io.milton.resource.MakeCollectionableResource; +import io.milton.resource.PutableResource; +import org.apache.jena.query.Dataset; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.util.Context; +import org.eclipse.jetty.server.Authentication; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import io.fairspace.saturn.PostgresAwareTest; +import io.fairspace.saturn.config.Config; +import io.fairspace.saturn.config.ConfigLoader; +import io.fairspace.saturn.config.ViewsConfig; +import io.fairspace.saturn.rdf.dao.DAO; +import io.fairspace.saturn.rdf.transactions.SimpleTransactions; +import io.fairspace.saturn.rdf.transactions.Transactions; +import io.fairspace.saturn.rdf.transactions.TxnIndexDatasetGraph; +import io.fairspace.saturn.services.maintenance.MaintenanceService; +import io.fairspace.saturn.services.metadata.MetadataPermissions; +import io.fairspace.saturn.services.metadata.MetadataService; +import io.fairspace.saturn.services.metadata.validation.ComposedValidator; +import io.fairspace.saturn.services.metadata.validation.UniqueLabelValidator; +import io.fairspace.saturn.services.users.User; +import io.fairspace.saturn.services.users.UserService; +import io.fairspace.saturn.services.views.*; +import io.fairspace.saturn.services.workspaces.Workspace; +import io.fairspace.saturn.services.workspaces.WorkspaceRole; +import io.fairspace.saturn.services.workspaces.WorkspaceService; +import io.fairspace.saturn.webdav.DavFactory; +import io.fairspace.saturn.webdav.blobstore.BlobInfo; +import io.fairspace.saturn.webdav.blobstore.BlobStore; + +import static io.fairspace.saturn.TestUtils.*; +import static io.fairspace.saturn.auth.RequestContext.getCurrentRequest; + +import static org.apache.jena.query.DatasetFactory.wrap; +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class JdbcFileSearchServiceTest extends PostgresAwareTest { + static final String BASE_PATH = "/api/webdav"; + static final String baseUri = ConfigLoader.CONFIG.publicUrl + BASE_PATH; + + @Mock + BlobStore store; + + @Mock + UserService userService; + + @Mock + private MetadataPermissions permissions; + + WorkspaceService workspaceService; + MetadataService api; + FileSearchService fileSearchService; + MaintenanceService maintenanceService; + + User user; + Authentication.User userAuthentication; + User workspaceManager; + Authentication.User workspaceManagerAuthentication; + User admin; + Authentication.User adminAuthentication; + private org.eclipse.jetty.server.Request request; + + private void selectRegularUser() { + lenient().when(request.getAuthentication()).thenReturn(userAuthentication); + lenient().when(userService.currentUser()).thenReturn(user); + } + + private void selectAdmin() { + lenient().when(request.getAuthentication()).thenReturn(adminAuthentication); + lenient().when(userService.currentUser()).thenReturn(admin); + } + + @Before + public void before() + throws SQLException, NotAuthorizedException, BadRequestException, ConflictException, IOException { + var viewDatabase = new Config.ViewDatabase(); + viewDatabase.url = postgres.getJdbcUrl(); + viewDatabase.username = postgres.getUsername(); + viewDatabase.password = postgres.getPassword(); + viewDatabase.maxPoolSize = 5; + ViewsConfig config = loadViewsConfig("src/test/resources/test-views.yaml"); + var viewStoreClientFactory = new ViewStoreClientFactory(config, viewDatabase, new Config.Search()); + + var dsg = new TxnIndexDatasetGraph(DatasetGraphFactory.createTxnMem(), viewStoreClientFactory); + Dataset ds = wrap(dsg); + Transactions tx = new SimpleTransactions(ds); + Model model = ds.getDefaultModel(); + var vocabulary = model.read("test-vocabulary.ttl"); + + var viewService = new ViewService(ConfigLoader.CONFIG, config, ds, viewStoreClientFactory, permissions); + + maintenanceService = new MaintenanceService(userService, ds, viewStoreClientFactory, viewService); + + workspaceService = new WorkspaceService(tx, userService); + + var context = new Context(); + + var davFactory = new DavFactory(model.createResource(baseUri), store, userService, context); + + fileSearchService = new JdbcFileSearchService( + ConfigLoader.CONFIG.search, + loadViewsConfig("src/test/resources/test-views.yaml"), + viewStoreClientFactory, + tx, + davFactory.root); + + when(permissions.canWriteMetadata(any())).thenReturn(true); + + api = new MetadataService(tx, vocabulary, new ComposedValidator(new UniqueLabelValidator()), permissions); + + userAuthentication = mockAuthentication("user"); + user = createTestUser("user", false); + new DAO(model).write(user); + workspaceManager = createTestUser("workspace-admin", false); + new DAO(model).write(workspaceManager); + workspaceManagerAuthentication = mockAuthentication("workspace-admin"); + adminAuthentication = mockAuthentication("admin"); + admin = createTestUser("admin", true); + new DAO(model).write(admin); + + setupRequestContext(); + request = getCurrentRequest(); + + selectAdmin(); + + var taxonomies = model.read("test-taxonomies.ttl"); + api.put(taxonomies, Boolean.TRUE); + + var workspace = workspaceService.createWorkspace( + Workspace.builder().code("Test").build()); + workspaceService.setUserRole(workspace.getIri(), workspaceManager.getIri(), WorkspaceRole.Manager); + workspaceService.setUserRole(workspace.getIri(), user.getIri(), WorkspaceRole.Member); + + when(request.getHeader("Owner")).thenReturn(workspace.getIri().getURI()); + when(request.getAttribute("BLOB")).thenReturn(new BlobInfo("id", 0, "md5")); + + var root = (MakeCollectionableResource) ((ResourceFactory) davFactory).getResource(null, BASE_PATH); + var coll1 = (PutableResource) root.createCollection("coll1"); + coll1.createNew("coffee.jpg", null, 0L, "image/jpeg"); + + selectRegularUser(); + + var coll2 = (PutableResource) root.createCollection("coll2"); + coll2.createNew("sample-s2-b-rna.fastq", null, 0L, "chemical/seq-na-fastq"); + + var coll3 = (PutableResource) root.createCollection("coll3"); + + coll3.createNew("sample-s2-b-rna_copy.fastq", null, 0L, "chemical/seq-na-fastq"); + + var testdata = model.read("testdata.ttl"); + api.put(testdata, Boolean.TRUE); + } + + @Test + public void testSearchFiles() { + var request = new FileSearchRequest(); + // There are two files with 'rna' in the file name in coll2. + request.setQuery("rna"); + var results = fileSearchService.searchFiles(request); + Assert.assertEquals(2, results.size()); + // Expect the results to be sorted by id + Assert.assertEquals("sample-s2-b-rna.fastq", results.get(0).getLabel()); + Assert.assertEquals("sample-s2-b-rna_copy.fastq", results.get(1).getLabel()); + } + + @Test + public void testSearchFilesRestrictsToAccessibleCollections() { + var request = new FileSearchRequest(); + // There is one file named coffee.jpg in coll1, not accessible by the regular user. + request.setQuery("coffee"); + var results = fileSearchService.searchFiles(request); + Assert.assertEquals(0, results.size()); + + selectAdmin(); + results = fileSearchService.searchFiles(request); + Assert.assertEquals(1, results.size()); + Assert.assertEquals("coffee.jpg", results.get(0).getLabel()); + } + + @Test + public void testSearchFilesRestrictsToAccessibleCollectionsAfterReindexing() { + maintenanceService.recreateIndex(); + var request = new FileSearchRequest(); + // There is one file named coffee.jpg in coll1, not accessible by the regular user. + request.setQuery("coffee"); + var results = fileSearchService.searchFiles(request); + Assert.assertEquals(0, results.size()); + + selectAdmin(); + results = fileSearchService.searchFiles(request); + Assert.assertEquals(1, results.size()); + Assert.assertEquals("coffee.jpg", results.get(0).getLabel()); + } + + @Test + public void testSearchFilesRestrictsToParentDirectory() { + selectAdmin(); + var request = new FileSearchRequest(); + // There is one file named coffee.jpg in coll1. + request.setQuery("coffee"); + + request.setParentIRI(ConfigLoader.CONFIG.publicUrl + "/api/webdav/coll1"); + var results = fileSearchService.searchFiles(request); + Assert.assertEquals(1, results.size()); + + request.setParentIRI(ConfigLoader.CONFIG.publicUrl + "/api/webdav/coll2"); + results = fileSearchService.searchFiles(request); + Assert.assertEquals(0, results.size()); + } + + @Test + public void testSearchFileDescription() { + selectAdmin(); + var request = new FileSearchRequest(); + // There is one file named sample-s2-b-rna.fastq with a description + request.setQuery("corona"); + + // request.setParentIRI(ConfigLoader.CONFIG.publicUrl + "/api/webdav/coll1"); + var results = fileSearchService.searchFiles(request); + Assert.assertEquals(1, results.size()); + } +} diff --git a/projects/saturn/src/test/java/io/fairspace/saturn/services/search/SparqlFileSearchServiceTest.java b/projects/saturn/src/test/java/io/fairspace/saturn/services/search/SparqlFileSearchServiceTest.java new file mode 100644 index 000000000..a561e3ad5 --- /dev/null +++ b/projects/saturn/src/test/java/io/fairspace/saturn/services/search/SparqlFileSearchServiceTest.java @@ -0,0 +1,180 @@ +package io.fairspace.saturn.services.search; + +import java.io.IOException; + +import io.milton.http.ResourceFactory; +import io.milton.http.exceptions.BadRequestException; +import io.milton.http.exceptions.ConflictException; +import io.milton.http.exceptions.NotAuthorizedException; +import io.milton.resource.MakeCollectionableResource; +import io.milton.resource.PutableResource; +import org.apache.jena.query.Dataset; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.sparql.core.DatasetGraphFactory; +import org.apache.jena.sparql.core.DatasetImpl; +import org.apache.jena.sparql.util.Context; +import org.eclipse.jetty.server.Authentication; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import io.fairspace.saturn.config.ConfigLoader; +import io.fairspace.saturn.rdf.dao.DAO; +import io.fairspace.saturn.rdf.search.FilteredDatasetGraph; +import io.fairspace.saturn.rdf.transactions.SimpleTransactions; +import io.fairspace.saturn.rdf.transactions.Transactions; +import io.fairspace.saturn.services.metadata.MetadataPermissions; +import io.fairspace.saturn.services.metadata.MetadataService; +import io.fairspace.saturn.services.metadata.validation.ComposedValidator; +import io.fairspace.saturn.services.metadata.validation.UniqueLabelValidator; +import io.fairspace.saturn.services.users.User; +import io.fairspace.saturn.services.users.UserService; +import io.fairspace.saturn.services.workspaces.Workspace; +import io.fairspace.saturn.services.workspaces.WorkspaceRole; +import io.fairspace.saturn.services.workspaces.WorkspaceService; +import io.fairspace.saturn.webdav.DavFactory; +import io.fairspace.saturn.webdav.blobstore.BlobInfo; +import io.fairspace.saturn.webdav.blobstore.BlobStore; + +import static io.fairspace.saturn.TestUtils.*; +import static io.fairspace.saturn.auth.RequestContext.getCurrentRequest; + +import static org.apache.jena.query.DatasetFactory.wrap; +import static org.junit.Assert.*; +import static org.mockito.Mockito.*; + +@RunWith(MockitoJUnitRunner.class) +public class SparqlFileSearchServiceTest { + static final String BASE_PATH = "/api/webdav"; + static final String baseUri = ConfigLoader.CONFIG.publicUrl + BASE_PATH; + + @Mock + BlobStore store; + + @Mock + UserService userService; + + @Mock + private MetadataPermissions permissions; + + WorkspaceService workspaceService; + MetadataService api; + FileSearchService fileSearchService; + + User user; + Authentication.User userAuthentication; + User user2; + Authentication.User user2Authentication; + User workspaceManager; + Authentication.User workspaceManagerAuthentication; + User admin; + Authentication.User adminAuthentication; + private org.eclipse.jetty.server.Request request; + + private void selectRegularUser() { + lenient().when(request.getAuthentication()).thenReturn(userAuthentication); + lenient().when(userService.currentUser()).thenReturn(user); + } + + private void selectAdmin() { + lenient().when(request.getAuthentication()).thenReturn(adminAuthentication); + lenient().when(userService.currentUser()).thenReturn(admin); + } + + private void setupUsers(Model model) { + userAuthentication = mockAuthentication("user"); + user = createTestUser("user", false); + user.setCanViewPublicMetadata(true); + new DAO(model).write(user); + user2Authentication = mockAuthentication("user2"); + user2 = createTestUser("user2", false); + new DAO(model).write(user2); + workspaceManager = createTestUser("workspace-admin", false); + new DAO(model).write(workspaceManager); + workspaceManagerAuthentication = mockAuthentication("workspace-admin"); + adminAuthentication = mockAuthentication("admin"); + admin = createTestUser("admin", true); + new DAO(model).write(admin); + } + + @Before + public void before() throws NotAuthorizedException, BadRequestException, ConflictException, IOException { + var dsg = DatasetGraphFactory.createTxnMem(); + Dataset ds = wrap(dsg); + Transactions tx = new SimpleTransactions(ds); + Model model = ds.getDefaultModel(); + var vocabulary = model.read("test-vocabulary.ttl"); + + workspaceService = new WorkspaceService(tx, userService); + + var context = new Context(); + var davFactory = new DavFactory(model.createResource(baseUri), store, userService, context); + var metadataPermissions = new MetadataPermissions(workspaceService, davFactory, userService); + var filteredDatasetGraph = new FilteredDatasetGraph(ds.asDatasetGraph(), metadataPermissions); + var filteredDataset = DatasetImpl.wrap(filteredDatasetGraph); + + fileSearchService = new SparqlFileSearchService(filteredDataset); + + when(permissions.canWriteMetadata(any())).thenReturn(true); + api = new MetadataService(tx, vocabulary, new ComposedValidator(new UniqueLabelValidator()), permissions); + + setupUsers(model); + + setupRequestContext(); + request = getCurrentRequest(); + + selectAdmin(); + + var taxonomies = model.read("test-taxonomies.ttl"); + api.put(taxonomies, Boolean.FALSE); + + var workspace = workspaceService.createWorkspace( + Workspace.builder().code("Test").build()); + workspaceService.setUserRole(workspace.getIri(), workspaceManager.getIri(), WorkspaceRole.Manager); + workspaceService.setUserRole(workspace.getIri(), user.getIri(), WorkspaceRole.Member); + + when(request.getHeader("Owner")).thenReturn(workspace.getIri().getURI()); + when(request.getAttribute("BLOB")).thenReturn(new BlobInfo("id", 0, "md5")); + + var root = (MakeCollectionableResource) ((ResourceFactory) davFactory).getResource(null, BASE_PATH); + var coll1 = (PutableResource) root.createCollection("coll1"); + coll1.createNew("coffee.jpg", null, 0L, "image/jpeg"); + + selectRegularUser(); + + var coll2 = (PutableResource) root.createCollection("coll2"); + coll2.createNew("sample-s2-b-rna.fastq", null, 0L, "chemical/seq-na-fastq"); + coll2.createNew("sample-s2-b-rna_copy.fastq", null, 0L, "chemical/seq-na-fastq"); + + var testdata = model.read("testdata.ttl"); + api.put(testdata, Boolean.FALSE); + } + + @Test + public void testRetrieveFilesForParent() { + selectAdmin(); + var request = new FileSearchRequest(); + request.setQuery("coffee"); + request.setParentIRI(baseUri + "/coll1"); + + var results = fileSearchService.searchFiles(request); + assertEquals(1, results.size()); + } + + @Test + public void testRetrieveFilesForInvalidParent() { + selectAdmin(); + var request = new FileSearchRequest(); + request.setQuery("coffee"); + request.setParentIRI(">; INSERT something"); + + Exception exception = + assertThrows(IllegalArgumentException.class, () -> fileSearchService.searchFiles(request)); + String expectedMessage = "Invalid IRI"; + String actualMessage = exception.getMessage(); + + assertTrue(actualMessage.contains(expectedMessage)); + } +} diff --git a/projects/saturn/src/test/java/io/fairspace/saturn/services/views/JdbcQueryServiceTest.java b/projects/saturn/src/test/java/io/fairspace/saturn/services/views/JdbcQueryServiceTest.java index 8053a7f25..d2d9629b7 100644 --- a/projects/saturn/src/test/java/io/fairspace/saturn/services/views/JdbcQueryServiceTest.java +++ b/projects/saturn/src/test/java/io/fairspace/saturn/services/views/JdbcQueryServiceTest.java @@ -37,7 +37,6 @@ import io.fairspace.saturn.services.metadata.MetadataService; import io.fairspace.saturn.services.metadata.validation.ComposedValidator; import io.fairspace.saturn.services.metadata.validation.UniqueLabelValidator; -import io.fairspace.saturn.services.search.FileSearchRequest; import io.fairspace.saturn.services.users.User; import io.fairspace.saturn.services.users.UserService; import io.fairspace.saturn.services.workspaces.Workspace; @@ -372,73 +371,4 @@ public void testCountResourceWithMaxDisplayCountLimitMoreThanTotalCount() { var result = sut.count(request); Assert.assertEquals(4, result.getCount()); } - - @Test - public void testSearchFiles() { - var request = new FileSearchRequest(); - // There are two files with 'rna' in the file name in coll2. - request.setQuery("rna"); - var results = sut.searchFiles(request); - Assert.assertEquals(2, results.size()); - // Expect the results to be sorted by id - Assert.assertEquals("sample-s2-b-rna.fastq", results.get(0).getLabel()); - Assert.assertEquals("sample-s2-b-rna_copy.fastq", results.get(1).getLabel()); - } - - @Test - public void testSearchFilesRestrictsToAccessibleCollections() { - var request = new FileSearchRequest(); - // There is one file named coffee.jpg in coll1, not accessible by the regular user. - request.setQuery("coffee"); - var results = sut.searchFiles(request); - Assert.assertEquals(0, results.size()); - - selectAdmin(); - results = sut.searchFiles(request); - Assert.assertEquals(1, results.size()); - Assert.assertEquals("coffee.jpg", results.get(0).getLabel()); - } - - @Test - public void testSearchFilesRestrictsToAccessibleCollectionsAfterReindexing() { - maintenanceService.recreateIndex(); - var request = new FileSearchRequest(); - // There is one file named coffee.jpg in coll1, not accessible by the regular user. - request.setQuery("coffee"); - var results = sut.searchFiles(request); - Assert.assertEquals(0, results.size()); - - selectAdmin(); - results = sut.searchFiles(request); - Assert.assertEquals(1, results.size()); - Assert.assertEquals("coffee.jpg", results.get(0).getLabel()); - } - - @Test - public void testSearchFilesRestrictsToParentDirectory() { - selectAdmin(); - var request = new FileSearchRequest(); - // There is one file named coffee.jpg in coll1. - request.setQuery("coffee"); - - request.setParentIRI(ConfigLoader.CONFIG.publicUrl + "/api/webdav/coll1"); - var results = sut.searchFiles(request); - Assert.assertEquals(1, results.size()); - - request.setParentIRI(ConfigLoader.CONFIG.publicUrl + "/api/webdav/coll2"); - results = sut.searchFiles(request); - Assert.assertEquals(0, results.size()); - } - - @Test - public void testSearchFileDescription() { - selectAdmin(); - var request = new FileSearchRequest(); - // There is one file named sample-s2-b-rna.fastq with a description - request.setQuery("corona"); - - // request.setParentIRI(ConfigLoader.CONFIG.publicUrl + "/api/webdav/coll1"); - var results = sut.searchFiles(request); - Assert.assertEquals(1, results.size()); - } } diff --git a/projects/saturn/src/test/java/io/fairspace/saturn/services/views/SparqlQueryServiceTest.java b/projects/saturn/src/test/java/io/fairspace/saturn/services/views/SparqlQueryServiceTest.java index 7affd5628..e929441fd 100644 --- a/projects/saturn/src/test/java/io/fairspace/saturn/services/views/SparqlQueryServiceTest.java +++ b/projects/saturn/src/test/java/io/fairspace/saturn/services/views/SparqlQueryServiceTest.java @@ -22,7 +22,6 @@ import io.fairspace.saturn.rdf.transactions.*; import io.fairspace.saturn.services.metadata.*; import io.fairspace.saturn.services.metadata.validation.*; -import io.fairspace.saturn.services.search.FileSearchRequest; import io.fairspace.saturn.services.users.*; import io.fairspace.saturn.services.workspaces.*; import io.fairspace.saturn.webdav.*; @@ -289,31 +288,6 @@ public void testRetrieveSamplesForInvalidLocation() { assertTrue(actualMessage.contains(expectedMessage)); } - @Test - public void testRetrieveFilesForParent() { - selectAdmin(); - var request = new FileSearchRequest(); - request.setQuery("coffee"); - request.setParentIRI(baseUri + "/coll1"); - - var results = queryService.searchFiles(request); - assertEquals(1, results.size()); - } - - @Test - public void testRetrieveFilesForInvalidParent() { - selectAdmin(); - var request = new FileSearchRequest(); - request.setQuery("coffee"); - request.setParentIRI(">; INSERT something"); - - Exception exception = assertThrows(IllegalArgumentException.class, () -> queryService.searchFiles(request)); - String expectedMessage = "Invalid IRI"; - String actualMessage = exception.getMessage(); - - assertTrue(actualMessage.contains(expectedMessage)); - } - @Test public void testRetrieveSamplePageForUnaccessibleCollection() { var request = new ViewRequest();