diff --git a/src/main/java/io/quarkus/search/app/SearchService.java b/src/main/java/io/quarkus/search/app/SearchService.java index 8c367df3..3e020c55 100644 --- a/src/main/java/io/quarkus/search/app/SearchService.java +++ b/src/main/java/io/quarkus/search/app/SearchService.java @@ -19,6 +19,7 @@ import io.quarkus.search.app.entity.Guide; import io.quarkus.search.app.entity.Language; import io.quarkus.search.app.entity.QuarkusVersionAndLanguageRoutingBinder; +import io.quarkus.search.app.quarkusio.QuarkusIO; import io.quarkus.runtime.LaunchMode; @@ -62,6 +63,7 @@ public void init(@Observes Router router) { public SearchResult search(@RestQuery @DefaultValue(QuarkusVersions.LATEST) String version, @RestQuery List categories, @RestQuery String q, + @RestQuery String origin, @RestQuery @DefaultValue("en") Language language, @RestQuery @DefaultValue("highlighted") String highlightCssClass, @RestQuery @DefaultValue("0") @Min(0) int page, @@ -83,6 +85,9 @@ public SearchResult search(@RestQuery @DefaultValue(QuarkusVersi if (categories != null && !categories.isEmpty()) { root.add(f.terms().field("categories").matchingAny(categories)); } + if (origin != null && !origin.isEmpty()) { + root.add(f.match().field("origin").matching(origin)); + } if (q != null && !q.isBlank()) { root.add(f.bool().must(f.simpleQueryString() @@ -101,7 +106,8 @@ public SearchResult search(@RestQuery @DefaultValue(QuarkusVersi // we also add phrase flag so that entire phrases could be searched as well, e.g.: "hibernate search" .flags(SimpleQueryFlag.AND, SimpleQueryFlag.OR, SimpleQueryFlag.PHRASE) .defaultOperator(BooleanOperator.AND)) - .should(f.match().field("origin").matching("quarkus").boost(50.0f)) + .should(f.match().field("origin").matching(QuarkusIO.QUARKUS_ORIGIN).constantScore() + .boost(1000.0f)) .should(f.not(f.match().field(language.addSuffix("topics")) .matching("compatibility", ValueConvert.NO)) .boost(50.0f))); diff --git a/src/main/java/io/quarkus/search/app/entity/I18nData.java b/src/main/java/io/quarkus/search/app/entity/I18nData.java index 8a78caa6..48af8955 100644 --- a/src/main/java/io/quarkus/search/app/entity/I18nData.java +++ b/src/main/java/io/quarkus/search/app/entity/I18nData.java @@ -43,6 +43,14 @@ public void set(Language language, T value) { } } + public void set(T value) { + en = value; + es = value; + pt = value; + cn = value; + ja = value; + } + public T get(Language language) { return switch (language) { case ENGLISH -> en; diff --git a/src/main/java/io/quarkus/search/app/entity/QuarkusVersionAndLanguageRoutingBinder.java b/src/main/java/io/quarkus/search/app/entity/QuarkusVersionAndLanguageRoutingBinder.java index 4f3ffc44..36e370b7 100644 --- a/src/main/java/io/quarkus/search/app/entity/QuarkusVersionAndLanguageRoutingBinder.java +++ b/src/main/java/io/quarkus/search/app/entity/QuarkusVersionAndLanguageRoutingBinder.java @@ -2,6 +2,9 @@ import java.util.List; +import io.quarkus.search.app.quarkiverseio.QuarkiverseIO; +import io.quarkus.search.app.quarkusio.QuarkusIO; + import org.hibernate.search.mapper.pojo.bridge.RoutingBridge; import org.hibernate.search.mapper.pojo.bridge.binding.RoutingBindingContext; import org.hibernate.search.mapper.pojo.bridge.mapping.programmatic.RoutingBinder; @@ -10,14 +13,23 @@ public class QuarkusVersionAndLanguageRoutingBinder implements RoutingBinder { private static String key(String version, Language language) { - if (language == null) { - return version; + return key(version, language, QuarkusIO.QUARKUS_ORIGIN); + } + + private static String key(String version, Language language, String origin) { + StringBuilder key = new StringBuilder(); + key.append(origin); + if (version != null) { + key.append("/").append(version); + } + if (language != null) { + key.append("/").append(language.code); } - return version + "/" + language.code; + return key.toString(); } public static List searchKeys(String version, Language language) { - return List.of(key(version, language), key(version, null)); + return List.of(key(version, language), key(version, null), key(null, null, QuarkiverseIO.QUARKIVERSE_ORIGIN)); } @Override @@ -34,7 +46,11 @@ public static class GuideRoutingBridge implements RoutingBridge { @Override public void route(DocumentRoutes routes, Object entityIdentifier, Guide entity, RoutingBridgeRouteContext context) { - routes.addRoute().routingKey(key(entity.quarkusVersion, entity.language)); + if (QuarkiverseIO.QUARKIVERSE_ORIGIN.equals(entity.origin)) { + routes.addRoute().routingKey(key(null, null, QuarkiverseIO.QUARKIVERSE_ORIGIN)); + } else { + routes.addRoute().routingKey(key(entity.quarkusVersion, entity.language)); + } } @Override diff --git a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java index b2ca59cc..6d84908c 100644 --- a/src/main/java/io/quarkus/search/app/fetching/FetchingService.java +++ b/src/main/java/io/quarkus/search/app/fetching/FetchingService.java @@ -18,6 +18,8 @@ import io.quarkus.search.app.entity.Language; import io.quarkus.search.app.indexing.FailureCollector; +import io.quarkus.search.app.quarkiverseio.QuarkiverseIO; +import io.quarkus.search.app.quarkiverseio.QuarkiverseIOConfig; import io.quarkus.search.app.quarkusio.QuarkusIO; import io.quarkus.search.app.quarkusio.QuarkusIOConfig; import io.quarkus.search.app.util.CloseableDirectory; @@ -41,9 +43,16 @@ public class FetchingService { @Inject QuarkusIOConfig quarkusIOConfig; + @Inject + QuarkiverseIOConfig quarkiverseIOConfig; + private final Map detailsCache = new ConcurrentHashMap<>(); private final Set tempDirectories = new ConcurrentHashSet<>(); + public QuarkiverseIO fetchQuarkiverseIo(FailureCollector failureCollector) { + return new QuarkiverseIO(quarkiverseIOConfig, failureCollector); + } + public QuarkusIO fetchQuarkusIo(FailureCollector failureCollector) { CompletableFuture main = null; Map> localized = new LinkedHashMap<>(); diff --git a/src/main/java/io/quarkus/search/app/indexing/IndexableGuides.java b/src/main/java/io/quarkus/search/app/indexing/IndexableGuides.java new file mode 100644 index 00000000..f811ce48 --- /dev/null +++ b/src/main/java/io/quarkus/search/app/indexing/IndexableGuides.java @@ -0,0 +1,10 @@ +package io.quarkus.search.app.indexing; + +import java.io.IOException; +import java.util.stream.Stream; + +import io.quarkus.search.app.entity.Guide; + +public interface IndexableGuides { + Stream guides() throws IOException; +} diff --git a/src/main/java/io/quarkus/search/app/indexing/IndexingService.java b/src/main/java/io/quarkus/search/app/indexing/IndexingService.java index 926d240c..7e77f1b6 100644 --- a/src/main/java/io/quarkus/search/app/indexing/IndexingService.java +++ b/src/main/java/io/quarkus/search/app/indexing/IndexingService.java @@ -9,6 +9,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Stream; import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; @@ -18,6 +19,7 @@ import io.quarkus.search.app.ReferenceService; import io.quarkus.search.app.fetching.FetchingService; +import io.quarkus.search.app.quarkiverseio.QuarkiverseIO; import io.quarkus.search.app.quarkusio.QuarkusIO; import io.quarkus.search.app.util.SimpleExecutor; @@ -230,8 +232,9 @@ private void createIndexes() { private void indexAll(FailureCollector failureCollector) { Log.info("Indexing..."); try (Rollover rollover = Rollover.start(searchMapping)) { - try (QuarkusIO quarkusIO = fetchingService.fetchQuarkusIo(failureCollector)) { - indexQuarkusIo(quarkusIO); + try (QuarkusIO quarkusIO = fetchingService.fetchQuarkusIo(failureCollector); + QuarkiverseIO quarkiverseIO = fetchingService.fetchQuarkiverseIo(failureCollector)) { + indexQuarkusIo(quarkusIO, quarkiverseIO); } // Refresh BEFORE committing the rollover, @@ -248,9 +251,9 @@ private void indexAll(FailureCollector failureCollector) { } } - private void indexQuarkusIo(QuarkusIO quarkusIO) throws IOException { - Log.info("Indexing quarkus.io..."); - try (var guideStream = quarkusIO.guides(); + private void indexQuarkusIo(IndexableGuides quarkus, IndexableGuides quarkiverse) throws IOException { + Log.info("Indexing quarkus.io/quarkiverse.io..."); + try (var guideStream = Stream.concat(quarkus.guides(), quarkiverse.guides()); var executor = new SimpleExecutor(indexingConfig.parallelism())) { indexAll(executor, guideStream.iterator()); } diff --git a/src/main/java/io/quarkus/search/app/quarkiverseio/QuarkiverseIO.java b/src/main/java/io/quarkus/search/app/quarkiverseio/QuarkiverseIO.java new file mode 100644 index 00000000..12d570fc --- /dev/null +++ b/src/main/java/io/quarkus/search/app/quarkiverseio/QuarkiverseIO.java @@ -0,0 +1,162 @@ +package io.quarkus.search.app.quarkiverseio; + +import java.io.Closeable; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import java.util.stream.Stream; + +import jakarta.ws.rs.core.UriBuilder; + +import io.quarkus.search.app.entity.Guide; +import io.quarkus.search.app.hibernate.InputProvider; +import io.quarkus.search.app.indexing.FailureCollector; +import io.quarkus.search.app.indexing.IndexableGuides; +import io.quarkus.search.app.util.CloseableDirectory; + +import io.quarkus.logging.Log; + +import org.hibernate.search.util.common.impl.Closer; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +public class QuarkiverseIO implements IndexableGuides, Closeable { + + public static final String QUARKIVERSE_ORIGIN = "quarkiverse-hub"; + + private final URI quarkiverseDocsIndex; + private final FailureCollector failureCollector; + + private final List quarkiverseGuides = new ArrayList<>(); + private final boolean enabled; + private final CloseableDirectory guideHtmls; + + public QuarkiverseIO(QuarkiverseIOConfig config, FailureCollector failureCollector) { + this.quarkiverseDocsIndex = config.webUri(); + this.enabled = config.enabled(); + this.failureCollector = failureCollector; + try { + guideHtmls = CloseableDirectory.temp("quarkiverse_htmls_"); + } catch (IOException e) { + throw new IllegalStateException("Failed to fetch quarkiverse guides: %s".formatted(e.getMessage()), e); + } + } + + public void parseGuides() { + Document index = null; + try { + index = Jsoup.connect(quarkiverseDocsIndex.toString()).get(); + } catch (IOException e) { + failureCollector.critical(FailureCollector.Stage.PARSING, "Unable to fetch the Quarkiverse Docs index page.", e); + // no point in doing anything else here: + return; + } + + // find links to quarkiverse extension docs: + Elements quarkiverseGuideIndexLinks = index.select("ul.components li.component a.title"); + + for (Element quarkiverseGuideIndexLink : quarkiverseGuideIndexLinks) { + Guide guide = new Guide(); + String topLevelTitle = quarkiverseGuideIndexLink.text(); + guide.title.set(topLevelTitle); + + Document extensionIndex = null; + try { + extensionIndex = readGuide(guide, quarkiverseGuideIndexLink.absUrl("href"), Optional.empty()); + } catch (URISyntaxException | IOException e) { + failureCollector.warning(FailureCollector.Stage.PARSING, + "Unable to fetch guide: " + topLevelTitle, e); + continue; + } + + quarkiverseGuides.add(guide); + + // find other sub-pages on the left side + Map indexLinks = new HashMap<>(); + Elements extensionSubGuides = extensionIndex.select("nav.nav-menu .nav-item a"); + for (Element element : extensionSubGuides) { + String href = element.absUrl("href"); + URI uri = UriBuilder.fromUri(href).replaceQuery(null).fragment(null).build(); + indexLinks.computeIfAbsent(uri, u -> element.text()); + } + + for (Map.Entry entry : indexLinks.entrySet()) { + Guide sub = new Guide(); + sub.title.set(entry.getValue()); + try { + readGuide(sub, entry.getKey().toString(), Optional.of(topLevelTitle)); + } catch (URISyntaxException | IOException e) { + failureCollector.warning(FailureCollector.Stage.PARSING, + "Unable to fetch guide: " + topLevelTitle, e); + continue; + } + quarkiverseGuides.add(sub); + } + } + } + + private Document readGuide(Guide guide, String link, Optional titlePrefix) throws URISyntaxException, IOException { + guide.url = new URI(link); + guide.type = "reference"; + guide.origin = QUARKIVERSE_ORIGIN; + + Document extensionIndex = Jsoup.connect(link).get(); + Elements content = extensionIndex.select("div.content"); + + String title = content.select("h1.page").text(); + if (!title.isBlank()) { + String actualTitle = titlePrefix.map(prefix -> "%s: %s".formatted(prefix, title)).orElse(title); + guide.title.set(actualTitle); + } + guide.summary.set(content.select("div#preamble").text()); + guide.htmlFullContentProvider.set(new FileInputProvider(link, dumpHtmlToFile(content.html()))); + + Log.debugf("Parsed guide: %s", guide.url); + return extensionIndex; + } + + private Path dumpHtmlToFile(String html) throws IOException { + Path path = guideHtmls.path().resolve(UUID.randomUUID().toString()); + try (FileOutputStream fos = new FileOutputStream(path.toFile())) { + fos.write(html.getBytes(StandardCharsets.UTF_8)); + } + return path; + } + + public Stream guides() { + if (enabled) { + parseGuides(); + } + return quarkiverseGuides.stream(); + } + + @Override + public void close() throws IOException { + try (var closer = new Closer()) { + closer.push(CloseableDirectory::close, guideHtmls); + closer.push(List::clear, quarkiverseGuides); + } + } + + private record FileInputProvider(String link, Path content) implements InputProvider { + + @Override + public InputStream open() throws IOException { + return new FileInputStream(content.toFile()); + } + } +} diff --git a/src/main/java/io/quarkus/search/app/quarkiverseio/QuarkiverseIOConfig.java b/src/main/java/io/quarkus/search/app/quarkiverseio/QuarkiverseIOConfig.java new file mode 100644 index 00000000..e463619f --- /dev/null +++ b/src/main/java/io/quarkus/search/app/quarkiverseio/QuarkiverseIOConfig.java @@ -0,0 +1,18 @@ +package io.quarkus.search.app.quarkiverseio; + +import java.net.URI; + +import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithDefault; + +@ConfigMapping(prefix = "quarkiverseio") +public interface QuarkiverseIOConfig { + String WEB_URI_DEFAULT_STRING = "https://docs.quarkiverse.io/index/explore/index.html"; + URI WEB_URI_DEFAULT = URI.create(WEB_URI_DEFAULT_STRING); + + @WithDefault(WEB_URI_DEFAULT_STRING) + URI webUri(); + + @WithDefault("true") + boolean enabled(); +} diff --git a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java index 52cea4ca..f275abbd 100644 --- a/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java +++ b/src/main/java/io/quarkus/search/app/quarkusio/QuarkusIO.java @@ -28,11 +28,11 @@ import io.quarkus.search.app.entity.I18nData; import io.quarkus.search.app.entity.Language; import io.quarkus.search.app.indexing.FailureCollector; +import io.quarkus.search.app.indexing.IndexableGuides; import io.quarkus.search.app.util.CloseableDirectory; import io.quarkus.search.app.util.GitCloneDirectory; import io.quarkus.search.app.util.GitInputProvider; import io.quarkus.search.app.util.GitUtils; -import io.quarkus.search.app.util.UrlInputProvider; import org.hibernate.search.util.common.impl.Closer; @@ -44,10 +44,9 @@ import org.fedorahosted.tennera.jgettext.PoParser; import org.yaml.snakeyaml.Yaml; -public class QuarkusIO implements Closeable { +public class QuarkusIO implements IndexableGuides, Closeable { public static final String QUARKUS_ORIGIN = "quarkus"; - private static final String QUARKIVERSE_ORIGIN = "quarkiverse"; public static final GitCloneDirectory.Branches MAIN_BRANCHES = new GitCloneDirectory.Branches( "develop", "master"); public static final GitCloneDirectory.Branches LOCALIZED_BRANCHES = new GitCloneDirectory.Branches( @@ -149,9 +148,7 @@ public void close() throws IOException { } public Stream guides() throws IOException { - return Stream.concat( - Stream.concat(versionedGuides(), legacyGuides()), - Stream.concat(quarkiverseGuides(), legacyQuarkiverseGuides())); + return Stream.concat(versionedGuides(), legacyGuides()); } // guides based on the info from the _data/versioned/[version]/index/ @@ -215,69 +212,10 @@ private Stream legacyGuides() { }); } - private Stream quarkiverseGuides() { - Language language = Language.ENGLISH; - GitCloneDirectory cloneDirectory = allSites.get(language); - - return cloneDirectory.sourcesFileStream("_data/versioned", path -> path.endsWith("quarkiverse.yaml")) - .map(QuarkusIO::extractQuarkiverseVersion) - .flatMap(quarkusVersion -> { - String quarkus = quarkusVersion.path(); - - Map translations = createTranslations( - lang -> resolveTranslationPath(quarkusVersion.versionDirectory(), "quarkiverse.yaml", lang)); - - try (InputStream file = cloneDirectory.sourcesFile(quarkus)) { - return parseYamlQuarkiverseMetadata(file, quarkusVersion.version(), translations); - } catch (IOException e) { - throw new IllegalStateException( - "Unable to load %s: %s".formatted(quarkusVersion.path(), e.getMessage()), - e); - } - }); - } - - private Stream legacyQuarkiverseGuides() { - Language language = Language.ENGLISH; - GitCloneDirectory cloneDirectory = allSites.get(language); - - return cloneDirectory.sourcesFileStream("_data", path -> path.matches("_data/guides-\\d+-\\d+\\.yaml")) - .map(QuarkusIO::extractLegacyVersion) - .flatMap(quarkusVersion -> { - String quarkus = quarkusVersion.path(); - - Map translations = createTranslations( - lang -> resolveLegacyTranslationPath(quarkusVersion.versionDirectory(), lang)); - - try (InputStream file = cloneDirectory.sourcesFile(quarkus)) { - return parseQuarkiverseYamlLegacyMetadata(cloneDirectory, file, quarkusVersion.version(), translations); - } catch (IOException e) { - throw new IllegalStateException( - "Unable to load %s: %s".formatted(quarkusVersion.path(), e.getMessage()), - e); - } - }); - } - - private Map createTranslations(Function pathCreator) { - Map translations = new HashMap<>(); - for (Map.Entry entry : allSites.entrySet()) { - Language lang = entry.getKey(); - translations.put(lang, translations( - entry.getValue().git().getRepository(), entry.getValue().sourcesTranslationTree(), - pathCreator.apply(lang))); - } - return translations; - } - private static VersionAndPaths extractVersion(String relativePath) { return extractVersion(relativePath, "quarkus.yaml"); } - private static VersionAndPaths extractQuarkiverseVersion(String relativePath) { - return extractVersion(relativePath, "quarkiverse.yaml"); - } - private static VersionAndPaths extractVersion(String relativePath, String filename) { String versionDirectory = relativePath.replace("_data/versioned/", "") .replace("/index/", "").replace(filename, ""); @@ -331,48 +269,6 @@ private Stream parseYamlMetadata(GitCloneDirectory cloneDirectory, InputS }); } - @SuppressWarnings("unchecked") - private Stream parseYamlQuarkiverseMetadata(InputStream quarkusYamlPath, String quarkusVersion, - Map translations) { - return parse(quarkusYamlPath, quarkusYaml -> { - Set parsed = new HashSet<>(); - for (Map.Entry>> type : ((Map>>) quarkusYaml - .get("types")).entrySet()) { - for (Map parsedGuide : type.getValue()) { - Guide guide = createQuarkiverseGuide(quarkusVersion, type.getKey(), parsedGuide, "summary"); - guide.categories = toSet(parsedGuide.get("categories")); - - // Quarkiverse guides have a single URL, because content is not translated, - // so there is a single Guide instance with translated maps for some of its metadata. - translateAllForSameGuide(guide.title, translations); - translateAllForSameGuide(guide.summary, translations); - translateAllForSameGuide(guide.keywords, translations); - - parsed.add(guide); - } - } - - return parsed.stream(); - }); - } - - private Stream parseQuarkiverseYamlLegacyMetadata(GitCloneDirectory cloneDirectory, InputStream quarkusYamlPath, - String version, - Map translations) { - return parseYamlLegacyMetadata( - cloneDirectory, quarkusYamlPath, version, Language.ENGLISH, translations.get(Language.ENGLISH), - (guide, guides) -> { - if (guide.language == null) { - translateAllForSameGuide(guide.title, translations); - translateAllForSameGuide(guide.summary, translations); - translateAllForSameGuide(guide.keywords, translations); - guide.topics.forEach(topics -> translateAllForSameGuide(topics, translations)); - return guides.put(guide.url, guide); - } - return guide; - }); - } - private Stream parseYamlLegacyMetadata(GitCloneDirectory cloneDirectory, InputStream quarkusYamlPath, String version, Language language, Catalog translations) { return parseYamlLegacyMetadata( @@ -433,17 +329,6 @@ private Catalog translations(Repository repository, RevTree sources, String path } } - private static void translateAllForSameGuide(I18nData data, Map translations) { - String key = data.get(Language.ENGLISH); - if (key == null) { - // No translation - return; - } - for (Map.Entry entry : translations.entrySet()) { - data.set(entry.getKey(), translate(entry.getValue(), key)); - } - } - private static String translate(Catalog messages, String key) { if (key == null || key.isBlank()) { return key; @@ -457,13 +342,6 @@ private static String translate(Catalog messages, String key) { : message.getMsgstr() == null || message.getMsgstr().isBlank() ? key : message.getMsgstr(); } - private static void putIfNotNull(Map map, Language language, String value) { - if (value == null) { - return; - } - map.put(language, value); - } - private static Set combine(Set a, Set b) { if (a == null) { return b; @@ -491,7 +369,7 @@ private Guide createGuide(GitCloneDirectory cloneDirectory, String quarkusVersio String parsedUrl = toString(parsedGuide.get("url")); if (parsedUrl.startsWith("http")) { // we are looking at a quarkiverse guide: - return createQuarkiverseGuide(quarkusVersion, type, parsedGuide, summaryKey); + return null; } else { return createCoreGuide(cloneDirectory, quarkusVersion, type, parsedGuide, summaryKey, language, messages); } @@ -531,26 +409,6 @@ private Guide createCoreGuide(GitCloneDirectory cloneDirectory, String quarkusVe return guide; } - private Guide createQuarkiverseGuide(String quarkusVersion, String type, Map parsedGuide, - String summaryKey) { - Guide guide = new Guide(); - guide.quarkusVersion = quarkusVersion; - // This is on purpose and will lead to the same guide instance being used for all languages - guide.language = null; - guide.origin = toString(parsedGuide.get("origin")); - if (guide.origin == null) { - guide.origin = QUARKIVERSE_ORIGIN; - } - guide.type = type; - guide.title.set(Language.ENGLISH, renderMarkdown(toString(parsedGuide.get("title")))); - guide.summary.set(Language.ENGLISH, renderMarkdown(toString(parsedGuide.get(summaryKey)))); - String parsedUrl = toString(parsedGuide.get("url")); - guide.url = httpUrl(quarkusVersion, parsedUrl); - guide.htmlFullContentProvider.set(Language.ENGLISH, - new UrlInputProvider(prefetchedGuides, guide.url, failureCollector)); - return guide; - } - private static String toString(Object value) { return value == null ? null : value.toString(); } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 9a7c2efb..969cf7e5 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -53,7 +53,7 @@ indexing.batch-size=10 # More secure HTTP defaults ######################## quarkus.http.cors=true -quarkus.http.cors.origins=https://quarkus.io,/https://.*\\\\.quarkus\\\\.io/,/https://quarkus-(.+-)?pr-.*-preview\\\\.surge\\\\.sh/ +quarkus.http.cors.origins=https://quarkus.io,/https://.*\\\\.quarkus\\\\.io/,/https://quarkus-(.+-)?pr-.*-preview\\\\.surge\\\\.sh/,https://docs.quarkiverse.io/ quarkus.http.cors.methods=GET quarkus.http.header."X-Content-Type-Options".value=nosniff quarkus.http.header."X-Frame-Options".value=deny @@ -125,6 +125,8 @@ quarkus.hibernate-search-orm.elasticsearch.max-connections=90 # Allow localhost in particular %dev,staging.quarkus.http.cors.origins=/.*/ %dev,staging.quarkus.http.header."Access-Control-Allow-Private-Network".value=true +# disable indexing and fetching of html quarkiverse guides in tests/dev +%dev,test.quarkiverseio.enabled=false ######################## diff --git a/src/main/resources/web/app/qs-form.ts b/src/main/resources/web/app/qs-form.ts index cf205fd9..b288ba23 100644 --- a/src/main/resources/web/app/qs-form.ts +++ b/src/main/resources/web/app/qs-form.ts @@ -49,6 +49,7 @@ export class QsForm extends LitElement { @property({type: String}) language: String = "en"; @property({type: String, attribute: 'quarkus-version'}) quarkusversion?: string; @property({type: String, attribute: 'local-search'}) localSearch: boolean = false; + @property({type: String, attribute: 'origin-filter'}) originFilter: string = ''; @state({ hasChanged(newVal: any, oldVal: any) { @@ -61,11 +62,13 @@ export class QsForm extends LitElement { private _page: number = 0; private _currentHitCount: number = 0; private _abortController?: AbortController = null; + private _initialQueryStringPresent: boolean; constructor() { super(); const searchParams = new URLSearchParams(window.location.hash.substring(1)); if (searchParams.size > 0) { + this._initialQueryStringPresent = true; const formElements = this._getFormElements(); for (const formElement of formElements) { const value = searchParams.get(formElement.name); @@ -73,7 +76,6 @@ export class QsForm extends LitElement { formElement.value = value; } } - this._handleInputChange(null); } } @@ -86,6 +88,10 @@ export class QsForm extends LitElement { } update(changedProperties: Map) { + if (this._initialQueryStringPresent) { + this._initialQueryStringPresent = false; + this._handleInputChange(null); + } window.location.hash = this._browserData ? (new URLSearchParams(this._browserData)).toString() : ''; if (!this._backendData) { this._clearSearch(); @@ -181,6 +187,9 @@ export class QsForm extends LitElement { if (this.quarkusversion) { formData['version'] = this.quarkusversion; } + if (this.originFilter) { + formData['origin'] = this.originFilter; + } var elements = 0; for (let el: HTMLFormElement of formElements) { if (this._isInput(el) && (el.value.length === 0 || el.value.length < this.minChars)) { diff --git a/src/main/resources/web/app/qs-target.ts b/src/main/resources/web/app/qs-target.ts index 362b96c4..b73f7668 100644 --- a/src/main/resources/web/app/qs-target.ts +++ b/src/main/resources/web/app/qs-target.ts @@ -43,6 +43,10 @@ export class QsTarget extends LitElement { background: var(--empty-background-color, #F0CA4D); } + .search-result-title { + margin-top: 2.5rem; + font-weight: var(--heading-font-weight); + } qs-guide { grid-column: span 4; @@ -64,6 +68,7 @@ export class QsTarget extends LitElement { `; + @property({type: String, attribute: 'search-results-title'}) searchResultsTitle: string = ''; @property({type: String}) private type: string = "guide"; @state() private _result: QsResult | undefined; @state() private _loading = true; @@ -97,6 +102,7 @@ export class QsTarget extends LitElement { } const result = this._result.hits.map(i => this._renderHit(i)); return html` + ${this.searchResultsTitle === '' ? '' : html`

${this.searchResultsTitle}

`}
${result}
@@ -187,4 +193,4 @@ export class QsTarget extends LitElement { private _loadingEnd = () => { this._loading = false; } -} \ No newline at end of file +} diff --git a/src/test/java/io/quarkus/search/app/SearchServiceQuarkiverseTest.java b/src/test/java/io/quarkus/search/app/SearchServiceQuarkiverseTest.java new file mode 100644 index 00000000..a472827b --- /dev/null +++ b/src/test/java/io/quarkus/search/app/SearchServiceQuarkiverseTest.java @@ -0,0 +1,126 @@ +package io.quarkus.search.app; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; + +import io.quarkus.search.app.dto.GuideSearchHit; +import io.quarkus.search.app.dto.SearchResult; +import io.quarkus.search.app.testsupport.GuideRef; +import io.quarkus.search.app.testsupport.QuarkusIOSample; +import io.quarkus.search.app.testsupport.SetupUtil; + +import io.quarkus.test.common.http.TestHTTPEndpoint; +import io.quarkus.test.junit.QuarkusTest; +import io.quarkus.test.junit.QuarkusTestProfile; +import io.quarkus.test.junit.TestProfile; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import org.assertj.core.api.InstanceOfAssertFactories; + +import io.restassured.RestAssured; +import io.restassured.common.mapper.TypeRef; +import io.restassured.filter.log.LogDetail; + +@QuarkusTest +@TestHTTPEndpoint(SearchService.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@TestProfile(SearchServiceQuarkiverseTest.Profile.class) +@QuarkusIOSample.Setup(filter = QuarkusIOSample.SearchServiceFilterDefinition.class) +class SearchServiceQuarkiverseTest { + public static class Profile implements QuarkusTestProfile { + @Override + public Map getConfigOverrides() { + return Map.of("quarkiverseio.enabled", "true"); + } + } + + private static final TypeRef> SEARCH_RESULT_SEARCH_HITS = new TypeRef<>() { + }; + private static final String GUIDES_SEARCH = "/guides/search"; + + private SearchResult search(String term) { + return given() + .queryParam("q", term) + .when().get(GUIDES_SEARCH) + .then() + .statusCode(200) + .extract().body().as(SEARCH_RESULT_SEARCH_HITS); + } + + @BeforeAll + void setup() { + SetupUtil.waitForIndexing(getClass()); + RestAssured.enableLoggingOfRequestAndResponseIfValidationFails(LogDetail.BODY); + } + + @Test + void version() { + var result = given() + .queryParam("q", "orm") + .queryParam("version", QuarkusVersions.MAIN) + .when().get(GUIDES_SEARCH) + .then() + .statusCode(200) + .extract().body().as(SEARCH_RESULT_SEARCH_HITS); + assertThat(result.hits()) + .isNotEmpty() + .allSatisfy(hit -> assertThat(hit).extracting(GuideSearchHit::url, InstanceOfAssertFactories.URI_TYPE) + .asString() + .satisfiesAnyOf( + uri -> assertThat(uri).startsWith("https://quarkus.io/version/" + + QuarkusVersions.MAIN + "/guides/"), + uri -> assertThat(uri).startsWith("https://docs.quarkiverse.io/"))); + result = given() + .queryParam("q", "orm") + .queryParam("version", "main") + .when().get(GUIDES_SEARCH) + .then() + .statusCode(200) + .extract().body().as(SEARCH_RESULT_SEARCH_HITS); + assertThat(result.hits()) + .isNotEmpty() + .allSatisfy(hit -> assertThat(hit).extracting(GuideSearchHit::url, InstanceOfAssertFactories.URI_TYPE) + .asString() + .satisfiesAnyOf( + uri -> assertThat(uri).startsWith("https://quarkus.io/version/main/guides/"), + uri -> assertThat(uri).startsWith("https://docs.quarkiverse.io/"))); + } + + @Test + void quarkiverse() { + var result = given() + .queryParam("q", "amazon") + .queryParam("version", QuarkusVersions.MAIN) + .when().get(GUIDES_SEARCH) + .then() + .statusCode(200) + .extract().body().as(SEARCH_RESULT_SEARCH_HITS); + assertThat(result.hits()).extracting(GuideSearchHit::url) + .satisfiesOnlyOnce( + uri -> assertThat(uri).asString().contains(GuideRef.QUARKIVERSE_AMAZON_S3.nameBeforeRestRenaming())) + .satisfiesOnlyOnce( + uri -> assertThat(uri).asString() + .contains(GuideRef.HIBERNATE_SEARCH_ORM_ELASTICSEARCH.nameBeforeRestRenaming())); + } + + @Test + @Disabled("Since quarkiverse guides are now fetched directly from their site there is no translation for them available anymore") + void language_quarkiverse() { + var result = given() + .queryParam("q", "クラウドストレージ") // means "Cloud storage" + .queryParam("language", "ja") + .queryParam("version", "main") + .when().get(GUIDES_SEARCH) + .then() + .statusCode(200) + .extract().body().as(SEARCH_RESULT_SEARCH_HITS); + assertThat(result.hits()).extracting(GuideSearchHit::url).satisfiesExactlyInAnyOrder( + uri -> assertThat(uri.toString()).startsWith(GuideRef.QUARKIVERSE_AMAZON_S3.name())); + } +} diff --git a/src/test/java/io/quarkus/search/app/SearchServiceTest.java b/src/test/java/io/quarkus/search/app/SearchServiceTest.java index c674a26b..790ee01a 100644 --- a/src/test/java/io/quarkus/search/app/SearchServiceTest.java +++ b/src/test/java/io/quarkus/search/app/SearchServiceTest.java @@ -268,10 +268,9 @@ void version() { .isNotEmpty() .allSatisfy(hit -> assertThat(hit).extracting(GuideSearchHit::url, InstanceOfAssertFactories.URI_TYPE) .asString() - .satisfiesAnyOf( + .satisfies( uri -> assertThat(uri).startsWith("https://quarkus.io/version/" - + QuarkusVersions.MAIN + "/guides/"), - uri -> assertThat(uri).startsWith("https://quarkiverse.github.io/quarkiverse-docs"))); + + QuarkusVersions.MAIN + "/guides/"))); result = given() .queryParam("q", "orm") .queryParam("version", "main") @@ -283,26 +282,7 @@ void version() { .isNotEmpty() .allSatisfy(hit -> assertThat(hit).extracting(GuideSearchHit::url, InstanceOfAssertFactories.URI_TYPE) .asString() - .satisfiesAnyOf( - uri -> assertThat(uri).startsWith("https://quarkus.io/version/main/guides/"), - uri -> assertThat(uri).startsWith("https://quarkiverse.github.io/quarkiverse-docs"))); - } - - @Test - void quarkiverse() { - var result = given() - .queryParam("q", "amazon") - .queryParam("version", QuarkusVersions.MAIN) - .when().get(GUIDES_SEARCH) - .then() - .statusCode(200) - .extract().body().as(SEARCH_RESULT_SEARCH_HITS); - assertThat(result.hits()).extracting(GuideSearchHit::url) - .satisfiesOnlyOnce( - uri -> assertThat(uri).asString().contains(GuideRef.QUARKIVERSE_AMAZON_S3.nameBeforeRestRenaming())) - .satisfiesOnlyOnce( - uri -> assertThat(uri).asString() - .contains(GuideRef.HIBERNATE_SEARCH_ORM_ELASTICSEARCH.nameBeforeRestRenaming())); + .satisfies(uri -> assertThat(uri).startsWith("https://quarkus.io/version/main/guides/"))); } @Test @@ -405,20 +385,6 @@ void language() { "Hibernate Search ガイド"); } - @Test - void language_quarkiverse() { - var result = given() - .queryParam("q", "クラウドストレージ") // means "Cloud storage" - .queryParam("language", "ja") - .queryParam("version", "main") - .when().get(GUIDES_SEARCH) - .then() - .statusCode(200) - .extract().body().as(SEARCH_RESULT_SEARCH_HITS); - assertThat(result.hits()).extracting(GuideSearchHit::url).satisfiesExactlyInAnyOrder( - uri -> assertThat(uri.toString()).startsWith(GuideRef.QUARKIVERSE_AMAZON_S3.name())); - } - @Test void quoteEmptyQuoteTitleTranslation() { var result = given() diff --git a/src/test/java/io/quarkus/search/app/testsupport/GuideRef.java b/src/test/java/io/quarkus/search/app/testsupport/GuideRef.java index ff3504fb..80040f68 100644 --- a/src/test/java/io/quarkus/search/app/testsupport/GuideRef.java +++ b/src/test/java/io/quarkus/search/app/testsupport/GuideRef.java @@ -31,7 +31,7 @@ public record GuideRef(String nameAfterRestRenaming, String nameBeforeRestRenami public static final GuideRef ALL_CONFIG = create("all-config"); public static final GuideRef ALL_BUILDITEMS = create("all-builditems"); public static final GuideRef QUARKIVERSE_AMAZON_S3 = createQuarkiverse( - "https://quarkiverse.github.io/quarkiverse-docs/quarkus-amazon-services/dev/amazon-s3.html"); + "https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-s3.html"); // NOTE: when adding new constants here, don't forget to run the main() method in QuarkusIOSample // to update the QuarkusIO sample in src/test/resources.