Transitive dependency tree resolution for plugins and modules (#1185)

* Clarify parsing of v1 dependencies in <depends>
* Clarify parsing of v2 dependencies in <dependencies>
* Clarify dependency parsing
* Introduce type-specific classes for <module> and <plugin> dependency
* Use type-safe plugin dependency instances in v2 model
* Use 'isModule' in hashCode and equals
* Introduce an interface to provide plugins by ID and module ID
* Introduce transitive dependency tree calculator
* When resolving core plugin, search in plugin.xml first
* Implement STaX-based XML filtering
* Introduce IDE dumper
* Add dump of IU-243.12818.47 for tests

intellij-plugin-structure/structure-ide/build.gradle.kts

@@ -8,4 +8,6 @@ dependencies {
   implementation(libs.platform.jps.model.core)
   implementation(libs.platform.jps.model.impl)
   implementation(libs.platform.jps.model.serialization)
+
+  testImplementation(sharedLibs.junit)
 }

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/ide/Ide.java

@@ -5,6 +5,7 @@
 package com.jetbrains.plugin.structure.ide;
 
 import com.jetbrains.plugin.structure.intellij.plugin.IdePlugin;
+import com.jetbrains.plugin.structure.intellij.plugin.PluginProvider;
 import com.jetbrains.plugin.structure.intellij.version.IdeVersion;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
@@ -17,7 +18,7 @@
  * An IDE instance consisting of the class-files and plugins.
  * IDE can be created via {@link IdeManager#createIde(java.nio.file.Path)}.
  */
-public abstract class Ide {
+public abstract class Ide implements PluginProvider {
   /**
    * Returns the IDE version either from 'build.txt' or specified with {@link IdeManager#createIde(java.nio.file.Path, IdeVersion)}
    *
@@ -41,7 +42,8 @@ public abstract class Ide {
    * @return bundled plugin with the specified id, or null if such plugin is not found
    */
   @Nullable
-  final public IdePlugin getPluginById(@NotNull String pluginId) {
+  @Override
+  final public IdePlugin findPluginById(@NotNull String pluginId) {
     for (IdePlugin plugin : getBundledPlugins()) {
       String id = plugin.getPluginId() != null ? plugin.getPluginId() : plugin.getPluginName();
       if (Objects.equals(id, pluginId))
@@ -57,7 +59,8 @@ final public IdePlugin getPluginById(@NotNull String pluginId) {
    * @return bundled plugin with definition of the module, or null if such plugin is not found
    */
   @Nullable
-  final public IdePlugin getPluginByModule(@NotNull String moduleId) {
+  @Override
+  final public IdePlugin findPluginByModule(@NotNull String moduleId) {
     for (IdePlugin plugin : getBundledPlugins()) {
       if (plugin.getDefinedModules().contains(moduleId)) {
         return plugin;

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/ide/IdeManagerImpl.kt

@@ -201,7 +201,7 @@ class IdeManagerImpl : IdeManager() {
     ideVersion: IdeVersion
   ): List<IdePlugin> {
     val platformPlugins = arrayListOf<IdePlugin>()
-    val descriptorPaths = listOf(product.platformPrefix + "Plugin.xml", IdePluginManager.PLUGIN_XML, PLATFORM_PLUGIN_XML)
+    val descriptorPaths = listOf(IdePluginManager.PLUGIN_XML, product.platformPrefix + "Plugin.xml", PLATFORM_PLUGIN_XML)
 
     for (jarFile in jarFiles) {
       val descriptorPath = FileSystems.newFileSystem(jarFile, IdeManagerImpl::class.java.classLoader).use { jarFs ->

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/ide/ProductInfoBasedIdeManager.kt

@@ -41,11 +41,9 @@ private val VERSION_FROM_PRODUCT_INFO: IdeVersion? = null
 
 private val LOG: Logger = LoggerFactory.getLogger(ProductInfoBasedIdeManager::class.java)
 
-class ProductInfoBasedIdeManager : IdeManager() {
+class ProductInfoBasedIdeManager(private val excludeMissingProductInfoLayoutComponents: Boolean = true) : IdeManager() {
   private val productInfoParser = ProductInfoParser()
 
-  private val excludeMissingProductInfoLayoutComponents = true
-
   /**
    * Problem level remapping used for bundled plugins.
    */

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/ide/dependencies/PluginXmlDependencyFilter.kt

@@ -0,0 +1,96 @@
+/*
+ * Copyright 2000-2024 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+ */
+
+package com.jetbrains.plugin.structure.ide.dependencies
+
+import com.jetbrains.plugin.structure.base.utils.closeAll
+import com.jetbrains.plugin.structure.xml.CloseableXmlEventReader
+import com.jetbrains.plugin.structure.xml.CountingXmlEventWriter
+import com.jetbrains.plugin.structure.xml.ElementNamesFilter
+import com.jetbrains.plugin.structure.xml.EventTypeExcludingEventFilter
+import com.jetbrains.plugin.structure.xml.LogicalAndXmlEventFilter
+import com.jetbrains.plugin.structure.xml.newEventWriter
+import com.jetbrains.plugin.structure.xml.newFilteredEventReader
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.io.ByteArrayOutputStream
+import java.io.Closeable
+import java.io.IOException
+import java.io.InputStream
+import java.io.OutputStream
+import javax.xml.stream.EventFilter
+import javax.xml.stream.XMLInputFactory
+import javax.xml.stream.XMLOutputFactory
+import javax.xml.stream.XMLStreamException
+import javax.xml.stream.events.XMLEvent
+import javax.xml.stream.events.XMLEvent.COMMENT
+import javax.xml.stream.events.XMLEvent.START_DOCUMENT
+
+private val LOG: Logger = LoggerFactory.getLogger(PluginXmlDependencyFilter::class.java)
+
+class PluginXmlDependencyFilter(private val ignoreComments: Boolean = true, private val ignoreXmlDeclaration: Boolean = true) {
+  private val allowedElements = listOf("idea-plugin", "id", "depends", "dependencies", "plugin",
+    "module", "content", "/idea-plugin/dependencies/module", "/idea-plugin/content/module")
+
+  @Throws(IOException::class)
+  fun filter(pluginXmlInputStream: InputStream, pluginXmlOutputStream: OutputStream) {
+    val closeables = mutableListOf<Closeable>()
+    try {
+      val inputFactory: XMLInputFactory = newXmlInputFactory()
+      val outputFactory: XMLOutputFactory = XMLOutputFactory.newInstance()
+
+      val elementNameFilter = ElementNamesFilter(allowedElements)
+      val eventFilter = mutableListOf<EventFilter>().apply {
+        add(elementNameFilter)
+        if (ignoreXmlDeclaration) add(EventTypeExcludingEventFilter(START_DOCUMENT))
+        if (ignoreComments) add(EventTypeExcludingEventFilter(COMMENT))
+      }.let { LogicalAndXmlEventFilter(it) }
+
+      val eventReader = inputFactory
+        .newFilteredEventReader(pluginXmlInputStream, eventFilter)
+        .also { closeables += it }
+      val eventWriter = newEventWriter(outputFactory, pluginXmlOutputStream).also { closeables += it }
+
+      while (eventReader.hasNextEvent()) {
+        val event: XMLEvent = eventReader.nextEvent()
+        eventWriter.add(event)
+      }
+
+    } catch (e: Exception) {
+      throw IOException("Cannot filter plugin descriptor input stream", e)
+    } finally {
+      closeables.closeAll()
+    }
+  }
+
+  private fun newEventWriter(outputFactory: XMLOutputFactory, outputStream: OutputStream): CountingXmlEventWriter {
+    return CountingXmlEventWriter(outputFactory.newEventWriter(outputStream))
+  }
+
+  private fun CloseableXmlEventReader.hasNextEvent(): Boolean {
+    return try {
+      hasNext()
+    } catch (e: XMLStreamException) {
+      LOG.atError().log("Cannot retrieve next event", e)
+      false
+    } catch (e: RuntimeException) {
+      LOG.atError().log("Cannot retrieve next event", e)
+      false
+    }
+  }
+
+  private fun newXmlInputFactory() = XMLInputFactory.newInstance().apply {
+    setProperty(XMLInputFactory.IS_REPLACING_ENTITY_REFERENCES, false)
+    setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false)
+  }
+
+  companion object {
+    fun PluginXmlDependencyFilter.toByteArray(pluginXmlInputStream: InputStream): ByteArray {
+      return ByteArrayOutputStream().use {
+        filter(pluginXmlInputStream, it)
+        it.toByteArray()
+      }
+    }
+  }
+}

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/xml/CountingXmlEventWriter.kt

@@ -0,0 +1,65 @@
+/*
+ * Copyright 2000-2024 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+ */
+
+package com.jetbrains.plugin.structure.xml
+
+import com.jetbrains.plugin.structure.ide.dependencies.PluginXmlDependencyFilter
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.io.Closeable
+import javax.xml.stream.XMLEventWriter
+import javax.xml.stream.XMLStreamException
+import javax.xml.stream.events.XMLEvent
+
+private val LOG: Logger = LoggerFactory.getLogger(PluginXmlDependencyFilter::class.java)
+
+/**
+ * STaX Event Writer that counts occurrences of STaX events.
+ * It handles various peculiarities of underlying STaX implementations that prevent correct filtering
+ * of semi-well-formed documents in the Platform.
+ */
+class CountingXmlEventWriter(private val delegate: XMLEventWriter) : XMLEventWriter by delegate, Closeable {
+  private val eventCounter = hashMapOf<XmlEventType, Int>()
+
+  override fun add(event: XMLEvent) {
+    delegate.add(event)
+    val type = event.eventType
+    eventCounter[type] = (eventCounter[type] ?: 0) + 1
+  }
+
+  private fun count(type: XmlEventType) = eventCounter[type] ?: 0
+
+  private fun processingInstructions(): Int {
+    return count(XMLEvent.PROCESSING_INSTRUCTION)
+  }
+
+  private fun startDocuments(): Int {
+    return count(XMLEvent.START_DOCUMENT)
+  }
+
+  @Throws(XMLStreamException::class)
+  override fun close() {
+    if ((eventCounter.size == 1 && startDocuments() == 1)
+      || (eventCounter.size == 1 && processingInstructions() > 0)
+      || eventCounter.isEmpty()) {
+      // closing without an actual document being written
+      try {
+        delegate.close()
+      } catch (e: Exception) {
+        when (e) {
+          is RuntimeException, is XMLStreamException -> {
+            LOG.atError().log("Failed to close delegate XML event writer: {}", e.message)
+            return
+          }
+        }
+      }
+    } else {
+      try {
+        delegate.close()
+      } catch (e: Exception) {
+        throw e
+      }
+    }
+  }
+}

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/xml/ElementNamesFilter.kt

@@ -0,0 +1,101 @@
+/*
+ * Copyright 2000-2024 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+ */
+
+package com.jetbrains.plugin.structure.xml
+
+import com.jetbrains.plugin.structure.xml.ElementNamesFilter.EventProcessing.Seen
+import com.jetbrains.plugin.structure.xml.ElementNamesFilter.EventProcessing.Unseen
+import javax.xml.namespace.QName
+import javax.xml.stream.EventFilter
+import javax.xml.stream.events.EndElement
+import javax.xml.stream.events.StartElement
+import javax.xml.stream.events.XMLEvent
+
+class ElementNamesFilter(private val elementLocalNames: List<String>) : EventFilter {
+
+  private var isAccepting = true
+
+  private val eventStack = ElementStack()
+
+  private var lastEvent: EventProcessing = Unseen
+
+  override fun accept(event: XMLEvent): Boolean {
+    event.onAlreadySeen { return it }
+
+    return doAccept(event).also {
+      lastEvent = Seen(event, it)
+    }
+  }
+
+  private fun doAccept(event: XMLEvent): Boolean = when (event) {
+      is StartElement -> {
+        eventStack.push(event)
+        isAccepting = if (isRoot) true else supports(event.name)
+        isAccepting
+      }
+
+      is EndElement -> {
+        isAccepting = supports(event.name)
+        eventStack.popIf(currentEvent = event)
+        isAccepting
+      }
+
+      else -> {
+        isAccepting
+      }
+    }
+
+  private fun supports(elementName: QName): Boolean {
+    return elementLocalNames.any { elementPath ->
+      if (elementPath.contains("/")) {
+        // stack contains the elementName on top
+        elementPath == eventStack.toPath()
+      } else {
+        elementName.localPart == elementPath
+      }
+    }
+  }
+
+  private sealed class EventProcessing {
+    object Unseen : EventProcessing()
+    data class Seen(val event: XMLEvent, val resolution: Boolean) : EventProcessing()
+  }
+
+  private inline fun XMLEvent.onAlreadySeen(seenHandler: (Boolean) -> Boolean): Boolean {
+    return when(val lastEvent = this@ElementNamesFilter.lastEvent) {
+      is Seen -> if (lastEvent.event === this) seenHandler(lastEvent.resolution) else false
+      is Unseen -> false
+    }
+  }
+
+  private val isRoot: Boolean get() = eventStack.size == 1
+
+  private class ElementStack {
+    private val stack = ArrayDeque<StartElement>()
+
+    val size: Int get() = stack.size
+
+    fun push(event: StartElement) {
+      if (isEmpty() || peek() !== event) {
+        // no need to push the same event twice
+        stack.addLast(event)
+      }
+    }
+
+    fun popIf(currentEvent: EndElement) {
+      if (isEmpty()) return
+      val peek = stack.last()
+      if (peek.name == currentEvent.name) {
+        stack.removeLast()
+      }
+    }
+
+    fun isEmpty() = stack.isEmpty()
+
+    @Throws(NoSuchElementException::class)
+    fun peek(): StartElement = stack.last()
+
+    fun toPath() = stack.joinToString(prefix = "/", separator = "/") { it.name.localPart }
+  }
+}

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/xml/EventTypeExcludingEventFilter.kt

@@ -0,0 +1,16 @@
+/*
+ * Copyright 2000-2024 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file.
+ */
+
+package com.jetbrains.plugin.structure.xml
+
+import javax.xml.stream.EventFilter
+import javax.xml.stream.events.XMLEvent
+
+typealias XmlEventType = Int
+
+class EventTypeExcludingEventFilter(private val excludedElementTypes: Set<XmlEventType>) : EventFilter {
+  constructor(vararg elementTypes: XmlEventType) : this(elementTypes.toSet())
+
+  override fun accept(event: XMLEvent) = event.eventType !in excludedElementTypes
+}

intellij-plugin-structure/structure-ide/src/main/java/com/jetbrains/plugin/structure/xml/LogicalAndXmlEventFilter.kt

@@ -0,0 +1,17 @@
+package com.jetbrains.plugin.structure.xml
+
+import javax.xml.stream.EventFilter
+import javax.xml.stream.events.XMLEvent
+
+class LogicalAndXmlEventFilter(private val filters: List<EventFilter>) : EventFilter {
+  constructor(vararg filters: EventFilter) : this(filters.toList())
+
+  override fun accept(event: XMLEvent): Boolean {
+    for (filter in filters) {
+      if (!filter.accept(event)) {
+        return false
+      }
+    }
+    return true
+  }
+}