Add feature tests for checking plugin with addon enabled

cli.php

@@ -34,16 +34,29 @@
 
 // Create the CLI command instance and add to WP CLI.
 $plugin_command = new Plugin_Check_Command( $context );
-WP_CLI::add_command( 'plugin', $plugin_command );
-
+WP_CLI::add_command(
+	'plugin',
+	$plugin_command,
+	array(
+		'after_invoke' => function () {
+			if (
+				file_exists( ABSPATH . 'wp-content/object-cache.php' ) &&
+				false !== strpos( file_get_contents( ABSPATH . 'wp-content/object-cache.php' ), 'WP_PLUGIN_CHECK_OBJECT_CACHE_DROPIN_VERSION' )
+			) {
+				unlink( ABSPATH . 'wp-content/object-cache.php' );
+			}
+		},
+	)
+);
 
-/**
- * Adds hook to set up the object-cache.php drop-in file.
+/*
+ * Add hook to set up the object-cache.php drop-in file.
  *
- * @since 1.0.0
+ * Runs after wp-config.php is loaded and thus ABSPATH is defined,
+ * but before any plugins are actually loaded.
  */
 WP_CLI::add_hook(
-	'before_wp_load',
+	'after_wp_config_load',
 	function () {
 		if ( CLI_Runner::is_plugin_check() ) {
 			if ( ! file_exists( ABSPATH . 'wp-content/object-cache.php' ) ) {

includes/Checker/CLI_Runner.php

@@ -32,12 +32,15 @@
 	 * @return bool Returns true if is an CLI request for the plugin check else false.
 	 */
 	public static function is_plugin_check() {
+		if ( ! defined( 'WP_CLI' ) || ! WP_CLI ) {
+			return false;
+		}
+
 		if ( empty( $_SERVER['argv'] ) || 3 > count( $_SERVER['argv'] ) ) {
 			return false;
 		}
 
 		if (
-			'wp' === substr( $_SERVER['argv'][0], -2 ) &&
 			'plugin' === $_SERVER['argv'][1] &&
 			'check' === $_SERVER['argv'][2]
 		) {

tests/behat/features/plugin-check.feature

@@ -374,3 +374,141 @@ Feature: Test that the WP-CLI command works.
       """
       Invalid plugin slug
       """
+
+  Scenario: Check a plugin with static checks from an add-on
+    Given a WP install with the Plugin Check plugin
+    And a Plugin Check add-on being installed
+
+    And a wp-content/plugins/foo-sample/foo-sample.php file:
+      """
+      <?php
+      /**
+       * Plugin Name: Foo Sample
+       * Plugin URI: https://example.com
+       * Description: Sample plugin.
+       * Version: 0.1.0
+       * Author: WordPress Performance Team
+       * Author URI: https://make.wordpress.org/performance/
+       * License: GPL-2.0+
+       * License URI: http://www.gnu.org/licenses/gpl-2.0.txt
+       */
+
+      $text = 'I am bad'; // This should trigger the error.
+      """
+    And I run the WP-CLI command `plugin activate foo-sample`
+
+    # The two checks from pcp-addon should be available.
+    When I run the WP-CLI command `plugin list-checks --fields=slug,category,stability --format=csv`
+    Then STDOUT should contain:
+      """
+      example_static,new_category,stable
+      """
+    And STDOUT should contain:
+      """
+      example_runtime,new_category,stable
+      """
+
+    # The new check category should therefore also be available.
+    When I run the WP-CLI command `plugin list-check-categories --fields=slug,name --format=csv`
+    Then STDOUT should contain:
+      """
+      new_category,"New Category"
+      """
+
+    # Running static checks, including the one from pcp-addon
+    When I run the WP-CLI command `plugin check foo-sample --fields=code,type --format=csv`
+    Then STDOUT should contain:
+      """
+      prohibited_text_detected,ERROR
+      """
+
+    # Same again, but after filtering only to the new categories from pcp-addon
+    When I run the WP-CLI command `plugin check foo-sample --fields=code,type --format=csv --categories=new_category`
+    Then STDOUT should contain:
+      """
+      prohibited_text_detected,ERROR
+      """
+
+    # Running only the check from pcp-addon
+    When I run the WP-CLI command `plugin check foo-sample --checks=example_static --fields=code,type --format=csv`
+    Then STDOUT should contain:
+      """
+      prohibited_text_detected,ERROR
+      """
+
+  Scenario: Check a plugin with runtime checks from an add-on
+    Given a WP install with the Plugin Check plugin
+    And a Plugin Check add-on being installed
+
+    And a wp-content/plugins/foo-sample/foo-sample.php file:
+      """
+      <?php
+      /**
+       * Plugin Name: Foo Sample
+       * Plugin URI: https://example.com
+       * Description: Sample plugin.
+       * Version: 0.1.0
+       * Author: WordPress Performance Team
+       * Author URI: https://make.wordpress.org/performance/
+       * License: GPL-2.0+
+       * License URI: http://www.gnu.org/licenses/gpl-2.0.txt
+       */
+
+      // This should trigger the error.
+      add_action(
+        'wp_enqueue_scripts',
+        function() {
+          wp_enqueue_script( 'test', plugin_dir_url( __FILE__ ) . 'test.js', array(), '1.0' );
+        }
+      );
+      """
+    And I run the WP-CLI command `plugin activate foo-sample`
+
+    # Running runtime checks, including the one from pcp-addon
+    When I run the WP-CLI command `plugin check foo-sample --fields=code,type --format=csv --require=./wp-content/plugins/plugin-check/cli.php`
+    Then STDOUT should contain:
+      """
+      Setting up runtime environment.
+      """
+    And STDOUT should contain:
+      """
+      Cleaning up runtime environment.
+      """
+    And STDOUT should contain:
+      """
+      WordPress.WP.EnqueuedResourceParameters.NotInFooter,WARNING
+      """
+# This doesn't currently work, because we are not actually loading any other plugins, including pcp-addon.
+#    And STDOUT should contain:
+#      """
+#      ExampleRuntimeCheck.ForbiddenScript,WARNING
+#      """
+
+    # Same again, to verify object-cache.php was properly cleared again
+    When I run the WP-CLI command `plugin check foo-sample --fields=code,type --format=csv --require=./wp-content/plugins/plugin-check/cli.php`
+    Then STDOUT should contain:
+      """
+      Setting up runtime environment.
+      """
+    And STDOUT should contain:
+      """
+      Cleaning up runtime environment.
+      """
+    And STDOUT should contain:
+      """
+      WordPress.WP.EnqueuedResourceParameters.NotInFooter,WARNING
+      """
+
+# This doesn't currently work, because we are not actually loading any other plugins, including pcp-addon.
+#    And STDOUT should contain:
+#      """
+#      ExampleRuntimeCheck.ForbiddenScript,WARNING
+#      """
+
+    # This doesn't currently work, because we are not actually loading any other plugins, including pcp-addon.
+    # Run only the runtime check from pcp-addon, no others
+#    When I run the WP-CLI command `plugin check foo-sample --checks=example_runtime --fields=code,type --format=csv --require=./wp-content/plugins/plugin-check/cli.php`
+#    Then STDOUT should contain:
+#      """
+#      ExampleRuntimeCheck.ForbiddenScript,WARNING
+#      """

tests/behat/includes/FeatureContext.php

@@ -81,6 +81,19 @@ public function given_a_wp_installation_with_plugin_check() {
 		// Activate the plugin.
 		$this->proc( 'wp plugin activate plugin-check' )->run_check();
 	}
+	/**
+	 * @Given a Plugin Check add-on being installed
+	 */
+	public function given_a_plugin_check_addon_being_installed() {
+		// Symlink the pcp-addon folder into the WP folder as a plugin.
+		$src_dir = realpath( self::get_vendor_dir() . '/../tests/behat/testdata/pcp-addon' );
+		$plugin_dir  = $this->variables['RUN_DIR'] . '/wp-content/plugins';
+		$this->ensure_dir_exists( $plugin_dir );
+		$this->proc( "ln -s {$src_dir} {$plugin_dir}/pcp-addon" )->run_check();
+
+		// Activate the plugin.
+		$this->proc( 'wp plugin activate pcp-addon' )->run_check();
+	}
 
 	/**
 	 * @When /^I (run|try) the WP-CLI command `([^`]+)`$/

tests/behat/testdata/pcp-addon/Example_Runtime_Check.php

@@ -0,0 +1,85 @@
+<?php
+
+use WordPress\Plugin_Check\Checker\Check_Result;
+use WordPress\Plugin_Check\Checker\Checks\Abstract_Runtime_Check;
+use WordPress\Plugin_Check\Traits\Amend_Check_Result;
+use WordPress\Plugin_Check\Traits\Stable_Check;
+use WordPress\Plugin_Check\Traits\URL_Aware;
+
+class Example_Runtime_Check extends Abstract_Runtime_Check {
+
+	use Amend_Check_Result;
+	use Stable_Check;
+	use URL_Aware;
+
+	public function get_categories() {
+		return array( 'new_category' );
+	}
+
+	public function prepare() {
+		$orig_scripts = isset( $GLOBALS['wp_scripts'] ) ? $GLOBALS['wp_scripts'] : null;
+
+		// Backup the original values for the global state.
+		$this->backup_globals();
+
+		return function () use ( $orig_scripts ) {
+			if ( is_null( $orig_scripts ) ) {
+				unset( $GLOBALS['wp_scripts'] );
+			} else {
+				$GLOBALS['wp_scripts'] = $orig_scripts;
+			}
+
+			$this->restore_globals();
+		};
+	}
+
+	public function run( Check_Result $result ) {
+		$this->run_for_urls(
+			array( home_url() ),
+			function ( $url ) use ( $result ) {
+				$this->check_url( $result, $url );
+			}
+		);
+	}
+
+	protected function check_url( Check_Result $result, $url ) {
+		// Reset the WP_Scripts instance.
+		unset( $GLOBALS['wp_scripts'] );
+
+		// Run the 'wp_enqueue_script' action, wrapped in an output buffer in case of any callbacks printing scripts
+		// directly. This is discouraged, but some plugins or themes are still doing it.
+		ob_start();
+		wp_enqueue_scripts();
+		wp_scripts()->do_head_items();
+		wp_scripts()->do_footer_items();
+		ob_end_clean();
+
+		foreach ( wp_scripts()->done as $handle ) {
+			$script = wp_scripts()->registered[ $handle ];
+
+			if ( strpos( $script->src, $result->plugin()->url() ) !== 0 ) {
+				continue;
+			}
+
+			$script_path = str_replace( $result->plugin()->url(), $result->plugin()->path(), $script->src );
+
+			$this->add_result_warning_for_file(
+				$result,
+				sprintf(
+					'Not allowed to enqueue scripts. Found script handle "%s"',
+					$handle
+				),
+				'ExampleRuntimeCheck.ForbiddenScript',
+				$script_path
+			);
+		}
+	}
+
+	public function get_description(): string {
+		return '';
+	}
+
+	public function get_documentation_url(): string {
+		return '';
+	}
+}

tests/behat/testdata/pcp-addon/Example_Static_Check.php

@@ -0,0 +1,41 @@
+<?php
+
+use WordPress\Plugin_Check\Checker\Check_Result;
+use WordPress\Plugin_Check\Checker\Checks\Abstract_File_Check;
+use WordPress\Plugin_Check\Traits\Amend_Check_Result;
+use WordPress\Plugin_Check\Traits\Stable_Check;
+
+class Example_Static_Check extends Abstract_File_Check {
+
+	use Amend_Check_Result;
+	use Stable_Check;
+
+	public function get_categories() {
+		return array( 'new_category' );
+	}
+
+	protected function check_files( Check_Result $result, array $files ) {
+		$php_files = self::filter_files_by_extension( $files, 'php' );
+		$file      = self::file_preg_match( '#I\sam\sbad#', $php_files );
+		if ( $file ) {
+			$this->add_result_error_for_file(
+				$result,
+				__( 'Prohibited text found.', 'pcp-addon' ),
+				'prohibited_text_detected',
+				$file,
+				0,
+				0,
+				'',
+				8
+			);
+		}
+	}
+
+	public function get_description(): string {
+		return '';
+	}
+
+	public function get_documentation_url(): string {
+		return '';
+	}
+}

tests/behat/testdata/pcp-addon/pcp-addon.php

@@ -0,0 +1,35 @@
+<?php
+/**
+ * Plugin Name: PCP Addon
+ * Plugin URI: https://example.com
+ * Description: Plugin Check addon.
+ * Version: 0.1.0
+ * Author: WordPress Performance Team
+ * Author URI: https://make.wordpress.org/performance/
+ * License: GPL-2.0+
+ * License URI: http://www.gnu.org/licenses/gpl-2.0.txt
+ * Requires Plugins: plugin-check
+ */
+
+add_filter(
+	'wp_plugin_check_categories',
+	function ( array $categories ) {
+		return array_merge( $categories, array( 'new_category' => esc_html__( 'New Category', 'pcp-addon' ) ) );
+	}
+);
+
+add_filter(
+	'wp_plugin_check_checks',
+	function ( array $checks ) {
+		require_once plugin_dir_path( __FILE__ ) . 'Example_Static_Check.php';
+		require_once plugin_dir_path( __FILE__ ) . 'Example_Runtime_Check.php';
+
+		return array_merge(
+			$checks,
+			array(
+				'example_static'  => new Example_Static_Check(),
+				'example_runtime' => new Example_Runtime_Check(),
+			)
+		);
+	}
+);

tests/phpunit/tests/Checker/CLI_Runner_Tests.php

@@ -28,7 +28,13 @@ public function tear_down() {
 		parent::tear_down();
 	}
 
+	/**
+	 * @runInSeparateProcess
+	 * @preserveGlobalState disabled
+	 */
 	public function test_is_plugin_check_returns_true() {
+		define( 'WP_CLI', true );
+
 		$_SERVER['argv'] = array(
 			'wp',
 			'plugin',