Merge pull request #11216 from vimeo/scan_threads

Add support for --scan-threads CLI and config flag

Author: danog

config.xsd

@@ -81,6 +81,7 @@
         <xs:attribute name="disableSuppressAll" type="xs:boolean" default="false" />
         <xs:attribute name="triggerErrorExits" type="TriggerErrorExitsType" default="default" />
         <xs:attribute name="threads" type="xs:integer" />
+        <xs:attribute name="scanThreads" type="xs:integer" />
         <xs:anyAttribute processContents="skip" />
     </xs:complexType>
 

docs/running_psalm/configuration.md

@@ -438,6 +438,14 @@ Allows you to hard-code a compressor for Psalm's cache. By default, Psalm uses `
 ```
 Allows you to hard-code the number of threads Psalm will use (similar to `--threads` on the command line). This value will be used in place of detecting threads from the host machine, but will be overridden by using `--threads` or `--debug` (which sets threads to 1) on the command line
 
+#### scanThreads
+```xml
+<psalm
+        scanThreads="[int]"
+>
+```
+Allows you to hard-code the number of threads Psalm will use during the scan phase (as opposed to the analysis phase, controlled by the `threads` field) (similar to `--scan-threads` on the command line). This value will be used in place of detecting threads from the host machine, but will be overridden by using `--scan-threads` or `--debug` (which sets threads to 1) on the command line.
+
 #### maxStringLength
 ```xml
 <psalm

psalm-baseline.xml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<files psalm-version="dev-master@e9e270f363bd05ea3790890de3f04f6726ea5643">
+<files psalm-version="6.x-dev@d3c7ca5430b3a75d8fab6439af1eefe119d12512">
   <file src="examples/TemplateChecker.php">
     <PossiblyUndefinedIntArrayOffset>
       <code><![CDATA[$comment_block->tags['variablesfrom'][0]]]></code>
@@ -1106,7 +1106,6 @@
       <code><![CDATA[!$paths_to_check]]></code>
       <code><![CDATA[$baseline_file_path]]></code>
       <code><![CDATA[$cache_directory]]></code>
-      <code><![CDATA[$config->threads]]></code>
       <code><![CDATA[$find_references_to]]></code>
       <code><![CDATA[empty($baselineFile)]]></code>
       <code><![CDATA[getenv('PSALM_SHEPHERD')]]></code>
@@ -2006,6 +2005,9 @@
     </RiskyTruthyFalsyComparison>
   </file>
   <file src="src/Psalm/IssueBuffer.php">
+    <PossiblyUnusedMethod>
+      <code><![CDATA[getServer]]></code>
+    </PossiblyUnusedMethod>
     <RiskyTruthyFalsyComparison>
       <code><![CDATA[!$report_options->output_path]]></code>
       <code><![CDATA[$parent_issue_type]]></code>

src/Psalm/Config.php

@@ -86,6 +86,7 @@
 use function libxml_clear_errors;
 use function libxml_get_errors;
 use function libxml_use_internal_errors;
+use function max;
 use function mkdir;
 use function phpversion;
 use function preg_match;
@@ -463,7 +464,10 @@ final class Config
      */
     public array $internal_stubs = [];
 
+    /** @var ?int<1, max> */
     public ?int $threads = null;
+    /** @var ?int<1, max> */
+    public ?int $scan_threads = null;
 
     /**
      * A list of php extensions supported by Psalm.
@@ -1379,12 +1383,25 @@ private static function fromXmlAndPaths(
         }
 
         if (isset($config_xml['threads'])) {
-            $config->threads = (int)$config_xml['threads'];
+            $config->threads = max(1, (int)$config_xml['threads']);
+            $config->scan_threads = $config->threads;
+        }
+
+        if (isset($config_xml['scanThreads'])) {
+            $config->scan_threads = max(1, (int)$config_xml['scanThreads']);
         }
 
         return $config;
     }
 
+    /**
+     * @psalm-suppress PossiblyUnusedMethod, PropertyTypeCoercion
+     * @internal */
+    public static function setInstance(self $config): void
+    {
+        self::$instance = $config;
+    }
+
     public static function getInstance(): Config
     {
         if (self::$instance) {
@@ -2428,9 +2445,7 @@ public function visitComposerAutoloadFiles(ProjectAnalyzer $project_analyzer, ?P
             // as they might be autoloadable once we require the autoloader below
             $codebase->classlikes->forgetMissingClassLikes();
 
-            $this->include_collector->runAndCollect(
-                $this->requireAutoloader(...),
-            );
+            $this->include_collector->runAndCollect($this->requireAutoloader(...));
         }
 
         $this->collectPredefinedConstants();

src/Psalm/Internal/Analyzer/ProjectAnalyzer.php

@@ -4,8 +4,6 @@
 
 namespace Psalm\Internal\Analyzer;
 
-use Fidry\CpuCoreCounter\CpuCoreCounter;
-use Fidry\CpuCoreCounter\NumberOfCpuCoreNotFound;
 use InvalidArgumentException;
 use Psalm\Codebase;
 use Psalm\Config;
@@ -65,11 +63,9 @@
 use function array_shift;
 use function clearstatcache;
 use function count;
-use function defined;
 use function dirname;
 use function end;
 use function explode;
-use function extension_loaded;
 use function file_exists;
 use function fwrite;
 use function implode;
@@ -204,6 +200,7 @@ public function __construct(
         public ?ReportOptions $stdout_report_options = null,
         public array $generated_report_options = [],
         public int $threads = 1,
+        public int $scanThreads = 1,
         ?Progress $progress = null,
         ?Codebase $codebase = null,
     ) {
@@ -364,15 +361,6 @@ public function serverMode(LanguageServer $server): void
         $this->file_reference_provider->loadReferenceCache();
         $this->codebase->enterServerMode();
 
-        $cpu_count = self::getCpuCount();
-
-        // let's not go crazy
-        $usable_cpus = $cpu_count - 2;
-
-        if ($usable_cpus > 1) {
-            $this->threads = $usable_cpus;
-        }
-
         $server->logInfo("Initializing: Initialize Plugins...");
         $this->config->initializePlugins($this);
 
@@ -484,7 +472,7 @@ public function check(string $base_dir, bool $is_diff = false): void
 
             $this->config->initializePlugins($this);
 
-            $this->codebase->scanFiles($this->threads);
+            $this->codebase->scanFiles($this->scanThreads);
 
             $this->codebase->infer_types_from_usage = true;
         } else {
@@ -508,7 +496,7 @@ public function check(string $base_dir, bool $is_diff = false): void
 
                     $this->config->initializePlugins($this);
 
-                    $this->codebase->scanFiles($this->threads);
+                    $this->codebase->scanFiles($this->scanThreads);
                 } else {
                     $diff_no_files = true;
                 }
@@ -885,7 +873,7 @@ public function checkDir(string $dir_name): void
 
         $this->config->initializePlugins($this);
 
-        $this->codebase->scanFiles($this->threads);
+        $this->codebase->scanFiles($this->scanThreads);
 
         $this->config->visitStubFiles($this->codebase, $this->progress);
 
@@ -995,7 +983,7 @@ public function checkFile(string $file_path): void
 
         $this->config->initializePlugins($this);
 
-        $this->codebase->scanFiles($this->threads);
+        $this->codebase->scanFiles($this->scanThreads);
 
         $this->config->visitStubFiles($this->codebase, $this->progress);
 
@@ -1039,7 +1027,7 @@ public function checkPaths(array $paths_to_check): void
         $this->config->initializePlugins($this);
 
 
-        $this->codebase->scanFiles($this->threads);
+        $this->codebase->scanFiles($this->scanThreads);
 
         $this->config->visitStubFiles($this->codebase, $this->progress);
 
@@ -1311,28 +1299,6 @@ public function getFunctionLikeAnalyzer(
         return $function_analyzer;
     }
 
-    /**
-     * Adapted from https://gist.github.com/divinity76/01ef9ca99c111565a72d3a8a6e42f7fb
-     * returns number of cpu cores
-     * Copyleft 2018, license: WTFPL
-     *
-     * @throws NumberOfCpuCoreNotFound
-     */
-    public static function getCpuCount(): int
-    {
-        if (defined('PHP_WINDOWS_VERSION_MAJOR')) {
-            // No support desired for Windows at the moment
-            return 1;
-        }
-
-        if (!extension_loaded('pcntl')) {
-            // Psalm requires pcntl for multi-threads support
-            return 1;
-        }
-
-        return (new CpuCoreCounter())->getCount();
-    }
-
     /**
      * @return array<int, string>
      * @psalm-pure

src/Psalm/Internal/Cli/Psalm.php

@@ -5,6 +5,7 @@
 namespace Psalm\Internal\Cli;
 
 use Composer\Autoload\ClassLoader;
+use Fidry\CpuCoreCounter\CpuCoreCounter;
 use Psalm\Config;
 use Psalm\Config\Creator;
 use Psalm\ErrorBaseline;
@@ -146,6 +147,7 @@ final class Psalm
         'show-snippet:',
         'stats',
         'threads:',
+        'scan-threads:',
         'update-baseline',
         'use-baseline:',
         'use-ini-defaults',
@@ -271,11 +273,12 @@ public static function run(array $argv): void
             $options['long-progress'] = true;
         }
 
-        $threads = self::detectThreads($options, $config, $in_ci);
+        $threads = self::getThreads($options, $config, $in_ci, false);
+        $scanThreads = self::getThreads($options, $config, $in_ci, true);
 
         $progress = self::initProgress($options, $config, $in_ci);
 
-        self::restart($options, $threads, $progress);
+        self::restart($options, $threads, $scanThreads, $progress);
 
         if (isset($options['debug-emitted-issues'])) {
             $config->debug_emitted_issues = true;
@@ -353,6 +356,7 @@ public static function run(array $argv): void
                     : true,
             ),
             $threads,
+            $scanThreads,
             $progress,
         );
 
@@ -412,6 +416,32 @@ public static function run(array $argv): void
         }
     }
 
+    public static function getThreads(array $options, Config $config, bool $in_ci, bool $for_scan): int
+    {
+        if ($for_scan) {
+            if (isset($options['scanThreads'])) {
+                $threads = (int)$options['scanThreads'];
+            } elseif (isset($options['debug']) || $in_ci) {
+                $threads = 1;
+            } elseif ($config->scan_threads) {
+                $threads = $config->scan_threads;
+            } else {
+                $threads = max(1, (new CpuCoreCounter())->getCount());
+            }
+        } else {
+            if (isset($options['threads'])) {
+                $threads = (int)$options['threads'];
+            } elseif (isset($options['debug']) || $in_ci) {
+                $threads = 1;
+            } elseif ($config->threads) {
+                $threads = $config->threads;
+            } else {
+                $threads = max(1, (new CpuCoreCounter())->getCount());
+            }
+        }
+        return $threads;
+    }
+
     private static function initOutputFormat(array $options): string
     {
         return isset($options['output-format']) && is_string($options['output-format'])
@@ -880,7 +910,7 @@ private static function getCurrentDir(array $options): string
         return $current_dir;
     }
 
-    private static function restart(array $options, int $threads, Progress $progress): void
+    private static function restart(array $options, int $threads, int $scanThreads, Progress $progress): void
     {
         $ini_handler = new PsalmRestarter('PSALM');
 
@@ -897,7 +927,7 @@ private static function restart(array $options, int $threads, Progress $progress
             }
         }
 
-        if ($threads > 1
+        if (($threads > 1 || $scanThreads > 1)
             && extension_loaded('grpc')
             && (ini_get('grpc.enable_fork_support') === '1' && ini_get('grpc.poll_strategy') === 'epoll1') === false
         ) {
@@ -957,20 +987,6 @@ private static function restart(array $options, int $threads, Progress $progress
         }
     }
 
-    private static function detectThreads(array $options, Config $config, bool $in_ci): int
-    {
-        if (isset($options['threads'])) {
-            $threads = (int)$options['threads'];
-        } elseif (isset($options['debug']) || $in_ci) {
-            $threads = 1;
-        } elseif ($config->threads) {
-            $threads = $config->threads;
-        } else {
-            $threads = max(1, ProjectAnalyzer::getCpuCount() - 1);
-        }
-        return $threads;
-    }
-
     /** @psalm-suppress UnusedParam $argv is being reported as unused */
     private static function forwardCliCall(array $options, array $argv): void
     {
@@ -1303,7 +1319,10 @@ private static function getHelpText(): string
                 If set, requires JIT acceleration to be available in order to run Psalm, exiting immediately if it cannot be enabled.
 
             --threads=INT
-                If greater than one, Psalm will run analysis on multiple threads, speeding things up.
+                If greater than one, Psalm will run the scan and analysis on multiple threads, speeding things up.
+
+            --scan-threads=INT
+                If greater than one, Psalm will run the scan on multiple threads, speeding things up (if specified, takes priority over the --threads flag).
 
             --no-diff
                 Turns off Psalm’s diff mode, checks all files regardless of whether they’ve changed.

src/Psalm/Internal/Cli/Psalter.php

@@ -48,7 +48,6 @@
 use function in_array;
 use function is_array;
 use function is_dir;
-use function is_numeric;
 use function is_string;
 use function microtime;
 use function pathinfo;
@@ -87,7 +86,7 @@ final class Psalter
     private const LONG_OPTIONS = [
         'help', 'debug', 'debug-by-line', 'debug-emitted-issues', 'config:', 'file:', 'root:',
         'plugin:', 'issues:', 'list-supported-issues', 'php-version:', 'dry-run', 'safe-types',
-        'find-unused-code', 'threads:', 'codeowner:',
+        'find-unused-code', 'threads:', 'scan-threads:', 'codeowner:',
         'allow-backwards-incompatible-changes:',
         'add-newline-between-docblock-annotations:',
         'no-cache',
@@ -263,8 +262,11 @@ public static function run(array $argv): void
             $current_dir = $config->base_dir;
             chdir($current_dir);
         }
+        
+        $in_ci = CliUtils::runningInCI();
 
-        $threads = isset($options['threads']) && is_numeric($options['threads']) ? (int)$options['threads'] : 1;
+        $threads = Psalm::getThreads($options, $config, $in_ci, false);
+        $scanThreads = Psalm::getThreads($options, $config, $in_ci, true);
 
         if (isset($options['no-cache'])) {
             $providers = new Providers(
@@ -304,6 +306,7 @@ public static function run(array $argv): void
             $stdout_report_options,
             [],
             $threads,
+            $scanThreads,
             $progress,
         );