Skip to content

Commit

Permalink
More detailed logging&improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
danog committed Sep 3, 2023
1 parent 535f1b7 commit f79012d
Show file tree
Hide file tree
Showing 13 changed files with 93 additions and 99 deletions.
40 changes: 38 additions & 2 deletions src/Psalm/Codebase.php
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

namespace Psalm;

use Amp\Parallel\Worker\WorkerPool;
use Exception;
use InvalidArgumentException;
use LanguageServerProtocol\Command;
Expand Down Expand Up @@ -38,7 +37,6 @@
use Psalm\Internal\Codebase\TaintFlowGraph;
use Psalm\Internal\DataFlow\TaintSink;
use Psalm\Internal\DataFlow\TaintSource;
use Psalm\Internal\Fork\Pool;
use Psalm\Internal\LanguageServer\PHPMarkdownContent;
use Psalm\Internal\LanguageServer\Reference;
use Psalm\Internal\MethodIdentifier;
Expand Down Expand Up @@ -98,6 +96,7 @@

use const PHP_VERSION_ID;

/** @psalm-import-type PoolData from Scanner */
final class Codebase
{
/**
Expand Down Expand Up @@ -394,6 +393,43 @@ private function loadAnalyzer(): void
);
}

/**
* @internal
* @param PoolData $pool_data
*/
public function addThreadData(array $pool_data): void
{
IssueBuffer::addIssues($pool_data['issues']);

$this->statements_provider->addChangedMembers(
$pool_data['changed_members'],
);
$this->statements_provider->addUnchangedSignatureMembers(
$pool_data['unchanged_signature_members'],
);
$this->statements_provider->addDiffMap(
$pool_data['diff_map'],
);
$this->statements_provider->addDeletionRanges(
$pool_data['deletion_ranges'],
);
$this->statements_provider->addErrors($pool_data['errors']);

if ($this->taint_flow_graph && $pool_data['taint_data']) {
$this->taint_flow_graph->addGraph($pool_data['taint_data']);
}

$this->file_storage_provider->addMore($pool_data['file_storage']);
$this->classlike_storage_provider->addMore($pool_data['classlike_storage']);

$this->classlikes->addThreadData($pool_data['classlikes_data']);

if ($this->statements_provider->parser_cache_provider) {
$this->statements_provider->parser_cache_provider->addNewFileContentHashes(
$pool_data['new_file_content_hashes'],
);
}
}
/**
* @param array<string> $candidate_files
*/
Expand Down
2 changes: 1 addition & 1 deletion src/Psalm/Internal/Analyzer/ProjectAnalyzer.php
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@
use function explode;
use function file_exists;
use function fwrite;
use function get_object_vars;
use function implode;
use function in_array;
use function is_dir;
Expand All @@ -77,7 +78,6 @@
use function mkdir;
use function number_format;
use function preg_match;
use function Psl\Type\array_key;
use function rename;
use function sprintf;
use function strlen;
Expand Down
8 changes: 3 additions & 5 deletions src/Psalm/Internal/Codebase/Analyzer.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@
use Psalm\Internal\FileManipulation\FunctionDocblockManipulator;
use Psalm\Internal\FileManipulation\PropertyDocblockManipulator;
use Psalm\Internal\Fork\AnalyzerTask;
use Psalm\Internal\Fork\InitAnalyzerTask;
use Psalm\Internal\Fork\Pool;
use Psalm\Internal\Fork\ShutdownAnalyzerTask;
use Psalm\Internal\Provider\FileProvider;
use Psalm\Internal\Provider\FileStorageProvider;
Expand Down Expand Up @@ -337,14 +335,14 @@ private function doAnalysis(ProjectAnalyzer $project_analyzer): void

$this->progress->debug('Forking analysis' . "\n");

$forked_pool_data = $project_analyzer->pool->run(
$project_analyzer->pool->run(
$new_file_paths,
new InitAnalyzerTask(),
AnalyzerTask::class,
new ShutdownAnalyzerTask,
$this->taskDoneClosure(...),
);

$forked_pool_data = $project_analyzer->pool->runAll(new ShutdownAnalyzerTask);

$this->progress->debug('Collecting forked analysis results' . "\n");

foreach ($forked_pool_data as $pool_data) {
Expand Down
47 changes: 8 additions & 39 deletions src/Psalm/Internal/Codebase/Scanner.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,12 @@
use Psalm\Internal\Analyzer\ProjectAnalyzer;
use Psalm\Internal\ErrorHandler;
use Psalm\Internal\Fork\InitScannerTask;
use Psalm\Internal\Fork\Pool;
use Psalm\Internal\Fork\ScannerTask;
use Psalm\Internal\Fork\ShutdownScannerTask;
use Psalm\Internal\Provider\FileProvider;
use Psalm\Internal\Provider\FileReferenceProvider;
use Psalm\Internal\Provider\FileStorageProvider;
use Psalm\Internal\Scanner\FileScanner;
use Psalm\IssueBuffer;
use Psalm\Progress\Progress;
use Psalm\Storage\ClassLikeStorage;
use Psalm\Storage\FileStorage;
Expand All @@ -27,12 +25,10 @@
use function array_filter;
use function array_merge;
use function array_pop;
use function ceil;
use function count;
use function error_reporting;
use function explode;
use function file_exists;
use function min;
use function realpath;
use function strtolower;
use function substr;
Expand Down Expand Up @@ -314,48 +310,21 @@ private function scanFilePaths(): bool
$pool_size = $this->is_forked ? 1 : $project_analyzer->threads;

if ($pool_size > 1) {
$this->progress->debug('Forking process for scanning' . PHP_EOL);
$cnt = count($files_to_scan);
$this->progress->debug("Sending {$cnt} files to pool for scanning" . PHP_EOL);

$forked_pool_data = ProjectAnalyzer::getInstance()->pool->run(
$pool = ProjectAnalyzer::getInstance()->pool;
$pool->runAll(new InitScannerTask);

$pool->run(
$files_to_scan,
new InitScannerTask(),
ScannerTask::class,
new ShutdownScannerTask,
);

$forked_pool_data = $pool->runAll(new ShutdownScannerTask);
foreach ($forked_pool_data as $pool_data) {
IssueBuffer::addIssues($pool_data['issues']);

$this->codebase->statements_provider->addChangedMembers(
$pool_data['changed_members'],
);
$this->codebase->statements_provider->addUnchangedSignatureMembers(
$pool_data['unchanged_signature_members'],
);
$this->codebase->statements_provider->addDiffMap(
$pool_data['diff_map'],
);
$this->codebase->statements_provider->addDeletionRanges(
$pool_data['deletion_ranges'],
);
$this->codebase->statements_provider->addErrors($pool_data['errors']);

if ($this->codebase->taint_flow_graph && $pool_data['taint_data']) {
$this->codebase->taint_flow_graph->addGraph($pool_data['taint_data']);
}

$this->codebase->file_storage_provider->addMore($pool_data['file_storage']);
$this->codebase->classlike_storage_provider->addMore($pool_data['classlike_storage']);

$this->codebase->classlikes->addThreadData($pool_data['classlikes_data']);

$this->codebase->addThreadData($pool_data);
$this->addThreadData($pool_data['scanner_data']);

if ($this->codebase->statements_provider->parser_cache_provider) {
$this->codebase->statements_provider->parser_cache_provider->addNewFileContentHashes(
$pool_data['new_file_content_hashes'],
);
}
}
} else {
$i = 0;
Expand Down
2 changes: 1 addition & 1 deletion src/Psalm/Internal/Codebase/TaintFlowGraph.php
Original file line number Diff line number Diff line change
Expand Up @@ -521,7 +521,7 @@ private function getSpecializedSources(DataFlowNode $source): array

return array_filter(
$generated_sources,
fn ($new_source) => isset($this->forward_edges[$new_source->id])
fn($new_source) => isset($this->forward_edges[$new_source->id])
);
}
}
29 changes: 9 additions & 20 deletions src/Psalm/Internal/Fork/InitAnalyzerTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -6,31 +6,20 @@
use Amp\Parallel\Worker\Task;
use Amp\Sync\Channel;
use Psalm\Internal\Analyzer\ProjectAnalyzer;
use Psalm\Internal\Codebase\TaintFlowGraph;
use Psalm\Internal\Codebase\Scanner;

/** @psalm-import-type PoolData from Scanner */
final class InitAnalyzerTask implements Task
{
/** @var PoolData */
private readonly array $data;
public function __construct()
{
$this->data = ShutdownScannerTask::getPoolData();
}
public function run(Channel $channel, Cancellation $cancellation): mixed
{
$project_analyzer = ProjectAnalyzer::getInstance();
$codebase = $project_analyzer->getCodebase();

$file_reference_provider = $codebase->file_reference_provider;

if ($codebase->taint_flow_graph) {
$codebase->taint_flow_graph = new TaintFlowGraph();
}

$file_reference_provider->setNonMethodReferencesToClasses([]);
$file_reference_provider->setCallingMethodReferencesToClassMembers([]);
$file_reference_provider->setCallingMethodReferencesToClassProperties([]);
$file_reference_provider->setFileReferencesToClassMembers([]);
$file_reference_provider->setFileReferencesToClassProperties([]);
$file_reference_provider->setCallingMethodReferencesToMissingClassMembers([]);
$file_reference_provider->setFileReferencesToMissingClassMembers([]);
$file_reference_provider->setReferencesToMixedMemberNames([]);
$file_reference_provider->setMethodParamUses([]);

ProjectAnalyzer::getInstance()->getCodebase()->addThreadData($this->data);
return null;
}
}
14 changes: 0 additions & 14 deletions src/Psalm/Internal/Fork/InitScannerTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,9 @@
use Amp\Cancellation;
use Amp\Parallel\Worker\Task;
use Amp\Sync\Channel;
use Psalm\Config;
use Psalm\Internal\Analyzer\ProjectAnalyzer;
use Psalm\Internal\CliUtils;
use Psalm\Internal\ErrorHandler;
use Psalm\Internal\Provider\ClassLikeStorageProvider;
use Psalm\Internal\Provider\FileStorageProvider;
use Psalm\Internal\VersionUtils;
use Psalm\IssueBuffer;

use function cli_get_process_title;
use function cli_set_process_title;
use function define;
use function function_exists;
use function gc_collect_cycles;
use function gc_disable;
use function ini_get;
use function ini_set;

use const PHP_EOL;

Expand Down
9 changes: 2 additions & 7 deletions src/Psalm/Internal/Fork/InitStartupTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -9,22 +9,17 @@
use Psalm\Internal\Analyzer\ProjectAnalyzer;
use Psalm\Internal\CliUtils;
use Psalm\Internal\ErrorHandler;
use Psalm\Internal\Provider\ClassLikeStorageProvider;
use Psalm\Internal\Provider\FileStorageProvider;
use Psalm\Internal\VersionUtils;
use Psalm\IssueBuffer;

use function cli_get_process_title;
use function cli_set_process_title;
use function define;
use function function_exists;
use function gc_collect_cycles;
use function gc_disable;
use function ini_get;
use function ini_set;

use const PHP_EOL;

final class InitStartupTask implements Task
{
private readonly string $memoryLimit;
Expand Down Expand Up @@ -60,9 +55,9 @@ final public function run(Channel $channel, Cancellation $cancellation): mixed
ProjectAnalyzer::$instance = $this->analyzer;
Config::setInstance($this->analyzer->getConfig());

if (function_exists('cli_set_process_title') && $this->processTitle !== null) {
/*if (function_exists('cli_set_process_title') && $this->processTitle !== null) {
@cli_set_process_title($this->processTitle);
}
}*/

return null;
}
Expand Down
29 changes: 22 additions & 7 deletions src/Psalm/Internal/Fork/Pool.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,17 @@
use AssertionError;
use Closure;
use Psalm\Internal\Analyzer\ProjectAnalyzer;
use Psalm\Progress\Progress;
use Throwable;

use function Amp\Future\await;
use function array_map;
use function count;
use function extension_loaded;
use function gc_collect_cycles;

use const PHP_BINARY;
use const PHP_EOL;

/**
* Adapted with relatively few changes from
Expand All @@ -34,6 +38,7 @@
final class Pool
{
private readonly WorkerPool $pool;
private readonly Progress $progress;
/**
* @param int<2, max> $threads
*/
Expand All @@ -57,6 +62,9 @@ public function __construct(private readonly int $threads, ProjectAnalyzer $proj
);

$this->runAll(new InitStartupTask($project_analyzer));
$this->runAll(new InitScannerTask());

$this->progress = $project_analyzer->progress;
}
/**
* @template TFinalResult
Expand All @@ -77,31 +85,38 @@ public function __construct(private readonly int $threads, ProjectAnalyzer $proj
*/
public function run(
array $process_task_data_iterator,
Task $startup_task,
string $main_task,
Task $shutdown_task,
?Closure $task_done_closure = null
): array {
$this->runAll($startup_task);
): void {
$total = count($process_task_data_iterator);
$this->progress->debug("Processing ".$total." tasks...");

$cnt = 0;

$results = [];
foreach ($process_task_data_iterator as $file) {
$results []= $f = $this->pool->submit(new $main_task($file))->getFuture();
if ($task_done_closure) {
$f->map($task_done_closure);
}
$f->catch(fn(Throwable $e) => throw $e);
$f->map(function () use (&$cnt, $total): void {
$cnt++;
if (!($cnt % 10)) {
$percent = (int) (($cnt*100) / $total);
$this->progress->debug("Processing tasks: $cnt/$total ($percent%)...".PHP_EOL);
}
});
}
await($results);

return $this->runAll($shutdown_task);
}

/**
* @template T
* @param Task<void, void, T> $task
* @return list<T>
*/
private function runAll(Task $task): array
public function runAll(Task $task): array
{
if ($this->pool->getIdleWorkerCount() !== $this->pool->getWorkerCount()) {
throw new AssertionError("Some workers are busy!");
Expand Down
1 change: 0 additions & 1 deletion src/Psalm/Internal/Fork/ScannerTask.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
use Amp\Cancellation;
use Amp\Parallel\Worker\Task;
use Amp\Sync\Channel;
use AssertionError;
use Psalm\Internal\Analyzer\ProjectAnalyzer;

/** @internal */
Expand Down
Loading

0 comments on commit f79012d

Please sign in to comment.