Skip to content

Commit

Permalink
Merge pull request #1816 from hydephp/history-graph-tree
Browse files Browse the repository at this point in the history 
Internal: Clean up the history graph generator script
  • Loading branch information
@caendesilva
caendesilva authored Jul 9, 2024
2 parents ae44b59 + 000a29f commit 35d33a3
Showing 1 changed file with 148 additions and 147 deletions.
295 changes: 148 additions & 147 deletions monorepo/scripts/build-git-history-graph.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,56 +5,86 @@
/**
* @internal This script is used to build the Git history graphs.
*
* @todo The HTML page has serious memory issues and may need pagination.
*
* @link https://hydephp.github.io/develop/master/git/history-graph.html
* @link https://hydephp.github.io/develop/master/git/history-graph.txt
*/
const CHUNK_SIZE = 100000;
const GRAPHS_DIR = __DIR__.'/graphs';
const TEMP_FILE = 'chunks.temp';

echo 'Building the Git history graph...'.PHP_EOL;

if (! file_exists(__DIR__.'/graphs')) {
mkdir(__DIR__.'/graphs');
}
echo 'Building the plaintext Git history graph... (This may take a while)'.PHP_EOL;
$text = shell_exec('git log --graph --oneline --all');
echo 'Saving the plaintext Git history graph...'.PHP_EOL;
file_put_contents(__DIR__.'/graphs/history-graph.txt', $text);
unset($text); // Free up memory

echo 'Building the HTML Git history graph... (This may take a while)'.PHP_EOL;
$html = shell_exec('git log --graph --oneline --all --color=always');
echo 'Converting ANSI color codes to HTML...'.PHP_EOL;
$html = processHtml($html);
echo 'Generating header...';
$graph = file_get_contents(__DIR__.'/graphs/history-graph.txt');
$header = generateHeader($graph);
unset($graph); // Free up memory
echo ' Done.'.PHP_EOL;
echo 'Wrapping the HTML...';
$html = wrapHtml($html, $header);
echo ' Done.'.PHP_EOL;
echo 'Saving the HTML Git history graph...'.PHP_EOL;
file_put_contents(__DIR__.'/graphs/history-graph.html', $html);
unset($html); // Free up memory
createGraphsDirectoryIfNotExists();
buildPlainTextGraph();
buildHtmlGraph();

echo 'Git history graphs built successfully!'.PHP_EOL;

echo $header;
function createGraphsDirectoryIfNotExists(): void
{
if (! file_exists(GRAPHS_DIR)) {
mkdir(GRAPHS_DIR);
}
}

function buildPlainTextGraph(): void
{
echo 'Building the plaintext Git history graph... (This may take a while)'.PHP_EOL;
$text = cached_shell_exec('git log --graph --oneline --all');
echo 'Saving the plaintext Git history graph...'.PHP_EOL;
file_put_contents(GRAPHS_DIR.'/history-graph.txt', $text);
}

function buildHtmlGraph(): void
{
echo 'Building the HTML Git history graph... (This may take a while)'.PHP_EOL;
$html = cached_shell_exec('git log --graph --oneline --all --color=always');
echo 'Converting ANSI color codes to HTML...'.PHP_EOL;
$html = processHtml($html);
echo 'Generating header...';
$graph = file_get_contents(GRAPHS_DIR.'/history-graph.txt');
$header = generateHeader($graph);
echo ' Done.'.PHP_EOL;
echo 'Wrapping the HTML...';
$html = wrapHtml($html, $header);
echo ' Done.'.PHP_EOL;
echo 'Saving the HTML Git history graph...'.PHP_EOL;
file_put_contents(GRAPHS_DIR.'/history-graph.html', $html);
}

function processHtml(string $html): string
{
// We need to run the ANSI to HTML conversion in chunks to prevent memory issues
$chunks = chunkHtml($html);
$message = 'Processing '.count($chunks).' chunks...';
echo $message;

$html = explode("\n", $html);
foreach ($chunks as $index => $chunk) {
echo "\033[0K\rProcessing chunk ".($index + 1).' of '.count($chunks).'...';
$chunkHtml = ansiToHtml($chunk);
$chunkHtml = postProcessChunk($chunkHtml);

$chunks = [];
// Since this process takes so much memory, we store the chunks on disk instead of memory.
file_put_contents(TEMP_FILE, $chunkHtml, FILE_APPEND);
}

$chunk = '';
$html = file_get_contents(TEMP_FILE);
unlink(TEMP_FILE);
echo PHP_EOL;

$chunkSize = 100000;
return $html;
}

foreach ($html as $line) {
$chunk .= $line."\n";
function chunkHtml(string $html): array
{
$lines = explode("\n", $html);
$chunks = [];
$chunk = '';

if (strlen($chunk) > $chunkSize) {
foreach ($lines as $line) {
$chunk .= $line."\n";
if (strlen($chunk) > CHUNK_SIZE) {
$chunks[] = $chunk;
$chunk = '';
}
Expand All @@ -64,60 +94,27 @@ function processHtml(string $html): string
$chunks[] = $chunk;
}

$message = 'Processing '.count($chunks).' chunks...';
echo $message;

foreach ($chunks as $index => $chunk) {
// Progress indicator
echo "\033[0K\rProcessing chunk ".($index + 1).' of '.count($chunks).'...';

$chunkHtml = ansiToHtml($chunk);

$chunkHtml = postProcessChunk($chunkHtml);

file_put_contents('chunks.temp', $chunkHtml, FILE_APPEND);

// Free up memory
unset($chunk);
unset($chunkHtml);
}

$html = file_get_contents('chunks.temp');
unlink('chunks.temp');

echo PHP_EOL;

return $html;
return $chunks;
}

function ansiToHtml(string $ansi): string
{
$ansi = preg_replace('/\x1b\[(\d+)(;\d+)*m/', '</span><span style="color: $1">', $ansi);

$colors = [
1 => '#800000',
30 => '#000000',
31 => '#800000',
32 => '#008000',
33 => '#808000',
34 => '#000080',
35 => '#800080',
36 => '#008080',
37 => '#c0c0c0',
90 => '#808080',
91 => '#ff0000',
92 => '#00ff00',
93 => '#ffff00',
94 => '#0000ff',
95 => '#ff00ff',
96 => '#00ffff',
97 => '#ffffff',
1 => '#C50F1F',
30 => '#0C0C0C',
31 => '#C50F1F',
32 => '#13A10E',
33 => '#C19C00',
34 => '#0037DA',
35 => '#881798',
36 => '#3A96DD',
37 => '#CCCCCC',
];

$ansi = preg_replace_callback('/<span style="color: (\d+)">/', function ($matches) use ($colors) {
$ansi = preg_replace('/\x1b\[(\d+)(;\d+)*m/', '</span><span style="color: $1">', $ansi);
$ansi = preg_replace_callback('/<span style="color: (\d+)">/', function (array $matches) use ($colors): string {
return '<span style="color: '.$colors[$matches[1]].'">';
}, $ansi);

$ansi = str_replace("\033[m", '</span>', $ansi);

return trim($ansi);
Expand All @@ -133,7 +130,7 @@ function wrapHtml(string $html, string $header): string
<title>Git History Graph</title>
<style>
body {
background-color: #000;
background-color: #121212;
color: #fff;
}
header, main {
Expand All @@ -154,8 +151,6 @@ function wrapHtml(string $html, string $header): string
function generateHeader(string $text): string
{
$commitCount = countCommitLines($text);
unset($text); // Free up memory

$head = trim(shell_exec('git rev-parse --short HEAD'));
$date = date('Y-m-d H:i:s');

Expand All @@ -175,12 +170,12 @@ function countCommitLines(string $text): int
{
$count = 0;
$lines = explode("\n", $text);

foreach ($lines as $line) {
if (str_starts_with(ltrim($line, ' /\\|'), '*')) {
$count++;
}
}
unset($lines); // Free up memory

return $count;
}
Expand All @@ -189,79 +184,85 @@ function postProcessChunk(string $chunk): string
{
$lines = explode("\n", $chunk);

$ignore = [
'Upload documentation preview for PR',
'Upload live reports from test suite run',
'Upload site preview from test suite run with ref',
'Upload coverage report from test suite run with ref',
'Upload API documentation from test suite run with ref',
];

foreach ($lines as $index => $line) {
// Remove stray closing span tags
// Replace </span> </span> with </span>
$line = str_replace('</span></span>', '</span>', $line);
$line = str_replace('</span> </span>', '</span> ', $line);
$line = str_replace('</span> </span>', '</span> ', $line);
$line = str_replace('</span> </span>', '</span> ', $line);

// replace </span> * </span> with </span> *
$line = str_replace('</span> * </span>', '</span> * ', $line);
$line = str_replace('</span> * </span>', '</span> * ', $line);

// If str starts with </span> then remove it
if (str_starts_with($line, '</span>')) {
$line = substr($line, 7);
}
if (str_starts_with($line, ' </span>')) {
$line = substr($line, 8).' ';
}
$line = cleanSpanTags($line);
$line = cleanAsterisk($line);
// assertValidLine($line, $index);

// If str starts with * </span> just use *
if (str_starts_with($line, '* </span>')) {
$line = '* '.substr($line, 9);
}
if (str_starts_with($line, '* </span>')) {
$line = '* '.substr($line, 10);
}
if (str_starts_with($line, '* </span>')) {
$line = '* '.substr($line, 11);
if (shouldIgnoreLine($line, $ignore)) {
$line = '';
}

// assert trimmed line does not start with closing span tag
$strStartsWith = str_starts_with(trim($line), '</span>');
if ($strStartsWith) {
echo PHP_EOL.$line.PHP_EOL;
$lines[$index] = trim($line);
}

// write chunk to disk
$nextThreeLines = array_slice($lines, $index, 4);
unset($nextThreeLines[0]);
file_put_contents('chunk.html', $chunk);
file_put_contents('failed-chunk.html', sprintf("<pre>%s\n<u>%s</u>\n^^^\n%s</pre>", implode("\n", $lines), $line, implode("\n", $nextThreeLines)));
}
assert(! $strStartsWith, "Line $index starts with closing span tag");
return implode("\n", array_filter($lines));
}

// assert line has equal number of opening and closing span tags
$open = substr_count($line, '<span');
$close = substr_count($line, '</span>');
if ($open !== $close) {
echo PHP_EOL.$line.PHP_EOL;
}
assert($open === $close, "Line $index has $open opening and $close closing span tags");

// if line contains any of these, ignore:
$ignore = [
'Upload documentation preview for PR',
'Upload live reports from test suite run',
'Upload site preview from test suite run with ref',
'Upload coverage report from test suite run with ref',
'Upload API documentation from test suite run with ref',
];

foreach ($ignore as $str) {
if (str_contains($line, $str)) {
$line = '';
break;
}
}
function cleanSpanTags(string $line): string
{
$line = str_replace(
['</span></span>', '</span> </span>', '</span> </span>', '</span> </span>'],
['</span>', '</span> ', '</span> ', '</span> '],
$line
);

if (str_starts_with($line, '</span>')) {
$line = substr($line, 7);
}

$lines[$index] = trim($line);
if (str_starts_with($line, ' </span>')) {
$line = substr($line, 8).' ';
}

return $line;
}

function cleanAsterisk(string $line): string
{
if (preg_match('/^\*\s*<\/span>/', $line)) {
$line = '* '.preg_replace('/^\*\s*<\/span>/', '', $line);
}

return $line;
}

function assertValidLine(string $line, int $index): void
{
$trimmedLine = trim($line);
assert(! str_starts_with($trimmedLine, '</span>'), "Line $index starts with closing span tag");

$openTags = substr_count($line, '<span');
$closeTags = substr_count($line, '</span>');
assert($openTags === $closeTags, "Line $index has $openTags opening and $closeTags closing span tags");
}

function shouldIgnoreLine(string $line, array $ignore): bool
{
foreach ($ignore as $str) {
if (str_contains($line, $str)) {
return true;
}
}

unset($chunk);
$lines = array_filter($lines);
return false;
}

function cached_shell_exec(string $command): string
{
$cacheFile = __DIR__.'/cache/'.sha1($command).'.txt';
$cache = file_exists($cacheFile) ? file_get_contents($cacheFile) : '';
$output = $cache ?: shell_exec($command);

file_put_contents($cacheFile, $output);

return implode("\n", $lines);
return $output;
}

0 comments on commit 35d33a3

Please sign in to comment.