-
Notifications
You must be signed in to change notification settings - Fork 1
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
ENH Update reference to supported modules data #45
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,7 @@ | ||
<?php | ||
|
||
use Panlatent\CronExpressionDescriptor\ExpressionDescriptor; | ||
use SilverStripe\SupportedModules\MetaData; | ||
|
||
// These functions in scripts can be used in scripts | ||
|
||
|
@@ -243,8 +244,14 @@ function is_docs() | |
*/ | ||
function is_gha_repository() | ||
{ | ||
global $MODULE_DIR; | ||
return !is_module() && strpos($MODULE_DIR, '/gha-') !== false; | ||
global $GITHUB_REF; | ||
return in_array( | ||
$GITHUB_REF, | ||
array_column( | ||
MetaData::getAllRepositoryMetaData()[MetaData::CATEGORY_WORKFLOW], | ||
'github' | ||
) | ||
); | ||
} | ||
|
||
/** | ||
|
@@ -257,8 +264,8 @@ function is_gha_repository() | |
*/ | ||
function module_name() | ||
{ | ||
global $MODULE_DIR; | ||
$parts = explode('/', $MODULE_DIR); | ||
global $GITHUB_REF; | ||
$parts = explode('/', $GITHUB_REF); | ||
return end($parts); | ||
} | ||
|
||
|
@@ -270,15 +277,15 @@ function module_name() | |
*/ | ||
function module_is_one_of($repos) | ||
{ | ||
global $MODULE_DIR; | ||
global $GITHUB_REF; | ||
if (!is_array($repos)) { | ||
error('repos is not an array'); | ||
} | ||
foreach ($repos as $repo) { | ||
if (!is_string($repo)) { | ||
error('repo is not a string'); | ||
} | ||
if (strpos($MODULE_DIR, "/$repo") !== false) { | ||
if (strpos($GITHUB_REF, "/$repo") !== false) { | ||
return true; | ||
} | ||
} | ||
|
@@ -293,11 +300,8 @@ function module_is_one_of($repos) | |
*/ | ||
function module_account() | ||
{ | ||
$s = read_file('.git/config'); | ||
if (!preg_match('#github.com:([^/]+)/#', $s, $matches)) { | ||
error('Could not determine github account'); | ||
} | ||
return $matches[1]; | ||
global $GITHUB_REF; | ||
return explode('/', $GITHUB_REF)[0]; | ||
Comment on lines
-296
to
+304
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No need to check |
||
} | ||
|
||
/** | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,7 @@ | ||
<?php | ||
|
||
use SilverStripe\SupportedModules\BranchLogic; | ||
use SilverStripe\SupportedModules\MetaData; | ||
use Symfony\Component\Console\Output\NullOutput; | ||
use Symfony\Component\Console\Input\ArgvInput; | ||
use Symfony\Component\Console\Style\SymfonyStyle; | ||
|
@@ -46,85 +48,6 @@ function write_file($path, $contents) | |
info("Wrote to $path"); | ||
} | ||
|
||
/** | ||
* Returns all the supported modules for a particular cms major version | ||
* Will download the list if it doesn't exist | ||
*/ | ||
function supported_modules($cmsMajor) | ||
{ | ||
$filename = "_data/modules-cms$cmsMajor.json"; | ||
if (!file_exists($filename)) { | ||
$url = "https://raw.githubusercontent.com/silverstripe/supported-modules/$cmsMajor/modules.json"; | ||
info("Downloading $url to $filename"); | ||
$contents = file_get_contents($url); | ||
file_put_contents($filename, $contents); | ||
} | ||
$json = json_decode(file_get_contents($filename), true); | ||
if (is_null($json)) { | ||
$lastError = json_last_error(); | ||
error("Could not parse from $filename - last error was $lastError"); | ||
} | ||
$modules = []; | ||
foreach ($json as $module) { | ||
$ghrepo = $module['github']; | ||
$modules[] = [ | ||
'ghrepo' => $ghrepo, | ||
'account' => explode('/', $ghrepo)[0], | ||
'repo' => explode('/', $ghrepo)[1], | ||
'cloneUrl' => "[email protected]:$ghrepo.git", | ||
]; | ||
} | ||
return $modules; | ||
} | ||
|
||
/** | ||
* Hardcoded list of non-supported, additional repositories to standardise (e.g. silverstripe/gha-*) | ||
* | ||
* Repositories in this list should only have a single supported major version | ||
* This will only be included if the $cmsMajor is the CURRENT_CMS_MAJOR | ||
*/ | ||
function extra_repositories() | ||
{ | ||
$importantRepos = [ | ||
'silverstripe/markdown-php-codesniffer', | ||
'silverstripe/silverstripe-standards', | ||
'silverstripe/documentation-lint', | ||
'silverstripe/.github', | ||
]; | ||
Comment on lines
-86
to
-93
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The set of "extra" repos we're gonna end up with after this change is a lot bigger than we were getting before. I've added a note to silverstripe/.github#242 that when this gets run next we should take care to check if there's anything we want to skip and add an exclude list if there is. |
||
$modules = []; | ||
// iterating to page 10 will be enough to get all the repos well into the future | ||
for ($i = 0; $i < 10; $i++) { | ||
$path = "_data/extra_repositories-$i.json"; | ||
if (file_exists($path)) { | ||
info("Reading local data from $path"); | ||
$json = json_decode(file_get_contents($path), true); | ||
} else { | ||
$json = github_api("https://api.github.com/orgs/silverstripe/repos?per_page=100&page=$i"); | ||
file_put_contents($path, json_encode($json, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT)); | ||
} | ||
if (empty($json)) { | ||
break; | ||
} | ||
foreach ($json as $repo) { | ||
if ($repo['archived']) { | ||
continue; | ||
} | ||
$ghrepo = $repo['full_name']; | ||
// Only include repos we care about | ||
if (!in_array($ghrepo, $importantRepos) && strpos($ghrepo, '/gha-') === false) { | ||
continue; | ||
} | ||
$modules[] = [ | ||
'ghrepo' => $ghrepo, | ||
'account' => explode('/', $ghrepo)[0], | ||
'repo' => explode('/', $ghrepo)[1], | ||
'cloneUrl' => "[email protected]:$ghrepo.git", | ||
]; | ||
} | ||
} | ||
return $modules; | ||
} | ||
|
||
/** | ||
* Returns a list of all scripts files to run against a particular cms major version | ||
*/ | ||
|
@@ -349,85 +272,45 @@ function branch_to_checkout($branches, $defaultBranch, $currentBranch, $currentB | |
} | ||
|
||
/** | ||
* Uses composer.json to workout the current branch cms major version | ||
* | ||
* If composer.json does not exist then it's assumed to be CURRENT_CMS_MAJOR | ||
* Works out the current branch cms major version | ||
*/ | ||
function current_branch_cms_major( | ||
// this param is only used for unit testing | ||
string $composerJson = '' | ||
) { | ||
global $MODULE_DIR; | ||
global $MODULE_DIR, $GITHUB_REF; | ||
|
||
// Some repositories don't have a valid matching CMS major | ||
$ignoreCMSMajor = [ | ||
'/silverstripe-simple', | ||
'/markdown-php-codesniffer', | ||
]; | ||
foreach ($ignoreCMSMajor as $ignore) { | ||
if (strpos($MODULE_DIR, $ignore) !== false) { | ||
return CURRENT_CMS_MAJOR; | ||
} | ||
// This repo matches every major and matches start at the lowest - but we only want the highest stable. | ||
if ($GITHUB_REF === 'silverstripe/silverstripe-simple') { | ||
return MetaData::HIGHEST_STABLE_CMS_MAJOR; | ||
} | ||
Comment on lines
-362
to
286
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. markdown-php-codesniffer will get correctly identified - but the theme still needs some help. |
||
|
||
if ($composerJson) { | ||
$contents = $composerJson; | ||
} elseif (check_file_exists('composer.json')) { | ||
$contents = read_file('composer.json'); | ||
} else { | ||
return CURRENT_CMS_MAJOR; | ||
} | ||
|
||
// special logic for developer-docs | ||
if (strpos($MODULE_DIR, '/developer-docs') !== false) { | ||
$currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR); | ||
if (!preg_match('#^(pulls/)?([0-9]+)(\.[0-9]+)?(/|$)#', $currentBranch, $matches)) { | ||
error("Could not work out current major for developer-docs from branch $currentBranch"); | ||
} | ||
return $matches[2]; | ||
} | ||
|
||
$json = json_decode($contents); | ||
if (is_null($json)) { | ||
$composerJson = json_decode($contents); | ||
if (is_null($composerJson)) { | ||
$lastError = json_last_error(); | ||
error("Could not parse from composer.json - last error was $lastError"); | ||
} | ||
$matchedOnBranchThreeLess = false; | ||
$version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/framework'} ?? ''); | ||
if (!$version) { | ||
$version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/cms'} ?? ''); | ||
} | ||
if (!$version) { | ||
$version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/mfa'} ?? ''); | ||
} | ||
if (!$version) { | ||
$version = preg_replace('#[^0-9\.]#', '', $json->require->{'silverstripe/assets'} ?? ''); | ||
if ($version) { | ||
$matchedOnBranchThreeLess = true; | ||
} | ||
} | ||
if (!$version) { | ||
$version = preg_replace('#[^0-9\.]#', '', $json->require->{'cwp/starter-theme'} ?? ''); | ||
if ($version) { | ||
$version += 1; | ||
} | ||
} | ||
$cmsMajor = ''; | ||
if (preg_match('#^([0-9]+)+\.?[0-9]*$#', $version, $matches)) { | ||
$cmsMajor = $matches[1]; | ||
if ($matchedOnBranchThreeLess) { | ||
$cmsMajor += 3; | ||
} | ||
} else { | ||
$phpVersion = $json->require->{'php'} ?? ''; | ||
if (substr($phpVersion,0, 4) === '^7.4') { | ||
$cmsMajor = 4; | ||
} elseif (substr($phpVersion,0, 4) === '^8.1') { | ||
$cmsMajor = 5; | ||
|
||
$repoData = MetaData::getMetaDataForRepository($GITHUB_REF); | ||
$branchMajor = ''; | ||
// If we're running unit tests, $MODULE_DIR will be some fake value with causes errors here | ||
if (!running_unit_tests()) { | ||
$currentBranch = cmd('git rev-parse --abbrev-ref HEAD', $MODULE_DIR); | ||
if (preg_match('#^(pulls/)?([0-9]+)(\.[0-9]+)?(/|$)#', $currentBranch, $matches)) { | ||
$branchMajor = $matches[2]; | ||
} | ||
} | ||
$cmsMajor = BranchLogic::getCmsMajor($repoData, $branchMajor, $composerJson, true); | ||
|
||
if ($cmsMajor === '') { | ||
error('Could not work out what the current CMS major version is'); | ||
// The supported modules metadata has a bunch of repos with no specific major version mapping. | ||
// Just assume they're on the highest major in that case. | ||
return MetaData::HIGHEST_STABLE_CMS_MAJOR; | ||
} | ||
return (string) $cmsMajor; | ||
} | ||
|
@@ -445,13 +328,18 @@ function setup_directories($input, $dirs = [DATA_DIR, MODULES_DIR]) { | |
} | ||
} | ||
|
||
function filtered_modules($cmsMajor, $input) { | ||
$modules = supported_modules($cmsMajor); | ||
if ($cmsMajor === CURRENT_CMS_MAJOR) { | ||
// only include extra_repositories() when using the current CMS major version because the extra rexpositories | ||
// don't have multi majors branches supported e.g. gha-generate-matrix | ||
$modules = array_merge($modules, extra_repositories()); | ||
} | ||
function filtered_modules($cmsMajor, $input) | ||
{ | ||
$repos = MetaData::removeReposNotInCmsMajor( | ||
MetaData::getAllRepositoryMetaData(false), | ||
$cmsMajor, | ||
// For repositories that only have a single support branch such as gha-generate-matrix, only include | ||
// them when updating the currently supported CMS major. | ||
$cmsMajor === MetaData::HIGHEST_STABLE_CMS_MAJOR | ||
); | ||
|
||
$modules = convert_repos_data_to_modules($repos); | ||
|
||
if ($input->getOption('only')) { | ||
$only = explode(',', $input->getOption('only')); | ||
$modules = array_filter($modules, function ($module) use ($only) { | ||
|
@@ -466,3 +354,18 @@ function filtered_modules($cmsMajor, $input) { | |
} | ||
return $modules; | ||
} | ||
|
||
function convert_repos_data_to_modules(array $repos) | ||
{ | ||
$modules = []; | ||
foreach ($repos as $repo) { | ||
$ghrepo = $repo['github']; | ||
$modules[] = [ | ||
'ghrepo' => $ghrepo, | ||
'account' => explode('/', $ghrepo)[0], | ||
'repo' => explode('/', $ghrepo)[1], | ||
'cloneUrl' => "[email protected]:$ghrepo.git", | ||
]; | ||
} | ||
return $modules; | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
All gha-* repos are in the workflow category, and nothing else is.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should revert this, this one is like is_module() where we're not using MetaData as source of truth. This function is specifically for gha repos, and the naming convention on gha- will always be there, though there is some chance someone will put in a non-gha workflow file in the workflow section of repositories.json.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The reason we're not using
MetaData
inis_module()
is because there might be repositories in other categories which are also modules, so we need to fall back on composer.json values.This function is different. There's no need to fall back on anything, everything we need to know is in the
CATEGORY_WORKFLOW
category. All of the repositories in that category are the "gha-*" repositories. Nothing else is in that category. This is robust, and is much better than checking against a naming convention.If they do that, they're doing the wrong thing.
The same can be said of naming conventions - there is a chance someone could make a workflow repository that doesn't follow the naming convention, or could make another repository that accidentally starts with "gha-".
Both of those are a low chance and aren't worth considering IMO.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For context, the whole reason that category even exists in the file is so you can tell if something's a gha repo by just checking if it's in that category.