From fe52d510b58d67b39748ef27721e11bcddc04e12 Mon Sep 17 00:00:00 2001 From: Jonathan Lelievre Date: Fri, 31 May 2024 10:02:50 +0200 Subject: [PATCH 1/4] Optimise composer install in multiple workflows --- .docker/docker_run_git.sh | 2 +- .github/workflows/create_diff_for_autoupgrade_pr_label.yml | 2 +- .github/workflows/cron_js_routing.yml | 2 +- .github/workflows/cron_php_update_modules.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.docker/docker_run_git.sh b/.docker/docker_run_git.sh index a3cebf729c830..450cd3af6908b 100755 --- a/.docker/docker_run_git.sh +++ b/.docker/docker_run_git.sh @@ -56,7 +56,7 @@ if [ "${DISABLE_MAKE}" != "1" ]; then runuser -g www-data -u www-data -- php -r "copy('https://getcomposer.org/installer', '/tmp/composer-setup.php');" && php /tmp/composer-setup.php --no-ansi --install-dir=/usr/local/bin --filename=composer && rm -rf /tmp/composer-setup.php echo "\n* Running composer ..."; - COMPOSER_PROCESS_TIMEOUT=600 runuser -g www-data -u www-data -- /usr/local/bin/composer install --no-interaction + COMPOSER_PROCESS_TIMEOUT=600 runuser -g www-data -u www-data -- /usr/local/bin/composer install --ansi --prefer-dist --no-interaction --no-progress if [ $? -ne 0]; then echo Composer install failed exit 1 diff --git a/.github/workflows/create_diff_for_autoupgrade_pr_label.yml b/.github/workflows/create_diff_for_autoupgrade_pr_label.yml index c862d1c927406..1665881e76cd6 100644 --- a/.github/workflows/create_diff_for_autoupgrade_pr_label.yml +++ b/.github/workflows/create_diff_for_autoupgrade_pr_label.yml @@ -30,7 +30,7 @@ jobs: run: docker compose build prestashop-git - name: Fetch dependencies - run: docker compose run --rm prestashop-git composer install --ansi --prefer-dist --no-interaction --no-progress + run: COMPOSER_PROCESS_TIMEOUT=600 docker compose run --rm prestashop-git composer install --ansi --prefer-dist --no-interaction --no-progress --no-dev - name: Waiting for database run: ./.docker/wait-for-it.sh --timeout=60 --strict localhost:3306 diff --git a/.github/workflows/cron_js_routing.yml b/.github/workflows/cron_js_routing.yml index 78059e639768d..5d970fa62a38f 100644 --- a/.github/workflows/cron_js_routing.yml +++ b/.github/workflows/cron_js_routing.yml @@ -84,7 +84,7 @@ jobs: ${{ runner.os }}-composer- - name: Install Composer dependencies - run: COMPOSER_PROCESS_TIMEOUT=600 composer install --prefer-dist + run: COMPOSER_PROCESS_TIMEOUT=600 composer install --ansi --prefer-dist --no-interaction --no-progress - name: Build assets run: make assets diff --git a/.github/workflows/cron_php_update_modules.yml b/.github/workflows/cron_php_update_modules.yml index a8c79f0dcd750..527d8bf1dfa6d 100644 --- a/.github/workflows/cron_php_update_modules.yml +++ b/.github/workflows/cron_php_update_modules.yml @@ -46,7 +46,7 @@ jobs: - name: Install Composer dependencies run: | composer self-update --stable - COMPOSER_PROCESS_TIMEOUT=600 composer install --prefer-dist + COMPOSER_PROCESS_TIMEOUT=600 composer install --ansi --prefer-dist --no-interaction --no-progress - name: Install NPM dependencies working-directory: ./tests/UI From a6880ef9bac3d2384b5a715f3de6e45f4af7d63a Mon Sep 17 00:00:00 2001 From: Jonathan Lelievre Date: Fri, 31 May 2024 10:02:26 +0200 Subject: [PATCH 2/4] Improve wait for it script for mysql check --- .docker/wait-for-it.sh | 42 +++++++------------ .../create_diff_for_autoupgrade_pr_label.yml | 2 +- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/.docker/wait-for-it.sh b/.docker/wait-for-it.sh index ba2e73cfddc4f..36ee8af4a5ae7 100755 --- a/.docker/wait-for-it.sh +++ b/.docker/wait-for-it.sh @@ -32,16 +32,25 @@ wait_for() start_ts=$(date +%s) while : do - if [[ $ISBUSY -eq 1 ]]; then + # Special check for mysql + if [[ "$PORT" = "3306" ]]; then + echoerr "$CMD_NAME: Fetching status from docker mysql" + mysql -h$HOST -uroot -pprestashop -e "status" + result=$? + elif [[ $ISBUSY -eq 1 ]]; then nc -z $HOST $PORT result=$? else (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 result=$? fi + end_ts=$(date +%s) + ELAPSED_TIME=$((end_ts - start_ts)) if [[ $result -eq 0 ]]; then - end_ts=$(date +%s) - echoerr "$CMD_NAME: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + echoerr "$CMD_NAME: $HOST:$PORT is available after $ELAPSED_TIME seconds" + break + elif [[ $TIMEOUT -gt 0 && $ELAPSED_TIME -gt $TIMEOUT ]]; then + echoerr "$CMD_NAME: $HOST:$PORT is still not available after $ELAPSED_TIME seconds" break fi sleep 1 @@ -49,24 +58,6 @@ wait_for() return $result } -wait_for_wrapper() -{ - # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 - if [[ $QUIET -eq 1 ]]; then - timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - else - timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & - fi - PID=$! - trap "kill -INT -$PID" INT - wait $PID - RESULT=$? - if [[ $RESULT -ne 0 ]]; then - echoerr "$CMD_NAME: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" - fi - return $RESULT -} - # process arguments while [[ $# -gt 0 ]] do @@ -157,13 +148,8 @@ if [[ $CHILD -gt 0 ]]; then RESULT=$? exit $RESULT else - if [[ $TIMEOUT -gt 0 ]]; then - wait_for_wrapper - RESULT=$? - else - wait_for - RESULT=$? - fi + wait_for + RESULT=$? fi if [[ $CLI != "" ]]; then diff --git a/.github/workflows/create_diff_for_autoupgrade_pr_label.yml b/.github/workflows/create_diff_for_autoupgrade_pr_label.yml index 1665881e76cd6..f2d248eccfc62 100644 --- a/.github/workflows/create_diff_for_autoupgrade_pr_label.yml +++ b/.github/workflows/create_diff_for_autoupgrade_pr_label.yml @@ -33,7 +33,7 @@ jobs: run: COMPOSER_PROCESS_TIMEOUT=600 docker compose run --rm prestashop-git composer install --ansi --prefer-dist --no-interaction --no-progress --no-dev - name: Waiting for database - run: ./.docker/wait-for-it.sh --timeout=60 --strict localhost:3306 + run: docker compose run --rm prestashop-git /tmp/wait-for-it.sh --timeout=60 --strict mysql:3306 - name: Create base database run: docker compose run --rm mysql mysql -hmysql -uroot -pprestashop -e "CREATE DATABASE presta_${{ matrix.branch }};" From f352ea7f456a668c07594cfc12976598ded64949 Mon Sep 17 00:00:00 2001 From: Jonathan Lelievre Date: Fri, 31 May 2024 10:57:42 +0200 Subject: [PATCH 3/4] Wait for composer install to be finished in wait for assets script --- .docker/docker_run_git.sh | 8 ++++---- config/config.inc.php | 2 +- install-dev/init.php | 2 +- tools/assets/wait-build.sh | 22 ++++++++++++++++++++++ 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.docker/docker_run_git.sh b/.docker/docker_run_git.sh index 450cd3af6908b..0d6218fc22da6 100755 --- a/.docker/docker_run_git.sh +++ b/.docker/docker_run_git.sh @@ -54,13 +54,13 @@ if [ "${DISABLE_MAKE}" != "1" ]; then mkdir -p /var/www/.composer chown -R www-data:www-data /var/www/.composer runuser -g www-data -u www-data -- php -r "copy('https://getcomposer.org/installer', '/tmp/composer-setup.php');" && php /tmp/composer-setup.php --no-ansi --install-dir=/usr/local/bin --filename=composer && rm -rf /tmp/composer-setup.php + if [ ! -f /usr/local/bin/composer ]; then + echo Composer installation failed + exit 1 + fi echo "\n* Running composer ..."; COMPOSER_PROCESS_TIMEOUT=600 runuser -g www-data -u www-data -- /usr/local/bin/composer install --ansi --prefer-dist --no-interaction --no-progress - if [ $? -ne 0]; then - echo Composer install failed - exit 1 - fi echo "\n* Build assets ..."; runuser -g www-data -u www-data -- /usr/bin/make assets diff --git a/config/config.inc.php b/config/config.inc.php index 3994f806ce45c..bde1f8d3528c8 100644 --- a/config/config.inc.php +++ b/config/config.inc.php @@ -46,7 +46,7 @@ /* in dev mode - check if composer was executed */ if (is_dir(_PS_ROOT_DIR_ . DIRECTORY_SEPARATOR . 'admin-dev') && (!is_dir(_PS_ROOT_DIR_ . DIRECTORY_SEPARATOR . 'vendor') || !file_exists(_PS_ROOT_DIR_ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'autoload.php'))) { - die('Error : please install composer. Then run "php composer.phar install"'); + die('Config check Error : please install composer. Then run "php composer.phar install"'); } /* No settings file? goto installer... */ diff --git a/install-dev/init.php b/install-dev/init.php index e20f37652eecc..7a2e69833665b 100644 --- a/install-dev/init.php +++ b/install-dev/init.php @@ -83,7 +83,7 @@ /* in dev mode - check if composer was executed */ if ((!is_dir(_PS_CORE_DIR_ . DIRECTORY_SEPARATOR . 'vendor') || !file_exists(_PS_CORE_DIR_ . DIRECTORY_SEPARATOR . 'vendor' . DIRECTORY_SEPARATOR . 'autoload.php'))) { - die('Error : please install composer. Then run "php composer.phar install"'); + die('Init Install Error : please install composer. Then run "php composer.phar install"'); } require_once _PS_CORE_DIR_ . '/config/defines.inc.php'; diff --git a/tools/assets/wait-build.sh b/tools/assets/wait-build.sh index d49332bf2de4b..48ec2af14a9f5 100755 --- a/tools/assets/wait-build.sh +++ b/tools/assets/wait-build.sh @@ -14,6 +14,8 @@ if [[ ! -d $ADMIN_DIR ]]; then return 1 fi +neededFiles="" +buildLocks="" if test $# -gt 0; then case $1 in admin-default) @@ -32,9 +34,14 @@ if test $# -gt 0; then echo ">>> Waiting for classic theme assets..." buildLocks="$PROJECT_PATH/themes/classic/_dev/buildLock" ;; + composer) + echo ">>> Waiting for composer install..." + neededFiles="$PROJECT_PATH/vendor/autoload.php" + ;; all) echo ">>> Waiting for all assets..." buildLocks="$ADMIN_DIR/themes/default/buildLock $ADMIN_DIR/themes/new-theme/buildLock $PROJECT_PATH/themes/classic/_dev/buildLock $PROJECT_PATH/themes/buildLock" + neededFiles="$PROJECT_PATH/vendor/autoload.php" ;; *) echo "Unknown asset to wait $1" @@ -44,9 +51,11 @@ if test $# -gt 0; then else echo ">>> Waiting for all assets..." buildLocks="$ADMIN_DIR/themes/default/buildLock $ADMIN_DIR/themes/new-theme/buildLock $PROJECT_PATH/themes/classic/_dev/buildLock $PROJECT_PATH/themes/buildLock" + neededFiles="$PROJECT_PATH/vendor/autoload.php" fi echo Checking for all these lock files $buildLocks +# Wait for lock files to disappear for lockFile in $buildLocks; do if [ -f $lockFile ]; then echo Wait for $lockFile to be removed @@ -58,3 +67,16 @@ for lockFile in $buildLocks; do fi echo $lockFile is no longer present done + +# Wait for needed files to be present +for neededFile in $neededFiles; do + if [ ! -f $neededFile ]; then + echo Wait for $neededFile to be generated + sleep 1 + while [ ! -f $neededFile ]; do + echo $neededFile still not present wait a bit more + sleep 1 + done + fi + echo $neededFile is present +done From 440941eb43e81dcf0bf405a2855e51996a134620 Mon Sep 17 00:00:00 2001 From: Jonathan Lelievre Date: Fri, 31 May 2024 15:56:40 +0200 Subject: [PATCH 4/4] Export composer env variable in docker init --- .docker/docker_run_git.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.docker/docker_run_git.sh b/.docker/docker_run_git.sh index 0d6218fc22da6..90f88fc80a5ff 100755 --- a/.docker/docker_run_git.sh +++ b/.docker/docker_run_git.sh @@ -60,7 +60,8 @@ if [ "${DISABLE_MAKE}" != "1" ]; then fi echo "\n* Running composer ..."; - COMPOSER_PROCESS_TIMEOUT=600 runuser -g www-data -u www-data -- /usr/local/bin/composer install --ansi --prefer-dist --no-interaction --no-progress + export COMPOSER_PROCESS_TIMEOUT=600 + runuser -g www-data -u www-data -- /usr/local/bin/composer install --ansi --prefer-dist --no-interaction --no-progress echo "\n* Build assets ..."; runuser -g www-data -u www-data -- /usr/bin/make assets