diff --git a/.github/actions/build_analytics/action.yml b/.github/actions/build_analytics/action.yml new file mode 100644 index 000000000000..2813fb4b62ac --- /dev/null +++ b/.github/actions/build_analytics/action.yml @@ -0,0 +1,51 @@ +name: build_analytics +description: Gather and send build analytics +inputs: + build_preset: + type: string + default: "" + build_target: + type: string + default: "" + secs: + type: string + default: "" + vars: + type: string + default: "" +runs: + using: "composite" + steps: + - name: Prepare s3cmd + uses: ./.github/actions/s3cmd + with: + s3_bucket: ${{ fromJSON( inputs.vars ).AWS_BUCKET }} + s3_endpoint: ${{ fromJSON( inputs.vars ).AWS_ENDPOINT }} + s3_key_id: ${{ fromJSON( inputs.secs ).AWS_KEY_ID }} + s3_key_secret: ${{ fromJSON( inputs.secs ).AWS_KEY_VALUE }} + folder_prefix: ya- + build_preset: ${{ inputs.build_preset }} + + - name: Run analytics tools + shell: bash + run: | + set -ex + # FIXME: target name may be not the same as dir name + export TARGET_NAME=`basename ${{ inputs.build_target }}` + export TARGET_DIR=${{ inputs.build_target }} + ./ya tool bloat --linker-map $TARGET_DIR/$TARGET_NAME.map.lld --input $TARGET_DIR/$TARGET_NAME --save-html ya_bloat_html + ./ydb/ci/build_bloat/main.py --build-dir . --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact + + - name: Upload results + shell: bash + run: | + set -ex + s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "ya_bloat_html/" "$S3_BUCKET_PATH/ya_bloat_html/" + echo "ya_bloat=$S3_URL_PREFIX/ya_bloat_html/index.html" >> $GITHUB_OUTPUT + + s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_cpp_impact/" "$S3_BUCKET_PATH/html_cpp_impact/" + echo "cpp_impact=$S3_URL_PREFIX/html_cpp_impact/index.html" >> $GITHUB_OUTPUT + + s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_headers_impact/" "$S3_BUCKET_PATH/html_headers_impact/" + echo "headers_impact=$S3_URL_PREFIX/html_headers_impact/index.html" >> $GITHUB_OUTPUT + diff --git a/.github/workflows/build_analytics.yml b/.github/workflows/build_analytics.yml index 0cf32d2ef3f6..95fbc54cbbfc 100644 --- a/.github/workflows/build_analytics.yml +++ b/.github/workflows/build_analytics.yml @@ -29,7 +29,7 @@ on: jobs: main: - name: Build and test ydb/tests/acceptance + name: Build and gather build analytics runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ] steps: - name: Checkout @@ -55,3 +55,14 @@ jobs: secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }} vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}', vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }} + + + - name: Gather and send build analytics + uses: ./.github/actions/build_analytics + with: + build_preset: ${{ inputs.build_preset || 'relwithdebinfo'}} + build_target: ${{ inputs.build_target || 'ydb/apps/ydbd'}} + secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}', + secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }} + vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}', + vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }} diff --git a/ydb/ci/build_bloat/main.py b/ydb/ci/build_bloat/main.py index fa8f606933aa..cc7a81906cb7 100755 --- a/ydb/ci/build_bloat/main.py +++ b/ydb/ci/build_bloat/main.py @@ -299,11 +299,14 @@ def main(): if args.html_dir_cpp: actions.append(("header build time impact", generate_header_bloat, args.html_dir_headers)) + current_script_dir = os.path.dirname(os.path.realpath(__file__)) + html_dir = os.path.join(current_script_dir, "html") + for description, fn, output_path in actions: print("Performing '{}'".format(description)) tree = fn(args.build_dir) - shutil.copytree("html", output_path, dirs_exist_ok=True) + shutil.copytree(html_dir, output_path, dirs_exist_ok=True) with open(os.path.join(output_path, "bloat.json"), "w") as f: f.write("var kTree = ") json.dump(tree, f, indent=4)