Build analytics (#4896)

.github/actions/build_analytics/action.yml

@@ -0,0 +1,51 @@
+name: build_analytics
+description: Gather and send build analytics 
+inputs:
+  build_preset:
+    type: string
+    default: ""
+  build_target:
+    type: string
+    default: ""
+  secs:
+    type: string
+    default: ""
+  vars:
+    type: string
+    default: ""
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare s3cmd
+      uses: ./.github/actions/s3cmd
+      with:
+        s3_bucket: ${{ fromJSON( inputs.vars ).AWS_BUCKET }}
+        s3_endpoint: ${{ fromJSON( inputs.vars ).AWS_ENDPOINT }}
+        s3_key_id: ${{ fromJSON( inputs.secs ).AWS_KEY_ID }}
+        s3_key_secret: ${{ fromJSON( inputs.secs ).AWS_KEY_VALUE }}
+        folder_prefix: ya-
+        build_preset: ${{ inputs.build_preset }}
+
+    - name: Run analytics tools
+      shell: bash
+      run: |
+        set -ex
+        # FIXME: target name may be not the same as dir name
+        export TARGET_NAME=`basename ${{ inputs.build_target }}`
+        export TARGET_DIR=${{ inputs.build_target }}
+        ./ya tool bloat --linker-map $TARGET_DIR/$TARGET_NAME.map.lld --input $TARGET_DIR/$TARGET_NAME --save-html ya_bloat_html
+        ./ydb/ci/build_bloat/main.py --build-dir . --html-dir-cpp html_cpp_impact --html-dir-headers html_headers_impact
+
+    - name: Upload results
+      shell: bash
+      run: |
+        set -ex
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "ya_bloat_html/" "$S3_BUCKET_PATH/ya_bloat_html/"
+        echo "ya_bloat=$S3_URL_PREFIX/ya_bloat_html/index.html" >> $GITHUB_OUTPUT
+
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_cpp_impact/" "$S3_BUCKET_PATH/html_cpp_impact/"
+        echo "cpp_impact=$S3_URL_PREFIX/html_cpp_impact/index.html" >> $GITHUB_OUTPUT
+
+        s3cmd sync -r --acl-public --stats --no-progress --no-mime-magic --guess-mime-type --no-check-md5 "html_headers_impact/" "$S3_BUCKET_PATH/html_headers_impact/"
+        echo "headers_impact=$S3_URL_PREFIX/html_headers_impact/index.html" >> $GITHUB_OUTPUT
+

.github/workflows/build_analytics.yml

@@ -29,7 +29,7 @@ on:
 
 jobs:
   main:
-    name: Build and test ydb/tests/acceptance
+    name: Build and gather build analytics
     runs-on: [ self-hosted, "${{ inputs.runner_label || 'auto-provisioned' }}", "${{ format('build-preset-{0}', inputs.build_preset || 'relwithdebinfo') }}" ]
     steps:
     - name: Checkout
@@ -55,3 +55,14 @@ jobs:
           secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
         vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
           vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}
+
+
+    - name: Gather and send build analytics
+      uses: ./.github/actions/build_analytics
+      with:
+        build_preset: ${{ inputs.build_preset || 'relwithdebinfo'}}
+        build_target: ${{ inputs.build_target || 'ydb/apps/ydbd'}}
+        secs: ${{ format('{{"TESTMO_TOKEN":"{0}","AWS_KEY_ID":"{1}","AWS_KEY_VALUE":"{2}","REMOTE_CACHE_USERNAME":"{3}","REMOTE_CACHE_PASSWORD":"{4}"}}', 
+          secrets.TESTMO_TOKEN, secrets.AWS_KEY_ID, secrets.AWS_KEY_VALUE, secrets.REMOTE_CACHE_USERNAME, secrets.REMOTE_CACHE_PASSWORD ) }}
+        vars: ${{ format('{{"AWS_BUCKET":"{0}","AWS_ENDPOINT":"{1}","REMOTE_CACHE_URL":"{2}","TESTMO_URL":"{3}","TESTMO_PROJECT_ID":"{4}"}}',
+          vars.AWS_BUCKET, vars.AWS_ENDPOINT, vars.REMOTE_CACHE_URL_YA, vars.TESTMO_URL, vars.TESTMO_PROJECT_ID ) }}

ydb/ci/build_bloat/main.py

@@ -299,11 +299,14 @@ def main():
     if args.html_dir_cpp:
         actions.append(("header build time impact", generate_header_bloat, args.html_dir_headers))
 
+    current_script_dir = os.path.dirname(os.path.realpath(__file__))
+    html_dir = os.path.join(current_script_dir, "html")
+
     for description, fn, output_path in actions:
         print("Performing '{}'".format(description))
         tree = fn(args.build_dir)
 
-        shutil.copytree("html", output_path, dirs_exist_ok=True)
+        shutil.copytree(html_dir, output_path, dirs_exist_ok=True)
         with open(os.path.join(output_path, "bloat.json"), "w") as f:
             f.write("var kTree = ")
             json.dump(tree, f, indent=4)