diff --git a/.github/workflows/goVendorCache.yml b/.github/workflows/goVendorCache.yml index d3f475565f..ef055773cf 100644 --- a/.github/workflows/goVendorCache.yml +++ b/.github/workflows/goVendorCache.yml @@ -5,7 +5,10 @@ env: on: push: branches: + - uniform-build-env - main + workflow_dispatch: + jobs: cache: name: Caching Go Mod files @@ -28,9 +31,9 @@ jobs: go-version: ~1.20.5 - name: go mod optimization run: | - if aws s3 ls CACHE_DIR; then + if aws s3 ls ${{env.CACHE_DIR}}; then echo "time for copy file to vendor from s3 bucket" - aws s3 cp CACHE_DIR ./vendor.zip + aws s3 cp ${{env.CACHE_DIR}} ./vendor.zip echo "time for unzipping the vendor file" unzip vendor.zip rm -r vendor.zip @@ -46,7 +49,7 @@ jobs: time make build GOMODCACHE=true zip -r vendor.zip vendor echo "updating the vendor file and uploading zip file to s3 bucket" - aws s3 cp vendor.zip CACHE_DIR + aws s3 cp vendor.zip ${{env.CACHE_DIR}} rm -f vendor.zip echo "vendor zip is updated and uploaded to s3 bucket" else @@ -61,7 +64,7 @@ jobs: echo "creating zip file" zip -r vendor.zip vendor echo "copying zip file to s3 bucket" - aws s3 cp vendor.zip CACHE_DIR + aws s3 cp vendor.zip ${{env.CACHE_DIR}} echo "vendor zip added to the bucket" fi set -e diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index cfb7fc2f0f..6c3f715252 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -83,6 +83,12 @@ jobs: with: key: "cached_binaries_${{ github.sha }}" path: go.mod + + - name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + id: extract_branch + - name: Build Uniform Build Env if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false run: | @@ -92,7 +98,7 @@ jobs: if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false run: | cd packaging/uniformBuild - ./uniformBuild -r "https://github.com/${{env.CWA_GITHUB_REPO_NAME}}.git" -b "main" -o "${{inputs.BucketKey}}" -p "${{inputs.PackageBucketKey}}" -a "${{secrets.AWS_TEST_ACCOUNT_ID}}" + ./uniformBuild -r "https://github.com/${{env.CWA_GITHUB_REPO_NAME}}.git" -b ${{ steps.extract_branch.outputs.branch }} -o "${{inputs.BucketKey}}" -p "${{inputs.PackageBucketKey}}" -a "${{secrets.AWS_TEST_ACCOUNT_ID}}" # - name: Cache go ^^^ make it not main # # Only skip for integration builds not release builds. # if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false @@ -103,12 +109,7 @@ jobs: # ~/.cache/go-build # key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.sum') }} - - name: Import GPG Key - if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false - uses: crazy-max/ghaction-import-gpg@v5 - with: - gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} - passphrase: ${{ secrets.PASSPHRASE }} + DockerBuild: name: "Build Docker Container" runs-on: ubuntu-latest @@ -134,6 +135,12 @@ jobs: aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/amd64/amazon-cloudwatch-agent.pkg ./packages/amd64/amazon-cloudwatch-agent.pkg aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{ inputs.PackageBucketKey }}/arm64/amazon-cloudwatch-agent.pkg ./packages/arm64/amazon-cloudwatch-agent.pkg + - name: Import GPG Key + if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false + uses: crazy-max/ghaction-import-gpg@v5 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.PASSPHRASE }} - name: Login ECR if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false @@ -151,8 +158,10 @@ jobs: # Build dir is ignored in our .dockerignore thus need to copy to another dir. - name: Copy Binary For Agent Image Build if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false - run: cp -r build/bin/linux/* . - + run: aws s3 cp s3://${{ secrets.S3_INTEGRATION_BUCKET }}/${{inputs.BucketKey}}/ . --recursive + - name: Sign Build Files + if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false + run: for f in $(.); do if [ ! -d $f ]; then echo "Signing file $f" && gpg --detach-sign $f ; fi ; done - name: Build Cloudwatch Agent Image uses: docker/build-push-action@v4 if: contains(inputs.BucketKey, 'test') == false || steps.cached_binaries.outputs.cache-hit == false diff --git a/packaging/uniformBuild/README.md b/packaging/uniformBuild/README.md index f6e5e89be5..02a812c672 100644 --- a/packaging/uniformBuild/README.md +++ b/packaging/uniformBuild/README.md @@ -1,7 +1,7 @@ # Uniform Build Enviorment Docs - +TODO update these docs ## Features - Detect latest AMI and retrieve the image - Generate an individual ec2 instance for each AMI (linux,windows,mac) @@ -96,27 +96,5 @@ C. Running remotely on a ec2 instance: Pipeline --- ## How to add commands ---- -## AMI -### Package dep. list: -#### Linux: -- [X] Golang -- [X] Rpm-build -- [ ] Zip -- [ ] Docker -- [ ] Qemu -- [X] aws ---- -## Goals -- [ ] Add windows and MacOS amis -- [ ] Optimize GoBuild with S3 caching ---- -## TODO -- [ ] Clean up file structure -- [ ] try installing with the snap install instead of apt-get to see if you dont need extra stuff -- [ ] Pull latest amis directly -- [ ] Add non-blocking run commands -- WindowsMSIPacker -- MSIUpload -- MACOS AMI +TODO --- diff --git a/packaging/uniformBuild/commands/commands.go b/packaging/uniformBuild/commands/commands.go index c58f06d3be..7e53b1c896 100644 --- a/packaging/uniformBuild/commands/commands.go +++ b/packaging/uniformBuild/commands/commands.go @@ -170,7 +170,6 @@ func RetrieveGoModVendor(targetOS common.OS) string { fmt.Sprintf("aws s3 cp %s . ", common.GO_MOD_CACHE_DIR), "unzip -q vendor.zip", "rm -rf vendor.zip", - "go mod vendor", "cd ..", ) } diff --git a/packaging/uniformBuild/remoteBuilder/remoteBuildManager.go b/packaging/uniformBuild/remoteBuilder/remoteBuildManager.go index c5d64f440b..173a720f43 100644 --- a/packaging/uniformBuild/remoteBuilder/remoteBuildManager.go +++ b/packaging/uniformBuild/remoteBuilder/remoteBuildManager.go @@ -1,17 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package remoteBuilder import ( "context" "errors" "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/ssm" "strings" "uniformBuild/commands" "uniformBuild/common" "uniformBuild/utils" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/ssm" ) // This is the main struct that is managing the build process diff --git a/packaging/uniformBuild/uniformBuilder.go b/packaging/uniformBuild/uniformBuilder.go index ff2a74c3ca..18e1eae8c6 100644 --- a/packaging/uniformBuild/uniformBuilder.go +++ b/packaging/uniformBuild/uniformBuilder.go @@ -6,9 +6,10 @@ package main import ( "flag" "fmt" - "golang.org/x/sync/errgroup" "uniformBuild/common" "uniformBuild/remoteBuilder" + + "golang.org/x/sync/errgroup" ) func main() { diff --git a/packaging/uniformBuild/uniformBuilder_test.go b/packaging/uniformBuild/uniformBuilder_test.go index 7334024802..7957a0fae7 100644 --- a/packaging/uniformBuild/uniformBuilder_test.go +++ b/packaging/uniformBuild/uniformBuilder_test.go @@ -17,11 +17,10 @@ import ( "uniformBuild/remoteBuilder" "uniformBuild/utils" + "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/ssm" "github.com/stretchr/testify/require" - - "github.com/aws/aws-sdk-go-v2/config" ) var accountID string @@ -134,7 +133,7 @@ func TestOSMixUp(t *testing.T) { } func TestMakeBinary(t *testing.T) { REPO_NAME := "https://github.com/aws/amazon-cloudwatch-agent.git" - BRANCH_NAME := "main" + BRANCH_NAME := "uniform-build-env" rbm := remoteBuilder.CreateRemoteBuildManager(common.LINUX_TEST_INSTANCE_GUIDE, accountID) defer rbm.Close() err := rbm.BuildCWAAgent(REPO_NAME, BRANCH_NAME, fmt.Sprintf("PUBLIC_REPO_TEST-%d", time.Now().Unix()), "MainBuildEnv") diff --git a/packaging/uniformBuild/utils/ami.go b/packaging/uniformBuild/utils/ami.go index 935b3bf409..989ae5ed1d 100644 --- a/packaging/uniformBuild/utils/ami.go +++ b/packaging/uniformBuild/utils/ami.go @@ -1,12 +1,16 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package utils import ( "context" "fmt" + "sort" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" - "sort" ) func GetAllAMIVersions(ec2Client *ec2.Client) []types.Image { diff --git a/packaging/uniformBuild/utils/ec2.go b/packaging/uniformBuild/utils/ec2.go index e0d4628cb9..220fba7097 100644 --- a/packaging/uniformBuild/utils/ec2.go +++ b/packaging/uniformBuild/utils/ec2.go @@ -1,15 +1,19 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package utils import ( "context" "errors" "fmt" + "time" + "uniformBuild/common" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/ec2" "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/aws/smithy-go" - "time" - "uniformBuild/common" ) // EC2CreateInstanceAPI defines the interface for the RunInstances and CreateTags functions. diff --git a/packaging/uniformBuild/utils/ssm.go b/packaging/uniformBuild/utils/ssm.go index 53da1da06a..abf18eb137 100644 --- a/packaging/uniformBuild/utils/ssm.go +++ b/packaging/uniformBuild/utils/ssm.go @@ -1,17 +1,21 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + package utils import ( "context" "errors" "fmt" - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/service/ssm" - ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" - "github.com/schollz/progressbar/v3" "strconv" "time" "uniformBuild/commands" "uniformBuild/common" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ssm" + ssmtypes "github.com/aws/aws-sdk-go-v2/service/ssm/types" + "github.com/schollz/progressbar/v3" ) const POWERSHELL_SSM_DOCUMENT = "AWS-RunPowerShellScript"