diff --git a/.taskcluster.yml b/.taskcluster.yml index 5565131e2b7..0203103d524 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,268 +1,486 @@ version: 1 +reporting: checks-v1 policy: pullRequests: collaborators tasks: -############################################################################### -# Task: Pull requests -# -# Triggered whenever a pull request is opened or updated. -# -# - Build the app (all flavors) -# - Run unit tests -# - Run code quality tools (spotbugs, lint, checkstyle etc.) -############################################################################### - - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' - then: - created: {$fromNow: ''} - deadline: {$fromNow: '2 hours'} - schedulerId: taskcluster-github - provisionerId: mobile-1 - workerType: b-linux # We don't use decision because PRs just schedule a single task - scopes: [] - routes: [] - payload: - maxRunTime: 7200 - image: mozillamobile/focus-android:1.7 - command: - - /bin/bash - - --login - - -cx - - >- - git fetch ${event.pull_request.head.repo.clone_url} ${event.pull_request.head.ref} - && git submodule update --init - && git config advice.detachedHead false - && git checkout ${event.pull_request.head.sha} - && echo "--" > .adjust_token - && ./gradlew --no-daemon clean assembleFocusDebug assembleKlarNightly assembleRelease detekt ktlint lintFocusDebug lintKlarNightly assembleFocusDebugAndroidTest testFocusDebugUnitTest testKlarNightlyUnitTest - && pip install "compare-locales>=5.0.2,<6.0" - && compare-locales --validate l10n.toml . - artifacts: - public: - type: directory - path: /opt/focus-android/app/build/reports - expires: {$fromNow: '1 week'} - metadata: - name: Focus for Android - Build - Pull Request - description: Building Focus for Android (via Gradle) - triggered by a pull request. - owner: ${event.pull_request.user.login}@users.noreply.github.com - source: ${event.repository.url} -############################################################################### -# Task: Builds on branches of the main repo -# -# Triggered whenever something is pushed/merged to the mozilla-mobile repo. -# -# Creates the following task pipeline: -# -# Build -+--> Unit tests ----+- UI tests -# \-> Code quality -/ -# -############################################################################### - - $if: 'tasks_for == "github-push"' - then: - $if: 'event.repository.fork == false' - then: - taskId: {$eval: as_slugid("decision_task")} - # The next line won't be needed anymore after https://github.com/taskcluster/taskcluster-github/pull/273 - taskGroupId: {$eval: as_slugid("decision_task")} - created: {$fromNow: ''} - deadline: {$fromNow: '2 hours'} - schedulerId: taskcluster-github - provisionerId: mobile-3 - workerType: decision - routes: - - 'notify.irc-channel.#android-ci.on-any' - scopes: + - $let: + taskgraph: + branch: taskgraph + revision: b975005a3219b65ad0dd073263a19ec6c94d8b73 + trustDomain: mobile + in: $let: - short_head_branch: - $if: 'event.ref[:11] == "refs/heads/"' - then: {$eval: 'event.ref[11:]'} - else: ${event.ref} + # Github events have this stuff in different places... + ownerEmail: + $if: 'tasks_for in ["cron", "action"]' + then: '${tasks_for}@noreply.mozilla.org' + else: + $if: 'tasks_for == "github-push"' + then: '${event.pusher.name}@users.noreply.github.com' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.user.login}@users.noreply.github.com' + else: + $if: 'tasks_for == "github-release"' + then: '${event.sender.login}@users.noreply.github.com' + baseRepoUrl: + $if: 'tasks_for in ["github-push", "github-release"]' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.base.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + repoUrl: + $if: 'tasks_for in ["github-push", "github-release"]' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + project: + $if: 'tasks_for in ["github-push", "github-release"]' + then: '${event.repository.name}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.name}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.project}' + head_branch: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.ref} + else: + $if: 'tasks_for == "github-push"' + then: ${event.ref} + else: + $if: 'tasks_for == "github-release"' + then: '${event.release.target_commitish}' + else: + $if: 'tasks_for in ["action", "cron"]' + then: '${push.branch}' + head_sha: + $if: 'tasks_for == "github-push"' + then: '${event.after}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.sha}' + else: + $if: 'tasks_for == "github-release"' + then: '${event.release.tag_name}' + else: + $if: 'tasks_for in ["action", "cron"]' + then: '${push.revision}' + + + head_tag: + $if: 'tasks_for == "github-release"' + then: '${event.release.tag_name}' + else: '' + ownTaskId: + $if: '"github" in tasks_for' + then: {$eval: as_slugid("decision_task")} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${ownTaskId}' + pullRequestAction: + $if: 'tasks_for == "github-pull-request"' + then: ${event.action} + else: 'UNDEFINED' + releaseAction: + $if: 'tasks_for == "github-release"' + then: ${event.action} + else: 'UNDEFINED' in: - - 'assume:repo:github.com/mozilla-mobile/focus-android:branch:${short_head_branch}' - payload: - maxRunTime: 7200 - image: mozillamobile/focus-android:1.7 - features: - taskclusterProxy: true - env: - TASK_ID: {$eval: as_slugid("decision_task")} - MOBILE_HEAD_REPOSITORY: ${event.repository.clone_url} - MOBILE_HEAD_BRANCH: ${event.ref} - MOBILE_HEAD_REV: ${event.after} - command: - - /bin/bash - - --login - - -cx - - >- - git fetch origin - && git submodule update --init - && git config advice.detachedHead false - && git checkout ${event.after} - && python tools/taskcluster/schedule-main-build.py - artifacts: - public: - type: directory - path: /opt/focus-android/test_artifacts - expires: {$fromNow: '1 week'} - metadata: - name: (Focus for Android) Schedule tasks - description: Scheduling tasks for main branch push - owner: ${event.pusher.name}@users.noreply.github.com - source: ${event.repository.url} -############################################################################### -# Task: Release builds -# -# Triggered when a new tag or release is published (in any branch) -# -# - Builds release versions of Focus and Klar -# - Signs the builds with the release key -# - Uploads the builds to the "alpha" track on Google Play -############################################################################### - - $if: 'tasks_for == "github-release" && event["action"] == "published"' - then: - $let: - decision_task_id: {$eval: as_slugid("decision_task")} - expires_in: {$fromNow: '1 year'} - repository: https://github.com/mozilla-mobile/focus-android - scheduler_id: taskcluster-github - in: - taskId: ${decision_task_id} - taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust - created: {$fromNow: ''} - deadline: {$fromNow: '2 hours'} - expires: ${expires_in} - schedulerId: ${scheduler_id} # Must be explicit because of Chain of Trust - provisionerId: mobile-3 - workerType: decision - requires: all-completed # Must be explicit because of Chain of Trust - priority: highest - retries: 5 - scopes: - - assume:repo:github.com/mozilla-mobile/focus-android:release - routes: - - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust - payload: - maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks - image: mozillamobile/focus-android:1.7 - features: - taskclusterProxy: true - chainOfTrust: true - env: - TASK_ID: ${decision_task_id} - SCHEDULER_ID: ${scheduler_id} - MOBILE_HEAD_REPOSITORY: ${repository} - MOBILE_HEAD_BRANCH: ${event.release.target_commitish} - MOBILE_HEAD_REV: ${event.release.tag_name} - MOBILE_TRIGGERED_BY: ${event.sender.login} - command: - - /bin/bash - - --login - - -cx - - >- - git fetch origin --tags - && git submodule update --init - && git config advice.detachedHead false - && git checkout ${event.release.tag_name} - && python tools/taskcluster/release.py \ - --tag ${event.release.tag_name} \ - --channel alpha \ - --commit \ - --output /opt/focus-android/app/build/outputs/apk \ - --apk focus/release/app-focus-armeabi-v7a-release-unsigned.apk \ - --apk focus/release/app-focus-arm64-v8a-release-unsigned.apk \ - --apk klar/release/app-klar-armeabi-v7a-release-unsigned.apk \ - --apk klar/release/app-klar-arm64-v8a-release-unsigned.apk - artifacts: - public/task-graph.json: - type: file - path: /opt/focus-android/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /opt/focus-android/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /opt/focus-android/parameters.yml - expires: ${expires_in} - extra: - tasks_for: ${tasks_for} - metadata: - name: (Focus for Android) Decision task (${event.release.tag_name}) - description: Scheduling tasks for releasing Focus/Klar - owner: ${event.sender.login}@users.noreply.github.com - source: ${repository}/raw/${event.release.tag_name}/.taskcluster.yml -# Nightly builds - - $if: 'tasks_for == "cron"' - then: - $let: - decision_task_id: {$eval: as_slugid("decision_task")} - expires_in: {$fromNow: '1 year'} - repository: https://github.com/mozilla-mobile/focus-android - scheduler_id: taskcluster-github - in: - taskId: ${decision_task_id} - taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust - created: {$fromNow: ''} - deadline: {$fromNow: '2 hours'} - expires: ${expires_in} - schedulerId: ${scheduler_id} # Must be explicit because of Chain of Trust - provisionerId: mobile-3 - workerType: decision - requires: all-completed # Must be explicit because of Chain of Trust - priority: medium - retries: 5 - scopes: - - assume:repo:github.com/mozilla-mobile/focus-android:cron:nightly - routes: - - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust - payload: - maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks - image: mozillamobile/focus-android:1.7 - features: - taskclusterProxy: true - chainOfTrust: true - env: - TASK_ID: ${decision_task_id} - SCHEDULER_ID: ${scheduler_id} - MOBILE_HEAD_REPOSITORY: ${repository} - MOBILE_HEAD_BRANCH: ${event.release.target_commitish} - MOBILE_HEAD_REV: ${event.release.tag_name} - MOBILE_TRIGGERED_BY: ${event.sender.login} - command: - - /bin/bash - - --login - - -cx - - >- - git fetch origin - && git reset --hard origin/main - && git submodule update --init - && python tools/taskcluster/release.py \ - --channel nightly \ - --commit \ - --output /opt/focus-android/app/build/outputs/apk \ - --apk focus/release/app-focus-armeabi-v7a-release-unsigned.apk \ - --apk focus/release/app-focus-arm64-v8a-release-unsigned.apk \ - --apk klar/release/app-klar-armeabi-v7a-release-unsigned.apk \ - --apk klar/release/app-klar-arm64-v8a-release-unsigned.apk - artifacts: - public/task-graph.json: - type: file - path: /opt/focus-android/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /opt/focus-android/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /opt/focus-android/parameters.yml - expires: ${expires_in} - extra: - cron: {$json: {$eval: 'cron'}} - tasks_for: ${tasks_for} - metadata: - name: (Focus for Android) Focus/Klar Nightly Builds (Public) - description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) - owner: ${event.sender.login}@users.noreply.github.com - source: ${repository}/raw/${event.release.tag_name}/.taskcluster.yml + $mergeDeep: + # Remove this block when releases have been moved to taskgraph + - $if: 'tasks_for == "github-release" && event["action"] == "published"' + then: + $let: + decision_task_id: {$eval: as_slugid("decision_task")} + expires_in: {$fromNow: '1 year'} + repository: https://github.com/mozilla-mobile/focus-android + scheduler_id: taskcluster-github + in: + taskId: ${decision_task_id} + taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust + created: {$fromNow: ''} + deadline: {$fromNow: '2 hours'} + expires: ${expires_in} + schedulerId: ${scheduler_id} # Must be explicit because of Chain of Trust + provisionerId: mobile-3 + workerType: decision + requires: all-completed # Must be explicit because of Chain of Trust + priority: highest + retries: 5 + scopes: + - assume:repo:github.com/mozilla-mobile/focus-android:release + routes: + - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust + payload: + maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks + image: mozillamobile/focus-android:1.7 + features: + taskclusterProxy: true + chainOfTrust: true + env: + TASK_ID: ${decision_task_id} + SCHEDULER_ID: ${scheduler_id} + MOBILE_HEAD_REPOSITORY: ${repository} + MOBILE_HEAD_BRANCH: ${event.release.target_commitish} + MOBILE_HEAD_REV: ${event.release.tag_name} + MOBILE_TRIGGERED_BY: ${event.sender.login} + command: + - /bin/bash + - --login + - -cx + - >- + git fetch origin --tags + && git submodule update --init + && git config advice.detachedHead false + && git checkout ${event.release.tag_name} + && python tools/taskcluster/release.py \ + --tag ${event.release.tag_name} \ + --channel alpha \ + --commit \ + --output /opt/focus-android/app/build/outputs/apk \ + --apk focus/release/app-focus-armeabi-v7a-release-unsigned.apk \ + --apk focus/release/app-focus-arm64-v8a-release-unsigned.apk \ + --apk klar/release/app-klar-armeabi-v7a-release-unsigned.apk \ + --apk klar/release/app-klar-arm64-v8a-release-unsigned.apk + artifacts: + public/task-graph.json: + type: file + path: /opt/focus-android/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /opt/focus-android/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /opt/focus-android/parameters.yml + expires: ${expires_in} + extra: + tasks_for: ${tasks_for} + metadata: + name: (Focus for Android) Decision task (${event.release.tag_name}) + description: Scheduling tasks for releasing Focus/Klar + owner: ${event.sender.login}@users.noreply.github.com + source: ${repository}/raw/${event.release.tag_name}/.taskcluster.yml + # Remove this block when nightlies have been moved to taskgraph + - $if: 'tasks_for == "cron"' + then: + $let: + decision_task_id: {$eval: as_slugid("decision_task")} + expires_in: {$fromNow: '1 year'} + repository: https://github.com/mozilla-mobile/focus-android + scheduler_id: taskcluster-github + in: + taskId: ${decision_task_id} + taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust + created: {$fromNow: ''} + deadline: {$fromNow: '2 hours'} + expires: ${expires_in} + schedulerId: ${scheduler_id} # Must be explicit because of Chain of Trust + provisionerId: mobile-3 + workerType: decision + requires: all-completed # Must be explicit because of Chain of Trust + priority: medium + retries: 5 + scopes: + - assume:repo:github.com/mozilla-mobile/focus-android:cron:nightly + routes: + - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust + payload: + maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks + image: mozillamobile/focus-android:1.7 + features: + taskclusterProxy: true + chainOfTrust: true + env: + TASK_ID: ${decision_task_id} + SCHEDULER_ID: ${scheduler_id} + MOBILE_HEAD_REPOSITORY: ${repository} + MOBILE_HEAD_BRANCH: ${event.release.target_commitish} + MOBILE_HEAD_REV: ${event.release.tag_name} + MOBILE_TRIGGERED_BY: ${event.sender.login} + command: + - /bin/bash + - --login + - -cx + - >- + git fetch origin + && git reset --hard origin/main + && git submodule update --init + && python tools/taskcluster/release.py \ + --channel nightly \ + --commit \ + --output /opt/focus-android/app/build/outputs/apk \ + --apk focus/release/app-focus-armeabi-v7a-release-unsigned.apk \ + --apk focus/release/app-focus-arm64-v8a-release-unsigned.apk \ + --apk klar/release/app-klar-armeabi-v7a-release-unsigned.apk \ + --apk klar/release/app-klar-arm64-v8a-release-unsigned.apk + artifacts: + public/task-graph.json: + type: file + path: /opt/focus-android/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /opt/focus-android/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /opt/focus-android/parameters.yml + expires: ${expires_in} + extra: + cron: {$json: {$eval: 'cron'}} + tasks_for: ${tasks_for} + metadata: + name: (Focus for Android) Focus/Klar Nightly Builds (Public) + description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) + owner: ${event.sender.login}@users.noreply.github.com + source: ${repository}/raw/${event.release.tag_name}/.taskcluster.yml + # cron commented out until we port it to taskgraph + #tasks_for in ["action", "cron"] + # release commented out until we port it to taskgraph + #|| (tasks_for == "github-release" && releaseAction == "published" && (ownerEmail != "mozilla-release-automation-bot@users.noreply.github.com") && (ownerEmail != "mozilla-release-automation-bot-staging@users.noreply.github.com")) + - $if: > + tasks_for in ["action"] + || (tasks_for == "github-pull-request" && pullRequestAction in ["opened", "reopened", "synchronize"]) + || (tasks_for == "github-push" && head_branch[:10] != "refs/tags/") && (head_branch != "staging.tmp") && (head_branch != "trying.tmp") + then: + $let: + level: + $if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/focus-android"' + then: '3' + else: '1' + + short_head_branch: + $if: 'head_branch[:11] == "refs/heads/"' + then: {$eval: 'head_branch[11:]'} + in: + $mergeDeep: + - $if: 'tasks_for != "action"' + then: + taskId: '${ownTaskId}' + - taskGroupId: + $if: 'tasks_for == "action"' + then: + '${action.taskGroupId}' + else: + '${ownTaskId}' # same as taskId; this is how automation identifies a decision task + schedulerId: '${trustDomain}-level-${level}' + created: {$fromNow: ''} + deadline: {$fromNow: '1 day'} + expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors + metadata: + $merge: + - owner: "${ownerEmail}" + source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml' + - $if: 'tasks_for in ["github-push", "github-pull-request", "github-release"]' + then: + name: "Decision Task" + description: 'The task that creates all of the other tasks in the task graph' + else: + $if: 'tasks_for == "action"' + then: + name: "Action: ${action.title}" + description: | + ${action.description} + + Action triggered by clientID `${clientId}` + else: + name: "Decision Task for cron job ${cron.job_name}" + description: 'Created by a [cron task](https://firefox-ci-tc.services.mozilla.com/tasks/${cron.task_id})' + provisionerId: "mobile-${level}" + workerType: "decision" + tags: + $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' + then: + kind: decision-task + else: + $if: 'tasks_for == "action"' + then: + kind: 'action-callback' + else: + $if: 'tasks_for == "cron"' + then: + kind: cron-task + routes: + $flattenDeep: + - checks + - $if: 'level == "3"' + then: + - tc-treeherder.v2.${project}.${head_sha} + # TODO Bug 1601928: Make this scope fork-friendly once ${project} is better defined. This will enable + # staging release promotion on forks. + - $if: 'tasks_for == "github-push"' + then: + - index.mobile.v2.${project}.branch.${short_head_branch}.latest.taskgraph.decision + - index.mobile.v2.${project}.branch.${short_head_branch}.revision.${head_sha}.taskgraph.decision + - index.mobile.v2.${project}.revision.${head_sha}.taskgraph.decision + - $if: 'tasks_for == "cron"' + then: + # cron context provides ${head_branch} as a short one + - index.mobile.v2.${project}.branch.${head_branch}.latest.taskgraph.decision-${cron.job_name} + - index.mobile.v2.${project}.branch.${head_branch}.revision.${head_sha}.taskgraph.decision-${cron.job_name} + - index.mobile.v2.${project}.branch.${head_branch}.revision.${head_sha}.taskgraph.cron.${ownTaskId} + scopes: + $if: 'tasks_for == "github-push"' + then: + # `https://` is 8 characters so, ${repoUrl[8:]} is the repository without the protocol. + - 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}' + else: + $if: 'tasks_for == "github-pull-request"' + then: + - 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request' + else: + $if: 'tasks_for == "github-release"' + then: + - 'assume:repo:${repoUrl[8:]}:release' + else: + $if: 'tasks_for == "action"' + then: + # when all actions are hooks, we can calculate this directly rather than using a variable + - '${action.repo_scope}' + else: + - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}' + + requires: all-completed + priority: lowest + retries: 5 + + payload: + env: + # run-task uses these to check out the source; the inputs + # to `mach taskgraph decision` are all on the command line. + $merge: + - MOBILE_BASE_REPOSITORY: '${baseRepoUrl}' + MOBILE_HEAD_REPOSITORY: '${repoUrl}' + MOBILE_HEAD_REF: '${head_branch}' + MOBILE_HEAD_REV: '${head_sha}' + MOBILE_HEAD_TAG: '${head_tag}' + MOBILE_REPOSITORY_TYPE: git + TASKGRAPH_BASE_REPOSITORY: https://hg.mozilla.org/ci/taskgraph + TASKGRAPH_HEAD_REPOSITORY: https://hg.mozilla.org/ci/${taskgraph.branch} + TASKGRAPH_HEAD_REV: ${taskgraph.revision} + TASKGRAPH_REPOSITORY_TYPE: hg + REPOSITORIES: {$json: {mobile: "Focus-Android", taskgraph: "Taskgraph"}} + HG_STORE_PATH: /builds/worker/checkouts/hg-store + ANDROID_SDK_ROOT: /builds/worker/android-sdk + - $if: 'tasks_for in ["github-pull-request"]' + then: + MOBILE_PULL_REQUEST_NUMBER: '${event.pull_request.number}' + - $if: 'tasks_for == "action"' + then: + ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task + ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) + ACTION_INPUT: {$json: {$eval: 'input'}} + ACTION_CALLBACK: '${action.cb_name}' + - $if: 'tasks_for == "github-release"' + then: + MOBILE_HEAD_TAG: '${event.release.tag_name}' + features: + taskclusterProxy: true + chainOfTrust: true + # Note: This task is built server side without the context or tooling that + # exist in tree so we must hard code the hash + image: + mozillareleases/taskgraph:decision-mobile-0e1cefb4ec46060d53bcd0eb2c0dd296210fa969998af468d693d1c6922efd1d@sha256:7cfb913bee636f1ab1477a09f8ad746e1f8c68fab0e2a3e1408c985d2ddc27f7 + + maxRunTime: 1800 + + command: + - /usr/local/bin/run-task + - '--mobile-checkout=/builds/worker/checkouts/src' + - '--taskgraph-checkout=/builds/worker/checkouts/taskgraph' + - '--task-cwd=/builds/worker/checkouts/src' + - '--' + - bash + - -cx + - $let: + extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''} + in: + $if: 'tasks_for == "action"' + then: > + PIP_IGNORE_INSTALLED=0 pip3 install --user /builds/worker/checkouts/taskgraph && + PIP_IGNORE_INSTALLED=0 pip3 install --user mozilla-version && + taskcluster/scripts/decision-install-sdk.sh && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph action-callback + else: > + PIP_IGNORE_INSTALLED=0 pip3 install --user /builds/worker/checkouts/taskgraph && + PIP_IGNORE_INSTALLED=0 pip3 install --user mozilla-version && + taskcluster/scripts/decision-install-sdk.sh && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph decision + --pushlog-id='0' + --pushdate='0' + --project='${project}' + --message="" + --owner='${ownerEmail}' + --level='${level}' + --base-repository="$MOBILE_BASE_REPOSITORY" + --head-repository="$MOBILE_HEAD_REPOSITORY" + --head-ref="$MOBILE_HEAD_REF" + --head-rev="$MOBILE_HEAD_REV" + --head-tag="$MOBILE_HEAD_TAG" + --repository-type="$MOBILE_REPOSITORY_TYPE" + --tasks-for='${tasks_for}' + ${extraArgs} + + artifacts: + 'public': + type: 'directory' + path: '/builds/worker/artifacts' + expires: {$fromNow: '1 year'} + 'public/docker-contexts': + type: 'directory' + path: '/builds/worker/checkouts/src/docker-contexts' + # This needs to be at least the deadline of the + # decision task + the docker-image task deadlines. + # It is set to a week to allow for some time for + # debugging, but they are not useful long-term. + expires: {$fromNow: '7 day'} + + extra: + $merge: + - treeherder: + $merge: + - machine: + platform: gecko-decision + - $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + symbol: D + else: + $if: 'tasks_for == "github-release"' + then: + symbol: 'ship_focus_android' + else: + $if: 'tasks_for == "action"' + then: + groupName: 'action-callback' + groupSymbol: AC + symbol: "${action.symbol}" + else: + groupSymbol: cron + symbol: "${cron.job_symbol}" + - $if: 'tasks_for == "action"' + then: + parent: '${action.taskGroupId}' + action: + name: '${action.name}' + context: + taskGroupId: '${action.taskGroupId}' + taskId: {$eval: 'taskId'} + input: {$eval: 'input'} + clientId: {$eval: 'clientId'} + - $if: 'tasks_for == "cron"' + then: + cron: {$json: {$eval: 'cron'}} + - tasks_for: '${tasks_for}' diff --git a/app/build.gradle b/app/build.gradle index 9485949941e..ba6c275451e 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -9,6 +9,7 @@ apply plugin: 'kotlin-android-extensions' apply from: "$project.rootDir/tools/gradle/versionCode.gradle" import com.android.build.OutputFile +import groovy.json.JsonOutput android { compileSdkVersion 30 @@ -539,3 +540,30 @@ if (gradle.hasProperty('localProperties.autoPublish.application-services.dir')) ext.appServicesSrcDir = gradle."localProperties.autoPublish.application-services.dir" apply from: "../${appServicesSrcDir}/build-scripts/substitute-local-appservices.gradle" } + +// ------------------------------------------------------------------------------------------------- +// Task for printing APK information for the requested variant +// Taskgraph Usage: "./gradlew printVariants +// ------------------------------------------------------------------------------------------------- +tasks.register('printVariants') { + doLast { + def variants = android.applicationVariants.collect { variant -> [ + apks: variant.outputs.collect { output -> [ + abi: output.getFilter(com.android.build.VariantOutput.FilterType.ABI), + fileName: output.outputFile.name + ]}, + build_type: variant.buildType.name, + name: variant.name, + ]} + // AndroidTest is a special case not included above + variants.add([ + apks: [[ + abi: 'noarch', + fileName: 'app-focus-debug-androidTest.apk', + ]], + build_type: 'debug', + name: 'FocusDebugAndroidTest', + ]) + println 'variants: ' + JsonOutput.toJson(variants) + } +} \ No newline at end of file diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml new file mode 100644 index 00000000000..8bc6703e1e1 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -0,0 +1,66 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: focus_android_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # The number of times a test execution should be re-attempted if one or more failures occur. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 1 + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - package org.mozilla.focus.activity + - package org.mozilla.focus.privacy + - package org.mozilla.focus.screenshots + + device: + - model: Pixel2 + version: 28 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. default: 1 + max-test-shards: -1 + # num-test-runs: the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + num-test-runs: 1 + ### Output Style flag + ## Output style of execution status. May be one of [verbose, multi, single, compact]. + ## For runs with only one test execution the default value is 'verbose', in other cases + ## 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles + ## which don't support ansi codes, to avoid corrupted output use single or verbose. + ## The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. + output-style: compact + ### Full Junit Result flag + ## Enable create additional local junit result on local storage with failure nodes on passed flaky tests. + full-junit-result: true diff --git a/automation/taskcluster/androidTest/parse-ui-test.py b/automation/taskcluster/androidTest/parse-ui-test.py new file mode 100644 index 00000000000..18893f6cac7 --- /dev/null +++ b/automation/taskcluster/androidTest/parse-ui-test.py @@ -0,0 +1,63 @@ +#!/usr/bin/python3 + +from __future__ import print_function + +import sys +import argparse +from pathlib import Path +import json +import yaml + +def parse_args(cmdln_args): + parser = argparse.ArgumentParser(description="Parse UI test logs an results") + parser.add_argument( + "--output-md", + type=argparse.FileType("w", encoding="utf-8"), + help="Output markdown file.", + required=True, + ) + parser.add_argument( + "--log", + type=argparse.FileType("r", encoding="utf-8"), + help="Log output of flank.", + required=True, + ) + parser.add_argument( + "--results", type=Path, help="Directory containing flank results", required=True + ) + parser.add_argument( + "--exit-code", type=int, help="Exit code of flank.", required=True + ) + parser.add_argument("--device-type", help="Type of device ", required=True) + return parser.parse_args(args=cmdln_args) + + +def extract_android_args(log): + return yaml.safe_load(log.split("AndroidArgs\n")[1].split("RunTests\n")[0]) + + +def main(): + args = parse_args(sys.argv[1:]) + + log = args.log.read() + matrix_ids = json.loads(args.results.joinpath("matrix_ids.json").read_text()) + #with args.results.joinpath("flank.yml") as f: + # flank_config = yaml.safe_load(f) + + android_args = extract_android_args(log) + + print = args.output_md.write + + print("# Devices\n") + print(yaml.safe_dump(android_args["gcloud"]["device"])) + + print("# Results\n") + print("| matrix | result | logs | details \n") + print("| --- | --- | --- | --- |\n") + for matrix, matrix_result in matrix_ids.items(): + print("| {matrixId} | {outcome} | [logs]({webLink}) | {axes[0][details]}\n".format(**matrix_result)) + + +if __name__ == "__main__": + main() + diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh new file mode 100755 index 00000000000..8e02e4be05b --- /dev/null +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script does the following: +# 1. Retrieves gcloud service account token +# 2. Activates gcloud service account +# 3. Connects to google Firebase (using TestArmada's Flank tool) +# 4. Executes UI tests +# 5. Puts test artifacts into the test_artifacts folder + +# NOTE: +# Flank supports sharding across multiple devices at a time, but gcloud API +# only supports 1 defined APK per test run. + + +# If a command fails then do not proceed and fail this script too. +set -e + +######################### +# The command line help # +######################### +display_help() { + echo "Usage: $0 Build_Variant [Number_Shards...]" + echo + echo "Examples:" + echo "To run UI tests on ARM device shard (1 test / shard)" + echo "$ ui-test.sh arm64-v8a -1" + echo + echo "To run UI tests on X86 device (on 3 shards)" + echo "$ ui-test.sh x86 3" + echo +} + +get_abs_filename() { + relative_filename="$1" + echo "$(cd "$(dirname "$relative_filename")" && pwd)/$(basename "$relative_filename")" +} + + +# Basic parameter check +if [[ $# -lt 1 ]]; then + echo "Error: please provide at least one build variant (arm|x86)" + display_help + exit 1 +fi + +device_type="$1" # arm64-v8a | armeabi-v7a | x86_64 | x86 +APK_APP="$2" +APK_TEST="$3" +if [[ ! -z "$4" ]]; then + num_shards=$4 +fi + +JAVA_BIN="/usr/bin/java" +PATH_TEST="./automation/taskcluster/androidTest" +FLANK_BIN="/builds/worker/test-tools/flank.jar" +ARTIFACT_DIR="/builds/worker/artifacts" +RESULTS_DIR="${ARTIFACT_DIR}/results" + +echo +echo "ACTIVATE SERVICE ACCT" +echo +# this is where the Google Testcloud project ID is set +gcloud config set project "$GOOGLE_PROJECT" +echo + +gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS" +echo +echo + +# Disable exiting on error. If the tests fail we want to continue +# and try to download the artifacts. We will exit with the actual error code later. +set +e + +if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then + flank_template="${PATH_TEST}/flank-${device_type}.yml" +elif [[ "${device_type}" == "x86-start-test" ]]; then + flank_template="${PATH_TEST}/flank-x86-start-test.yml" +elif [[ "${device_type}" == "arm-start-test" ]]; then + flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml" +elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then + flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml" +elif [[ "${device_type}" == "x86-beta-tests" ]]; then + flank_template="${PATH_TEST}/flank-x86-beta.yml" +else + echo "FAILURE: flank config file not found!" + exitcode=1 +fi + +APK_APP="$(get_abs_filename $APK_APP)" +APK_TEST="$(get_abs_filename $APK_TEST)" +echo "device_type: ${device_type}" +echo "APK_APP: ${APK_APP}" +echo "APK_TEST: ${APK_TEST}" + +# function to exit script with exit code from test run. +# (Only 0 if all test executions passed) +function failure_check() { + echo + echo + if [[ $exitcode -ne 0 ]]; then + echo "FAILURE: UI test run failed, please check above URL" + else + echo "All UI test(s) have passed!" + fi + + echo + echo "RESULTS" + echo + + mkdir -p /builds/worker/artifacts/github + chmod +x $PATH_TEST/parse-ui-test.py + $PATH_TEST/parse-ui-test.py \ + --exit-code "${exitcode}" \ + --log flank.log \ + --results "${RESULTS_DIR}" \ + --output-md "${ARTIFACT_DIR}/github/customCheckRunText.md" \ + --device-type "${device_type}" +} + +echo +echo "FLANK VERSION" +echo +$JAVA_BIN -jar $FLANK_BIN --version +echo +echo + +echo +echo "EXECUTE TEST(S)" +echo +# Note that if --local-results-dir is "results", timestamped sub-directory will +# contain the results. For any other value, the directory itself will have the results. +set -o pipefail && $JAVA_BIN -jar $FLANK_BIN android run \ + --config=$flank_template \ + --max-test-shards=$num_shards \ + --app=$APK_APP --test=$APK_TEST \ + --local-result-dir="${RESULTS_DIR}" \ + --project=$GOOGLE_PROJECT \ + | tee flank.log + +exitcode=$? +failure_check + +exit $exitcode diff --git a/build.gradle b/build.gradle index b09789b8be2..380efa067cb 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,7 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. +import org.mozilla.focus.gradle.tasks.GithubDetailsTask + buildscript { ext.espresso_version = '3.1.0-alpha4' ext.coroutines_version = '1.4.2' @@ -98,3 +100,15 @@ task ktlintFormat(type: JavaExec, group: "formatting") { main = "com.pinterest.ktlint.Main" args "-F", "app/**/*.kt", "!**/build/**/*.kt", "buildSrc/**/*.kt" } + +tasks.register("githubTestDetails", GithubDetailsTask) { + text = "### [Unit Test Results]({reportsUrl}/test/testFocusDebugUnitTest/index.html)" +} + +tasks.register("githubLintDetektDetails", GithubDetailsTask) { + text = "### [Detekt Results]({reportsUrl}/detekt.html)" +} + +tasks.register("githubLintAndroidDetails", GithubDetailsTask) { + text = "### [Android Lint Results]({reportsUrl}/lint-results-debug.html)" +} \ No newline at end of file diff --git a/buildSrc/src/main/java/org/mozilla/focus/gradle/tasks/GithubDetailsTask.kt b/buildSrc/src/main/java/org/mozilla/focus/gradle/tasks/GithubDetailsTask.kt new file mode 100644 index 00000000000..9a7aad1bab6 --- /dev/null +++ b/buildSrc/src/main/java/org/mozilla/focus/gradle/tasks/GithubDetailsTask.kt @@ -0,0 +1,38 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.focus.gradle.tasks + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction +import java.io.File + +/** + * Helper to write to the "customCheckRunText.md" file for Taskcluster. + * Taskcluster uses this file to populate the "Details" section in the GitHub Checks panel UI. + */ +open class GithubDetailsTask : DefaultTask() { + + /** + * Text to display in the Github Checks panel under "Details". Any markdown works here. + * The text is written to a markdown file which is used by Taskcluster. + * Links are automatically rewritten to point to the correct Taskcluster URL. + */ + @Input + var text: String = "" + + private val detailsFile = File("/builds/worker/github/customCheckRunText.md") + private val suffix = "\n\n_(404 if compilation failed)_" + + @TaskAction + fun writeFile() { + val taskId = System.getenv("TASK_ID") + val reportsUrl = "https://firefoxci.taskcluster-artifacts.net/$taskId/0/public/reports" + val replaced = text.replace("{reportsUrl}", reportsUrl) + + project.mkdir("/builds/worker/github") + detailsFile.writeText(replaced + suffix) + } +} diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml new file mode 100644 index 00000000000..d6b9da452dc --- /dev/null +++ b/taskcluster/ci/build/kind.yml @@ -0,0 +1,104 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - focus_android_taskgraph.transforms.build:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - toolchain + +job-defaults: + apk-artifact-template: + type: file + name: 'public/build/{fileName}' + path: '/builds/worker/checkouts/src/app/build/outputs/apk/{gradle_build}/{gradle_build_type}/{fileName}' + fetches: + toolchain: + - android-sdk-linux + - android-gradle-dependencies + run: + using: gradlew + use-caches: false + workdir: '/builds/worker' + run-on-tasks-for: [] + treeherder: + kind: build + symbol: B + platform: android-all/opt + tier: 1 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 7200 + chain-of-trust: true + +jobs: + focus-debug: + description: 'Focus debug build from source code' + run-on-tasks-for: [github-pull-request, github-push] + run: + gradle-build-type: debug + gradle-build-name: focusDebug + gradle-build: focus + treeherder: + symbol: debug(Bf) + + klar-debug: + description: 'Klar debug build from source code' + run-on-tasks-for: [github-pull-request, github-push] + run: + gradle-build-type: debug + gradle-build-name: klarDebug + gradle-build: klar + treeherder: + symbol: debug(Bkl) + + focus-release: + description: 'Release Focus build for debugging' + run-on-tasks-for: [github-pull-request] + run: + gradle-build-type: release + gradle-build-name: focusRelease + gradle-build: focus + treeherder: + symbol: release(Bf) + + klar-release: + description: 'Release Klar build for debugging' + run-on-tasks-for: [github-pull-request] + run: + gradle-build-type: release + gradle-build-name: klarRelease + gradle-build: klar + treeherder: + symbol: release(Bkl) + + nightly: + description: 'Nightly focus build for debugging' + run-on-tasks-for: [github-pull-request] + run: + gradle-build-type: nightly + gradle-build-name: focusNightly + gradle-build: focus + treeherder: + symbol: nightly(B) + + beta: + description: 'Beta focus build for debugging' + run-on-tasks-for: [github-pull-request] + run: + gradle-build-type: beta + gradle-build-name: focusBeta + gradle-build: focus + treeherder: + symbol: beta(B) + + + # TODO modify release, nightly and beta (focus only for the latter two) for signing and push-apk + # Add androidTest as a separate task + # Note: nightly and beta will not push to google plaly store \ No newline at end of file diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml new file mode 100644 index 00000000000..196a21fad32 --- /dev/null +++ b/taskcluster/ci/config.yml @@ -0,0 +1,87 @@ +--- +trust-domain: mobile +treeherder: + group-names: + 'beta': 'Beta-related tasks with same APK configuration as Fennec' + 'bump': 'Bump dependencies' + 'debug': 'Builds made for testing' + 'Fetch': 'Fetch and store content' + 'I': 'Docker Image Builds' + 'nightly': 'Nightly-related tasks' + 'release': 'Production-related tasks with same APK configuration as Focus' + 'TL': 'Toolchain builds for Linux 64-bits' + +task-priority: highest + +taskgraph: + register: focus_android_taskgraph:register + repositories: + mobile: + name: "Focus-Android" + cached-task-prefix: mobile.v2.focus-android + +workers: + aliases: + b-android: + provisioner: 'mobile-{level}' + implementation: docker-worker + os: linux + worker-type: b-linux + b-android-large: + provisioner: 'mobile-{level}' + implementation: docker-worker + os: linux + worker-type: b-linux-large + dep-signing: + provisioner: scriptworker-k8s + implementation: scriptworker-signing + os: scriptworker + worker-type: mobile-t-signing + github: + provisioner: scriptworker-k8s + implementation: scriptworker-github + os: scriptworker + worker-type: 'mobile-{level}-github' + images: + provisioner: 'mobile-{level}' + implementation: docker-worker + os: linux + worker-type: 'images' + # misc is used by taskgraph to generate tasks with more than 10 routes + misc: + provisioner: 'mobile-{level}' + implementation: docker-worker + os: linux + worker-type: 'b-linux' + push-apk: + provisioner: scriptworker-k8s + implementation: scriptworker-pushapk + os: scriptworker + worker-type: 'mobile-{level}-pushapk' + ship-it: + provisioner: scriptworker-k8s + implementation: scriptworker-shipit + os: scriptworker + worker-type: 'mobile-{level}-shipit' + signing: + provisioner: scriptworker-k8s + implementation: scriptworker-signing + os: scriptworker + worker-type: + by-level: + "3": mobile-3-signing + default: mobile-t-signing + tree: + provisioner: scriptworker-k8s + implementation: scriptworker-tree + os: scriptworker + worker-type: 'mobile-{level}-tree' + +scriptworker: + scope-prefix: project:mobile:focus-android:releng + + +release-promotion: + flavors: + ship: + target-tasks-method: release diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 00000000000..211cba58ed3 --- /dev/null +++ b/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,21 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.docker_image:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + +jobs: + android-build: + symbol: I(agb) + parent: base + base: + symbol: I(base) + ui-tests: + symbol: I(ui-tests) + parent: base diff --git a/taskcluster/ci/fetch/kind.yml b/taskcluster/ci/fetch/kind.yml new file mode 100644 index 00000000000..f49abb5f3bb --- /dev/null +++ b/taskcluster/ci/fetch/kind.yml @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.fetch:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + docker-image: {in-tree: base} + +jobs: + android-sdk-3859397: + description: Android SDK + fetch: + type: static-url + url: https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip + artifact-name: sdk-tools-linux.zip + sha256: 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0 + size: 136964098 + artifact-prefix: mobile/android-sdk + fetch-alias: android-sdk diff --git a/taskcluster/ci/lint/kind.yml b/taskcluster/ci/lint/kind.yml new file mode 100644 index 00000000000..0e4b0d4646b --- /dev/null +++ b/taskcluster/ci/lint/kind.yml @@ -0,0 +1,75 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - toolchain + + +job-defaults: + attributes: + code-review: true + retrigger: true + fetches: + toolchain: + - android-sdk-linux + - android-gradle-dependencies + run: + use-caches: false + run-on-tasks-for: [github-pull-request, github-push] + treeherder: + kind: test + platform: 'lint/opt' + tier: 1 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 7200 + artifacts: + - name: public/reports + path: /builds/worker/checkouts/src/build/reports + type: directory + - name: public/github + path: /builds/worker/github + type: directory + +jobs: + compare-locales: + description: 'Validate strings.xml with compare-locales' + run: + using: run-task + cwd: '{checkout}' + command: 'pip install --user "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .' + treeherder: + symbol: compare-locale + tier: 2 + detekt: + description: 'Running detekt over all modules' + run: + using: gradlew + gradlew: ['detekt', 'githubLintDetektDetails'] + treeherder: + symbol: detekt + + ktlint: + description: 'Running ktlint over all modules' + run: + using: gradlew + gradlew: ['ktlint'] + treeherder: + symbol: ktlint + + lint: + description: 'Running lint over all modules' + run: + using: gradlew + gradlew: ['lint', 'githubLintAndroidDetails'] + treeherder: + symbol: lint diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml new file mode 100644 index 00000000000..cb5b99f71a4 --- /dev/null +++ b/taskcluster/ci/test/kind.yml @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - toolchain + +job-defaults: + attributes: + retrigger: true + description: Unit tests + fetches: + toolchain: + - android-sdk-linux + - android-gradle-dependencies + run: + using: gradlew + use-caches: false + treeherder: + kind: test + tier: 2 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 7200 + +jobs: + debug: + attributes: + code-review: true + run: + gradlew: ['clean', 'test', 'githubTestDetails'] + dummy-secrets: + - content: "faketoken" + path: .adjust_token + treeherder: + platform: 'android-all/opt' + symbol: debug(T) + tier: 1 + worker: + artifacts: + - name: public/reports/index.html + path: /builds/worker/checkouts/src/app/build/reports/tests/testDebugUnitTest/index.html + type: file + - name: public/reports/test + path: /builds/worker/checkouts/src/app/build/reports/tests + type: directory + - name: public/github + path: /builds/worker/github + type: directory diff --git a/taskcluster/ci/toolchain/android.yml b/taskcluster/ci/toolchain/android.yml new file mode 100644 index 00000000000..2643d97245d --- /dev/null +++ b/taskcluster/ci/toolchain/android.yml @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: toolchain-script + treeherder: + kind: build + platform: toolchains/opt + tier: 1 + + +linux64-android-sdk-linux-repack: + attributes: + artifact_prefix: mobile/android-sdk + description: "Android SDK (Linux) repack toolchain build" + fetches: + fetch: + - android-sdk + run: + script: repack-android-sdk-linux.sh + resources: [] + toolchain-artifact: mobile/android-sdk/android-sdk-linux.tar.xz + toolchain-alias: android-sdk-linux + treeherder: + symbol: TL(android-sdk-linux) + worker: + docker-image: {in-tree: base} + max-run-time: 1800 + worker-type: b-android + + +linux64-android-gradle-dependencies: + description: "Android Gradle dependencies toolchain task" + fetches: + toolchain: + # Aliases aren't allowed for toolchains depending on toolchains. + - linux64-android-sdk-linux-repack + run: + script: android-gradle-dependencies.sh + sparse-profile: null + resources: + - taskcluster/scripts/toolchain/android-gradle-dependencies.sh + - taskcluster/scripts/toolchain/android-gradle-dependencies/** + - buildSrc/src/main/java/Dependencies.kt + - buildSrc/src/main/java/AndroidComponents.kt + toolchain-artifact: public/build/android-gradle-dependencies.tar.xz + toolchain-alias: android-gradle-dependencies + treeherder: + symbol: TL(gradle-dependencies) + worker: + docker-image: {in-tree: android-build} + env: + # TODO do no hardcode + ANDROID_SDK_ROOT: /builds/worker/fetches/android-sdk-linux + max-run-time: 14400 + artifacts: + - type: directory + name: public/logs/nexus + path: /opt/sonatype/nexus/logs + worker-type: b-android-large diff --git a/taskcluster/ci/toolchain/kind.yml b/taskcluster/ci/toolchain/kind.yml new file mode 100644 index 00000000000..ba9c02645ff --- /dev/null +++ b/taskcluster/ci/toolchain/kind.yml @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +kind-dependencies: + - fetch + +transforms: + - taskgraph.transforms.job:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + + +jobs-from: + - android.yml diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml new file mode 100644 index 00000000000..be5186a7e3d --- /dev/null +++ b/taskcluster/ci/ui-test/kind.yml @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - toolchain + - build + +job-defaults: + attributes: + build-type: debug + code-review: true + retrigger: true + fetches: + toolchain: + - android-sdk-linux + - android-gradle-dependencies + worker-type: b-android + worker: + docker-image: {in-tree: ui-tests} + max-run-time: 7200 + env: + GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json' + GOOGLE_PROJECT: 'moz-focus-android' + artifacts: + - name: public + path: /builds/worker/artifacts + type: directory + run-on-tasks-for: [github-pull-request, github-push] + treeherder: + kind: test + platform: 'ui-test/opt' + tier: 2 + run: + use-caches: false + using: gradlew + dummy-secrets: + - content: "faketoken" + path: .adjust_token + secrets: + - name: project/mobile/focus-android/firebase + key: firebaseToken + path: .firebase_token.json + json: true + # TODO retrieving these from two separate tasks means they're signed differently; need to create a signing + # dependency in order to get this working properly (signed with same key) + # - [wget, {artifact-reference: ''}, '-O', app.apk] + # - [wget, {artifact-reference: ''}, '-O', android-test.apk] + +jobs: + x86-debug: + description: 'UI tests with firebase' + run: + gradlew: ['clean', 'assembleFocusDebug', 'assembleFocusDebugAndroidTest'] + post-gradlew: + - ['automation/taskcluster/androidTest/ui-test.sh', 'x86', './app/build/outputs/apk/focus/debug/app-focus-x86-debug.apk', './app/build/outputs/apk/androidTest/focus/debug/app-focus-debug-androidTest.apk', '-1'] + treeherder: + symbol: debug(ui-test-x86) \ No newline at end of file diff --git a/taskcluster/docker/android-build/Dockerfile b/taskcluster/docker/android-build/Dockerfile new file mode 100644 index 00000000000..72089e3229d --- /dev/null +++ b/taskcluster/docker/android-build/Dockerfile @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM $DOCKER_IMAGE_PARENT + +MAINTAINER Release Engineering + +VOLUME /builds/worker/checkouts + +# Install Sonatype Nexus. Cribbed directly from +# https://github.com/sonatype/docker-nexus/blob/fffd2c61b2368292040910c055cf690c8e76a272/oss/Dockerfile. + +ENV NEXUS_ARCHIVE='nexus-bundle.tar.gz' \ + NEXUS_ROOT='/opt/sonatype/nexus' \ + NEXUS_SHA1SUM=1a9aaad8414baffe0a2fd46eed1f41b85f4049e6 \ + NEXUS_VERSION=2.12.0-01 \ + NEXUS_WORK=/builds/worker/workspace/nexus + +RUN mkdir -p "$NEXUS_ROOT" \ + && chown -R worker:worker "$NEXUS_ROOT" + +USER worker:worker + +RUN $CURL --output "$NEXUS_ARCHIVE" "https://download.sonatype.com/nexus/oss/nexus-${NEXUS_VERSION}-bundle.tar.gz" \ + && echo "$NEXUS_SHA1SUM $NEXUS_ARCHIVE" | sha1sum --check \ + && tar xzvf "$NEXUS_ARCHIVE" --strip-components=1 --directory="$NEXUS_ROOT" \ + && rm "$NEXUS_ARCHIVE" + +# run-task expects to run as root +USER root diff --git a/taskcluster/docker/base/Dockerfile b/taskcluster/docker/base/Dockerfile new file mode 100644 index 00000000000..5f287f78444 --- /dev/null +++ b/taskcluster/docker/base/Dockerfile @@ -0,0 +1,67 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# Inspired by: +# https://hub.docker.com/r/runmymind/docker-android-sdk/~/dockerfile/ + +FROM ubuntu:18.04 + +MAINTAINER Release Engineering + +# Add worker user +RUN mkdir /builds && \ + useradd -d /builds/worker -s /bin/bash -m worker && \ + chown worker:worker /builds/worker && \ + mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +WORKDIR /builds/worker/ + +# -- System ----------------------------------------------------------------------------- + +ENV CURL='curl --location --retry 5' \ + GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \ + LANG='en_US.UTF-8' \ + TERM='dumb' + +RUN apt-get update -qq \ + # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that + # we cannot navigate while building the Docker image. + && apt-get install -y tzdata \ + && apt-get install -y openjdk-8-jdk \ + openjdk-11-jdk \ + wget \ + expect \ + git \ + curl \ + python \ + python-pip \ + python3 \ + python3-yaml \ + locales \ + unzip \ + mercurial \ + && apt-get clean + + +RUN pip install --upgrade pip +RUN pip install taskcluster + +RUN locale-gen en_US.UTF-8 + +# Do not delete - this is magic code that will enable the run-task worker for Taskgraph +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH="/builds/worker/.local/bin:$PATH" + + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + + +# run-task expects to run as root +USER root + diff --git a/taskcluster/docker/ui-tests/Dockerfile b/taskcluster/docker/ui-tests/Dockerfile new file mode 100644 index 00000000000..a54a3aa5c70 --- /dev/null +++ b/taskcluster/docker/ui-tests/Dockerfile @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM $DOCKER_IMAGE_PARENT + +LABEL authors="Richard Pappalardo , Aaron Train " +LABEL maintainer="Richard Pappalardo " + +#---------------------------------------------------------------------------------------------------------------------- +#-- Test tools -------------------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------------------------------- + +RUN apt-get install -y jq \ + && apt-get clean + +USER worker:worker + +ENV GOOGLE_SDK_DOWNLOAD ./gcloud.tar.gz +ENV GOOGLE_SDK_VERSION 233 + +ENV TEST_TOOLS /builds/worker/test-tools +ENV PATH ${PATH}:${TEST_TOOLS}:${TEST_TOOLS}/google-cloud-sdk/bin + +RUN mkdir -p ${TEST_TOOLS} && \ + mkdir -p ${HOME}/.config/gcloud + +RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_SDK_VERSION}.0.0-linux-x86_64.tar.gz --output ${GOOGLE_SDK_DOWNLOAD} \ + && tar -xvf ${GOOGLE_SDK_DOWNLOAD} -C ${TEST_TOOLS} \ + && rm -f ${GOOGLE_SDK_DOWNLOAD} \ + && ${TEST_TOOLS}/google-cloud-sdk/install.sh --quiet \ + && ${TEST_TOOLS}/google-cloud-sdk/bin/gcloud --quiet components update + +# Flank v21.08.1 + +RUN URL_FLANK_BIN="$($CURL --silent 'https://api.github.com/repos/Flank/flank/releases/48276753' | jq -r '.assets[] | select(.browser_download_url | test("flank.jar")) .browser_download_url')" \ + && $CURL --output "${TEST_TOOLS}/flank.jar" "${URL_FLANK_BIN}" \ + && chmod +x "${TEST_TOOLS}/flank.jar" + +# run-task expects to run as root +USER root + + + +# FROM $DOCKER_IMAGE_PARENT + +# MAINTAINER Release Engineering + +# VOLUME /builds/worker/checkouts + +# # Install Google Cloud SDK for using Firebase Test Lab +# RUN cd /opt && curl --location --retry 5 --output gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-331.0.0-linux-x86_64.tar.gz \ +# && tar -xvf /opt/gcloud.tar.gz \ +# && rm -f gcloud.tar.gz \ +# && /opt/google-cloud-sdk/install.sh --quiet \ +# && /opt/google-cloud-sdk/bin/gcloud --quiet components update diff --git a/taskcluster/focus_android_taskgraph/__init__.py b/taskcluster/focus_android_taskgraph/__init__.py new file mode 100644 index 00000000000..a2ed1e3cfc7 --- /dev/null +++ b/taskcluster/focus_android_taskgraph/__init__.py @@ -0,0 +1,26 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from importlib import import_module + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + _import_modules([ + "job", + # parameters + # release_promotion + # "routes", + # "target_tasks", + # "worker_types", + ]) + + +def _import_modules(modules): + for module in modules: + import_module(f".{module}", package=__name__) diff --git a/taskcluster/focus_android_taskgraph/gradle.py b/taskcluster/focus_android_taskgraph/gradle.py new file mode 100644 index 00000000000..8977c879b1b --- /dev/null +++ b/taskcluster/focus_android_taskgraph/gradle.py @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import json +import subprocess + +from taskgraph.util.memoize import memoize + + + +def get_variant(build_type, build_name): + all_variants = _fetch_all_variants() + matching_variants = [ + variant for variant in all_variants + if variant["build_type"] == build_type and variant["name"] == build_name + ] + number_of_matching_variants = len(matching_variants) + if number_of_matching_variants == 0: + raise ValueError('No variant found for build type "{}"'.format( + build_type + )) + elif number_of_matching_variants > 1: + raise ValueError('Too many variants found for build type "{}"": {}'.format( + build_type, matching_variants + )) + + return matching_variants.pop() + + +@memoize +def _fetch_all_variants(): + output = _run_gradle_process('printVariants') + content = _extract_content_from_command_output(output, prefix='variants: ') + return json.loads(content) + + +def _run_gradle_process(gradle_command, **kwargs): + gradle_properties = [ + f'-P{property_name}={value}' + for property_name, value in kwargs.items() + ] + process = subprocess.Popen(["./gradlew", "--no-daemon", "--quiet", gradle_command] + gradle_properties, stdout=subprocess.PIPE, universal_newlines=True) + output, err = process.communicate() + exit_code = process.wait() + + if exit_code != 0: + raise RuntimeError(f"Gradle command returned error: {exit_code}") + + return output + + +def _extract_content_from_command_output(output, prefix): + variants_line = [line for line in output.split('\n') if line.startswith(prefix)][0] + return variants_line.split(' ', 1)[1] diff --git a/taskcluster/focus_android_taskgraph/job.py b/taskcluster/focus_android_taskgraph/job.py new file mode 100644 index 00000000000..5f65053216c --- /dev/null +++ b/taskcluster/focus_android_taskgraph/job.py @@ -0,0 +1,186 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run +from taskgraph.util import path +from taskgraph.util.schema import Schema, taskref_or_string +from voluptuous import Required, Optional +from six import text_type + +from pipes import quote as shell_quote + +secret_schema = { + Required("name"): str, + Required("path"): str, + Required("key"): str, + Optional("json"): bool, + Optional("decode"): bool, +} + +dummy_secret_schema = { + Required("content"): str, + Required("path"): str, + Optional("json"): bool, +} + +gradlew_schema = Schema({ + Required("using"): "gradlew", + Optional("pre-gradlew"): [[str]], + Required("gradlew"): [str], + Optional("post-gradlew"): [[str]], + # Base work directory used to set up the task. + Required("workdir"): str, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], +}) + +run_commands_schema = Schema({ + Required("using"): "run-commands", + Optional("pre-commands"): [[str]], + Required("commands"): [[taskref_or_string]], + Required("workdir"): str, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], + Optional("dummy-secrets"): [dummy_secret_schema], +}) + + +@run_job_using("docker-worker", "run-commands", schema=run_commands_schema) +def configure_run_commands_schema(config, job, taskdesc): + run = job["run"] + pre_commands = run.pop("pre-commands", []) + pre_commands += [ + _generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", []) + ] + pre_commands += [ + _generate_secret_command(secret) for secret in run.get("secrets", []) + ] + + all_commands = pre_commands + run.pop("commands", []) + + run["command"] = _convert_commands_to_string(all_commands) + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) + + +@run_job_using("docker-worker", "gradlew", schema=gradlew_schema) +def configure_gradlew(config, job, taskdesc): + run = job["run"] + worker = taskdesc["worker"] = job["worker"] + + fetches_dir = path.join(run["workdir"], worker["env"]["MOZ_FETCHES_DIR"]) + worker.setdefault("env", {}).update({ + "ANDROID_SDK_ROOT": path.join( + fetches_dir, "android-sdk-linux" + ) + }) + + run["command"] = _extract_gradlew_command(run, fetches_dir) + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) + + +def _extract_gradlew_command(run, fetches_dir): + pre_gradle_commands = run.pop("pre-gradlew", []) + pre_gradle_commands += [ + _generate_dummy_secret_command(secret) for secret in run.pop("dummy-secrets", []) + ] + pre_gradle_commands += [ + _generate_secret_command(secret) for secret in run.get("secrets", []) + ] + + maven_dependencies_dir = path.join(fetches_dir, "android-gradle-dependencies") + gradle_repos_args = [ + "-P{repo_name}Repo=file://{dir}/{repo_name}".format( + dir=maven_dependencies_dir, repo_name=repo_name + ) + for repo_name in ("google", "jcenter") + ] + gradle_command = ["./gradlew"] + gradle_repos_args + run.pop("gradlew") + post_gradle_commands = run.pop("post-gradlew", []) + + commands = pre_gradle_commands + [gradle_command] + post_gradle_commands + return _convert_commands_to_string(commands) + + +def _generate_secret_command(secret): + secret_command = [ + "taskcluster/scripts/get-secret.py", + "-s", secret["name"], + "-k", secret["key"], + "-f", secret["path"], + ] + if secret.get("json"): + secret_command.append("--json") + + if secret.get("decode"): + secret_command.append("--decode") + + return secret_command + + +def _generate_dummy_secret_command(secret): + secret_command = [ + "taskcluster/scripts/write-dummy-secret.py", + "-f", secret["path"], + "-c", secret["content"], + ] + if secret.get("json"): + secret_command.append("--json") + + return secret_command + + +def _convert_commands_to_string(commands): + should_artifact_reference = False + should_task_reference = False + + sanitized_commands = [] + for command in commands: + sanitized_parts = [] + for part in command: + if isinstance(part, dict): + if "artifact-reference" in part: + part_string = part["artifact-reference"] + should_artifact_reference = True + elif "task-reference" in part: + part_string = part["task-reference"] + should_task_reference = True + else: + raise ValueError(f'Unsupported dict: {part}') + else: + part_string = part + + sanitized_parts.append(part_string) + sanitized_commands.append(sanitized_parts) + + shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in sanitized_commands] + full_string_command = " && ".join(shell_quoted_commands) + + if should_artifact_reference and should_task_reference: + raise NotImplementedError('"arifact-reference" and "task-reference" cannot be both used') + elif should_artifact_reference: + return {"artifact-reference": full_string_command} + elif should_task_reference: + return {"task-reference": full_string_command} + else: + return full_string_command + + +def _inject_secrets_scopes(run, taskdesc): + secrets = run.pop("secrets", []) + scopes = taskdesc.setdefault("scopes", []) + new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets] + new_secret_scopes = list(set(new_secret_scopes)) # Scopes must not have any duplicates + scopes.extend(new_secret_scopes) + + +def _set_run_task_attributes(job): + run = job["run"] + run["cwd"] = "{checkout}" + run["using"] = "run-task" diff --git a/taskcluster/focus_android_taskgraph/loader/__init__.py b/taskcluster/focus_android_taskgraph/loader/__init__.py new file mode 100644 index 00000000000..afc72acb6a5 --- /dev/null +++ b/taskcluster/focus_android_taskgraph/loader/__init__.py @@ -0,0 +1,72 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy + + +# Define a collection of group_by functions +GROUP_BY_MAP = {} + + +def group_by(name): + def wrapper(func): + GROUP_BY_MAP[name] = func + return func + return wrapper + + + +def group_tasks(config, tasks): + group_by_fn = GROUP_BY_MAP[config['group-by']] + + groups = group_by_fn(config, tasks) + + for combinations in groups.values(): + dependencies = [copy.deepcopy(t) for t in combinations] + yield dependencies + + +@group_by('build-type') +def build_type_grouping(config, tasks): + groups = {} + kind_dependencies = config.get('kind-dependencies') + only_build_type = config.get('only-for-build-types') + + for task in tasks: + if task.kind not in kind_dependencies: + continue + + if only_build_type: + build_type = task.attributes.get('build-type') + if build_type not in only_build_type: + continue + + build_type = task.attributes.get('build-type') + + groups.setdefault(build_type, []).append(task) + + return groups + + +@group_by('attributes') +def attributes_grouping(config, tasks): + groups = {} + kind_dependencies = config.get('kind-dependencies') + only_attributes = config.get('only-for-attributes') + + for task in tasks: + if task.kind not in kind_dependencies: + continue + + group_attr = None + if only_attributes: + if not any(attr in task.attributes for attr in only_attributes): + continue + else: + continue + + groups.setdefault(task.label, []).append(task) + + return groups diff --git a/taskcluster/focus_android_taskgraph/loader/multi_dep.py b/taskcluster/focus_android_taskgraph/loader/multi_dep.py new file mode 100644 index 00000000000..18e1d30b13d --- /dev/null +++ b/taskcluster/focus_android_taskgraph/loader/multi_dep.py @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import copy + +from voluptuous import Required + +from taskgraph.task import Task +from taskgraph.util.attributes import sorted_unique_list +from taskgraph.util.schema import Schema + +from . import group_tasks + + +schema = Schema({ + Required('primary-dependency', 'primary dependency task'): Task, + Required( + 'dependent-tasks', + 'dictionary of dependent tasks, keyed by kind', + ): {str: Task}, +}) + + + +def loader(kind, path, config, params, loaded_tasks): + """ + Load tasks based on the jobs dependant kinds, designed for use as + multiple-dependent needs. + Required ``group-by-fn`` is used to define how we coalesce the + multiple deps together to pass to transforms, e.g. all kinds specified get + collapsed by platform with `platform` + Optional ``primary-dependency`` (ordered list or string) is used to determine + which upstream kind to inherit attrs from. See ``get_primary_dep``. + The `only-for-build-type` kind configuration, if specified, will limit + the build types for which a job will be created. + Optional ``job-template`` kind configuration value, if specified, will be used to + pass configuration down to the specified transforms used. + """ + job_template = config.get('job-template') + + for dep_tasks in group_tasks(config, loaded_tasks): + + kinds = [dep.kind for dep in dep_tasks] + kinds_occurrences = {kind: kinds.count(kind) for kind in kinds} + + dep_tasks_per_unique_key = { + dep.kind if kinds_occurrences[dep.kind] == 1 else dep.label: dep + for dep in dep_tasks + } + + job = {'dependent-tasks': dep_tasks_per_unique_key} + job['primary-dependency'] = get_primary_dep(config, dep_tasks_per_unique_key) + if job_template: + job.update(copy.deepcopy(job_template)) + + yield job + + +def get_primary_dep(config, dep_tasks): + """Find the dependent task to inherit attributes from. + If ``primary-dependency`` is defined in ``kind.yml`` and is a string, + then find the first dep with that task kind and return it. If it is + defined and is a list, the first kind in that list with a matching dep + is the primary dependency. If it's undefined, return the first dep. + """ + primary_dependencies = config.get('primary-dependency') + if isinstance(primary_dependencies, str): + primary_dependencies = [primary_dependencies] + if not primary_dependencies: + assert len(dep_tasks) == 1, "Must define a primary-dependency!" + return dep_tasks.values()[0] + primary_dep = None + for primary_kind in primary_dependencies: + for dep_kind in dep_tasks: + if dep_kind == primary_kind: + assert primary_dep is None, \ + "Too many primary dependent tasks in dep_tasks: {}!".format( + [t.label for t in dep_tasks] + ) + primary_dep = dep_tasks[dep_kind] + if primary_dep is None: + raise Exception("Can't find dependency of {}: {}".format( + config['primary-dependency'], config + )) + return primary_dep diff --git a/taskcluster/focus_android_taskgraph/routes.py b/taskcluster/focus_android_taskgraph/routes.py new file mode 100644 index 00000000000..9cae480df92 --- /dev/null +++ b/taskcluster/focus_android_taskgraph/routes.py @@ -0,0 +1,40 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import time + +from taskgraph.transforms.task import index_builder + +# Please ping the performance testing team (whawkins at the moment) if these routes change. +# In the future, notifying consumers may be easier (https://bugzilla.mozilla.org/show_bug.cgi?id=1548810), but +# we need to remember to tell users for the time being +SIGNING_ROUTE_TEMPLATES = [ + "index.{trust-domain}.v2.{project}.{variant}.latest.{abi}", + "index.{trust-domain}.v2.{project}.{variant}.{build_date}.revision.{head_rev}.{abi}", + "index.{trust-domain}.v2.{project}.{variant}.{build_date}.latest.{abi}", + "index.{trust-domain}.v2.{project}.{variant}.revision.{head_rev}.{abi}", +] + + +@index_builder("signing") +def add_signing_indexes(config, task): + if config.params["level"] != "3": + return task + + subs = config.params.copy() + subs["build_date"] = time.strftime( + "%Y.%m.%d", time.gmtime(config.params["build_date"]) + ) + subs["trust-domain"] = config.graph_config["trust-domain"] + subs["variant"] = task["attributes"]["build-type"] + + unique_routes = set() + for tpl in SIGNING_ROUTE_TEMPLATES: + for abi in task["attributes"]["apks"].keys(): + subs["abi"] = abi + unique_routes.add(tpl.format(**subs)) + + task.setdefault("routes", sorted(list(unique_routes))) + return task diff --git a/taskcluster/focus_android_taskgraph/target_tasks.py b/taskcluster/focus_android_taskgraph/target_tasks.py new file mode 100644 index 00000000000..9f400131412 --- /dev/null +++ b/taskcluster/focus_android_taskgraph/target_tasks.py @@ -0,0 +1,68 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.target_tasks import _target_task, filter_for_tasks_for + + +@_target_task('default') +def target_tasks_default(full_task_graph, parameters, graph_config): + """Target the tasks which have indicated they should be run on this project + via the `run_on_projects` attributes.""" + + filter = filter_for_tasks_for + return [l for l, t in full_task_graph.tasks.iteritems() if filter_for_tasks_for(t, parameters)] + + +@_target_task('release') +def target_tasks_default(full_task_graph, parameters, graph_config): + + # TODO Use shipping-phase once we retire github-releases + def filter(task, parameters): + # Mark-as-shipped is always red on github-release and it confuses people. + # This task cannot be green if we kick off a release through github-releases, so + # let's exlude that task there. + if task.kind == "mark-as-shipped" and parameters["tasks_for"] == "github-release": + return False + + return task.attributes.get("release-type", "") == parameters["release_type"] + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] + + +@_target_task("nightly") +def target_tasks_nightly(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build.""" + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] + + +def _filter_fennec(fennec_type, task, parameters): + return task.attributes.get("build-type", "") == f"fennec-{fennec_type}" + + +@_target_task("fennec-production") +def target_tasks_fennec_nightly(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a production build signed with the fennec key.""" + + return [l for l, t in full_task_graph.tasks.iteritems() if _filter_fennec("production", t, parameters)] + + +@_target_task("bump_android_components") +def target_tasks_bump_android_components(full_task_graph, parameters, graph_config): + """Select the set of tasks required to update android components.""" + + def filter(task, parameters): + return task.attributes.get("bump-type", "") == "android-components" + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] + + +@_target_task("screenshots") +def target_tasks_screnshots(full_task_graph, parameters, graph_config): + """Select the set of tasks required to generate screenshots on a real device.""" + + def filter(task, parameters): + return task.attributes.get("screenshots", False) + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] diff --git a/taskcluster/focus_android_taskgraph/transforms/build.py b/taskcluster/focus_android_taskgraph/transforms/build.py new file mode 100644 index 00000000000..3a5781bd4e5 --- /dev/null +++ b/taskcluster/focus_android_taskgraph/transforms/build.py @@ -0,0 +1,169 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + + +import datetime + +from taskgraph.transforms.base import TransformSequence +from focus_android_taskgraph.gradle import get_variant + + +transforms = TransformSequence() + + +@transforms.add +def add_variant_config(config, tasks): + for task in tasks: + attributes = task.setdefault("attributes", {}) + if not attributes.get("build-type"): + attributes["build-type"] = task["name"] + yield task + + +@transforms.add +def add_shippable_secrets(config, tasks): + for task in tasks: + secrets = task["run"].setdefault("secrets", []) + dummy_secrets = task["run"].setdefault("dummy-secrets", []) + + if task.pop("include-shippable-secrets", False) and config.params["level"] == "3": + build_type = task["attributes"]["build-type"] + gradle_build_type = task["run"]["gradle-build-type"] + secret_index = f'project/focus/{build_type}' + secrets.extend([{ + "key": key, + "name": secret_index, + "path": target_file, + } for key, target_file in ( + ('adjust', '.adjust_token'), + ('firebase', f'app/src/{gradle_build_type}/res/values/firebase.xml'), + ('sentry_dsn', '.sentry_token'), + ('mls', '.mls_token'), + ('nimbus_url', '.nimbus'), + )]) + else: + dummy_secrets.extend([{ + "content": fake_value, + "path": target_file, + } for fake_value, target_file in ( + ("faketoken", ".adjust_token"), + ("faketoken", ".mls_token"), + ("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"), + )]) + + yield task + + +@transforms.add +def build_gradle_command(config, tasks): + for task in tasks: + gradle_build_type = task["run"]["gradle-build-type"] + gradle_build_name = task["run"]["gradle-build-name"] + variant_config = get_variant(gradle_build_type, gradle_build_name) + variant_name = variant_config["name"][0].upper() + variant_config["name"][1:] + task["run"]["gradlew"] = [ + "clean", + "assemble{}".format(variant_name), + ] + yield task + +@transforms.add +def extra_gradle_options(config, tasks): + for task in tasks: + for extra in task["run"].pop("gradle-extra-options", []): + task["run"]["gradlew"].append(extra) + + yield task + +@transforms.add +def add_test_build_type(config, tasks): + for task in tasks: + test_build_type = task["run"].pop("test-build-type", "") + if test_build_type: + task["run"]["gradlew"].append( + f"-PtestBuildType={test_build_type}" + ) + yield task + + +@transforms.add +def add_disable_optimization(config, tasks): + for task in tasks: + if task.pop("disable-optimization", False): + task["run"]["gradlew"].append("-PdisableOptimization") + yield task + + +@transforms.add +def add_nightly_version(config, tasks): + for task in tasks: + if task.pop("include-nightly-version", False): + task["run"]["gradlew"].extend([ + # We only set the `official` flag here. The actual version name will be determined + # by Gradle (depending on the Gecko/A-C version being used) + '-Pofficial' + ]) + yield task + + +@transforms.add +def add_release_version(config, tasks): + for task in tasks: + if task.pop("include-release-version", False): + task["run"]["gradlew"].extend([ + '-PversionName={}'.format(config.params["version"]), + '-Pofficial' + ]) + yield task + + +@transforms.add +def add_artifacts(config, tasks): + for task in tasks: + gradle_build_type = task["run"].pop("gradle-build-type") + gradle_build_name = task["run"].pop("gradle-build-name") + gradle_build = task["run"].pop("gradle-build") + variant_config = get_variant(gradle_build_type, gradle_build_name) + artifacts = task.setdefault("worker", {}).setdefault("artifacts", []) + task["attributes"]["apks"] = apks = {} + + if "apk-artifact-template" in task: + artifact_template = task.pop("apk-artifact-template") + for apk in variant_config["apks"]: + apk_name = artifact_template["name"].format( + **apk + ) + artifacts.append({ + "type": artifact_template["type"], + "name": apk_name, + "path": artifact_template["path"].format( + gradle_build_type=gradle_build_type, + gradle_build=gradle_build, + **apk + ), + }) + # TODO figure out what to do here since Focus apk's already have an "abi" key + # apks[apk["abi"]] = { + # "name": apk_name, + # "github-name": artifact_template["github-name"].format( + # version=config.params["version"], + # **apk + # ) + # } + + yield task + + +@transforms.add +def filter_incomplete_translation(config, tasks): + for task in tasks: + if task.pop("filter-incomplete-translations", False): + # filter-release-translations modifies source, which could cause problems if we ever start caching source + pre_gradlew = task["run"].setdefault("pre-gradlew", []) + pre_gradlew.append(["python", "automation/taskcluster/l10n/filter-release-translations.py"]) + yield task diff --git a/taskcluster/focus_android_taskgraph/transforms/multi_dep.py b/taskcluster/focus_android_taskgraph/transforms/multi_dep.py new file mode 100644 index 00000000000..0a0b6638c16 --- /dev/null +++ b/taskcluster/focus_android_taskgraph/transforms/multi_dep.py @@ -0,0 +1,103 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the single_dep jobs. +""" + + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol + + +transforms = TransformSequence() + + +@transforms.add +def build_name_and_attributes(config, tasks): + for task in tasks: + task["dependencies"] = { + dep_key: dep.label + for dep_key, dep in _get_all_deps(task).items() + } + primary_dep = task["primary-dependency"] + copy_of_attributes = primary_dep.attributes.copy() + task.setdefault("attributes", copy_of_attributes) + # run_on_tasks_for is set as an attribute later in the pipeline + task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for']) + task["name"] = _get_dependent_job_name_without_its_kind(primary_dep) + + yield task + + +def _get_dependent_job_name_without_its_kind(dependent_job): + return dependent_job.label[len(dependent_job.kind) + 1:] + + +def _get_all_deps(task): + if task.get("dependent-tasks"): + return task["dependent-tasks"] + + return {task["primary-dependency"].kind: task["primary-dependency"]} + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + resolve_keyed_by( + task, + "treeherder.job-symbol", + item_name=task["name"], + **{ + 'build-type': task["attributes"]["build-type"], + 'level': config.params["level"], + } + ) + yield task + + +@transforms.add +def build_upstream_artifacts(config, tasks): + for task in tasks: + worker_definition = {} + + for dep in _get_all_deps(task).values(): + paths = sorted([ + apk_metadata["name"] + for apk_metadata in dep.attributes.get("apks", {}).values() + ]) + if paths: + worker_definition["upstream-artifacts"] = [{ + "taskId": {"task-reference": f"<{dep.kind}>"}, + "taskType": dep.kind, + "paths": paths, + }] + + task.setdefault("worker", {}).update(worker_definition) + yield task + + +@transforms.add +def build_treeherder_definition(config, tasks): + for task in tasks: + dep = task.pop("primary-dependency") + + task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep)) + job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?") + job_symbol = dep.task["extra"]["treeherder"].get("symbol", "?") + full_symbol = join_symbol(job_group, job_symbol) + task["treeherder"]["symbol"] = full_symbol + + yield task + + +@transforms.add +def remove_dependent_tasks(config, tasks): + for task in tasks: + try: + del task["dependent-tasks"] + except KeyError: + pass + + yield task diff --git a/taskcluster/focus_android_taskgraph/transforms/test.py b/taskcluster/focus_android_taskgraph/transforms/test.py new file mode 100644 index 00000000000..32bbeb811c2 --- /dev/null +++ b/taskcluster/focus_android_taskgraph/transforms/test.py @@ -0,0 +1,19 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from taskgraph.transforms.base import TransformSequence + + +transforms = TransformSequence() + + +@transforms.add +def add_pr_number(config, tasks): + for task in tasks: + include_pr = task.pop("include-pull-request-number") + if include_pr and config.params["pull_request_number"]: + task["worker"]["env"]["PULL_REQUEST_NUMBER"] = str(config.params["pull_request_number"]) + + yield task diff --git a/taskcluster/focus_android_taskgraph/util.py b/taskcluster/focus_android_taskgraph/util.py new file mode 100644 index 00000000000..01d2933660e --- /dev/null +++ b/taskcluster/focus_android_taskgraph/util.py @@ -0,0 +1,10 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import re + + +def upper_case_first_letter(string): + return string[0].upper() + string[1:] diff --git a/taskcluster/focus_android_taskgraph/worker_types.py b/taskcluster/focus_android_taskgraph/worker_types.py new file mode 100644 index 00000000000..4d3d8467bad --- /dev/null +++ b/taskcluster/focus_android_taskgraph/worker_types.py @@ -0,0 +1,201 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +from six import text_type + +from voluptuous import Any, Required, Optional + +from taskgraph.util.schema import taskref_or_string +from taskgraph.transforms.task import payload_builder + + +@payload_builder( + "scriptworker-signing", + schema={ + # the maximum time to run, in seconds + Required("max-run-time"): int, + Required("signing-type"): str, + # list of artifact URLs for the artifacts that should be signed + Required("upstream-artifacts"): [ + { + # taskId of the task with the artifact + Required("taskId"): taskref_or_string, + # type of signing task (for CoT) + Required("taskType"): str, + # Paths to the artifacts to sign + Required("paths"): [str], + # Signing formats to use on each of the paths + Required("formats"): [str], + } + ], + }, +) +def build_scriptworker_signing_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = { + "maxRunTime": worker["max-run-time"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + formats = set() + for artifacts in worker["upstream-artifacts"]: + formats.update(artifacts["formats"]) + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].append( + "{}:signing:cert:{}".format(scope_prefix, worker["signing-type"]) + ) + task_def["scopes"].extend( + [ + f"{scope_prefix}:signing:format:{format}" + for format in sorted(formats) + ] + ) + + +@payload_builder( + "scriptworker-pushapk", + schema={ + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + Required("certificate-alias"): str, + Required("channel"): str, + Required("commit"): bool, + Required("product"): str, + Required("dep"): bool, + }, +) +def build_push_apk_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = { + "certificate_alias": worker["certificate-alias"], + "channel": worker["channel"], + "commit": worker["commit"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].append( + "{}:googleplay:product:{}{}".format( + scope_prefix, worker["product"], ":dep" if worker["dep"] else "" + ) + ) + + +@payload_builder( + "scriptworker-shipit", + schema={ + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + Required("release-name"): str, + }, +) +def build_shipit_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def['payload'] = { + 'release_name': worker['release-name'] + } + + +@payload_builder( + "scriptworker-github", + schema={ + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): str, + Required("paths"): [str], + } + ], + Required("artifact-map"): [object], + Required("action"): str, + Required("git-tag"): str, + Required("git-revision"): str, + Required("github-project"): str, + Required("is-prerelease"): bool, + Required("release-name"): str, + }, +) +def build_github_release_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = { + "artifactMap": worker["artifact-map"], + "gitTag": worker["git-tag"], + "gitRevision": worker["git-revision"], + "isPrerelease": worker["is-prerelease"], + "releaseName": worker["release-name"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].extend([ + "{}:github:project:{}".format(scope_prefix, worker["github-project"]), + "{}:github:action:{}".format(scope_prefix, worker["action"]), + ]) + + +@payload_builder( + "scriptworker-tree", + schema={ + Optional("upstream-artifacts"): [ + { + Optional("taskId"): taskref_or_string, + Optional("taskType"): str, + Optional("paths"): [str], + } + ], + Required("bump"): bool, + Optional("bump-files"): [str], + Optional("push"): bool, + Optional("branch"): str, + }, +) +def build_version_bump_payload(config, task, task_def): + worker = task["worker"] + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def['payload'] = {'actions': []} + actions = task_def['payload']['actions'] + + if worker['bump']: + if not worker['bump-files']: + raise Exception("Version Bump requested without bump-files") + + bump_info = {} + bump_info["next_version"] = config.params["next_version"] + bump_info['files'] = worker['bump-files'] + task_def['payload']['version_bump_info'] = bump_info + actions.append('version_bump') + + if worker["push"]: + task_def['payload']['push'] = True + + if worker.get('force-dry-run'): + task_def['payload']['dry_run'] = True + + if worker.get("branch"): + task_def["payload"]["branch"] = worker["branch"] diff --git a/taskcluster/scripts/decision-install-sdk.sh b/taskcluster/scripts/decision-install-sdk.sh new file mode 100755 index 00000000000..0d2afac6aa8 --- /dev/null +++ b/taskcluster/scripts/decision-install-sdk.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ex + +export CURL='curl --location --retry 5' + +ANDROID_SDK_VERSION='3859397' +ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' +SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip" + +# For the Android build system we want Java 11. However this version of sdkmanager requires Java 8. +JAVA8PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/:$PATH" + +$CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" +echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check +unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" +rm "$SDK_ZIP_LOCATION" + +yes | PATH=$JAVA8PATH "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses + diff --git a/taskcluster/scripts/get-secret.py b/taskcluster/scripts/get-secret.py new file mode 100755 index 00000000000..1e2e697b40f --- /dev/null +++ b/taskcluster/scripts/get-secret.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import base64 +import errno +import json +import os +import taskcluster + + +def write_secret_to_file(path, data, key, base64decode=False, json_secret=False, append=False, prefix=''): + path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../' + path)) + try: + os.makedirs(os.path.dirname(path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise + print("Outputting secret to: {}".format(path)) + + with open(path, 'a' if append else 'w') as f: + value = data['secret'][key] + if base64decode: + value = base64.b64decode(value) + if json_secret: + value = json.dumps(value) + f.write(prefix + value) + + +def fetch_secret_from_taskcluster(name): + try: + secrets = taskcluster.Secrets({ + # BaseUrl is still needed for tasks that haven't migrated to taskgraph yet. + 'baseUrl': 'http://taskcluster/secrets/v1', + }) + except taskcluster.exceptions.TaskclusterFailure: + # taskcluster library >=5 errors out when `baseUrl` is used + secrets = taskcluster.Secrets({ + 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), + }) + + return secrets.get(name) + + +def main(): + parser = argparse.ArgumentParser( + description='Fetch a taskcluster secret value and save it to a file.') + + parser.add_argument('-s', dest="secret", action="store", help="name of the secret") + parser.add_argument('-k', dest='key', action="store", help='key of the secret') + parser.add_argument('-f', dest="path", action="store", help='file to save secret to') + parser.add_argument('--decode', dest="decode", action="store_true", default=False, help='base64 decode secret before saving to file') + parser.add_argument('--json', dest="json", action="store_true", default=False, help='serializes the secret to JSON format') + parser.add_argument('--append', dest="append", action="store_true", default=False, help='append secret to existing file') + parser.add_argument('--prefix', dest="prefix", action="store", default="", help='add prefix when writing secret to file') + + result = parser.parse_args() + + secret = fetch_secret_from_taskcluster(result.secret) + write_secret_to_file(result.path, secret, result.key, result.decode, result.json, result.append, result.prefix) + + +if __name__ == "__main__": + main() diff --git a/taskcluster/scripts/toolchain/android-gradle-dependencies.sh b/taskcluster/scripts/toolchain/android-gradle-dependencies.sh new file mode 100755 index 00000000000..abbd466f984 --- /dev/null +++ b/taskcluster/scripts/toolchain/android-gradle-dependencies.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +set -ex + +function get_abs_path { + local file_path="$1" + echo "$( cd "$(dirname "$file_path")" >/dev/null 2>&1 ; pwd -P )" +} + +CURRENT_DIR="$(get_abs_path $0)" +PROJECT_DIR="$(get_abs_path $CURRENT_DIR/../../../..)" + +pushd $PROJECT_DIR + +. taskcluster/scripts/toolchain/android-gradle-dependencies/before.sh + +NEXUS_PREFIX='http://localhost:8081/nexus/content/repositories' +GRADLE_ARGS="--parallel -PgoogleRepo=$NEXUS_PREFIX/google/ -PcentralRepo=$NEXUS_PREFIX/central/" +# We build everything to be sure to fetch all dependencies +./gradlew $GRADLE_ARGS assembleFocusDebug testFocusDebugUnitTest detekt ktlint clean + +# Some tests may be flaky, although they still download dependencies. So we let the following +# command fail, if needed. +set +e; ./gradlew $GRADLE_ARGS test; set -e + + +# ./gradlew lint is missing because of https://github.com/mozilla-mobile/fenix/issues/10439. So far, +# we're lucky and the dependencies it fetches are found elsewhere. + +. taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh + +popd diff --git a/taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh b/taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh new file mode 100644 index 00000000000..0dc8cc20871 --- /dev/null +++ b/taskcluster/scripts/toolchain/android-gradle-dependencies/after.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This is copy of +# https://searchfox.org/mozilla-central/rev/2cd2d511c0d94a34fb7fa3b746f54170ee759e35/taskcluster/scripts/misc/android-gradle-dependencies/after.sh. +# gradle-plugins was removed because it's not used in this project. + +set -x -e + +echo "running as" $(id) + +: WORKSPACE ${WORKSPACE:=/builds/worker/workspace} + +set -v + +# Package everything up. +pushd $WORKSPACE +mkdir -p android-gradle-dependencies /builds/worker/artifacts + +cp -R ${NEXUS_WORK}/storage/google android-gradle-dependencies +cp -R ${NEXUS_WORK}/storage/central android-gradle-dependencies + +tar cf - android-gradle-dependencies | xz > /builds/worker/artifacts/android-gradle-dependencies.tar.xz + +popd diff --git a/taskcluster/scripts/toolchain/android-gradle-dependencies/before.sh b/taskcluster/scripts/toolchain/android-gradle-dependencies/before.sh new file mode 100644 index 00000000000..4e860698280 --- /dev/null +++ b/taskcluster/scripts/toolchain/android-gradle-dependencies/before.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This is a copy of +# https://searchfox.org/mozilla-central/rev/2cd2d511c0d94a34fb7fa3b746f54170ee759e35/taskcluster/scripts/misc/android-gradle-dependencies/before.sh. +# `misc` was renamed into `toolchain` and `/builds/worker/workspace/build` was changed into +# `/builds/worker/checkouts/` + +set -x -e + +echo "running as" $(id) + +: WORKSPACE ${WORKSPACE:=/builds/worker/workspace} + +set -v + +mkdir -p ${NEXUS_WORK}/conf +cp /builds/worker/checkouts/src/taskcluster/scripts/toolchain/android-gradle-dependencies/nexus.xml ${NEXUS_WORK}/conf/nexus.xml + +PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/:$PATH" RUN_AS_USER=worker /opt/sonatype/nexus/bin/nexus restart + +# Wait "a while" for Nexus to actually start. Don't fail if this fails. +wget --quiet --retry-connrefused --waitretry=2 --tries=100 \ + http://localhost:8081/nexus/service/local/status || true +rm -rf status + +# It's helpful when debugging to see the "latest state". +curl http://localhost:8081/nexus/service/local/status || true + +# Verify Nexus has actually started. Fail if this fails. +curl --fail --silent --location http://localhost:8081/nexus/service/local/status | grep 'STARTED' + +# It's helpful when debugging to see the repository configurations. +curl http://localhost:8081/nexus/service/local/repositories || true diff --git a/taskcluster/scripts/toolchain/android-gradle-dependencies/nexus.xml b/taskcluster/scripts/toolchain/android-gradle-dependencies/nexus.xml new file mode 100644 index 00000000000..64e4c96335f --- /dev/null +++ b/taskcluster/scripts/toolchain/android-gradle-dependencies/nexus.xml @@ -0,0 +1,382 @@ + + + + + 2.8.0 + 2.12.0-01 + + 20000 + 3 + + + + 60000 + + + true + 8082 + strict + + + true + + + + gradle-plugins + Gradle Plugins + org.sonatype.nexus.proxy.repository.Repository + maven2 + IN_SERVICE + true + 1440 + true + true + true + READ_ONLY + true + true + + file + + + https://plugins.gradle.org/m2/ + + + RELEASE + STRICT + true + false + -1 + 1440 + 1440 + true + + + + google + google + org.sonatype.nexus.proxy.repository.Repository + maven2 + IN_SERVICE + true + 1440 + true + true + true + READ_ONLY + true + true + + file + + + https://maven.google.com/ + + + RELEASE + + STRICT_IF_EXISTS + true + false + -1 + 1440 + 1440 + true + + + + central + Central + org.sonatype.nexus.proxy.repository.Repository + maven2 + IN_SERVICE + true + 1440 + true + true + true + READ_ONLY + true + true + + file + + + https://repo1.maven.org/maven2/ + + + ALLOW + -1 + 1440 + false + false + WARN + RELEASE + + + + apache-snapshots + Apache Snapshots + org.sonatype.nexus.proxy.repository.Repository + maven2 + IN_SERVICE + true + 1440 + true + true + true + READ_ONLY + true + true + + file + + + https://repository.apache.org/snapshots/ + + + ALLOW + 1440 + 1440 + false + false + WARN + SNAPSHOT + + + + releases + Releases + org.sonatype.nexus.proxy.repository.Repository + maven2 + IN_SERVICE + 1440 + true + true + true + ALLOW_WRITE_ONCE + true + true + + file + + + ALLOW + -1 + 1440 + false + false + WARN + RELEASE + + + + snapshots + Snapshots + org.sonatype.nexus.proxy.repository.Repository + maven2 + IN_SERVICE + 1440 + true + true + true + ALLOW_WRITE + true + true + + file + + + ALLOW + 1440 + 1440 + false + false + WARN + SNAPSHOT + + + + thirdparty + 3rd party + org.sonatype.nexus.proxy.repository.Repository + maven2 + IN_SERVICE + 1440 + true + true + true + ALLOW_WRITE_ONCE + true + true + + file + + + ALLOW + -1 + 1440 + false + false + WARN + RELEASE + + + + central-m1 + Central M1 shadow + org.sonatype.nexus.proxy.repository.ShadowRepository + m2-m1-shadow + IN_SERVICE + 15 + true + true + true + READ_ONLY + + file + + + central + false + + + + public + Public Repositories + org.sonatype.nexus.proxy.repository.GroupRepository + maven2 + IN_SERVICE + 15 + true + true + true + READ_ONLY + true + + file + + + true + + releases + snapshots + thirdparty + central + + + + + + + + inhouse-stuff + * + inclusive + + ^/(com|org)/somecompany/.* + + + snapshots + releases + + + + apache-stuff + * + exclusive + + ^/org/some-oss/.* + + + releases + snapshots + + + + + + + 1 + All (Maven2) + maven2 + + .* + + + + 2 + All (Maven1) + maven1 + + .* + + + + 3 + All but sources (Maven2) + maven2 + + (?!.*-sources.*).* + + + + 4 + All Metadata (Maven2) + maven2 + + .*maven-metadata\.xml.* + + + + any + All (Any Repository) + any + + .* + + + + site + All (site) + site + + .* + + + + npm + All (npm) + npm + + .* + + + + nuget + All (nuget) + nuget + + .* + + + + rubygems + All (rubygems) + rubygems + + .* + + + + + smtp-host + 25 + smtp-username + {jyU2gDFaNz8HQ4ybBAIdtJ6KL+YB08GXQs7vLPnia3o=} + system@nexus.org + + + diff --git a/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh b/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh new file mode 100755 index 00000000000..7605741570d --- /dev/null +++ b/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +export ANDROID_SDK_ROOT=$MOZ_FETCHES_DIR + +# For the Android build system we want Java 11. However this version of sdkmanager requires Java 8. +JAVA8PATH="/usr/lib/jvm/java-8-openjdk-amd64/jre/bin/:$PATH" + +yes | PATH=$JAVA8PATH "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses + +# It's nice to have the build logs include the state of the world upon completion. +PATH=$JAVA8PATH "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --list + +tar cf - -C "$ANDROID_SDK_ROOT" . --transform 's,^\./,android-sdk-linux/,' | xz > "$UPLOAD_DIR/android-sdk-linux.tar.xz" diff --git a/taskcluster/scripts/write-dummy-secret.py b/taskcluster/scripts/write-dummy-secret.py new file mode 100755 index 00000000000..c90c5036ebf --- /dev/null +++ b/taskcluster/scripts/write-dummy-secret.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + + +import argparse +import errno +import os + + +def write_secret_to_file(path, secret): + path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../' + path)) + try: + os.makedirs(os.path.dirname(path)) + except OSError as error: + if error.errno != errno.EEXIST: + raise + + print("Outputting secret to: {}".format(path)) + + with open(path, 'w') as f: + f.write(secret) + + +def main(): + parser = argparse.ArgumentParser(description="Store a dummy secret to a file") + + parser.add_argument("-c", dest="content", action="store", help="content of the secret") + parser.add_argument("-f", dest="path", action="store", help="file to save secret to") + + result = parser.parse_args() + + write_secret_to_file(result.path, result.content) + + +if __name__ == "__main__": + main()