Merge branch 'master'

.gitattributes

@@ -0,0 +1,11 @@
+rust/trezor-client/src/protos/generated/* linguist-generated
+rust/trezor-client/src/messages/generated.rs linguist-generated
+
+core/embed/rust/src/translations/generated/*.rs linguist-generated
+
+core/src/all_modules.py linguist-generated
+core/src/trezor/messages.py linguist-generated
+core/src/trezor/enums/*.py linguist-generated
+core/src/apps/common/coininfo.py linguist-generated
+
+python/src/trezorlib/messages.py linguist-generated

.github/actions/environment/action.yml

@@ -0,0 +1,20 @@
+name: 'Download dependencies'
+description: 'Nixpkgs and poetry'
+inputs:
+  full-deps:
+    description: 'Pass --arg fullDeps true to nix-shell?'
+    required: false
+    default: false
+runs:
+  using: "composite"
+  steps:
+    - name: Install nix
+      uses: cachix/install-nix-action@v23
+      with:
+        nix_path: nixpkgs=channel:nixos-unstable
+    - name: Dependencies nixpkgs
+      run: nix-shell --arg fullDeps "${{ inputs.full-deps }}" --run "true"
+      shell: sh
+    - name: Dependencies poetry
+      run: nix-shell --arg fullDeps "${{ inputs.full-deps }}" --run "poetry install"
+      shell: sh

.github/actions/ui-comment/action.yml

@@ -0,0 +1,24 @@
+name: 'UI comment links'
+description: 'Post URLs of HTML test reports to pull request discussion'
+runs:
+  using: composite
+  steps:
+    - name: Find Comment
+      uses: peter-evans/find-comment@v3
+      id: fc
+      if: github.event_name == 'pull_request'
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: ui-comment-${{ github.workflow }}
+
+    - name: Create or update comment
+      uses: peter-evans/create-or-update-comment@v4
+      if: github.event_name == 'pull_request'
+      with:
+        comment-id: ${{ steps.fc.outputs.comment-id }}
+        issue-number: ${{ github.event.pull_request.number }}
+        body: |
+          <!-- ui-comment-${{ github.workflow }} -->
+          ${{ env.PULL_COMMENT }}
+        edit-mode: replace

.github/actions/ui-report/action.yml

@@ -0,0 +1,55 @@
+name: 'UI report'
+description: 'Prepare and upload HTML report of UI test results'
+inputs:
+  model:
+    description: 'Internal model name'
+    required: true
+  lang:
+    description: 'Firmware language code'
+    required: true
+  status:
+    description: 'Either "success" or "failure"'
+    required: false
+    default: "failure"
+runs:
+  using: composite
+  steps:
+    - name: Set AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: arn:aws:iam::538326561891:role/gh_actions_deploy_dev_firmware_data
+        aws-region: eu-west-1
+    - run: |
+        MODELJOB=${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}
+        OUTDIR=${{ github.run_id }}/$MODELJOB
+        mkdir -p $OUTDIR
+        nix-shell --run "poetry run python ci/prepare_ui_artifacts.py || true"
+        mv tests/ui_tests/reports/test/* $OUTDIR || true
+        mv tests/ui_tests/fixtures.*.json $OUTDIR || true
+        mv tests/trezor.log $OUTDIR || true
+        diff -u tests/ui_tests/fixtures.json tests/ui_tests/fixtures.suggestion.json || true
+        tar -cf screens_$MODELJOB.tar tests/ui_tests/screens || true
+
+        if [ "${{ inputs.status }}" = "success" ]; then
+          cp .github/actions/ui-report/success.png $OUTDIR/status.png
+        else
+          cp .github/actions/ui-report/failure.png $OUTDIR/status.png
+        fi
+      shell: sh
+    - name: Upload report
+      run: |
+        aws s3 sync --no-progress ${{ github.run_id }} s3://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}
+        echo "[UI test report](https://data.trezor.io/dev/firmware/ui_report/${{ github.run_id }}/${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}/index.html)" >> $GITHUB_STEP_SUMMARY
+      shell: sh
+    - name: Upload test screen recording
+      run: |
+        aws s3 sync --no-progress ci/ui_test_records s3://data.trezor.io/dev/firmware/ui_tests
+        # TODO: generate directory listing / autoindex
+      shell: sh
+    - uses: actions/upload-artifact@v4
+      with:
+        name: ui-records-${{ github.job }}-${{ strategy.job-index }}
+        path: |
+          # used by core_ui_main
+          screens_${{ inputs.model }}-${{ inputs.lang }}-${{ github.job }}.tar
+        retention-days: 1  # not useful after workflow finishes

.github/workflows/common.yml

@@ -0,0 +1,117 @@
+name: Common
+
+on:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    - cron: '14 23 * * *'  # every day @ 23:14
+
+jobs:
+  crypto_build:
+    name: Crypto library
+    runs-on: ubuntu-latest
+    env:
+      CC: gcc
+      ADDRESS_SANITIZER: 1
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: cachix/install-nix-action@v23
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: nix-shell --run "poetry install"
+      - run: cp -r crypto crypto_noasan
+      - run: nix-shell --run "poetry run make -C crypto"
+      - run: nix-shell --run "export ADDRESS_SANITIZER=0; poetry run make -C crypto_noasan"
+      - run: mv crypto_noasan/tests/test_check crypto/tests/test_check_noasan
+      - uses: actions/upload-artifact@v4
+        with:
+          name: crypto-build
+          path: |
+            crypto/tests/aestst
+            crypto/tests/libtrezor-crypto.so
+            crypto/tests/test_check
+            crypto/tests/test_check_noasan
+            crypto/tests/test_openssl
+          retention-days: 7
+
+  crypto_test:
+    name: Crypto test
+    needs: [crypto_build]
+    runs-on: ubuntu-latest
+    env:
+      ASAN_OPTIONS: "verify_asan_link_order=0"
+      CK_TIMEOUT_MULTIPLIER: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: cachix/install-nix-action@v23
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: nix-shell --run "poetry install"
+      - uses: actions/download-artifact@v4
+        with:
+          name: crypto-build
+          path: crypto/tests
+      - run: chmod +x crypto/tests/*
+      - run: ./crypto/tests/aestst
+      - run: ./crypto/tests/test_check
+      - run: ./crypto/tests/test_openssl 1000
+      - run: nix-shell --run "cd crypto && ITERS=10 poetry run pytest tests"
+      - run: nix-shell --run "CK_TIMEOUT_MULTIPLIER=20 valgrind -q --error-exitcode=1 ./crypto/tests/test_check_noasan"
+
+  python_test:
+    name: Python test
+    runs-on: ubuntu-latest
+    env:
+      LC_ALL: C.UTF-8
+      LANG: C.UTF-8
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: cachix/install-nix-action@v23
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: nix-shell --run "poetry install"
+      # Workaround for nixpkgs+tox integration failure which results in:
+      #     ModuleNotFoundError: No module named '_sysconfigdata__linux_x86_64-linux-gnu'
+      # The value of _PYTHON_SYSCONFIGDATA_NAME has changed between python 3.7 and 3.8 and with
+      # multiple versions in your environment the older pythons don't seem to work under tox.
+      # When the variable is unset the interpreter seems to do the right thing. Can be removed in
+      # july 2023 when python 3.7 is EOLed.
+      # See also:
+      # https://github.com/NixOS/nixpkgs/blob/b00c7c2d1d905eb63c81a0917f1a94b763a7843b/pkgs/development/interpreters/python/cpython/default.nix#L103
+      # https://github.com/NixOS/nixpkgs/pull/98915
+      - run: nix-shell --arg fullDeps true --run "unset _PYTHON_SYSCONFIGDATA_NAME && cd python && poetry run tox"
+
+  python_support_test:
+    name: Python support test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: cachix/install-nix-action@v23
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: nix-shell --run "poetry install"
+      - run: nix-shell --run "poetry run make python_support_check"
+
+  storage_test:
+    name: Storage test
+    # TODO: only for changes in storage/
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: cachix/install-nix-action@v23
+        with:
+          nix_path: nixpkgs=channel:nixos-unstable
+      - run: nix-shell --run "poetry install"
+      - run: unset PYTEST_TIMEOUT
+      - run: nix-shell --run "poetry run make -C storage/tests build"
+      - run: nix-shell --run "poetry run make -C storage/tests tests_all"

.github/workflows/core-hw.yml

@@ -0,0 +1,138 @@
+name: Hardware tests
+
+on:
+  schedule:
+    - cron: '15 23 * * *'  # every day @ 23:15
+  workflow_dispatch:
+
+# [Device tests](../tests/device-tests.md) that run against an actual physical Trezors.
+# The device needs to have special bootloader, found in `core/embed/bootloader_ci`, that
+# makes it possible to flash firmware without confirmation on the touchscreen.
+#
+# All hardware tests are run nightly on the `main` branch,
+# and also can be started manually.
+#
+# Currently it's not possible to run all regular TT tests without getting into
+# a state where the micropython heap is too fragmented and allocations fail
+# (often manifesting as a stuck test case). For that reason some tests are
+# skipped.
+# See also: https://github.com/trezor/trezor-firmware/issues/1371
+jobs:
+  core_device_test:
+    name: Device tests
+    runs-on:
+      - self-hosted
+      - ${{ matrix.model == 'T2B1' && 'runner3' || 'hw-t2t1' }}
+      # runner4 does not work at the moment
+      # - ${{ matrix.model == 'T2B1' && 'hw-t2b1' || 'hw-t2t1' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        model: [T2T1, T2B1]
+        coins: [universal, btconly]
+    env:
+      TREZOR_MODEL: ${{ matrix.model == 'T2T1' && 'T' || 'R' }}
+      TREZOR_PYTEST_SKIP_ALTCOINS: ${{ matrix.coins == 'btconly' && '1' || '0' }}
+      PYTEST_TIMEOUT: 1200
+      PYOPT: 0
+      DISABLE_OPTIGA: 1
+      BOOTLOADER_DEVEL: ${{ matrix.model == 'T2B1' && '1' || '0' }}
+      TESTOPTS: "-k 'not authenticate and not recovery and not lots'"
+      TT_UHUB_PORT: 1
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: ./.github/actions/environment
+      - run: nix-shell --arg hardwareTest true --run uhubctl
+      - run: nix-shell --run "poetry run make -C core build_firmware"
+      - run: nix-shell --arg hardwareTest true --run "poetry run python ci/hardware_tests/bootstrap.py ${{ matrix.model }} core/build/firmware/firmware.bin"
+      - run: nix-shell --run "poetry run trezorctl list"
+      - run: nix-shell --run "poetry run trezorctl get-features"
+      - run: |
+          # log serial console to file; sleep is used because tio needs stdin that is not /dev/null
+          nix-shell --arg hardwareTest true --run "sleep 8h | tio --no-autoconnect /dev/ttyTREZOR &> trezor.log" &
+          nix-shell --run "poetry run pytest -v tests/device_tests"
+      - run: tail -n50 trezor.log || true
+        if: failure()
+      - uses: actions/upload-artifact@v4
+        with:
+          name: core-hardware-${{ matrix.model }}-${{ matrix.coins }}
+          path: trezor.log
+          retention-days: 7
+        if: always()
+
+  core_monero_test:
+    name: Monero tests
+    runs-on:
+      - self-hosted
+      - ${{ matrix.model == 'T2B1' && 'runner3' || 'hw-t2t1' }}
+      # runner4 does not work at the moment
+      # - ${{ matrix.model == 'T2B1' && 'hw-t2b1' || 'hw-t2t1' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        model: [T2T1, T2B1]
+    env:
+      TREZOR_MODEL: ${{ matrix.model == 'T2T1' && 'T' || 'R' }}
+      PYTEST_TIMEOUT: 1200
+      PYOPT: 0
+      DISABLE_OPTIGA: 1
+      BOOTLOADER_DEVEL: ${{ matrix.model == 'T2B1' && '1' || '0' }}
+      TT_UHUB_PORT: 1
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: ./.github/actions/environment
+      - run: nix-shell --arg hardwareTest true --run uhubctl
+      - run: nix-shell --run "poetry run make -C core build_firmware"
+      - run: nix-shell --arg hardwareTest true --run "poetry run python ci/hardware_tests/bootstrap.py ${{ matrix.model }} core/build/firmware/firmware.bin"
+      - run: nix-shell --run "poetry run trezorctl list"
+      - run: nix-shell --run "poetry run trezorctl get-features"
+      - run: |
+          # log serial console to file; sleep is used because tio needs stdin that is not /dev/null
+          nix-shell --arg hardwareTest true --run "sleep 8h | tio --no-autoconnect /dev/ttyTREZOR &> trezor.log" &
+          nix-shell --arg fullDeps true --run "./core/tests/run_tests_device_emu_monero.sh $TESTOPTS"
+      - run: tail -n50 trezor.log || true
+        if: failure()
+      - uses: actions/upload-artifact@v4
+        with:
+          name: core-hardware-monero-${{ matrix.model }}
+          path: trezor.log
+          retention-days: 7
+        if: always()
+
+  legacy_device_test:
+    name: Device tests T1B1
+    runs-on:
+      - self-hosted
+      - hw-t1b1
+    strategy:
+      fail-fast: false
+      matrix:
+        coins: [universal, btconly]
+    env:
+      TREZOR_PYTEST_SKIP_ALTCOINS: ${{ matrix.coins == 'btconly' && '1' || '0' }}
+      PYTEST_TIMEOUT: 1200
+      T1_UHUB_LOCATION: 3-1
+      T1_UHUB_PORT: 2
+      T1_CAMERA: /dev/video0  # camera device
+      T1_ARDUINO_SERIAL: /dev/ttyTPMB  # arduino that pushes T1 buttons
+      BITCOIN_ONLY: ${{ matrix.coins == 'universal' && '0' || '1' }}
+      DEBUG_LINK: 1
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: ./.github/actions/environment
+      - run: nix-shell --arg hardwareTest true --run uhubctl
+      - run: nix-shell --run "poetry run legacy/script/setup"
+      - run: nix-shell --run "export PRODUCTION=0 && poetry run legacy/script/cibuild"
+      - run: nix-shell --arg hardwareTest true --run "ci/hardware_tests/t1_hw_test.sh"
+      - uses: actions/upload-artifact@v4
+        with:
+          name: legacy-hardware-${{ matrix.coins }}
+          path: ci/hardware_tests/*.mp4
+          retention-days: 7
+        if: always()