From d6de0fd2687a753c7f41a6e2c14d8109a1e5616d Mon Sep 17 00:00:00 2001 From: Lorenzo Gasparini Date: Tue, 19 Dec 2023 23:26:47 +0100 Subject: [PATCH] Add web installer --- .github/actions/build-esp/action.yaml | 40 ++++++++++++++ .github/workflows/ci.yaml | 53 +++++-------------- .github/workflows/docs.yaml | 2 + .pre-commit-config.yaml | 1 + docs/css/extra.css | 4 ++ docs/development.md | 7 ++- docs/installer.md | 6 +++ docs/usage.md | 20 ++++--- mkdocs.yml | 15 +++++- mkdocs/gen_index.py | 17 ++++++ mkdocs/gen_manifest.py | 49 +++++++++++++++++ .../theme_overrides}/main.html | 0 scripts/merge_firmware.sh | 11 ++++ 13 files changed, 174 insertions(+), 51 deletions(-) create mode 100644 .github/actions/build-esp/action.yaml create mode 100644 docs/css/extra.css create mode 100644 docs/installer.md create mode 100644 mkdocs/gen_index.py create mode 100644 mkdocs/gen_manifest.py rename {mkdocs_theme_overrides => mkdocs/theme_overrides}/main.html (100%) create mode 100755 scripts/merge_firmware.sh diff --git a/.github/actions/build-esp/action.yaml b/.github/actions/build-esp/action.yaml new file mode 100644 index 0000000..25d605c --- /dev/null +++ b/.github/actions/build-esp/action.yaml @@ -0,0 +1,40 @@ +name: Build ESP binaries +description: Build ESP binaries + +runs: + using: 'composite' + steps: + - uses: pnpm/action-setup@v2 + with: + version: latest + - name: Install node + uses: actions/setup-node@v4 + with: + node-version: '20.10.0' + cache: 'pnpm' + - name: Install frontend dependencies and build frontend + run: | + pnpm i + pnpm build + shell: bash + - name: Cache esp-idf build + uses: actions/cache@v3 + with: + # TODO This caching seems useless + path: 'build' + key: ${{ runner.os }}-build-v6 + - name: Cache managed components + uses: actions/cache@v3 + with: + path: 'managed_components' + key: ${{ runner.os }}-managed_components-${{ hashFiles('dependencies.lock') }}-v2 + - name: Address ESP-IDF component hash bug + run: | + rm -rf managed_components/bblanchon__arduinojson/.component_hash + shell: bash + - name: esp-idf build and merge firmware + run: | + docker run -t -e IDF_TARGET="esp32" -e GITHUB_ACTIONS=true -v "${GITHUB_WORKSPACE}:/app/${{ github.repository }}" \ + -w "/app/${{ github.repository }}" espressif/idf:release-v5.1 \ + /bin/bash -c 'git config --global --add safe.directory "*" && idf.py build && cd /app/${{ github.repository }}/scripts && ./merge_firmware.sh' + shell: bash diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6ec8de8..4aba634 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -86,54 +86,25 @@ jobs: uses: raven-actions/actionlint@v1 build_esp: - name: Build ESP binaries - timeout-minutes: 10 + name: Build and archive ESP firmware runs-on: ubuntu-latest steps: - - name: Checkout repo + - name: Checkout uses: actions/checkout@v4 - with: - submodules: 'recursive' - - uses: pnpm/action-setup@v2 - with: - version: latest - - name: Install node - uses: actions/setup-node@v4 - with: - node-version: '20.10.0' - cache: 'pnpm' - - name: Install frontend dependencies and build frontend - run: | - pnpm i - pnpm build - - name: Cache esp-idf build - uses: actions/cache@v3 - with: - path: 'build' - key: ${{ runner.os }}-build-v6 - - name: Cache managed components - uses: actions/cache@v3 - with: - path: 'managed_components' - key: ${{ runner.os }}-managed_components-${{ hashFiles('dependencies.lock') }}-v2 - - name: Address ESP-IDF component hash bug - run: | - rm -rf managed_components/bblanchon__arduinojson/.component_hash - - name: esp-idf build - run: | - docker run -t -e IDF_TARGET="esp32" -e GITHUB_ACTIONS=true -v "${GITHUB_WORKSPACE}:/app/${{ github.repository }}" \ - -w "/app/${{ github.repository }}" espressif/idf:release-v5.1 \ - /bin/bash -c 'git config --global --add safe.directory "*" && idf.py build' + - run: pwd + - run: ls -la . + - run: ls -la .github + - run: ls -la .github/actions + - run: ls -la .github/actions/build-esp + - name: Build ESP binaries + uses: ./.github/actions/build-esp - name: Archive build output artifacts uses: actions/upload-artifact@v4 with: - name: build - path: | - build/bootloader/bootloader.bin - build/partition_table/partition-table.bin - build/suntransit.bin - build/www.bin + name: merged_firmware + path: build/merged_firmware.bin + compression-level: 0 build_simulator: name: Build simulator binary diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 32a450b..90a01fe 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -26,6 +26,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + - name: Build ESP binaries + uses: ./.github/actions/build-esp - name: Setup Pages uses: actions/configure-pages@v4 - name: Setup Python diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 52b3f59..cd5a7f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,6 +16,7 @@ repos: exclude_types: - python - image + - shell entry: pnpm exec prettier --write - id: clang-format diff --git a/docs/css/extra.css b/docs/css/extra.css new file mode 100644 index 0000000..01862b1 --- /dev/null +++ b/docs/css/extra.css @@ -0,0 +1,4 @@ +body { + --esp-tools-button-color: var(--md-primary-fg-color); + --esp-tools-button-text-color: black; +} diff --git a/docs/development.md b/docs/development.md index e13a749..1b88e1d 100644 --- a/docs/development.md +++ b/docs/development.md @@ -6,15 +6,20 @@ The project is composed of three main parts: ## Dependencies -Install the Python dependencies via `pip install -r requirements.txt` (required to build docs & for the commit hooks). +Install the Python dependencies via `pip install -r requirements.txt`. Either install them globally or in a virtual environment. If you use `idf.py` from the command line, you might want to install them in the IDF virtual environment, so that they are available when activating the IDF virtual environment. +## ESP32 + +Install the [ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/) framework, preferably via the [VSCode Extension](https://github.com/espressif/vscode-esp-idf-extension/blob/master/docs/tutorial/install.md). + ## Simulator The simulator code lives in `simulator` and is used to develop the UI. The UI library is symlinked from the ESP folder. It is developed using the [PlatformIO](https://platformio.org/) framework. Follow [these instructions](https://platformio.org/install/ide?install=vscode) to get started using it within VSCode. +It uses [libsdl](https://github.com/libsdl-org/SDL), make sure to install it. ## Frontend diff --git a/docs/installer.md b/docs/installer.md new file mode 100644 index 0000000..a2e115b --- /dev/null +++ b/docs/installer.md @@ -0,0 +1,6 @@ +# Firmware Web Installer + +Click the button below to installer the latest development firmware onto your device. + + + diff --git a/docs/usage.md b/docs/usage.md index 5b53c21..21dac9f 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -4,25 +4,31 @@ You'll need a Sunton [3248S035C](https://www.openhasp.com/0.7.0/hardware/sunton/ You can get one [here](https://de.aliexpress.com/item/1005004632953455.html). You might want to enclose the board in a case. 3D models to print are available online ([example](https://cults3d.com/en/3d-model/gadget/sunon-esp32-3248s035-matsekberg)). -## Step 2: Development setup +## Step 2: Flash the firmware -Install the [ESP-IDF](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/get-started/) framework via the [VSCode Extension](https://github.com/espressif/vscode-esp-idf-extension/blob/master/docs/tutorial/install.md). +### Web installer -Install [pnpm](https://pnpm.io/installation), required to build the frontend. +See [web installer](installer.md). -## Step 3: Build and flash the firmware +### Compile it yourself + +Follow the instructions in [development](development.md) to install ESP-IDF and pnpm. + +Build and flash the firmware, either via the VSCode extension or `idf.py`. + +## Step : Build and flash the firmware Build the firmware, connect the board via USB and flash it either via the ESP-IDF VSCode extension or `idf.py`. -## Step 4: Configure WiFi +## Step 3: Configure WiFi The board needs to be connected to the Internet to retrieve the realtime departures information. After flash and reboot, a wizard should appear on the board with instructions on how to connect the board to a WiFi access point. This can be done via the "ESP SoftAP Provisioning" app ([Play Store](https://play.google.com/store/apps/details?id=com.espressif.provsoftap), [Apple Store](https://apps.apple.com/us/app/esp-softap-provisioning/id1474040630)). -## Step 5: Configure station +## Step 4: Configure station After the network connection is estabilished, the screen should show a message asking to configure the station to show departures from, along with the URL to connect to to do so ([http://suntransit.local](http://suntransit.local)). -## Step 6: Profit! +## Step 5: Profit! diff --git a/mkdocs.yml b/mkdocs.yml index c994515..1ed4c87 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,18 +5,26 @@ repo_url: https://github.com/joined/suntransit nav: - Home: index.md - Usage: usage.md + - Installer: installer.md - Development: development.md theme: name: material logo: icon.svg favicon: favicon.png - custom_dir: mkdocs_theme_overrides + custom_dir: mkdocs/theme_overrides palette: scheme: slate primary: yellow accent: deep orange features: - navigation.instant +extra_javascript: + - path: https://unpkg.com/esp-web-tools@9/dist/web/install-button.js?module + type: module +extra_css: + - css/extra.css +watch: + - mkdocs/ markdown_extensions: # Python Markdown - abbr @@ -50,7 +58,10 @@ markdown_extensions: custom_checkbox: true - pymdownx.tilde plugins: + - search - glightbox - gen-files: scripts: - - docs/gen_index.py + - mkdocs/gen_index.py +hooks: + - mkdocs/gen_manifest.py diff --git a/mkdocs/gen_index.py b/mkdocs/gen_index.py new file mode 100644 index 0000000..1a86495 --- /dev/null +++ b/mkdocs/gen_index.py @@ -0,0 +1,17 @@ +import mkdocs_gen_files + + +def generate_index(): + in_path = "README.md" + out_path = "index.md" + + with open(in_path, "r") as file: + filedata = file.read() + + filedata = filedata.replace('src="docs/', 'src="') + + with mkdocs_gen_files.open(out_path, "w") as file: + file.write(filedata) + + +generate_index() diff --git a/mkdocs/gen_manifest.py b/mkdocs/gen_manifest.py new file mode 100644 index 0000000..f5a4146 --- /dev/null +++ b/mkdocs/gen_manifest.py @@ -0,0 +1,49 @@ +import json +from pathlib import Path +import shutil + +IN_FIRMWARE_PATH = Path(__file__).parent.parent / "build" / "merged-firmware.bin" + + +def get_firmware_content(): + if not IN_FIRMWARE_PATH.exists(): + print( + f"Could not find firmware at {IN_FIRMWARE_PATH}, using dummy firmware" + ) + + return "{}" + + manifest = { + "name": "SunTransit", + # TODO Get version from somewhere, like Git (hash / hash-dirty) + "version": "0.1", + "new_install_prompt_erase": False, + "builds": [ + { + "chipFamily": "ESP32", + "parts": [{"path": IN_FIRMWARE_PATH.name, "offset": 0}], + } + ], + } + + return json.dumps(manifest, indent=4) + + +def generate_espwebtools_manifest(site_dir: str): + MANIFEST_PATH = Path(site_dir) / "installer" / "manifest.json" + + with open(MANIFEST_PATH, "w") as out_file: + out_file.write(get_firmware_content()) + +def copy_firmware(site_dir: str): + if not IN_FIRMWARE_PATH.exists(): + return + + OUT_FIRMWARE_PATH = Path(site_dir) / "installer" / IN_FIRMWARE_PATH.name + + shutil.copyfile(IN_FIRMWARE_PATH, OUT_FIRMWARE_PATH) + + +def on_post_build(config, **kwargs): + generate_espwebtools_manifest(config["site_dir"]) + copy_firmware(config["site_dir"]) diff --git a/mkdocs_theme_overrides/main.html b/mkdocs/theme_overrides/main.html similarity index 100% rename from mkdocs_theme_overrides/main.html rename to mkdocs/theme_overrides/main.html diff --git a/scripts/merge_firmware.sh b/scripts/merge_firmware.sh new file mode 100755 index 0000000..61966c3 --- /dev/null +++ b/scripts/merge_firmware.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +esptool.py --chip esp32 merge_bin \ + -o ../build/merged-firmware.bin \ + --flash_mode dio \ + --flash_freq 80m \ + --flash_size 4MB \ + 0x1000 ../build/bootloader/bootloader.bin \ + 0x8000 ../build/partition_table/partition-table.bin \ + 0x10000 ../build/suntransit.bin \ + 0x290000 ../build/www.bin \