Merge branch 'portexpander-keyboard' of https://github.com/meshtastic/firmware into portexpander-keyboard

.github/workflows/build_esp32.yml

@@ -12,52 +12,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Build base
-        id: base
-        uses: ./.github/actions/setup-base
-
-      - name: Pull web ui
-        uses: dsaltares/fetch-gh-release-asset@master
-        with:
-          repo: meshtastic/web
-          file: build.tar
-          target: build.tar
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Unpack web ui
-        run: |
-          tar -xf build.tar -C data/static
-          rm build.tar
-
-      - name: Remove debug flags for release
-        if: ${{ github.event_name == 'workflow_dispatch' }}
-        run: |
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32s2.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32s3.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32c3.ini
 
       - name: Build ESP32
-        run: bin/build-esp32.sh ${{ inputs.board }}
-
-      - name: Pull OTA Firmware
-        uses: dsaltares/fetch-gh-release-asset@master
+        id: build
+        uses: ./.github/actions/build-variant
         with:
-          repo: meshtastic/firmware-ota
-          file: firmware.bin
-          target: release/bleota.bin
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get release version string
-        shell: bash
-        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
-        id: version
-
-      - name: Store binaries as an artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip
-          overwrite: true
-          path: |
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          board: ${{ inputs.board }}
+          remove-debug-flags: >-
+            ./arch/esp32/esp32.ini
+            ./arch/esp32/esp32s2.ini
+            ./arch/esp32/esp32s3.ini
+            ./arch/esp32/esp32c3.ini
+          build-script-path: bin/build-esp32.sh
+          ota-firmware-source: firmware.bin
+          ota-firmware-target: release/bleota.bin
+          artifact-paths: |
             release/*.bin
             release/*.elf
+          include-web-ui: true
+          arch: esp32

.github/workflows/build_esp32_c3.yml

@@ -14,50 +14,22 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Build base
-        id: base
-        uses: ./.github/actions/setup-base
 
-      - name: Pull web ui
-        uses: dsaltares/fetch-gh-release-asset@master
+      - name: Build ESP32-C3
+        id: build
+        uses: ./.github/actions/build-variant
         with:
-          repo: meshtastic/web
-          file: build.tar
-          target: build.tar
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Unpack web ui
-        run: |
-          tar -xf build.tar -C data/static
-          rm build.tar
-      - name: Remove debug flags for release
-        if: ${{ github.event_name == 'workflow_dispatch' }}
-        run: |
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32s2.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32s3.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32c3.ini
-      - name: Build ESP32
-        run: bin/build-esp32.sh ${{ inputs.board }}
-
-      - name: Pull OTA Firmware
-        uses: dsaltares/fetch-gh-release-asset@master
-        with:
-          repo: meshtastic/firmware-ota
-          file: firmware-c3.bin
-          target: release/bleota-c3.bin
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get release version string
-        shell: bash
-        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
-        id: version
-
-      - name: Store binaries as an artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip
-          overwrite: true
-          path: |
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          board: ${{ inputs.board }}
+          remove-debug-flags: >-
+            ./arch/esp32/esp32.ini
+            ./arch/esp32/esp32s2.ini
+            ./arch/esp32/esp32s3.ini
+            ./arch/esp32/esp32c3.ini
+          build-script-path: bin/build-esp32.sh
+          ota-firmware-source: firmware-c3.bin
+          ota-firmware-target: release/bleota-c3.bin
+          artifact-paths: |
             release/*.bin
             release/*.elf
+          arch: esp32c3

.github/workflows/build_esp32_c6.yml

@@ -0,0 +1,36 @@
+name: Build ESP32-C6
+
+on:
+  workflow_call:
+    inputs:
+      board:
+        required: true
+        type: string
+
+permissions: read-all
+
+jobs:
+  build-esp32-c6:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build ESP32-C6
+        id: build
+        uses: ./.github/actions/build-variant
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          board: ${{ inputs.board }}
+          remove-debug-flags: >-
+            ./arch/esp32/esp32.ini
+            ./arch/esp32/esp32s2.ini
+            ./arch/esp32/esp32s3.ini
+            ./arch/esp32/esp32c3.ini
+            ./arch/esp32/esp32c6.ini
+          build-script-path: bin/build-esp32.sh
+          ota-firmware-source: firmware-c3.bin
+          ota-firmware-target: release/bleota-c3.bin
+          artifact-paths: |
+            release/*.bin
+            release/*.elf
+          arch: esp32c6

.github/workflows/build_esp32_s3.yml

@@ -12,50 +12,23 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Build base
-        id: base
-        uses: ./.github/actions/setup-base
 
-      - name: Pull web ui
-        uses: dsaltares/fetch-gh-release-asset@master
+      - name: Build ESP32-S3
+        id: build
+        uses: ./.github/actions/build-variant
         with:
-          repo: meshtastic/web
-          file: build.tar
-          target: build.tar
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Unpack web ui
-        run: |
-          tar -xf build.tar -C data/static
-          rm build.tar
-      - name: Remove debug flags for release
-        if: ${{ github.event_name == 'workflow_dispatch' }}
-        run: |
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32s2.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32s3.ini
-          sed -i '/DDEBUG_HEAP/d' ./arch/esp32/esp32c3.ini
-      - name: Build ESP32
-        run: bin/build-esp32.sh ${{ inputs.board }}
-
-      - name: Pull OTA Firmware
-        uses: dsaltares/fetch-gh-release-asset@master
-        with:
-          repo: meshtastic/firmware-ota
-          file: firmware-s3.bin
-          target: release/bleota-s3.bin
-          token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get release version string
-        shell: bash
-        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
-        id: version
-
-      - name: Store binaries as an artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip
-          overwrite: true
-          path: |
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          board: ${{ inputs.board }}
+          remove-debug-flags: >-
+            ./arch/esp32/esp32.ini
+            ./arch/esp32/esp32s2.ini
+            ./arch/esp32/esp32s3.ini
+            ./arch/esp32/esp32c3.ini
+          build-script-path: bin/build-esp32.sh
+          ota-firmware-source: firmware-s3.bin
+          ota-firmware-target: release/bleota-s3.bin
+          artifact-paths: |
             release/*.bin
             release/*.elf
+          include-web-ui: true
+          arch: esp32s3

.github/workflows/build_native.yml

@@ -10,10 +10,10 @@ jobs:
   build-native:
     runs-on: ubuntu-latest
     steps:
-      - name: Install libbluetooth
+      - name: Install libs needed for native build
         shell: bash
         run: |
-          sudo apt-get update
+          sudo apt-get update --fix-missing
           sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev
 
       - name: Checkout code
@@ -67,7 +67,7 @@ jobs:
       - name: Docker build and push tagged versions
         if: ${{ github.event_name == 'workflow_dispatch' }}
         continue-on-error: true # FIXME: Failing docker login auth
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile
@@ -77,7 +77,7 @@ jobs:
       - name: Docker build and push
         if: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request_target' && github.event_name != 'pull_request' }}
         continue-on-error: true # FIXME: Failing docker login auth
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile

.github/workflows/build_nrf52.yml

@@ -12,23 +12,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Build base
-        id: base
-        uses: ./.github/actions/setup-base
 
       - name: Build NRF52
-        run: bin/build-nrf52.sh ${{ inputs.board }}
-
-      - name: Get release version string
-        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
-        id: version
-
-      - name: Store binaries as an artifact
-        uses: actions/upload-artifact@v4
+        id: build
+        uses: ./.github/actions/build-variant
         with:
-          name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip
-          overwrite: true
-          path: |
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          board: ${{ inputs.board }}
+          build-script-path: bin/build-nrf52.sh
+          artifact-paths: |
+            release/*.hex
             release/*.uf2
             release/*.elf
             release/*.zip
+          arch: nrf52840

.github/workflows/build_raspbian.yml

@@ -13,6 +13,7 @@ jobs:
       - name: Install libbluetooth
         shell: bash
         run: |
+          apt-get update -y --fix-missing
           apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev
 
       - name: Checkout code

.github/workflows/build_raspbian_armv7l.yml

@@ -13,6 +13,7 @@ jobs:
       - name: Install libbluetooth
         shell: bash
         run: |
+          apt-get update -y --fix-missing
           apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev
 
       - name: Checkout code

.github/workflows/build_rpi2040.yml

@@ -12,22 +12,15 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Build base
-        id: base
-        uses: ./.github/actions/setup-base
 
       - name: Build Raspberry Pi 2040
-        run: ./bin/build-rpi2040.sh ${{ inputs.board }}
-
-      - name: Get release version string
-        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
-        id: version
-
-      - name: Store binaries as an artifact
-        uses: actions/upload-artifact@v4
+        id: build
+        uses: ./.github/actions/build-variant
         with:
-          name: firmware-${{ inputs.board }}-${{ steps.version.outputs.version }}.zip
-          overwrite: true
-          path: |
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          board: ${{ inputs.board }}
+          build-script-path: bin/build-rpi2040.sh
+          artifact-paths: |
             release/*.uf2
             release/*.elf
+          arch: rp2040

.github/workflows/build_stm32.yml

@@ -0,0 +1,27 @@
+name: Build STM32
+
+on:
+  workflow_call:
+    inputs:
+      board:
+        required: true
+        type: string
+
+jobs:
+  build-stm32:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build STM32WL
+        id: build
+        uses: ./.github/actions/build-variant
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          board: ${{ inputs.board }}
+          build-script-path: bin/build-stm32.sh
+          artifact-paths: |
+            release/*.hex
+            release/*.bin
+            release/*.elf
+          arch: stm32

.github/workflows/generate-userprefs.yml

@@ -0,0 +1,35 @@
+name: Generate UsersPrefs JSON manifest
+
+on:
+  push:
+    paths:
+      - userPrefs.h
+    branches:
+      - master
+
+jobs:
+  generate-userprefs:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Clang
+        run: sudo apt-get install -y clang
+
+      - name: Install trunk
+        run: curl https://get.trunk.io -fsSL | bash
+
+      - name: Generate userPrefs.jsom
+        run: python3 ./bin/build-userprefs-json.py
+
+      - name: Trunk format json
+        run: trunk format userPrefs.json
+
+      - name: Commit userPrefs.json
+        run: |
+          git config --global user.email "actions@github.com"
+          git config --global user.name "GitHub Actions"
+          git add userPrefs.json
+          git commit -m "Update userPrefs.json"
+          git push

.github/workflows/main_matrix.yml

@@ -1,7 +1,7 @@
 name: CI
-#concurrency:
-#  group: ${{ github.ref }}
-#  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+concurrency:
+  group: ci-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
 on:
   # # Triggers the workflow on push but only for the master branch
   push:
@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        arch: [esp32, esp32s3, esp32c3, nrf52840, rp2040, check]
+        arch: [esp32, esp32s3, esp32c3, esp32c6, nrf52840, rp2040, stm32, check]
     runs-on: ubuntu-latest
     steps:
       - id: checkout
@@ -32,15 +32,21 @@ jobs:
         name: Checkout base
       - id: jsonStep
         run: |
-          TARGETS=$(./bin/generate_ci_matrix.py ${{matrix.arch}})
-          echo "$TARGETS"
+          if [[ "${{ github.head_ref }}" == "" ]]; then
+            TARGETS=$(./bin/generate_ci_matrix.py ${{matrix.arch}})
+          else  
+            TARGETS=$(./bin/generate_ci_matrix.py ${{matrix.arch}} quick)
+          fi
+          echo "Name: ${{ github.ref_name }} Base: ${{ github.base_ref }} Head: ${{ github.head_ref }} Ref: ${{ github.ref }} Targets: $TARGETS"
           echo "${{matrix.arch}}=$(jq -cn --argjson environments "$TARGETS" '{board: $environments}')" >> $GITHUB_OUTPUT
     outputs:
       esp32: ${{ steps.jsonStep.outputs.esp32 }}
       esp32s3: ${{ steps.jsonStep.outputs.esp32s3 }}
       esp32c3: ${{ steps.jsonStep.outputs.esp32c3 }}
+      esp32c6: ${{ steps.jsonStep.outputs.esp32c6 }}
       nrf52840: ${{ steps.jsonStep.outputs.nrf52840 }}
       rp2040: ${{ steps.jsonStep.outputs.rp2040 }}
+      stm32: ${{ steps.jsonStep.outputs.stm32 }}
       check: ${{ steps.jsonStep.outputs.check }}
 
   check:
@@ -50,6 +56,7 @@ jobs:
       matrix: ${{ fromJson(needs.setup.outputs.check) }}
 
     runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'workflow_dispatch' }}
     steps:
       - uses: actions/checkout@v4
       - name: Build base
@@ -85,6 +92,15 @@ jobs:
     with:
       board: ${{ matrix.board }}
 
+  build-esp32-c6:
+    needs: setup
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup.outputs.esp32c6) }}
+    uses: ./.github/workflows/build_esp32_c6.yml
+    with:
+      board: ${{ matrix.board }}
+
   build-nrf52:
     needs: setup
     strategy:
@@ -103,6 +119,15 @@ jobs:
     with:
       board: ${{ matrix.board }}
 
+  build-stm32:
+    needs: setup
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup.outputs.stm32) }}
+    uses: ./.github/workflows/build_stm32.yml
+    with:
+      board: ${{ matrix.board }}
+
   package-raspbian:
     uses: ./.github/workflows/package_raspbian.yml
 
@@ -114,6 +139,7 @@ jobs:
 
   after-checks:
     runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'workflow_dispatch' }}
     needs: [check]
     steps:
       - name: Checkout code
@@ -126,17 +152,20 @@ jobs:
     permissions:
       contents: write
       pull-requests: write
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [esp32, esp32s3, esp32c3, esp32c6, nrf52840, rp2040, stm32]
     runs-on: ubuntu-latest
     needs:
       [
         build-esp32,
         build-esp32-s3,
         build-esp32-c3,
+        build-esp32-c6,
         build-nrf52,
         build-rpi2040,
-        package-raspbian,
-        package-raspbian-armv7l,
-        package-native
+        build-stm32,
       ]
     steps:
       - name: Checkout code
@@ -148,6 +177,7 @@ jobs:
       - uses: actions/download-artifact@v4
         with:
           path: ./
+          pattern: firmware-${{matrix.arch}}-*
           merge-multiple: true
 
       - name: Display structure of downloaded files
@@ -158,29 +188,28 @@ jobs:
         id: version
 
       - name: Move files up
-        run: mv -b -t ./ ./release/meshtasticd_linux_* ./bin/config-dist.yaml ./bin/device-*.sh ./bin/device-*.bat
+        run: mv -b -t ./ ./bin/device-*.sh ./bin/device-*.bat
 
       - name: Repackage in single firmware zip
         uses: actions/upload-artifact@v4
         with:
-          name: firmware-${{ steps.version.outputs.version }}
+          name: firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}
           overwrite: true
           path: |
             ./firmware-*.bin
             ./firmware-*.uf2
+            ./firmware-*.hex
             ./firmware-*-ota.zip
             ./device-*.sh
             ./device-*.bat
-            ./meshtasticd_linux_*
-            ./config-dist.yaml
             ./littlefs-*.bin
             ./bleota*bin
             ./Meshtastic_nRF52_factory_erase*.uf2
-          retention-days: 90
+          retention-days: 30
 
       - uses: actions/download-artifact@v4
         with:
-          name: firmware-${{ steps.version.outputs.version }}
+          name: firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}
           merge-multiple: true
           path: ./output
 
@@ -194,32 +223,35 @@ jobs:
           chmod +x ./output/device-update.sh
 
       - name: Zip firmware
-        run: zip -j -9 -r ./firmware-${{ steps.version.outputs.version }}.zip ./output
+        run: zip -j -9 -r ./firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip ./output
 
       - name: Repackage in single elfs zip
         uses: actions/upload-artifact@v4
         with:
-          name: debug-elfs-${{ steps.version.outputs.version }}.zip
+          name: debug-elfs-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip
           overwrite: true
           path: ./*.elf
           retention-days: 30
 
-      - name: Create request artifacts
-        continue-on-error: true # FIXME: Why are we getting 502, but things still work?
-        if: ${{ github.event_name == 'pull_request_target' || github.event_name == 'pull_request' }}
-        uses: gavv/pull-request-artifacts@v2.1.0
+      - uses: scruplelesswizard/comment-artifact@main
+        if: ${{ github.event_name == 'pull_request' }}
         with:
-          commit: ${{ (github.event.pull_request_target || github.event.pull_request).head.sha }}
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-          artifacts-token: ${{ secrets.ARTIFACTS_TOKEN }}
-          artifacts-repo: meshtastic/artifacts
-          artifacts-branch: device
-          artifacts: ./firmware-${{ steps.version.outputs.version }}.zip
+          name: firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}
+          description: "Download firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip. This artifact will be available for 90 days from creation"
+          github-token: ${{ secrets.GITHUB_TOKEN }}
 
   release-artifacts:
     runs-on: ubuntu-latest
     if: ${{ github.event_name == 'workflow_dispatch' }}
-    needs: [gather-artifacts, after-checks]
+    outputs:
+      upload_url: ${{ steps.create_release.outputs.upload_url }}
+    needs:
+      [
+        gather-artifacts,
+        package-raspbian,
+        package-raspbian-armv7l,
+        package-native,
+      ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -233,42 +265,6 @@ jobs:
         run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
         id: version
 
-      - uses: actions/download-artifact@v4
-        with:
-          name: firmware-${{ steps.version.outputs.version }}
-          merge-multiple: true
-          path: ./output
-
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: meshtasticd_${{ steps.version.outputs.version }}_*.deb
-          merge-multiple: true
-          path: ./output
-
-      - name: Display structure of downloaded files
-        run: ls -R
-
-      - name: Device scripts permissions
-        run: |
-          chmod +x ./output/device-install.sh
-          chmod +x ./output/device-update.sh
-
-      - name: Zip firmware
-        run: zip -j -9 -r ./firmware-${{ steps.version.outputs.version }}.zip ./output -x *.deb
-
-      - uses: actions/download-artifact@v4
-        with:
-          name: debug-elfs-${{ steps.version.outputs.version }}.zip
-          merge-multiple: true
-          path: ./elfs
-
-      - name: Zip Elfs
-        run: zip -j -9 -r ./debug-elfs-${{ steps.version.outputs.version }}.zip ./elfs
-
-      # For diagnostics
-      - name: Show artifacts
-        run: ls -lR
-
       - name: Create release
         uses: actions/create-release@v1
         id: create_release
@@ -282,25 +278,16 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
-      - name: Add bins to release
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
+      - name: Download deb files
+        uses: actions/download-artifact@v4
         with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./firmware-${{ steps.version.outputs.version }}.zip
-          asset_name: firmware-${{ steps.version.outputs.version }}.zip
-          asset_content_type: application/zip
+          pattern: meshtasticd_${{ steps.version.outputs.version }}_*.deb
+          merge-multiple: true
+          path: ./output
 
-      - name: Add debug elfs to release
-        uses: actions/upload-release-asset@v1
-        env:
-          GITHUB_TOKEN: ${{ github.token }}
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: ./debug-elfs-${{ steps.version.outputs.version }}.zip
-          asset_name: debug-elfs-${{ steps.version.outputs.version }}.zip
-          asset_content_type: application/zip
+      # For diagnostics
+      - name: Display structure of downloaded files
+        run: ls -lR
 
       - name: Add raspbian aarch64 .deb
         uses: actions/upload-release-asset@v1
@@ -337,7 +324,79 @@ jobs:
           bin/bump_version.py
 
       - name: Create version.properties pull request
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
+          title: Bump version.properties
           add-paths: |
             version.properties
+
+  release-firmware:
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [esp32, esp32s3, esp32c3, esp32c6, nrf52840, rp2040, stm32]
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name == 'workflow_dispatch' }}
+    needs: [release-artifacts]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+
+      - name: Get release version string
+        run: echo "version=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
+        id: version
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}
+          merge-multiple: true
+          path: ./output
+
+      - name: Display structure of downloaded files
+        run: ls -lR
+
+      - name: Device scripts permissions
+        run: |
+          chmod +x ./output/device-install.sh
+          chmod +x ./output/device-update.sh
+
+      - name: Zip firmware
+        run: zip -j -9 -r ./firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip ./output
+
+      - uses: actions/download-artifact@v4
+        with:
+          name: debug-elfs-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip
+          merge-multiple: true
+          path: ./elfs
+
+      - name: Zip firmware
+        run: zip -j -9 -r ./debug-elfs-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip ./elfs
+
+      # For diagnostics
+      - name: Display structure of downloaded files
+        run: ls -lR
+
+      - name: Add bins to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{needs.release-artifacts.outputs.upload_url}}
+          asset_path: ./firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip
+          asset_name: firmware-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip
+          asset_content_type: application/zip
+
+      - name: Add debug elfs to release
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        with:
+          upload_url: ${{needs.release-artifacts.outputs.upload_url}}
+          asset_path: ./debug-elfs-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip
+          asset_name: debug-elfs-${{matrix.arch}}-${{ steps.version.outputs.version }}.zip
+          asset_content_type: application/zip

.github/workflows/package_amd64.yml

@@ -13,7 +13,7 @@ jobs:
     uses: ./.github/workflows/build_native.yml
 
   package-native:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: build-native
     steps:
       - name: Checkout code

.github/workflows/package_raspbian.yml

@@ -13,7 +13,7 @@ jobs:
     uses: ./.github/workflows/build_raspbian.yml
 
   package-raspbian:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: build-raspbian
     steps:
       - name: Checkout code

.github/workflows/package_raspbian_armv7l.yml

@@ -13,7 +13,7 @@ jobs:
     uses: ./.github/workflows/build_raspbian_armv7l.yml
 
   package-raspbian_armv7l:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     needs: build-raspbian_armv7l
     steps:
       - name: Checkout code

.github/workflows/sec_sast_semgrep_cron.yml

@@ -12,7 +12,7 @@ jobs:
   semgrep-full:
     runs-on: ubuntu-latest
     container:
-      image: returntocorp/semgrep
+      image: semgrep/semgrep
 
     steps:
       # step 1

.github/workflows/sec_sast_semgrep_pull.yml

@@ -4,9 +4,9 @@ on: pull_request
 
 jobs:
   semgrep-diff:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     container:
-      image: returntocorp/semgrep
+      image: semgrep/semgrep
 
     steps:
       # step 1

.github/workflows/tests.yml

@@ -0,0 +1,106 @@
+name: End to end tests
+
+on:
+  schedule:
+    - cron: "0 0 * * *" # Run every day at midnight
+  workflow_dispatch: {}
+
+jobs:
+  test-simulator:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install libbluetooth
+        shell: bash
+        run: |
+          sudo apt-get update --fix-missing
+          sudo apt-get install -y libbluetooth-dev libgpiod-dev libyaml-cpp-dev openssl libssl-dev libulfius-dev liborcania-dev
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Upgrade python tools
+        shell: bash
+        run: |
+          python -m pip install --upgrade pip
+          pip install -U platformio adafruit-nrfutil
+          pip install -U meshtastic --pre
+
+      - name: Upgrade platformio
+        shell: bash
+        run: |
+          pio upgrade
+
+      - name: Build Native
+        run: bin/build-native.sh
+
+      # We now run integration test before other build steps (to quickly see runtime failures)
+      - name: Build for native
+        run: platformio run -e native
+
+      - name: Integration test
+        run: |
+          .pio/build/native/program & sleep 10 # 5 seconds was not enough
+          echo "Simulator started, launching python test..."
+          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
+
+      - name: PlatformIO Tests
+        run: platformio test -e native --junit-output-path testreport.xml
+
+      - name: Test Report
+        uses: dorny/test-reporter@v1.9.1
+        if: success() || failure() # run this step even if previous step failed
+        with:
+          name: PlatformIO Tests
+          path: testreport.xml
+          reporter: java-junit
+
+  hardware-tests:
+    runs-on: test-runner
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      # - uses: actions/setup-python@v5
+      #   with:
+      #     python-version: '3.10'
+
+      # pipx install "setuptools<72"
+      - name: Upgrade python tools
+        shell: bash
+        run: |
+          pipx install adafruit-nrfutil
+          pipx install poetry
+          pipx install meshtastic --pip-args=--pre
+
+      - name: Install PlatformIO from script
+        shell: bash
+        run: |
+          curl -fsSL -o get-platformio.py https://raw.githubusercontent.com/platformio/platformio-core-installer/master/get-platformio.py
+          python3 get-platformio.py
+
+      - name: Upgrade platformio
+        shell: bash
+        run: |
+          export PATH=$PATH:$HOME/.local/bin
+          pio upgrade
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+
+      - name: Setup pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: latest
+
+      - name: Install dependencies, setup devices and run
+        shell: bash
+        run: |
+          git submodule update --init --recursive
+          cd meshtestic/
+          pnpm install
+          pnpm run setup
+          pnpm run test

.github/workflows/update_protobufs.yml

@@ -1,4 +1,4 @@
-name: "Update protobufs and regenerate classes"
+name: Update protobufs and regenerate classes
 on: workflow_dispatch
 
 jobs:
@@ -17,17 +17,18 @@ jobs:
 
       - name: Download nanopb
         run: |
-          wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.4.8-linux-x86.tar.gz
-          tar xvzf nanopb-0.4.8-linux-x86.tar.gz
-          mv nanopb-0.4.8-linux-x86 nanopb-0.4.8
+          wget https://jpa.kapsi.fi/nanopb/download/nanopb-0.4.9-linux-x86.tar.gz
+          tar xvzf nanopb-0.4.9-linux-x86.tar.gz
+          mv nanopb-0.4.9-linux-x86 nanopb-0.4.9
 
       - name: Re-generate protocol buffers
         run: |
           ./bin/regen-protos.sh
 
       - name: Create pull request
-        uses: peter-evans/create-pull-request@v6
+        uses: peter-evans/create-pull-request@v7
         with:
+          title: Update protobufs and classes
           add-paths: |
             protobufs
             src/mesh