Wait for location to set time

* Mark implicit ACK for MQTT as MQTT transport

* TRUNK

* Fix build

* Make sure implicit ACKs from MQTT do not stop retransmissions in ReliableRouter

---------

Co-authored-by: Ben Meadors <benmmeadors@gmail.com>

.github/workflows/build_firmware.yml

@@ -56,16 +56,18 @@ jobs:
           ota_firmware_source: ${{ steps.ota_dir.outputs.src || '' }}
           ota_firmware_target: ${{ steps.ota_dir.outputs.tgt || '' }}
 
-      - name: Echo manifest from release/firmware-*.mt.json to job summary
-        if: ${{ always() }}
+      - name: Job summary
         env:
           PIO_ENV: ${{ inputs.pio_env }}
         run: |
-          echo "## Manifest: \`$PIO_ENV\`" >> $GITHUB_STEP_SUMMARY
+          echo "## $PIO_ENV" >> $GITHUB_STEP_SUMMARY
+          echo "<details><summary><strong>Manifest</strong></summary>" >> $GITHUB_STEP_SUMMARY
+          echo '' >> $GITHUB_STEP_SUMMARY
           echo '```json' >> $GITHUB_STEP_SUMMARY
           cat release/firmware-*.mt.json >> $GITHUB_STEP_SUMMARY
           echo '' >> $GITHUB_STEP_SUMMARY
           echo '```' >> $GITHUB_STEP_SUMMARY
+          echo "</details>" >> $GITHUB_STEP_SUMMARY
 
       - name: Store binaries as an artifact
         uses: actions/upload-artifact@v5

.github/workflows/main_matrix.yml

@@ -77,16 +77,21 @@ jobs:
       fail-fast: false
       matrix:
         check: ${{ fromJson(needs.setup.outputs.check) }}
-
-    runs-on: ubuntu-latest
+    # Use 'arctastic' self-hosted runner pool when checking in the main repo
+    runs-on: ${{ github.repository_owner == 'meshtastic' && 'arctastic' || 'ubuntu-latest' }}
     if: ${{ github.event_name != 'workflow_dispatch' && github.repository == 'meshtastic/firmware' }}
     steps:
       - uses: actions/checkout@v6
-      - name: Build base
-        id: base
-        uses: ./.github/actions/setup-base
+        with:
+          submodules: recursive
+          ref: ${{github.event.pull_request.head.ref}}
+          repository: ${{github.event.pull_request.head.repo.full_name}}
       - name: Check ${{ matrix.check.board }}
-        run: bin/check-all.sh ${{ matrix.check.board }}
+        uses: meshtastic/gh-action-firmware@main
+        with:
+          pio_platform: ${{ matrix.check.platform }}
+          pio_env: ${{ matrix.check.board }}
+          pio_target: check
 
   build:
     needs: [setup, version]

bin/build-esp32.sh

@@ -22,7 +22,7 @@ export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
-pio run --environment $1 # -v
+pio run --environment $1 -t mtjson # -v
 
 cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
 
@@ -32,20 +32,10 @@ cp $BUILDDIR/$basename.factory.bin $OUTDIR/$basename.factory.bin
 echo "Copying ESP32 update bin file"
 cp $BUILDDIR/$basename.bin $OUTDIR/$basename.bin
 
-echo "Building Filesystem for ESP32 targets"
-# If you want to build the webui, uncomment the following lines
-# pio run --environment $1 -t buildfs
-# cp .pio/build/$1/littlefs.bin $OUTDIR/littlefswebui-$1-$VERSION.bin
-# # Remove webserver files from the filesystem and rebuild
-# ls -l data/static # Diagnostic list of files
-# rm -rf data/static
-pio run --environment $1 -t buildfs --disable-auto-clean
+echo "Copying Filesystem for ESP32 targets"
 cp $BUILDDIR/littlefs-$1-$VERSION.bin $OUTDIR/littlefs-$1-$VERSION.bin
 cp bin/device-install.* $OUTDIR/
 cp bin/device-update.* $OUTDIR/
 
-# Generate the manifest file
-echo "Generating Meshtastic manifest"
-TIMEFORMAT="Generated manifest in %E seconds"
-time pio run --environment $1 -t mtjson --silent --disable-auto-clean
+echo "Copying manifest"
 cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/build-nrf52.sh

@@ -22,7 +22,7 @@ export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
-pio run --environment $1 # -v
+pio run --environment $1 -t mtjson # -v
 
 cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
 
@@ -47,8 +47,5 @@ if (echo $1 | grep -q "rak4631"); then
 	cp $SRCHEX $OUTDIR/
 fi
 
-# Generate the manifest file
-echo "Generating Meshtastic manifest"
-TIMEFORMAT="Generated manifest in %E seconds"
-time pio run --environment $1 -t mtjson --silent --disable-auto-clean
+echo "Copying manifest"
 cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/build-rp2xx0.sh

@@ -22,15 +22,12 @@ export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
-pio run --environment $1 # -v
+pio run --environment $1 -t mtjson # -v
 
 cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
 
 echo "Copying uf2 file"
 cp $BUILDDIR/$basename.uf2 $OUTDIR/$basename.uf2
 
-# Generate the manifest file
-echo "Generating Meshtastic manifest"
-TIMEFORMAT="Generated manifest in %E seconds"
-time pio run --environment $1 -t mtjson --silent --disable-auto-clean
+echo "Copying manifest"
 cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/build-stm32wl.sh

@@ -22,15 +22,12 @@ export APP_VERSION=$VERSION
 
 basename=firmware-$1-$VERSION
 
-pio run --environment $1 # -v
+pio run --environment $1 -t mtjson # -v
 
 cp $BUILDDIR/$basename.elf $OUTDIR/$basename.elf
 
 echo "Copying STM32 bin file"
 cp $BUILDDIR/$basename.bin $OUTDIR/$basename.bin
 
-# Generate the manifest file
-echo "Generating Meshtastic manifest"
-TIMEFORMAT="Generated manifest in %E seconds"
-time pio run --environment $1 -t mtjson --silent --disable-auto-clean
+echo "Copying manifest"
 cp $BUILDDIR/$basename.mt.json $OUTDIR/$basename.mt.json

bin/config-dist.yaml

@@ -184,8 +184,6 @@ Input:
 Logging:
   LogLevel: info # debug, info, warn, error
 #  TraceFile: /var/log/meshtasticd.json
-#  JSONFile: /packets.json # File location for JSON output of decoded packets
-#  JSONFilter: position # filter for packets to save to JSON file
 #  AsciiLogs: true     # default if not specified is !isatty() on stdout
 
 Webserver:

bin/platformio-custom.py

@@ -159,20 +159,22 @@ def load_boot_logo(source, target, env):
 
 # Load the boot logo on TFT builds
 if ("HAS_TFT", 1) in env.get("CPPDEFINES", []):
-    env.AddPreAction('$BUILD_DIR/littlefs.bin', load_boot_logo)
+    env.AddPreAction(f"$BUILD_DIR/{lfsbin}", load_boot_logo)
 
-# Rename (mv) littlefs.bin to include the PROGNAME
-# This ensures the littlefs.bin is named consistently with the firmware
-env.AddPostAction('$BUILD_DIR/littlefs.bin', env.VerboseAction(
-    f'mv $BUILD_DIR/littlefs.bin $BUILD_DIR/{lfsbin}',
-    f'Renaming littlefs.bin to {lfsbin}'
-))
+mtjson_deps = ["buildprog"]
+if platform.name == "espressif32":
+    # Build littlefs image as part of mtjson target
+    # Equivalent to `pio run -t buildfs`
+    target_lfs = env.DataToBin(
+        join("$BUILD_DIR", "${ESP32_FS_IMAGE_NAME}"), "$PROJECT_DATA_DIR"
+    )
+    mtjson_deps.append(target_lfs)
 
 env.AddCustomTarget(
     name="mtjson",
-    dependencies=None,
+    dependencies=mtjson_deps,
     actions=[manifest_gather],
     title="Meshtastic Manifest",
     description="Generating Meshtastic manifest JSON + Checksums",
-    always_build=True,
+    always_build=False,
 )

bin/platformio-pre.py

@@ -11,6 +11,9 @@ else:
     prefsLoc = env["PROJECT_DIR"] + "/version.properties"
     verObj = readProps(prefsLoc)
     env.Replace(PROGNAME=f"firmware-{env.get('PIOENV')}-{verObj['long']}")
+    env.Replace(ESP32_FS_IMAGE_NAME=f"littlefs-{env.get('PIOENV')}-{verObj['long']}")
 
 # Print the new program name for verification
 print(f"PROGNAME: {env.get('PROGNAME')}")
+if platform.name == "espressif32":
+    print(f"ESP32_FS_IMAGE_NAME: {env.get('ESP32_FS_IMAGE_NAME')}")

src/gps/GPS.cpp

@@ -1133,13 +1133,10 @@ int32_t GPS::runOnce()
         // if gps_update_interval is <=10s, GPS never goes off, so we treat that differently
         uint32_t updateInterval = Default::getConfiguredOrDefaultMs(config.position.gps_update_interval);
 
-        // 1. Got a time for the first time
+        // Got a time for the first time
         bool gotTime = (getRTCQuality() >= RTCQualityGPS);
-        if (!gotTime && lookForTime()) { // Note: we count on this && short-circuiting and not resetting the RTC time
-            gotTime = true;
-        }
 
-        // 2. Got a lock for the first time, or 3. Got a lock after turning back on
+        // Got a lock for the first time, or Got a lock after turning back on
         bool gotLoc = lookForLocation();
         if (gotLoc) {
 #ifdef GPS_DEBUG
@@ -1147,6 +1144,10 @@ int32_t GPS::runOnce()
                 LOG_DEBUG("hasValidLocation RISING EDGE");
             }
 #endif
+            if (!gotTime && lookForTime()) { // Note: we count on this && short-circuiting and not resetting the RTC time
+                gotTime = true;
+            }
+
             if (updateInterval <= GPS_UPDATE_ALWAYS_ON_THRESHOLD_MS) {
                 hasValidLocation = true;
                 shouldPublish = true;

src/mesh/MeshModule.h

@@ -225,4 +225,4 @@ class MeshModule
 /** set the destination and packet parameters of packet p intended as a reply to a particular "to" packet
  * This ensures that if the request packet was sent reliably, the reply is sent that way as well.
  */
-void setReplyTo(meshtastic_MeshPacket *p, const meshtastic_MeshPacket &to);
+void setReplyTo(meshtastic_MeshPacket *p, const meshtastic_MeshPacket &to);

src/mesh/ReliableRouter.cpp

@@ -150,7 +150,9 @@ void ReliableRouter::sniffReceived(const meshtastic_MeshPacket *p, const meshtas
         PacketId nakId = (c && c->error_reason != meshtastic_Routing_Error_NONE) ? p->decoded.request_id : 0;
 
         // We intentionally don't check wasSeenRecently, because it is harmless to delete non existent retransmission records
-        if (ackId || nakId) {
+        if ((ackId || nakId) &&
+            // Implicit ACKs from MQTT should not stop retransmissions
+            !(isFromUs(p) && p->transport_mechanism == meshtastic_MeshPacket_TransportMechanism_TRANSPORT_MQTT)) {
             LOG_DEBUG("Received a %s for 0x%x, stopping retransmissions", ackId ? "ACK" : "NAK", ackId);
             if (ackId) {
                 stopRetransmission(p->to, ackId);

src/mesh/Router.cpp

@@ -526,10 +526,6 @@ DecodeState perhapsDecode(meshtastic_MeshPacket *p)
 #elif ARCH_PORTDUINO
         if (portduino_config.traceFilename != "" || portduino_config.logoutputlevel == level_trace) {
             LOG_TRACE("%s", MeshPacketSerializer::JsonSerialize(p, false).c_str());
-        } else if (portduino_config.JSONFilename != "") {
-            if (portduino_config.JSONFilter == (_meshtastic_PortNum)0 || portduino_config.JSONFilter == p->decoded.portnum) {
-                JSONFile << MeshPacketSerializer::JsonSerialize(p, false) << std::endl;
-            }
         }
 #endif
         return DecodeState::DECODE_SUCCESS;

src/modules/RoutingModule.cpp

@@ -75,6 +75,12 @@ uint8_t RoutingModule::getHopLimitForResponse(uint8_t hopStart, uint8_t hopLimit
     return Default::getConfiguredOrDefaultHopLimit(config.lora.hop_limit); // Use the default hop limit
 }
 
+meshtastic_MeshPacket *RoutingModule::allocAckNak(meshtastic_Routing_Error err, NodeNum to, PacketId idFrom, ChannelIndex chIndex,
+                                                  uint8_t hopLimit)
+{
+    return MeshModule::allocAckNak(err, to, idFrom, chIndex, hopLimit);
+}
+
 RoutingModule::RoutingModule() : ProtobufModule("routing", meshtastic_PortNum_ROUTING_APP, &meshtastic_Routing_msg)
 {
     isPromiscuous = true;

src/modules/RoutingModule.h

@@ -16,6 +16,9 @@ class RoutingModule : public ProtobufModule<meshtastic_Routing>
     virtual void sendAckNak(meshtastic_Routing_Error err, NodeNum to, PacketId idFrom, ChannelIndex chIndex, uint8_t hopLimit = 0,
                             bool ackWantsAck = false);
 
+    meshtastic_MeshPacket *allocAckNak(meshtastic_Routing_Error err, NodeNum to, PacketId idFrom, ChannelIndex chIndex,
+                                       uint8_t hopLimit = 0);
+
     // Given the hopStart and hopLimit upon reception of a request, return the hop limit to use for the response
     uint8_t getHopLimitForResponse(uint8_t hopStart, uint8_t hopLimit);
 

src/mqtt/MQTT.cpp

@@ -87,10 +87,13 @@ inline void onReceiveProto(char *topic, byte *payload, size_t length)
         // Generate an implicit ACK towards ourselves (handled and processed only locally!) for this message.
         // We do this because packets are not rebroadcasted back into MQTT anymore and we assume that at least one node
         // receives it when we get our own packet back. Then we'll stop our retransmissions.
-        if (isFromUs(e.packet))
-            routingModule->sendAckNak(meshtastic_Routing_Error_NONE, getFrom(e.packet), e.packet->id, ch.index);
-        else
+        if (isFromUs(e.packet)) {
+            auto pAck = routingModule->allocAckNak(meshtastic_Routing_Error_NONE, getFrom(e.packet), e.packet->id, ch.index);
+            pAck->transport_mechanism = meshtastic_MeshPacket_TransportMechanism_TRANSPORT_MQTT;
+            router->sendLocal(pAck);
+        } else {
             LOG_INFO("Ignore downlink message we originally sent");
+        }
         return;
     }
     if (isFromUs(e.packet)) {

src/platform/nrf52/main-nrf52.cpp

@@ -337,7 +337,7 @@ void cpuDeepSleep(uint32_t msecToWake)
 #endif
 
 #ifdef TTGO_T_ECHO
-    // To power off the T-Echo, the display must be set 
+    // To power off the T-Echo, the display must be set
     // as an input pin; otherwise, there will be leakage current.
     pinMode(PIN_EINK_CS, INPUT);
     pinMode(PIN_EINK_DC, INPUT);

src/platform/portduino/PortduinoGlue.cpp

@@ -29,7 +29,6 @@
 
 portduino_config_struct portduino_config;
 std::ofstream traceFile;
-std::ofstream JSONFile;
 Ch341Hal *ch341Hal = nullptr;
 char *configPath = nullptr;
 char *optionMac = nullptr;
@@ -464,7 +463,6 @@ void portduinoSetup()
     if (portduino_config.lora_spi_dev != "" && portduino_config.lora_spi_dev != "ch341") {
         SPI.begin(portduino_config.lora_spi_dev.c_str());
     }
-
     if (portduino_config.traceFilename != "") {
         try {
             traceFile.open(portduino_config.traceFilename, std::ios::out | std::ios::app);
@@ -472,21 +470,6 @@ void portduinoSetup()
             std::cout << "*** traceFile Exception " << e.what() << std::endl;
             exit(EXIT_FAILURE);
         }
-        if (!traceFile.is_open()) {
-            std::cout << "*** traceFile open failure" << std::endl;
-            exit(EXIT_FAILURE);
-        }
-    } else if (portduino_config.JSONFilename != "") {
-        try {
-            JSONFile.open(portduino_config.JSONFilename, std::ios::out | std::ios::app);
-        } catch (std::ofstream::failure &e) {
-            std::cout << "*** JSONFile Exception " << e.what() << std::endl;
-            exit(EXIT_FAILURE);
-        }
-        if (!JSONFile.is_open()) {
-            std::cout << "*** JSONFile open failure" << std::endl;
-            exit(EXIT_FAILURE);
-        }
     }
     if (verboseEnabled && portduino_config.logoutputlevel != level_trace) {
         portduino_config.logoutputlevel = level_debug;
@@ -534,29 +517,6 @@ bool loadConfig(const char *configPath)
                 portduino_config.logoutputlevel = level_error;
             }
             portduino_config.traceFilename = yamlConfig["Logging"]["TraceFile"].as<std::string>("");
-            portduino_config.JSONFilename = yamlConfig["Logging"]["JSONFile"].as<std::string>("");
-            portduino_config.JSONFilter = (_meshtastic_PortNum)yamlConfig["Logging"]["JSONFilter"].as<int>(0);
-            if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "textmessage")
-                portduino_config.JSONFilter = meshtastic_PortNum_TEXT_MESSAGE_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "telemetry")
-                portduino_config.JSONFilter = meshtastic_PortNum_TELEMETRY_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "nodeinfo")
-                portduino_config.JSONFilter = meshtastic_PortNum_NODEINFO_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "position")
-                portduino_config.JSONFilter = meshtastic_PortNum_POSITION_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "waypoint")
-                portduino_config.JSONFilter = meshtastic_PortNum_WAYPOINT_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "neighborinfo")
-                portduino_config.JSONFilter = meshtastic_PortNum_NEIGHBORINFO_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "traceroute")
-                portduino_config.JSONFilter = meshtastic_PortNum_TRACEROUTE_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "detection")
-                portduino_config.JSONFilter = meshtastic_PortNum_DETECTION_SENSOR_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "paxcounter")
-                portduino_config.JSONFilter = meshtastic_PortNum_PAXCOUNTER_APP;
-            else if (yamlConfig["Logging"]["JSONFilter"].as<std::string>("") == "remotehardware")
-                portduino_config.JSONFilter = meshtastic_PortNum_REMOTE_HARDWARE_APP;
-
             if (yamlConfig["Logging"]["AsciiLogs"]) {
                 // Default is !isatty(1) but can be set explicitly in config.yaml
                 portduino_config.ascii_logs = yamlConfig["Logging"]["AsciiLogs"].as<bool>();

src/platform/portduino/PortduinoGlue.h

@@ -5,7 +5,6 @@
 
 #include "LR11x0Interface.h"
 #include "Module.h"
-#include "mesh/generated/meshtastic/mesh.pb.h"
 #include "platform/portduino/USBHal.h"
 #include "yaml-cpp/yaml.h"
 
@@ -47,8 +46,6 @@ struct pinMapping {
 };
 
 extern std::ofstream traceFile;
-extern std::ofstream JSONFile;
-
 extern Ch341Hal *ch341Hal;
 int initGPIOPin(int pinNum, std::string gpioChipname, int line);
 bool loadConfig(const char *configPath);
@@ -151,9 +148,6 @@ extern struct portduino_config_struct {
     bool ascii_logs = !isatty(1);
     bool ascii_logs_explicit = false;
 
-    std::string JSONFilename;
-    meshtastic_PortNum JSONFilter = (_meshtastic_PortNum)0;
-
     // Webserver
     std::string webserver_root_path = "";
     std::string webserver_ssl_key_path = "/etc/meshtasticd/ssl/private_key.pem";
@@ -419,29 +413,6 @@ extern struct portduino_config_struct {
         }
         if (traceFilename != "")
             out << YAML::Key << "TraceFile" << YAML::Value << traceFilename;
-        if (JSONFilename != "") {
-            out << YAML::Key << "JSONFile" << YAML::Value << JSONFilename;
-            if (JSONFilter == meshtastic_PortNum_TEXT_MESSAGE_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "textmessage";
-            else if (JSONFilter == meshtastic_PortNum_TELEMETRY_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "telemetry";
-            else if (JSONFilter == meshtastic_PortNum_NODEINFO_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "nodeinfo";
-            else if (JSONFilter == meshtastic_PortNum_POSITION_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "position";
-            else if (JSONFilter == meshtastic_PortNum_WAYPOINT_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "waypoint";
-            else if (JSONFilter == meshtastic_PortNum_NEIGHBORINFO_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "neighborinfo";
-            else if (JSONFilter == meshtastic_PortNum_TRACEROUTE_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "traceroute";
-            else if (JSONFilter == meshtastic_PortNum_DETECTION_SENSOR_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "detection";
-            else if (JSONFilter == meshtastic_PortNum_PAXCOUNTER_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "paxcounter";
-            else if (JSONFilter == meshtastic_PortNum_REMOTE_HARDWARE_APP)
-                out << YAML::Key << "JSONFilter" << YAML::Value << "remotehardware";
-        }
         if (ascii_logs_explicit) {
             out << YAML::Key << "AsciiLogs" << YAML::Value << ascii_logs;
         }