Merge pull request #737 from geeksville/eink

Eink

.github/ISSUE_TEMPLATE/bug-report-or-feature-proposal.md

@@ -0,0 +1,38 @@
+---
+name: Bug report or feature proposal
+about: Create a report to help us improve
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Please - if you just have a question (i.e. not a bug report or a feature proposal), post in our [forum](https://meshtastic.discourse.group/) instead.
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Device info:**
+ - Device model: [e.g. TBEAM]
+ - Software Version [e.g. 0.7.8]
+
+**Smartphone information (if relevant):**
+ - Device: [e.g. iPhone6]
+ - OS: [e.g. iOS8.1]
+ - App Version [e.g. 0.7.2]
+
+**Additional context**
+Add any other context about the problem here.

.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Thank you for sending in a pull request, here's some tips to get started!
+
+(Please delete all these tips and replace with your text)
+
+- Before starting on some new big chunk of code, it it is optional but highly recommended to open an issue first
+  to say "hey, I think this idea X should be implemented and I'm starting work on it. My general plan is Y, any feedback
+  is appreciated." This will allow other devs to potentially save you time by not accidentially duplicating work etc...
+- Please do not check in files that don't have real changes
+- Please do not reformat lines that you didn't have to change the code on
+- We recommend using the [Visual Studio Code](https://platformio.org/install/ide?install=vscode) editor,
+  because automatically follows our indentation rules and it's auto reformatting will not cause spurious changes to lines.
+- If your PR fixes a bug, mention "fixes #bugnum" somewhere in your pull request description.
+- If your other co-developers have comments on your PR please tweak as needed.

.github/workflows/main.yml

@@ -4,18 +4,33 @@ on:
   - pull_request
 
 jobs:
-  main:
-    name: Main
+  setup:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - name: Checkout code
+        uses: actions/checkout@v2
+        with:
+          submodules: true
       - name: Setup Python
-        uses: actions/setup-python@master
+        uses: actions/setup-python@v2
         with:
           python-version: 3.x
       - name: Install Platform IO
         run: |
           python -m pip install --upgrade pip
           pip install -U platformio
-      - name: Build
-        run: platformio run
+      - name: Install extra python tools
+        run: |
+          pip install -U adafruit-nrfutil
+      - name: Install libs needed for linux build
+        run: |
+          sudo apt install -y libpsocksxx-dev
+      - name: Build for tbeam
+        run: platformio run -e tbeam
+      - name: Build for heltec
+        run: platformio run -e heltec
+      - name: Build for lora-relay-v1
+        run: platformio run -e lora-relay-v1 
+      # Turn off linux for now
+      #- name: Build for linux
+      #  run: platformio run -e linux

.gitignore

@@ -7,4 +7,15 @@ main/credentials.h
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/extensions.json
-*.code-workspace
+*.code-workspace
+
+.DS_Store
+Thumbs.db
+.autotools
+.built
+.context
+.cproject
+.idea/*
+.vagrant
+nanopb*
+flash.uf2

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "sdk-nrfxlib"]
 	path = sdk-nrfxlib
 	url = https://github.com/nrfconnect/sdk-nrfxlib.git
+[submodule "design"]
+	path = design
+	url = https://github.com/meshtastic/meshtastic-design.git

.vscode/settings.json

@@ -47,18 +47,34 @@
         "memory_resource": "cpp",
         "optional": "cpp",
         "string_view": "cpp",
-        "cassert": "cpp"
+        "cassert": "cpp",
+        "iterator": "cpp",
+        "shared_mutex": "cpp",
+        "iostream": "cpp"
     },
     "cSpell.words": [
         "Blox",
+        "EINK",
         "HFSR",
         "Meshtastic",
         "NEMAGPS",
+        "NMEAGPS",
+        "RDEF",
         "Ublox",
         "bkpt",
         "cfsr",
         "descs",
         "ocrypto",
-        "protobufs"
-    ]
+        "protobufs",
+        "wifi"
+    ],
+    "C_Cpp.dimInactiveRegions": true,
+    "protoc": {
+        "compile_on_save": false,
+        "compile_all_path": "/home/kevinh/development/meshtastic/meshtastic-esp32/proto",
+        "options": [
+            "--java_out=/tmp",
+            "-I=/home/kevinh/development/meshtastic/meshtastic-esp32/proto"
+        ]
+    }    
 }

.vscode/tasks.json

@@ -0,0 +1,17 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "PlatformIO",
+			"task": "Build",
+			"problemMatcher": [
+				"$platformio"
+			],
+			"group": {
+				"kind": "build",
+				"isDefault": true
+			},
+			"label": "PlatformIO: Build"
+		}
+	]
+}

README.md

@@ -4,7 +4,7 @@ This is the device side code for the [meshtastic.org](https://www.meshtastic.org
 
 ![Continuous Integration](https://github.com/meshtastic/Meshtastic-esp32/workflows/Continuous%20Integration/badge.svg)
 
-Meshtastic™ is a project that lets you use
+Meshtastic® is a project that lets you use
 inexpensive GPS mesh radios as an extensible, super long battery life mesh GPS communicator. These radios are great for hiking, skiing, paragliding -
 essentially any hobby where you don't have reliable internet access. Each member of your private mesh can always see the location and distance of all other
 members and any text messages sent to your group chat.
@@ -22,24 +22,41 @@ This software is 100% open source and developed by a group of hobbyist experimen
 
 We currently support three models of radios.
 
-- TTGO T-Beam
-
-  - [T-Beam V1.0 w/ NEO-M8N](https://www.aliexpress.com/item/33047631119.html) (Recommended)
-  - [T-Beam V1.0 w/ NEO-6M](https://www.aliexpress.com/item/33050391850.html)
+- TTGO T-Beam (usually the recommended choice)
+  - [T-Beam V1.1 w/ NEO-6M - special Meshtastic version](https://www.aliexpress.com/item/4001178678568.html) (Includes built-in OLED display and they have **preinstalled** the meshtastic software)
+  - [T-Beam V1.1 w/ NEO-M8N](https://www.aliexpress.com/item/33047631119.html) (slightly better GPS)
+  - [T-Beam V1.1 w/ NEO-M8N /w SX1262](https://www.aliexpress.com/item/4001287221970.html) (slightly better GPS + LoRa)
+    - board labels "TTGO T22_V1.1 20191212"
+  - [T-Beam V0.7 w/ NEO-6M](https://www.aliexpress.com/item/4000574335430.html) (will work but **you must use the tbeam0.7 firmware ** - but the T-Beam V1.0 or later are better!)
+     - board labels "TTGO T22_V07 20180711"
   - 3D printable cases
-    - [T-Beam V0](https://www.thingiverse.com/thing:3773717)
-    - [T-Beam V1](https://www.thingiverse.com/thing:3830711)
+    - [T-Beam V0](https://www.thingiverse.com/thing:3773717) (GPS and LoRa antenna misaligned if GPS placed as pictured)
+    - [T-Beam V1 (SMA-antenna)](https://www.thingiverse.com/thing:3830711)
+    - [T-Beam V1 (SMA-antenna)](https://www.thingiverse.com/thing:4677388) (Mounting option for larger GPS antenna but LoRa antenna enclosed)
+    - [T-Beam V1 (IPEX-antenna)](https://www.thingiverse.com/thing:4587297) (GPS and LoRa antenna misaligned if GPS placed as pictured)
+    - [T-Beam V1 (IPEX-antenna)](https://www.thingiverse.com/thing:4589651)
+    - [T-Beam V1 (IPEX-antenna)](https://www.thingiverse.com/thing:4619981) (GPS and LoRa antenna misaligned if GPS placed as pictured)
+  - Laser-cut cases
+    - [T-Beam V1 (SMA-antenna)](https://www.thingiverse.com/thing:4552771)
 
 - [TTGO LORA32](https://www.aliexpress.com/item/4000211331316.html) - No GPS
+  - version 2.1
+    - board labels "TTGO T3_V1.6 20180606"
+  - 3D printable case
+    - [TTGO LORA32 v1](https://www.thingiverse.com/thing:3385109)
 
 - [Heltec LoRa 32](https://heltec.org/project/wifi-lora-32/) - No GPS
+  - [Official Heltec case](https://www.aliexpress.com/item/4001050707951.html)
   - [3D Printable case](https://www.thingiverse.com/thing:3125854)
+  
+Note: The GPS and LoRa stock antennas should be placed in a way, that the GPS antenna faces the sky and the LoRa antenna radiates 360 degrees horizontally. For better GPS reception you might want to [upgrade the GPS antenna](https://meshtastic.discourse.group/t/the-importance-of-gps-antennas-and-request-to-3d-case-documentation-people/1505) and to properly align the antennas you might want to upgrade to a LoRa antenna that can be adjusted to radiate into the right directions.
 
 **Make sure to get the frequency for your country**
 
-- US/JP/AU/NZ - 915MHz
+- US/JP/AU/NZ/CA - 915MHz
 - CN - 470MHz
 - EU - 868MHz, 433MHz
+- full list of LoRa frequencies per region is available [here](https://www.thethingsnetwork.org/docs/lorawan/frequencies-by-country.html)
 
 Getting a version that includes a screen is optional, but highly recommended.
 
@@ -47,6 +64,8 @@ Getting a version that includes a screen is optional, but highly recommended.
 
 Prebuilt binaries for the supported radios are available in our [releases](https://github.com/meshtastic/Meshtastic-esp32/releases). Your initial installation has to happen over USB from your Mac, Windows or Linux PC. Once our software is installed, all future software updates happen over bluetooth from your phone.
 
+Be **very careful** to install the correct load for your board. In particular the popular 'T-BEAM' radio from TTGO is not called 'TTGO-Lora' (that is a different board). So don't install the 'TTGO-Lora' build on a TBEAM, it won't work correctly.
+
 Please post comments on our [group chat](https://meshtastic.discourse.group/) if you have problems or successes.
 
 ### Installing from a GUI - Windows and Mac
@@ -90,10 +109,10 @@ Hard resetting via RTS pin...
 ```
 
 5. cd into the directory where the release zip file was expanded.
-6. Install the correct firmware for your board with `device-install.sh firmware-_board_-_country_.bin`.
-   - Example: `./device-install.sh firmware-HELTEC-US-0.0.3.bin`.
-7. To update run `device-update.sh firmware-_board_-_country_.bin`
-   - Example: `./device-update.sh firmware-HELTEC-US-0.0.3.bin`.
+6. Install the correct firmware for your board with `device-install.sh -f firmware-_board_-_country_.bin`.
+   - Example: `./device-install.sh -f firmware-HELTEC-US-0.0.3.bin`.
+7. To update run `device-update.sh -f firmware-_board_-_country_.bin`
+   - Example: `./device-update.sh -f firmware-HELTEC-US-0.0.3.bin`.
 
 Note: If you have previously installed meshtastic, you don't need to run this full script instead just run `esptool.py --baud 921600 write_flash 0x10000 firmware-_board_-_country_-_version_.bin`. This will be faster, also all of your current preferences will be preserved.
 

bin/build-all.sh

@@ -2,14 +2,19 @@
 
 set -e
 
-source bin/version.sh
+VERSION=`bin/buildinfo.py`
 
-COUNTRIES="US EU433 EU865 CN JP"
+# We now only do regionless builds
+COUNTRIES=""
+#COUNTRIES="US EU433 EU865 CN JP ANZ KR"
 #COUNTRIES=US
 #COUNTRIES=CN
 
-BOARDS="ttgo-lora32-v2 ttgo-lora32-v1 tbeam heltec"
-#BOARDS=tbeam
+BOARDS_ESP32="tlora-v2 tlora-v1 tlora-v2-1-1.6 tbeam heltec tbeam0.7"
+#BOARDS_ESP32=tbeam
+
+# FIXME note nrf52840dk build is for some reason only generating a BIN file but not a HEX file nrf52840dk-geeksville is fine
+BOARDS_NRF52="lora-relay-v1"
 
 OUTDIR=release/latest
 
@@ -18,25 +23,61 @@ ARCHIVEDIR=release/archive
 
 rm -f $OUTDIR/firmware*
 
-mkdir -p $OUTDIR/bins $OUTDIR/elfs
-rm -f $OUTDIR/bins/*
+mkdir -p $OUTDIR/bins $ARCHIVEDIR
+rm -r $OUTDIR/bins/*
+mkdir -p $OUTDIR/bins/universal $OUTDIR/elfs/universal
 
 # build the named environment and copy the bins to the release directory
-function do_build {
-    ENV_NAME=$1
-    echo "Building for $ENV_NAME with $PLATFORMIO_BUILD_FLAGS"
-    SRCBIN=.pio/build/$ENV_NAME/firmware.bin
-    SRCELF=.pio/build/$ENV_NAME/firmware.elf
-    rm -f $SRCBIN 
+function do_build() {
+	BOARD=$1
+	COUNTRY=$2
+	isNrf=$3
+	
+    echo "Building $COUNTRY for $BOARD with $PLATFORMIO_BUILD_FLAGS"
+    rm -f .pio/build/$BOARD/firmware.*
 
     # The shell vars the build tool expects to find
-    export HW_VERSION="1.0-$COUNTRY"
     export APP_VERSION=$VERSION
-    export COUNTRY
 
-    pio run --jobs 4 --environment $ENV_NAME # -v
-    cp $SRCBIN $OUTDIR/bins/firmware-$ENV_NAME-$COUNTRY-$VERSION.bin
-    cp $SRCELF $OUTDIR/elfs/firmware-$ENV_NAME-$COUNTRY-$VERSION.elf
+    # Are we building a universal/regionless rom?
+    if [ "x$COUNTRY" != "x" ]
+    then
+        export HW_VERSION="1.0-$COUNTRY"
+        export COUNTRY
+        basename=firmware-$BOARD-$COUNTRY-$VERSION
+    else
+        export HW_VERSION="1.0"
+        unset COUNTRY
+        basename=universal/firmware-$BOARD-$VERSION
+    fi
+
+    pio run --environment $BOARD # -v
+    SRCELF=.pio/build/$BOARD/firmware.elf
+    cp $SRCELF $OUTDIR/elfs/$basename.elf
+
+    if [ "$isNrf" = "false" ]
+    then
+        echo "Copying ESP32 bin file"
+        SRCBIN=.pio/build/$BOARD/firmware.bin
+        cp $SRCBIN $OUTDIR/bins/$basename.bin
+    else
+        echo "Generating NRF52 uf2 file"
+        SRCHEX=.pio/build/$BOARD/firmware.hex
+        bin/uf2conv.py $SRCHEX -c -o $OUTDIR/bins/$basename.uf2 -f 0xADA52840
+    fi
+}
+
+function do_boards() {
+	declare boards=$1
+	declare isNrf=$2
+	for board in $boards; do
+		for country in $COUNTRIES; do 
+		    do_build $board $country "$isNrf"   
+		done
+
+		# Build universal
+		do_build $board "" "$isNrf" 
+	done
 }
 
 # Make sure our submodules are current
@@ -45,14 +86,20 @@ git submodule update
 # Important to pull latest version of libs into all device flavors, otherwise some devices might be stale
 platformio lib update 
 
-for COUNTRY in $COUNTRIES; do 
-    for BOARD in $BOARDS; do
-        do_build $BOARD
-    done
-done
+do_boards "$BOARDS_ESP32" "false"
+do_boards "$BOARDS_NRF52" "true"
+
+echo "Building SPIFFS for ESP32 targets"
+pio run --environment tbeam -t buildfs
+cp .pio/build/tbeam/spiffs.bin $OUTDIR/bins/universal/spiffs-$VERSION.bin
 
 # keep the bins in archive also
-cp $OUTDIR/bins/firmware* $OUTDIR/elfs/firmware* $ARCHIVEDIR
+cp $OUTDIR/bins/universal/spiffs* $OUTDIR/bins/universal/firmware* $OUTDIR/elfs/universal/firmware* $ARCHIVEDIR
+
+echo Updating android bins $OUTDIR/forandroid
+rm -rf $OUTDIR/forandroid
+mkdir -p $OUTDIR/forandroid
+cp -a $OUTDIR/bins/universal/*.bin $OUTDIR/forandroid/
 
 cat >$OUTDIR/curfirmwareversion.xml <<XML
 <?xml version="1.0" encoding="utf-8"?>
@@ -62,11 +109,12 @@ release.  It is used by the android app for forcing software updates.  Do not ed
 Generated by bin/buildall.sh -->
 
 <resources>
-    <string name="cur_firmware_version">$VERSION</string>
+    <string name="cur_firmware_version" translatable="false">$VERSION</string>
 </resources>
 XML
 
+echo Generating $ARCHIVEDIR/firmware-$VERSION.zip
 rm -f $ARCHIVEDIR/firmware-$VERSION.zip
-zip --junk-paths $ARCHIVEDIR/firmware-$VERSION.zip $OUTDIR/bins/firmware-*-$VERSION.* images/system-info.bin bin/device-install.sh bin/device-update.sh
+zip --junk-paths $ARCHIVEDIR/firmware-$VERSION.zip $ARCHIVEDIR/spiffs-$VERSION.bin $OUTDIR/bins/universal/firmware-*-$VERSION.* images/system-info.bin bin/device-install.sh bin/device-update.sh
 
 echo BUILT ALL

bin/buildinfo.py

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+import configparser
+
+config = configparser.RawConfigParser()
+config.read('version.properties')
+
+version = dict(config.items('VERSION'))
+
+verStr = "{}.{}.{}".format(version["major"], version["minor"], version["build"])
+
+print(f"{verStr}")

bin/device-install.sh

@@ -1,11 +1,48 @@
-#!/bin/bash
+#!/bin/sh
 
 set -e
 
-FILENAME=$1
+# Usage info
+show_help() {
+cat << EOF
+Usage: ${0##*/} [-h] [-p ESPTOOL_PORT] [-f FILENAME]
+Flash image file to device, but first erasing and writing system information"
 
-echo "Trying to flash $FILENAME, but first erasing and writing system information"
-esptool.py --baud 921600 erase_flash
-esptool.py --baud 921600 write_flash 0x1000 system-info.bin
-esptool.py --baud 921600 write_flash 0x10000 $FILENAME
+    -h               Display this help and exit
+    -p ESPTOOL_PORT  Set the environment variable for ESPTOOL_PORT.  If not set, ESPTOOL iterates all ports (Dangerrous).
+    -f FILENAME         The .bin file to flash.  Custom to your device type and region.
+EOF
+}
 
+
+while getopts ":h:p:f:" opt; do
+    case "${opt}" in
+        h)
+            show_help
+            exit 0
+            ;;
+        p)  export ESPTOOL_PORT=${OPTARG}
+	    ;;
+        f)  FILENAME=${OPTARG}
+            ;;
+        *)
+ 	    echo "Invalid flag."
+            show_help >&2
+            exit 1
+            ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+if [ -f "${FILENAME}" ]; then
+	echo "Trying to flash ${FILENAME}, but first erasing and writing system information"
+	esptool.py --baud 921600 erase_flash
+	esptool.py --baud 921600 write_flash 0x1000 system-info.bin
+    esptool.py --baud 921600 write_flash 0x00390000 spiffs-*.bin
+	esptool.py --baud 921600 write_flash 0x10000 ${FILENAME}
+else
+	echo "Invalid file: ${FILENAME}"
+	show_help
+fi
+
+exit 0

bin/device-update.sh

@@ -1,8 +1,43 @@
-#!/bin/bash
+#!/bin/sh
 
-set -e
+# Usage info
+show_help() {
+cat << EOF
+Usage: ${0##*/} [-h] [-p ESPTOOL_PORT] -f FILENAME
+Flash image file to device, leave existing system intact."
 
-FILENAME=$1
+    -h               Display this help and exit
+    -p ESPTOOL_PORT  Set the environment variable for ESPTOOL_PORT.  If not set, ESPTOOL iterates all ports (Dangerrous).
+    -f FILENAME         The .bin file to flash.  Custom to your device type and region.
+EOF
+}
 
-echo "Trying to update $FILENAME"
-esptool.py --baud 921600 write_flash 0x10000 $FILENAME
+
+while getopts ":h:p:f:" opt; do
+    case "${opt}" in
+        h)
+            show_help
+            exit 0
+            ;;
+        p)  export ESPTOOL_PORT=${OPTARG}
+	    ;;
+        f)  FILENAME=${OPTARG}
+            ;;
+        *)
+ 	    echo "Invalid flag."
+            show_help >&2
+            exit 1
+            ;;
+    esac
+done
+shift "$((OPTIND-1))"
+
+if [ -f "${FILENAME}" ]; then
+	echo "Trying to flash update ${FILENAME}."
+	esptool.py --baud 921600 write_flash 0x10000 ${FILENAME}
+else
+	echo "Invalid file: ${FILENAME}"
+	show_help
+fi
+
+exit 0

bin/exception_decoder.py

@@ -241,7 +241,7 @@ def print_addr(name, value, resolver):
 def print_stack_full(lines, resolver):
     print("stack:")
     for line in lines:
-        print(line.offset + ":")
+        print(str(line.offset) + ":")
         for content in line.content:
             print("  " + resolver.resolve_stack_addr(content))
 

bin/gen-images.sh

@@ -0,0 +1,19 @@
+
+
+set -e 
+
+# regen the design bins first
+cd design
+bin/generate-pngs.sh
+cd ..
+
+# assumes 50 wide, 28 high
+convert design/logo/png/Mesh_Logo_Black_Small.png -background white -alpha Background src/graphics/img/icon.xbm
+
+inkscape --batch-process -o images/compass.png -w 48 -h 48 images/location_searching-24px.svg
+convert compass.png -background white -alpha Background src/graphics/img/compass.xbm
+
+inkscape --batch-process -o images/face.png -w 13 -h 13 images/face-24px.svg
+
+inkscape --batch-process -o images/pin.png -w 13 -h 13 images/room-24px.svg
+convert pin.png -background white -alpha Background src/graphics/img/pin.xbm

bin/install-bootloader.sh

@@ -0,0 +1,35 @@
+# You probably don't want to use this script, it programs a custom bootloader build onto a nrf52 board
+
+set -e 
+
+# dependencies
+# apt install srecord
+
+BOOTDIR=/home/kevinh/development/meshtastic/Adafruit_nRF52_Bootloader
+BOARD=othernet_ppr1
+BOOTVER=0.3.2
+BOOTNUM=128
+BOOTSHA=gc01b9ea
+SDCODE=s113
+SDVER=7.2.0
+PROJ=ppr1
+
+# FIXME for nRF52840 use 0xff000, for nRF52833 use 0x7f000
+BOOTSET=0x7f000
+
+nrfjprog --eraseall -f nrf52
+
+# this generates an intel hex file that can be programmed into a NRF52 to tell the adafruit bootloader that the current app image is valid
+# Bootloader settings are at BOOTLOADER_SETTINGS (rw) : ORIGIN = 0xFF000, LENGTH = 0x1000
+# first 4 bytes should be 0x01 to indicate valid app image
+# second 4 bytes should be 0x00 to indicate no CRC required for image
+echo "01 00 00 00 00 00 00 00" | xxd -r -p - >/tmp/bootconf.bin
+srec_cat /tmp/bootconf.bin -binary -offset $BOOTSET -output /tmp/bootconf.hex -intel   
+
+echo Generating merged hex file from .pio/build/$PROJ/firmware.hex
+mergehex -o ${BOARD}_full.hex -m $BOOTDIR/_build/build-$BOARD/${BOARD}_bootloader-$BOOTVER-$BOOTNUM-$BOOTSHA-dirty_${SDCODE}_$SDVER.hex .pio/build/$PROJ/firmware.hex /tmp/bootconf.hex
+
+echo Telling bootloader app region is valid and telling CPU to run
+nrfjprog --program ${BOARD}_full.hex -f nrf52 --reset
+
+# nrfjprog --readuicr /tmp/uicr.hex; objdump -s /tmp/uicr.hex | less

bin/install-eink.sh

@@ -0,0 +1,24 @@
+# You probably don't want to use this script, it programs a custom bootloader build onto a nrf52 board
+
+set -e 
+
+BOOTDIR=/home/kevinh/development/meshtastic/Adafruit_nRF52_Bootloader
+
+nrfjprog --eraseall -f nrf52
+
+# to get tool run "sudo apt-get install srecord"
+
+# this generates an intel hex file that can be programmed into a NRF52 to tell the adafruit bootloader that the current app image is valid
+# Bootloader settings are at BOOTLOADER_SETTINGS (rw) : ORIGIN = 0xFF000, LENGTH = 0x1000
+# first 4 bytes should be 0x01 to indicate valid app image
+# second 4 bytes should be 0x00 to indicate no CRC required for image
+echo "01 00 00 00 00 00 00 00" | xxd -r -p - >/tmp/bootconf.bin
+srec_cat /tmp/bootconf.bin -binary -offset 0xff000 -output /tmp/bootconf.hex -intel   
+
+echo Generating merged hex file 
+mergehex -m $BOOTDIR/_build/build-ttgo_eink/ttgo_eink_bootloader-0.3.2-213-gf67f592-dirty_s140_6.1.1.hex .pio/build/eink/firmware.hex /tmp/bootconf.hex -o ttgo_eink_full.hex
+
+echo Telling bootloader app region is valid and telling CPU to run
+nrfjprog --program ttgo_eink_full.hex -f nrf52 --reset
+
+# nrfjprog --readuicr /tmp/uicr.hex; objdump -s /tmp/uicr.hex | less

bin/nrf52832-gdbserver.sh

@@ -1,3 +1,3 @@
 
 
-JLinkGDBServerCLExe -if SWD -select USB -port 2331 -device NRF52840_XXAA
+JLinkGDBServerCLExe -if SWD -select USB -port 2331 -device NRF52832_XXAA

bin/nrf52833-gdbserver.sh

@@ -0,0 +1,3 @@
+
+
+JLinkGDBServerCLExe -if SWD -select USB -port 2331 -device NRF52833_XXAA

bin/nrf52840-gdbserver.sh

@@ -0,0 +1,3 @@
+
+
+JLinkGDBServerCLExe -if SWD -select USB -port 2331 -device NRF52840_XXAA -SuppressInfoUpdateFW -DisableAutoUpdateFW -rtos GDBServer/RTOSPlugin_FreeRTOS

bin/platformio-custom.py

@@ -0,0 +1,16 @@
+
+Import("projenv")
+
+import configparser
+prefsLoc = projenv["PROJECT_DIR"] + "/version.properties"
+config = configparser.RawConfigParser()
+config.read(prefsLoc)
+version = dict(config.items('VERSION'))
+verStr = "{}.{}.{}".format(version["major"], version["minor"], version["build"])
+
+print("Using meshtastic platform-custom.py, firmare version " + verStr)
+
+# General options that are passed to the C and C++ compilers
+projenv.Append(CCFLAGS=[
+    "-DAPP_VERSION=" + verStr
+    ])

bin/program-release-tbeam.sh

@@ -1,6 +1,8 @@
 
 set -e
 
-source bin/version.sh
+VERSION=`bin/buildinfo.py`
+FILENAME=release/latest/bins/universal/firmware-tbeam-$VERSION.bin
 
-esptool.py --baud 921600 write_flash 0x10000 release/latest/bins/firmware-tbeam-US-$VERSION.bin
+echo Installing $FILENAME
+esptool.py --baud 921600 write_flash 0x10000 $FILENAME

bin/program-release-universal.sh

@@ -0,0 +1,6 @@
+
+set -e
+
+source bin/version.sh
+
+esptool.py --baud 921600 write_flash 0x10000 release/latest/bins/universal/firmware-tbeam-$VERSION.bin

bin/qspi-flash-test.sh

@@ -0,0 +1,6 @@
+# You probably don't need this - it is a basic test of the serial flash on the TTGO eink board
+
+nrfjprog --qspiini nrf52/ttgo_eink_qpsi.ini --qspieraseall
+nrfjprog --qspiini nrf52/ttgo_eink_qpsi.ini --memwr 0x12000000 --val 0xdeadbeef --verify
+nrfjprog --qspiini nrf52/ttgo_eink_qpsi.ini --readqspi spi.hex
+objdump -s spi.hex | less

bin/regen-protos.sh

@@ -1,6 +1,15 @@
 #!/bin/bash
 
-echo "This script requires https://jpa.kapsi.fi/nanopb/download/ version 0.4.1"
+set -e
+
+echo "This script requires https://jpa.kapsi.fi/nanopb/download/ version 0.4.4 to be located in the"
+echo "meshtastic-device root directory if the following step fails, you should download the correct"
+echo "prebuilt binaries for your computer into nanopb-0.4.4"
+
 # the nanopb tool seems to require that the .options file be in the current directory!
 cd proto
-../../nanopb-0.4.1-linux-x86/generator-bin/protoc --nanopb_out=-v:../src/mesh -I=../proto mesh.proto
+../nanopb-0.4.4/generator-bin/protoc --nanopb_out=-v:../src/mesh/generated -I=../proto *.proto
+
+echo "Regenerating protobuf documentation - if you see an error message"
+echo "you can ignore it unless doing a new protobuf release to github."
+bin/regen-docs.sh

bin/uf2conv.py

@@ -0,0 +1,314 @@
+#!/usr/bin/env python3
+import sys
+import struct
+import subprocess
+import re
+import os
+import os.path
+import argparse
+
+
+UF2_MAGIC_START0 = 0x0A324655 # "UF2\n"
+UF2_MAGIC_START1 = 0x9E5D5157 # Randomly selected
+UF2_MAGIC_END    = 0x0AB16F30 # Ditto
+
+families = {
+    'SAMD21': 0x68ed2b88,
+    'SAML21': 0x1851780a,
+    'SAMD51': 0x55114460,
+    'NRF52': 0x1b57745f,
+    'STM32F0': 0x647824b6,
+    'STM32F1': 0x5ee21072,
+    'STM32F2': 0x5d1a0a2e,
+    'STM32F3': 0x6b846188,
+    'STM32F4': 0x57755a57,
+    'STM32F7': 0x53b80f00,
+    'STM32G0': 0x300f5633,
+    'STM32G4': 0x4c71240a,
+    'STM32H7': 0x6db66082,
+    'STM32L0': 0x202e3a91,
+    'STM32L1': 0x1e1f432d,
+    'STM32L4': 0x00ff6919,
+    'STM32L5': 0x04240bdf,
+    'STM32WB': 0x70d16653,
+    'STM32WL': 0x21460ff0,
+    'ATMEGA32': 0x16573617,
+    'MIMXRT10XX': 0x4FB2D5BD
+}
+
+INFO_FILE = "/INFO_UF2.TXT"
+
+appstartaddr = 0x2000
+familyid = 0x0
+
+
+def is_uf2(buf):
+    w = struct.unpack("<II", buf[0:8])
+    return w[0] == UF2_MAGIC_START0 and w[1] == UF2_MAGIC_START1
+
+def is_hex(buf):
+    try:
+        w = buf[0:30].decode("utf-8")
+    except UnicodeDecodeError:
+        return False
+    if w[0] == ':' and re.match(b"^[:0-9a-fA-F\r\n]+$", buf):
+        return True
+    return False
+
+def convert_from_uf2(buf):
+    global appstartaddr
+    numblocks = len(buf) // 512
+    curraddr = None
+    outp = b""
+    for blockno in range(numblocks):
+        ptr = blockno * 512
+        block = buf[ptr:ptr + 512]
+        hd = struct.unpack(b"<IIIIIIII", block[0:32])
+        if hd[0] != UF2_MAGIC_START0 or hd[1] != UF2_MAGIC_START1:
+            print("Skipping block at " + ptr + "; bad magic")
+            continue
+        if hd[2] & 1:
+            # NO-flash flag set; skip block
+            continue
+        datalen = hd[4]
+        if datalen > 476:
+            assert False, "Invalid UF2 data size at " + ptr
+        newaddr = hd[3]
+        if curraddr == None:
+            appstartaddr = newaddr
+            curraddr = newaddr
+        padding = newaddr - curraddr
+        if padding < 0:
+            assert False, "Block out of order at " + ptr
+        if padding > 10*1024*1024:
+            assert False, "More than 10M of padding needed at " + ptr
+        if padding % 4 != 0:
+            assert False, "Non-word padding size at " + ptr
+        while padding > 0:
+            padding -= 4
+            outp += b"\x00\x00\x00\x00"
+        outp += block[32 : 32 + datalen]
+        curraddr = newaddr + datalen
+    return outp
+
+def convert_to_carray(file_content):
+    outp = "const unsigned char bindata[] __attribute__((aligned(16))) = {"
+    for i in range(len(file_content)):
+        if i % 16 == 0:
+            outp += "\n"
+        outp += "0x%02x, " % ord(file_content[i])
+    outp += "\n};\n"
+    return outp
+
+def convert_to_uf2(file_content):
+    global familyid
+    datapadding = b""
+    while len(datapadding) < 512 - 256 - 32 - 4:
+        datapadding += b"\x00\x00\x00\x00"
+    numblocks = (len(file_content) + 255) // 256
+    outp = b""
+    for blockno in range(numblocks):
+        ptr = 256 * blockno
+        chunk = file_content[ptr:ptr + 256]
+        flags = 0x0
+        if familyid:
+            flags |= 0x2000
+        hd = struct.pack(b"<IIIIIIII",
+            UF2_MAGIC_START0, UF2_MAGIC_START1,
+            flags, ptr + appstartaddr, 256, blockno, numblocks, familyid)
+        while len(chunk) < 256:
+            chunk += b"\x00"
+        block = hd + chunk + datapadding + struct.pack(b"<I", UF2_MAGIC_END)
+        assert len(block) == 512
+        outp += block
+    return outp
+
+class Block:
+    def __init__(self, addr):
+        self.addr = addr
+        self.bytes = bytearray(256)
+
+    def encode(self, blockno, numblocks):
+        global familyid
+        flags = 0x0
+        if familyid:
+            flags |= 0x2000
+        hd = struct.pack("<IIIIIIII",
+            UF2_MAGIC_START0, UF2_MAGIC_START1,
+            flags, self.addr, 256, blockno, numblocks, familyid)
+        hd += self.bytes[0:256]
+        while len(hd) < 512 - 4:
+            hd += b"\x00"
+        hd += struct.pack("<I", UF2_MAGIC_END)
+        return hd
+
+def convert_from_hex_to_uf2(buf):
+    global appstartaddr
+    appstartaddr = None
+    upper = 0
+    currblock = None
+    blocks = []
+    for line in buf.split('\n'):
+        if line[0] != ":":
+            continue
+        i = 1
+        rec = []
+        while i < len(line) - 1:
+            rec.append(int(line[i:i+2], 16))
+            i += 2
+        tp = rec[3]
+        if tp == 4:
+            upper = ((rec[4] << 8) | rec[5]) << 16
+        elif tp == 2:
+            upper = ((rec[4] << 8) | rec[5]) << 4
+            assert (upper & 0xffff) == 0
+        elif tp == 1:
+            break
+        elif tp == 0:
+            addr = upper | (rec[1] << 8) | rec[2]
+            if appstartaddr == None:
+                appstartaddr = addr
+            i = 4
+            while i < len(rec) - 1:
+                if not currblock or currblock.addr & ~0xff != addr & ~0xff:
+                    currblock = Block(addr & ~0xff)
+                    blocks.append(currblock)
+                currblock.bytes[addr & 0xff] = rec[i]
+                addr += 1
+                i += 1
+    numblocks = len(blocks)
+    resfile = b""
+    for i in range(0, numblocks):
+        resfile += blocks[i].encode(i, numblocks)
+    return resfile
+
+def to_str(b):
+    return b.decode("utf-8")
+
+def get_drives():
+    drives = []
+    if sys.platform == "win32":
+        r = subprocess.check_output(["wmic", "PATH", "Win32_LogicalDisk",
+                                     "get", "DeviceID,", "VolumeName,",
+                                     "FileSystem,", "DriveType"])
+        for line in to_str(r).split('\n'):
+            words = re.split('\s+', line)
+            if len(words) >= 3 and words[1] == "2" and words[2] == "FAT":
+                drives.append(words[0])
+    else:
+        rootpath = "/media"
+        if sys.platform == "darwin":
+            rootpath = "/Volumes"
+        elif sys.platform == "linux":
+            tmp = rootpath + "/" + os.environ["USER"]
+            if os.path.isdir(tmp):
+                rootpath = tmp
+        for d in os.listdir(rootpath):
+            drives.append(os.path.join(rootpath, d))
+
+
+    def has_info(d):
+        try:
+            return os.path.isfile(d + INFO_FILE)
+        except:
+            return False
+
+    return list(filter(has_info, drives))
+
+
+def board_id(path):
+    with open(path + INFO_FILE, mode='r') as file:
+        file_content = file.read()
+    return re.search("Board-ID: ([^\r\n]*)", file_content).group(1)
+
+
+def list_drives():
+    for d in get_drives():
+        print(d, board_id(d))
+
+
+def write_file(name, buf):
+    with open(name, "wb") as f:
+        f.write(buf)
+    print("Wrote %d bytes to %s" % (len(buf), name))
+
+
+def main():
+    global appstartaddr, familyid
+    def error(msg):
+        print(msg)
+        sys.exit(1)
+    parser = argparse.ArgumentParser(description='Convert to UF2 or flash directly.')
+    parser.add_argument('input', metavar='INPUT', type=str, nargs='?',
+                        help='input file (HEX, BIN or UF2)')
+    parser.add_argument('-b' , '--base', dest='base', type=str,
+                        default="0x2000",
+                        help='set base address of application for BIN format (default: 0x2000)')
+    parser.add_argument('-o' , '--output', metavar="FILE", dest='output', type=str,
+                        help='write output to named file; defaults to "flash.uf2" or "flash.bin" where sensible')
+    parser.add_argument('-d' , '--device', dest="device_path",
+                        help='select a device path to flash')
+    parser.add_argument('-l' , '--list', action='store_true',
+                        help='list connected devices')
+    parser.add_argument('-c' , '--convert', action='store_true',
+                        help='do not flash, just convert')
+    parser.add_argument('-D' , '--deploy', action='store_true',
+                        help='just flash, do not convert')
+    parser.add_argument('-f' , '--family', dest='family', type=str,
+                        default="0x0",
+                        help='specify familyID - number or name (default: 0x0)')
+    parser.add_argument('-C' , '--carray', action='store_true',
+                        help='convert binary file to a C array, not UF2')
+    args = parser.parse_args()
+    appstartaddr = int(args.base, 0)
+
+    if args.family.upper() in families:
+        familyid = families[args.family.upper()]
+    else:
+        try:
+            familyid = int(args.family, 0)
+        except ValueError:
+            error("Family ID needs to be a number or one of: " + ", ".join(families.keys()))
+
+    if args.list:
+        list_drives()
+    else:
+        if not args.input:
+            error("Need input file")
+        with open(args.input, mode='rb') as f:
+            inpbuf = f.read()
+        from_uf2 = is_uf2(inpbuf)
+        ext = "uf2"
+        if args.deploy:
+            outbuf = inpbuf
+        elif from_uf2:
+            outbuf = convert_from_uf2(inpbuf)
+            ext = "bin"
+        elif is_hex(inpbuf):
+            outbuf = convert_from_hex_to_uf2(inpbuf.decode("utf-8"))
+        elif args.carray:
+            outbuf = convert_to_carray(inpbuf)
+            ext = "h"
+        else:
+            outbuf = convert_to_uf2(inpbuf)
+        print("Converting to %s, output size: %d, start address: 0x%x" %
+              (ext, len(outbuf), appstartaddr))
+        if args.convert or ext != "uf2":
+            drives = []
+            if args.output == None:
+                args.output = "flash." + ext
+        else:
+            drives = get_drives()
+
+        if args.output:
+            write_file(args.output, outbuf)
+        else:
+            if len(drives) == 0:
+                error("No drive to deploy.")
+        for d in drives:
+            print("Flashing %s (%s)" % (d, board_id(d)))
+            write_file(d + "/NEW.UF2", outbuf)
+
+
+if __name__ == "__main__":
+    main()

bin/upload-to-bootloader.sh

@@ -0,0 +1,4 @@
+
+echo "Converting to uf2 for NRF52 Adafruit bootloader"
+bin/uf2conv.py .pio/build/lora-relay-v2/firmware.hex -f 0xADA52840
+# cp flash.uf2 /media/kevinh/FTH*BOOT/

bin/version.sh

@@ -1,3 +0,0 @@
-
-
-export VERSION=0.7.7

bin/view-map.sh

@@ -0,0 +1,2 @@
+echo using amap tool to display memory map
+amap .pio/build/output.map

boards/eink.json

@@ -0,0 +1,61 @@
+{
+  "build": {
+    "arduino": {
+      "ldscript": "nrf52840_s140_v6.ld"
+    },
+    "core": "nRF5",
+    "cpu": "cortex-m4",
+    "extra_flags": "-DARDUINO_NRF52840_TTGO_EINK -DNRF52840_XXAA",
+    "f_cpu": "64000000L",
+    "hwids": [
+      [
+        "0x239A",
+        "0x4405"
+      ]
+    ],
+    "usb_product": "TTGO_eink",
+    "mcu": "nrf52840",
+    "variant": "eink",
+    "variants_dir": "variants",
+    "bsp": {
+      "name": "adafruit"
+    },
+    "softdevice": {
+      "sd_flags": "-DS140",
+      "sd_name": "s140",
+      "sd_version": "6.1.1",
+      "sd_fwid": "0x00B6"
+    },
+    "bootloader": {
+      "settings_addr": "0xFF000"
+    }
+  },
+  "connectivity": [
+    "bluetooth"
+  ],
+  "debug": {
+    "jlink_device": "nRF52840_xxAA",
+    "onboard_tools": [
+      "jlink"
+    ],
+    "svd_path": "nrf52840.svd"
+  },
+  "frameworks": [
+    "arduino"
+  ],
+  "name": "TTGO eink (Adafruit BSP)",
+  "upload": {
+    "maximum_ram_size": 248832,
+    "maximum_size": 815104,
+    "require_upload_port": true,
+    "speed": 115200,
+    "protocol": "jlink",
+    "protocols": [
+      "jlink",
+      "nrfjprog",
+      "stlink"
+    ]
+  },
+  "url": "FIXME",
+  "vendor": "TTGO"
+}

boards/eink0.1.json

@@ -0,0 +1,61 @@
+{
+  "build": {
+    "arduino": {
+      "ldscript": "nrf52840_s140_v6.ld"
+    },
+    "core": "nRF5",
+    "cpu": "cortex-m4",
+    "extra_flags": "-DARDUINO_NRF52840_TTGO_EINK -DNRF52840_XXAA",
+    "f_cpu": "64000000L",
+    "hwids": [
+      [
+        "0x239A",
+        "0x4405"
+      ]
+    ],
+    "usb_product": "TTGO_eink",
+    "mcu": "nrf52840",
+    "variant": "eink0.1",
+    "variants_dir": "variants",
+    "bsp": {
+      "name": "adafruit"
+    },
+    "softdevice": {
+      "sd_flags": "-DS140",
+      "sd_name": "s140",
+      "sd_version": "6.1.1",
+      "sd_fwid": "0x00B6"
+    },
+    "bootloader": {
+      "settings_addr": "0xFF000"
+    }
+  },
+  "connectivity": [
+    "bluetooth"
+  ],
+  "debug": {
+    "jlink_device": "nRF52840_xxAA",
+    "onboard_tools": [
+      "jlink"
+    ],
+    "svd_path": "nrf52840.svd"
+  },
+  "frameworks": [
+    "arduino"
+  ],
+  "name": "TTGO eink (Adafruit BSP)",
+  "upload": {
+    "maximum_ram_size": 248832,
+    "maximum_size": 815104,
+    "require_upload_port": true,
+    "speed": 115200,
+    "protocol": "jlink",
+    "protocols": [
+      "jlink",
+      "nrfjprog",
+      "stlink"
+    ]
+  },
+  "url": "FIXME",
+  "vendor": "TTGO"
+}

boards/lora-relay-v1.json

@@ -0,0 +1,46 @@
+{
+  "build": {
+    "arduino": {
+      "ldscript": "nrf52840_s140_v6.ld"
+    },
+    "core": "nRF5",
+    "cpu": "cortex-m4",
+    "extra_flags": "-DARDUINO_NRF52840_LORA_RELAY_V1 -DNRF52840_XXAA",
+    "f_cpu": "64000000L",
+    "hwids": [["0x239A", "0x4404"]],
+    "usb_product": "LORA_RELAY",
+    "mcu": "nrf52840",
+    "variant": "lora_relay_v1",
+    "variants_dir": "variants",
+    "bsp": {
+      "name": "adafruit"
+    },
+    "softdevice": {
+      "sd_flags": "-DS140",
+      "sd_name": "s140",
+      "sd_version": "6.1.1",
+      "sd_fwid": "0x00B6"
+    },
+    "bootloader": {
+      "settings_addr": "0xFF000"
+    }
+  },
+  "connectivity": ["bluetooth"],
+  "debug": {
+    "jlink_device": "nRF52840_xxAA",
+    "onboard_tools": ["jlink"],
+    "svd_path": "nrf52840.svd"
+  },
+  "frameworks": ["arduino"],
+  "name": "Meshtastic Lora Relay V1 (Adafruit BSP)",
+  "upload": {
+    "maximum_ram_size": 248832,
+    "maximum_size": 815104,
+    "require_upload_port": true,
+    "speed": 115200,
+    "protocol": "jlink",
+    "protocols": ["jlink", "nrfjprog", "stlink"]
+  },
+  "url": "https://github.com/BigCorvus/SX1262-LoRa-BLE-Relay",
+  "vendor": "BigCorvus"
+}

boards/lora-relay-v2.json

@@ -0,0 +1,46 @@
+{
+  "build": {
+    "arduino": {
+      "ldscript": "nrf52840_s140_v6.ld"
+    },
+    "core": "nRF5",
+    "cpu": "cortex-m4",
+    "extra_flags": "-DARDUINO_NRF52840_LORA_RELAY_V2 -DNRF52840_XXAA",
+    "f_cpu": "64000000L",
+    "hwids": [["0x239A", "0x4406"]],
+    "usb_product": "LORA_RELAY",
+    "mcu": "nrf52840",
+    "variant": "lora_relay_v2",
+    "variants_dir": "variants",
+    "bsp": {
+      "name": "adafruit"
+    },
+    "softdevice": {
+      "sd_flags": "-DS140",
+      "sd_name": "s140",
+      "sd_version": "6.1.1",
+      "sd_fwid": "0x00B6"
+    },
+    "bootloader": {
+      "settings_addr": "0xFF000"
+    }
+  },
+  "connectivity": ["bluetooth"],
+  "debug": {
+    "jlink_device": "nRF52840_xxAA",
+    "onboard_tools": ["jlink"],
+    "svd_path": "nrf52840.svd"
+  },
+  "frameworks": ["arduino"],
+  "name": "Meshtastic Lora Relay V1 (Adafruit BSP)",
+  "upload": {
+    "maximum_ram_size": 248832,
+    "maximum_size": 815104,
+    "require_upload_port": true,
+    "speed": 115200,
+    "protocol": "jlink",
+    "protocols": ["jlink", "nrfjprog", "stlink"]
+  },
+  "url": "https://github.com/BigCorvus/SX1262-LoRa-BLE-Relay",
+  "vendor": "BigCorvus"
+}

boards/nrf52840_dk.json

@@ -0,0 +1,47 @@
+{
+    "build": {
+      "arduino": {
+        "ldscript": "nrf52840_s140_v6.ld"
+      },
+      "core": "nRF5",
+      "cpu": "cortex-m4",
+      "extra_flags": "-DARDUINO_NRF52840_PCA10056 -DNRF52840_XXAA",
+      "f_cpu": "64000000L",
+      "hwids": [["0x239A", "0x4404"]],
+      "usb_product": "nrf52840dk",
+      "mcu": "nrf52840",
+      "variant": "pca10056",
+      "variants_dir": "variants",
+      "bsp": {
+        "name": "adafruit"
+      },
+      "softdevice": {
+        "sd_flags": "-DS140",
+        "sd_name": "s140",
+        "sd_version": "6.1.1",
+        "sd_fwid": "0x00B6"
+      },
+      "bootloader": {
+        "settings_addr": "0xFF000"
+      }
+    },
+    "connectivity": ["bluetooth"],
+    "debug": {
+      "jlink_device": "nRF52840_xxAA",
+      "onboard_tools": ["jlink"],
+      "svd_path": "nrf52840.svd"
+    },
+    "frameworks": ["arduino"],
+    "name": "A modified NRF52840-DK devboard (Adafruit BSP)",
+    "upload": {
+      "maximum_ram_size": 248832,
+      "maximum_size": 815104,
+      "require_upload_port": true,
+      "speed": 115200,
+      "protocol": "jlink",
+      "protocols": ["jlink", "nrfjprog", "stlink"]
+    },
+    "url": "https://meshtastic.org/",
+    "vendor": "Nordic Semi"
+  }
+  

boards/nrf52840_dk_modified.json

@@ -1,14 +1,14 @@
 {
   "build": {
     "arduino": {
-      "ldscript": "nrf52840_s140_v6.ld"
+      "ldscript": "nrf52840_s113_v7.ld"
     },
     "core": "nRF5",
     "cpu": "cortex-m4",
     "extra_flags": "-DARDUINO_NRF52840_PCA10056 -DNRF52840_XXAA",
     "f_cpu": "64000000L",
     "hwids": [["0x239A", "0x4404"]],
-    "usb_product": "SimPPR",
+    "usb_product": "nrf52840dk",
     "mcu": "nrf52840",
     "variant": "pca10056-rc-clock",
     "variants_dir": "variants",
@@ -16,9 +16,9 @@
       "name": "adafruit"
     },
     "softdevice": {
-      "sd_flags": "-DS140",
-      "sd_name": "s140",
-      "sd_version": "6.1.1",
+      "sd_flags": "-DS113",
+      "sd_name": "s113",
+      "sd_version": "7.2.0",
       "sd_fwid": "0x00B6"
     },
     "bootloader": {

boards/ppr1.json

@@ -0,0 +1,46 @@
+{
+  "build": {
+    "arduino": {
+      "ldscript": "nrf52833_s113_v7.ld"
+    },
+    "core": "nRF5",
+    "cpu": "cortex-m4",
+    "extra_flags": "-DARDUINO_NRF52833_PPR -DNRF52833_XXAA",
+    "f_cpu": "64000000L",
+    "hwids": [["0x239A", "0x4406"]],
+    "usb_product": "PPR",
+    "mcu": "nrf52833",
+    "variant": "ppr",
+    "variants_dir": "variants",
+    "bsp": {
+      "name": "adafruit"
+    },
+    "softdevice": {
+      "sd_flags": "-DS113",
+      "sd_name": "s113",
+      "sd_version": "7.2.0",
+      "sd_fwid": "0x00b6"
+    },
+    "bootloader": {
+      "settings_addr": "0xFF000"
+    }
+  },
+  "connectivity": ["bluetooth"],
+  "debug": {
+    "jlink_device": "nRF52833_xxAA",
+    "onboard_tools": ["jlink"],
+    "svd_path": "nrf52833.svd"
+  },
+  "frameworks": ["arduino"],
+  "name": "Meshtastic PPR1 (Adafruit BSP)",
+  "upload": {
+    "maximum_ram_size": 248832,
+    "maximum_size": 815104,
+    "require_upload_port": true,
+    "speed": 115200,
+    "protocol": "jlink",
+    "protocols": ["jlink", "nrfjprog", "stlink"]
+  },
+  "url": "https://meshtastic.org/",
+  "vendor": "Othernet"
+}

boards/rak815.json

@@ -0,0 +1,55 @@
+{
+  "build": {
+    "arduino":{
+      "ldscript": "nrf52832_s132_v6.ld"
+    },
+    "core": "nRF5",
+    "cpu": "cortex-m4",
+    "extra_flags": "-DNRF52832_XXAA -DNRF52",
+    "f_cpu": "64000000L",
+    "hwids": [
+      [
+        "0x10c4",
+        "0xea60"
+      ]
+    ],
+    "usb_product": "RAK815",
+    "mcu": "nrf52832",
+    "variant": "rak815",
+    "bsp": {
+      "name": "adafruit"
+    },
+    "softdevice": {
+      "sd_flags": "-DS132",
+      "sd_name": "s132",
+      "sd_version": "6.1.1",
+      "sd_fwid": "0x00B7"
+    }
+  },
+  "connectivity": [
+    "bluetooth"
+  ],
+  "debug": {
+    "jlink_device": "nRF52832_xxAA",
+    "svd_path": "nrf52.svd"
+  },
+  "frameworks": [
+    "arduino"
+  ],
+  "name": "RAK RAK815",
+  "upload": {
+    "maximum_ram_size": 65536,
+    "maximum_size": 524288,
+    "require_upload_port": true,
+    "speed": 115200,
+    "protocol": "nrfutil",
+    "protocols": [
+      "jlink",
+      "nrfjprog",
+      "nrfutil",
+      "stlink"
+    ]
+  },
+  "url": "https://store.rakwireless.com/products/rak815-hybrid-location-tracker",
+  "vendor": "RAK"
+}

data/static/basic.js

@@ -0,0 +1,43 @@
+var meshtasticClient;
+var connectionOne;
+
+
+// Important: the connect action must be called from a user interaction (e.g. button press), otherwise the browsers won't allow the connect
+function connect() {
+
+    // Create new connection
+    var httpconn = new meshtasticjs.IHTTPConnection();
+
+    // Set connection params
+    let sslActive;
+    if (window.location.protocol === 'https:') {
+        sslActive = true;
+    } else {
+        sslActive = false;
+    }
+    let deviceIp = window.location.hostname; // Your devices IP here
+
+
+    // Add event listeners that get called when a new packet is received / state of device changes
+    httpconn.addEventListener('fromRadio', function (packet) { console.log(packet) });
+
+    // Connect to the device async, then send a text message
+    httpconn.connect(deviceIp, sslActive)
+        .then(result => {
+
+            alert('device has been configured')
+            // This gets called when the connection has been established
+            // -> send a message over the mesh network. If no recipient node is provided, it gets sent as a broadcast
+            return httpconn.sendText('meshtastic is awesome');
+
+        })
+        .then(result => {
+
+            // This gets called when the message has been sucessfully sent
+            console.log('Message sent!');
+        })
+
+        .catch(error => { console.log(error); });
+
+}
+

data/static/index.html

@@ -0,0 +1,18 @@
+<!doctype html>
+<html class="no-js" lang="">
+
+<head>
+  <meta charset="utf-8">
+  <title></title>
+
+  <script src="/static/meshtastic.js"></script>
+  <script src="/static/basic.js"></script>
+</head>
+
+<body>
+
+  <button id="connect_button" onclick="connect()">Connect to Meshtastic device</button>
+ 
+</body>
+
+</html>

docs/README.md

@@ -1,13 +1,13 @@
 # What is Meshtastic?
 
-Meshtastic™ is a project that lets you use
+Meshtastic® is a project that lets you use
 inexpensive (\$30 ish) GPS radios as an extensible, long battery life, secure, mesh GPS communicator. These radios are great for hiking, skiing, paragliding - essentially any hobby where you don't have reliable internet access. Each member of your private mesh can always see the location and distance of all other members and any text messages sent to your group chat.
 
 The radios automatically create a mesh to forward packets as needed, so everyone in the group can receive messages from even the furthest member. The radios will optionally work with your phone, but no phone is required.
 
 Note: Questions after reading this? See our new [forum](https://meshtastic.discourse.group/).
 
-### Uses
+## Uses
 
 - Outdoor sports where cellular coverage is limited. (Hiking, Skiing, Boating, Paragliding, Gliders etc..)
 - Applications where closed source GPS communicators just won't cut it (it is easy to add features for glider pilots etc...)
@@ -17,7 +17,7 @@ Note: Questions after reading this? See our new [forum](https://meshtastic.disco
 
 [![Youtube video demo](desk-video-screenshot.png)](https://www.youtube.com/watch?v=WlNbMbVZlHI "Meshtastic early demo")
 
-### Features
+## Features
 
 Not all of these features are fully implemented yet - see **important** disclaimers below. But they should be in by the time we decide to call this project beta (three months?)
 
@@ -35,10 +35,21 @@ This project is currently in beta testing but it is fairly stable and feature co
 
 This software is 100% open source and developed by a group of hobbyist experimenters. No warranty is provided, if you'd like to improve it - we'd love your help. Please post in the [forum](https://meshtastic.discourse.group/).
 
+### Beginner's Guide
+
+For an detailed walk-through aimed at beginners, we recommend [meshtastic.letstalkthis.com](https://meshtastic.letstalkthis.com/).
+
+### Related Groups
+
+Telegram group for **Italy**-based users [t.me/meshtastic_italia](http://t.me/meshtastic_italia) (Italian language, unofficial).<br/>
+Telegram group for **Russian**-based users [t.me/meshtastic_russia](https://t.me/meshtastic_russia) (Russian language, unofficial).
+
 # Updates
 
 Note: Updates are happening almost daily, only major updates are listed below. For more details see our forum.
 
+- 09/14/2020 - 1.0.0 Now with over 1700 android users, over 2000 nodes and translated into 15 languages. This project will always be a "beta" experiment, but now quite usable. We are currently selecting 1.1 features in our discussion forum.
+- 06/24/2020 - 0.7.x Now with over 1000 android users, over 600 people using the radios and translated into 22 languages. Fairly stable and we are working through bugs to get to 1.0.
 - 06/04/2020 - 0.6.7 Beta releases of both the application and the device code are released. Features are fairly solid now with a sizable number of users.
 - 04/28/2020 - 0.6.0 [Python API](https://pypi.org/project/meshtastic/) released. Makes it easy to use meshtastic devices as "zero config / just works" mesh transport adapters for other projects.
 - 04/20/2020 - 0.4.3 Pretty solid now both for the android app and the device code. Many people have donated translations and code. Probably going to call it a beta soon.
@@ -61,18 +72,25 @@ The link above will return older more stable releases. We would prefer if you jo
 
 If you'd like to help with development, the source code is [on github](https://github.com/meshtastic/Meshtastic-Android).
 
+The app is also distributed for Amazon Fire devices via the Amazon appstore: [![Amazon appstore link](https://raw.githubusercontent.com/meshtastic/Meshtastic-device/master/images/amazon-fire-button.png)](https://www.amazon.com/Geeksville-Industries-Meshtastic/dp/B08CY9394Q)
+
 ## Supported hardware
 
-We currently support two brands of radios. The [TTGO T-Beam](https://www.aliexpress.com/item/4000119152086.html) and the [Heltec LoRa 32](https://heltec.org/project/wifi-lora-32/). Most people should buy the T-Beam and a 18650 battery (total cost less than \$35). Make
-sure to buy the frequency range which is legal for your country. For the USA, you should buy the 915MHz version. Getting a version that include a screen is optional, but highly recommended.
+We currently support two brands of radios. The [TTGO T-Beam](https://www.aliexpress.com/item/4001178678568.html) and the [Heltec LoRa 32](https://heltec.org/project/wifi-lora-32/). Most people should buy the T-Beam and a 18650 battery (total cost less than \$35). Also, the version of the T-Beam we link to is shipped with Meshtastic **preinstalled** by TTGO, so you don't have to install it yourself.
+
+Make sure to buy the frequency range which is legal for your country. For the USA, you should buy the 915MHz version. Getting a version that include a screen is optional, but highly recommended.
 
 Instructions for installing prebuilt firmware can be found [here](https://github.com/meshtastic/Meshtastic-esp32/blob/master/README.md).
 
-For a nice printable cases:
+For a nice looking cases:
 
-1. TTGO T-Beam V0 see this [design](https://www.thingiverse.com/thing:3773717) by [bsiege](https://www.thingiverse.com/bsiege).
-2. TTGO T_Beam V1 see this [design](https://www.thingiverse.com/thing:3830711) by [rwanrooy](https://www.thingiverse.com/rwanrooy) or this [remix](https://www.thingiverse.com/thing:3949330) by [8ung](https://www.thingiverse.com/8ung)
-3. Heltec Lora32 see this [design](https://www.thingiverse.com/thing:3125854) by [ornotermes](https://www.thingiverse.com/ornotermes).
+- 3D printable cases
+  1. TTGO T-Beam V0 see this [design](https://www.thingiverse.com/thing:3773717) by [bsiege](https://www.thingiverse.com/bsiege).
+  2. TTGO T_Beam V1 (SMA) see this [design](https://www.thingiverse.com/thing:3830711) by [rwanrooy](https://www.thingiverse.com/rwanrooy) or this [remix](https://www.thingiverse.com/thing:3949330) by [8ung](https://www.thingiverse.com/8ung)
+  3. TTGO T_Beam V1 (IPEX) see this [design](https://www.thingiverse.com/thing:4587297) by [drewsed](https://www.thingiverse.com/drewsed)
+  4. Heltec Lora32 see this [design](https://www.thingiverse.com/thing:3125854) by [ornotermes](https://www.thingiverse.com/ornotermes).
+- Laser-cut cases
+  1. TTGO T_Beam V1 (SMA) see this [design](https://www.thingiverse.com/thing:4552771) by [jefish](https://www.thingiverse.com/jefish)
 
 # IMPORTANT DISCLAIMERS AND FAQ
 

docs/SupportedHardware.md

@@ -0,0 +1,8 @@
+| Vendor  | Product line  | Version  | Board labels  | Notes  | URL |
+|---|---|---|---|---|---|
+| TTGO  | T-Beam  | 0.7  | T22_V07 20180711  | LoRa 433/470MHz *OR* LoRa 868/915MHz , <br/>GPS ublox NEO-6M , <br/>battery holder for Li-Ion 18650  |  [buy](https://www.aliexpress.com/item/4000574335430.html)  |
+| TTGO  | T-Beam  | 1.0  |   |   |  [buy](https://www.aliexpress.com/item/4001178678568.html)  |
+| TTGO  | T-Beam  | 1.1  | T22_V11 20191212 | LoRa 433/470MHz *OR* LoRa 868/915MHz *OR* LoRa 923MHz , <br/>GPS ublox NEO-M8N , <br/>battery holder for Li-Ion 18650  | [buy](https://www.aliexpress.com/item/4001178678568.html) |
+| TTGO  | Lora32  | 2.0  | *missing*  | LoRa 433/470MHz *OR* LoRa 868/915MHz , <br/>OLED SSD1306 , <br/>SD card holder | [buy](https://www.aliexpress.com/item/4000211331316.html) |
+| TTGO  | Lora32  | 2.1  | T3_V1.6 20180606  | LoRa 32 (V2) , <br/>SD card holder | [buy](https://www.aliexpress.com/item/4000119208093.html) |
+| Heltec | Lora 32 | V2  | V2 | LoRa 433/470MHz *OR* LoRa 868/915MHz | [buy](https://heltec.org/project/wifi-lora-32/) |

docs/_config.yml

@@ -1,7 +1,7 @@
 theme: jekyll-theme-cayman
 
 title: Meshtastic
-description: An opensource hiking, pilot, skiing, Signal-App-extending GPS mesh communicator
+description: An opensource hiking, pilot, skiing, secure GPS mesh communicator
 google_analytics: G-DRZ5H5EXHV
 
 include: [".well-known"]

docs/faq.md

@@ -4,7 +4,7 @@ This project is still pretty young but moving at a pretty good pace. Not all fea
 Most of these problems should be solved by the beta release (within three months):
 
 - We don't make these devices and they haven't been tested by UL or the FCC. If you use them you are experimenting and we can't promise they won't burn your house down ;-)
-- The encryption [implementation](software/crypto.md) has not been reviewed by an expert. (Are you an expert? Please help us)
+- The encryption implementation is good but see this list of [caveats](software/crypto.md#summary-of-strengthsweaknesses-of-our-current-implementation) to determine risks you might face.
 - A number of (straightforward) software work items have to be completed before battery life matches our measurements, currently battery life is about three days. Join us on chat if you want the spreadsheet of power measurements/calculations.
 - The Android API needs to be documented better
 - No one has written an iOS app yet. But some good souls [are talking about it](https://github.com/meshtastic/Meshtastic-esp32/issues/14) ;-)

docs/hardware/corvus.md

@@ -0,0 +1,35 @@
+# Notes on @BigCorvus boards
+
+## Board version 1.1
+
+variant name lora_relay_v1
+
+### Remaining TODOs
+
+- power hold for the ST7735
+- look at example sketch
+- turn on xmit boost
+
+## Recommendations for future boards
+
+@BigCorvus your board is **really** nice. Here's some ideas for the future:
+
+- make the SWDIO header more standard (the small ARM 2x5 micro footprint?) or at least througholes so it is easy to solder a header
+
+## How to program bootloader
+
+Download from here: https://github.com/adafruit/Adafruit_nRF52_Bootloader/releases
+
+```
+nrfjprog -f nrf52 --eraseall
+Erasing user available code and UICR flash areas.
+Applying system reset.
+
+nrfjprog -f nrf52 --program feather_nrf52840_express_bootloader-0.3.2_s140_6.1.1.hex
+Parsing hex file.
+Reading flash area to program to guarantee it is erased.
+Checking that the area to write is not protected.
+Programming device.
+```
+
+Then reboot the board, if all went well it now shows up as a mountable filesystem on your USB bus.

docs/radio-settings.md

@@ -2,13 +2,13 @@
 
 We use the same channel maps as LoRaWAN (though this is not LoRaWAN).
 
-![freq table](images/LoRa-Frequency-Bands.jpg)
+![freq table](/images/LoRa-Frequency-Bands.jpg)
 
 See [this site](https://www.rfwireless-world.com/Tutorials/LoRa-channels-list.html) for more information.
 
 ## LoRaWAN Europe Frequency Band
 
-The maximum power allowed is +14dBM.
+The maximum power allowed is +14dBm ERP (Effective Radiated Power, see [this site](https://en.wikipedia.org/wiki/Effective_radiated_power) for more information).
 
 ### 433 MHz
 
@@ -24,7 +24,82 @@ Channel zero starts at 865.20 MHz
 
 LoRaWAN defines 64, 125 kHz channels from 902.3 to 914.9 MHz increments.
 
-The maximum output power for North America is +30 dBM.
+The maximum output power for North America is +30 dBm ERP.
 
 The band is from 902 to 928 MHz. It mentions channel number and its respective channel frequency. All the 13 channels are separated by 2.16 MHz with respect to the adjacent channels.  
-Channel zero starts at 903.08 MHz center frequency.
+Channel zero starts at 903.08 MHz center frequency.
+
+## Data-rates
+
+### About
+
+Various data-rates are selectable when configuring a channel and are inversely proportional to the theoretical range of the devices.
+
+Considerations:
+
+* Spreading Factor - How much we "spread" our data over time.
+* * Each step up in Spreading Factor dobules the airtime to transmit.
+* * Each step up in Spreading Factor adds about 2.5db extra link budget.
+* Bandwidth - How big of a slice of the spectrum we use.
+* * Each doubling of the bandwidth is almost 3db less link budget.
+* * Bandwidths less than 31 may be unstable unless you have a high quality Crystal Ossilator.
+* Coding Rate - How much redundency we encode to resist noise.
+* * Increasing coding rate increases reliability while decrasing data-rate.
+* * 4/5 - 1.25x overhead
+* * 4/6 - 1.5x overhead
+* * 4/7 - 1.75x overhead
+* * 4/8 - 2x overhead
+
+
+### Pre-Defined
+
+We have four predefined channels. These are the most common settings and have been proven to work well:
+
+| Channel setting            | Alt Channel Name | Data-rate            | SF / Symbols | Coding Rate | Bandwidth | Link Budget |
+|:---------------------------|:-----------------|:---------------------|:-------------|:------------|:----------|:------------|
+| Short range (but fast)     | Short Fast       | 21.875 kbps          | 7 / 128      | 4/5         | 125       | 134dB       |
+| Medium range (but fast)    | Medium           | 5.469 kbps           | 7 / 128      | 4/5         | 500       | 140dB       |
+| Long range (but slower)    | Long Alt         | 0.275 kbps           | 9 / 512      | 4/8         | 31        | 153dB       |
+| Very long range (but slow) | Long Slow        | 0.183 kbps (default) | 12 / 4096    | 4/8         | 125       | 154dB       |
+
+The link budget used by these calculations assumes a transmit power of 17dBm and an antenna with 0dB gain. Adjust your link budget assumptions based on your actual devices.
+
+### Custom Settings
+
+You may want to select other channels for your usage. The other settings can be set by using the Python API.
+
+> meshtastic --setchan spread_factor 10 --setchan coding_rate 8 --setchan bandwidth 125
+
+After applying the settings, you will need to restart the device. After your device is restarted, it will generate a new crypto key and you will need to share the newly generated QR Code or URL to all your other devices.
+
+Some example settings:
+
+| Data-rate            | SF / Symbols | Coding Rate | Bandwidth | Link Budget | Note |
+|:---------------------|:-------------|:------------|:----------|:------------|:-----|
+| 37.50 kbps           | 6 / 64       | 4/5         | 500       | 129dB       | Fastest possible speed |
+| 3.125 kbps           | 8 / 256      | 4/5         | 125       | 143dB       | |
+| 1.953 kbps           | 8 / 256      | 4/8         | 125       | 143dB       | |
+| 1.343 kbps           | 11 / 2048    | 4/8         | 500       | 145dB       | | 
+| 1.099 kbps           | 9 / 512      | 4/8         | 125       | 146dB       | |
+| 0.814 kbps           | 10 / 1024    | 4/6         | 125       | 149dB       | |
+| 0.610 kbps           | 10 / 1024    | 4/8         | 125       | 149dB       | |
+| 0.488 kbps           | 11 / 2048    | 4/6         | 125       | 152dB       | |
+| 0.336 kbps           | 11 / 2048    | 4/8         | 125       | 152dB       | |
+| 0.073 kbps           | 12 / 4096    | 4/5         | 31        | 160dB       | Twice the range and/or coverage of "Long Slow", low resliance to noise |
+| 0.046 kbps           | 12 / 4096    | 4/8         | 31        | 160dB       | Twice the range and/or coverage of "Long Slow", high resliance to noise |
+
+The link budget used by these calculations assumes a transmit power of 17dBm and an antenna with 0dB gain. Adjust your link budget assumptions based on your actual devices.
+
+These channel settings may have not been tested. Use at your own discression. Share on https://meshtastic.discourse.group with your successes or failure.
+
+## Cryptography
+
+The preshared key used by the devices can be modified.
+
+* 0 = No crypto
+* 1 = Default channel key
+* 2 - 10 = The default channel key, except with 1 through 9 added to the last byte
+
+Use of cryptography can also be modified. To disable cryptography (maybe useful if you have HAM radio license):
+
+> meshtastic --setchan psk 0

docs/software/TODO.md

@@ -2,40 +2,242 @@
 
 You probably don't care about this section - skip to the next one.
 
-- make new android release
-- check in our modified arduino binaries
-- post bug on esp32-arduino
-- router mode
-- let users set arbitary params in android
-- encryption review findings writeup
-- NRF52 BLE support
-- DSR
+## 1.2 cleanup & multichannel support:
+
+* cdcacm bug on nrf52: emittx thinks it emitted but client sees nothing.  works again later
+* nrf52: segger logs have errors in formatting that should be impossible (because not going through serial, try stalling on segger)
+* DONE call RouterPlugin for *all* packets - not just Router packets
+* DONE generate channel hash from the name of the channel+the psk (not just one or the other)
+* DONE send a hint that can be used to select which channel to try and hash against with each message
+* DONE remove deprecated
+* DONE fix setchannel in phoneapi.cpp
+* DONE set mynodeinfo.max_channels
+* DONE set mynodeinfo.num_bands (formerly num_channels)
+* DONE fix sniffing of non Routing packets
+* DONE enable remote setttings access by moving settings operations into a regular plugin (move settings ops out of PhoneAPI)
+* DONE move portnum up?
+* DONE remove region specific builds from the firmware
+* DONE test single channel without python
+* DONE Use "default" for name if name is empty
+* DONE fix python data packet receiving (nothing showing in log?)
+* DONE implement 'get channels' Admin plugin operation
+* DONE use get-channels from python
+* DONE use get channels & get settings from android
+* DONE use set-channel from python
+* DONE make settings changes from python work
+* DONE pthon should stop fetching channels once we've reached our first empty channel definition (hasSettings == true)
+* DONE add check for old devices with new API library
+* DONE release python api
+* DONE release protobufs
+* DONE release to developers
+* DONE fix setch-fast in python tool
+* turn off fault 8: https://github.com/meshtastic/Meshtastic-device/issues/734
+* age out pendingrequests in the python API
+* DONE stress test channel download from python, sometimes it seems like we don't get all replies, bug was due to simultaneous android connection
+* DONE combine acks and responses in a single message if possible (do routing plugin LAST and drop ACK if someone else has already replied)
+* DONE don't send packets we received from the phone BACK TOWARDS THE PHONE (possibly use fromnode 0 for packets the phone sends?)
+* fix 1.1.50 android debug panel display
+* DONE test android channel setting
+* DONE release to users
+* DONE warn in android app about unset regions
+* DONE use set-channel from android
+* DONE add gui in android app for setting region
+* DONE clean up python channel usage
+* DONE use bindToChannel to limit admin access for remote nodes
+* DONE move channels and radio config out of device settings
+* test remote info and remote settings changes
+* make python tests more exhaustive
+* pick default random admin key
+* exclude admin channels from URL?
+* make a way to share just secondary channels via URL
+* use single byte 'well known' channel names for the four default channel names (longslow etc), and for admin, gpio, etc...
+* use presence of gpio channel to enable gpio ops, same for serial etc...
+* restrict gpio & serial & settings operations to the admin channel (unless local to the current node)
+* add channel restrictions for plugins (and restrict routing plugin to the "control" channel)
+* stress test multi channel
+* investigate @mc-hamster report of heap corruption
+* DONE use set-user from android
+* generalize the concept of "shortstrings" use it for both PSKs and well known channel names.  Possibly use a ShortString class.
+* use portuino TCP connection to debug with python API
+* document the relationship between want_response (indicating remote node received it) and want_ack (indicating that this message should be sent reliably - and also get acks from the first rx node and naks if it is never delivered)
+* DONE android should stop fetching channels once we've reached our first empty channel definition (hasSettings == true)
+* DONE warn in python api if we are too new to talk to the device code
+* DONE make a post warning about 1.2, telling how to stay on old android & python clients.  link to this from the android dialog message and python version warning.
+* DONE "FIXME - move the radioconfig/user/channel READ operations into SettingsMessage as well"
+* DONE scrub protobufs to make sure they are absoloute minimum wiresize (in particular Data, ChannelSets and positions)
+* DONE change syncword (now ox2b)
+* allow chaning packets in single transmission - to increase airtime efficiency and amortize packet overhead
+* DONE move most parts of meshpacket into the Data packet, so that we can chain multiple Data for sending when they all have a common destination and key.
+* when selecting a MeshPacket for transmit, scan the TX queue for any Data packets we can merge together as a WirePayload.  In the low level send/rx code expand that into multiple MeshPackets as needed (thus 'hiding' from MeshPacket that over the wire we send multiple datapackets
+* confirm we are still calling the plugins for messages inbound from the phone (or generated locally)
+* confirm we are still multi hop routing flood broadcasts
+* confirm we are still doing resends on unicast reliable packets
+* add history to routed packets: https://meshtastic.discourse.group/t/packet-source-tracking/2764/2
+* add support for full DSR unicast delivery
+* DONE move acks into routing
+* DONE make all subpackets different versions of data
+* DONE move routing control into a data packet
+* have phoneapi done via plugin (will allow multiple simultaneous API clients - stop disabling BLE while using phone API)
+* DONE figure out how to add micro_delta to position, make it so that phone apps don't need to understand it?
+* only send battery updates a max of once a minute
+* add python channel selection for sending
+* DONE record recevied channel in meshpacket
+* test remote settings operations (confirm it works 3 hops away)
+* DONE make a primaryChannel global and properly maintain it when the phone sends setChannel
+* DONE move setCrypto call into packet send and packet decode code
+* implement 'small location diffs' change
+* move battery level out of position? 
+* consider "A special exception (FIXME, not sure if this is a good idea) - packets that arrive on the local interface 
+  are allowed on any channel (this lets the local user do anything)."  Probably by adding a "secure_local_interface" settings bool.
+* DOUBLE CHECK android app can still upgrade 1.1 and 1.0 loads
+ 
+eink:
+
+* DONE check email of reported issues
+* DONE turn off vbus driving (in bootloader)
+* new battery level sensing
+* current draw no good
+* DONE: fix backlight
+* DONE - USB is busted because of power enable mode?
+* test CPU voltage? something is bad with RAM (removing eink module does not help)
+* test that board leaves bootloader always
+* test USB - works in bootloader
+* test LEDs
+* Test BME280
+* test gps
+* check GPS fast locking
+* tested! dlora
+* test eink backlight
+* tested! eink
+* test buttons
+* test battery charging
+* test serial flash
+* send updated app and bootloader image
+* OHH BME280!  THAT IS GREAT!
+* make new screen work, ask for datasheet
+* say I think you could ship this
+* leds seem busted
+* fix hw_model: "nrf52unknown"
+* use larger icon for meshtastic logo
+* send email about variants & faster flash programming - https://github.com/geeksville/Meshtastic-esp32/commit/f110225173a77326aac029321cdb6491bfa640f6
+* send PR for bootloader
+* fix nrf52 time/date
+* send new master bin file
+* send email about low power mode problems
+* support new flash chip in appload, possibly use low power mode
+* swbug! stuck busy tx occurred!
+  
+For app cleanup:
+
+* use structured logging to kep logs in ram.  Also send logs as packets to api clients
+* DONE writeup nice python options docs (common cases, link to protobuf docs)
+* have android app link to user manual
+* DONE only do wantReplies once per packet type, if we change network settings force it again
+* update positions and nodeinfos based on packets we just merely witness on the mesh.  via isPromsciousPort bool, remove sniffing
+* DONE make device build always have a valid version
+* DONE do fixed position bug https://github.com/meshtastic/Meshtastic-device/issues/536
+* DONE check build guide
+* DONE write devapi user guide
+* DONE update android code: https://developer.android.com/topic/libraries/view-binding/migration
+* DONE test GPIO watch
+* DONE set --set-chan-fast, --set-chan-default
+* writeup docs on gpio 
+* DONE make python ping command
+* DONE make hello world example service
+* DONE have python tool check max packet size before sending to device
+* DONE if request was sent reliably, send reply reliably
+* DONE require a recent python api to talk to these new device loads
+* DONE require a recent android app to talk to these new device loads
+* DONE fix handleIncomingPosition
+* DONE move want_replies handling into plugins
+* DONE on android for received positions handle either old or new positions / user messages
+* DONE on android side send old or new positions as needed / user messages
+* DONE test python side handle new position/user messages
+* DONE make a gpio example. --gpiowrb 4 1, --gpiord 0x444, --gpiowatch 0x3ff
+* DONE fix position sending to use new plugin
+* DONE Add SinglePortNumPlugin - as the new most useful baseclass
+* DONE move positions into regular data packets (use new app framework)
+* DONE move user info into regular data packets (use new app framework)
+* DONE test that positions, text messages and user info still work
+* DONE test that position, text messages and user info work properly with new android app and old device code
+* DONE do UDP tunnel
+* DONE fix the RTC drift bug
+* move python ping functionality into device, reply with rxsnr info
+* use channels for gpio security https://github.com/meshtastic/Meshtastic-device/issues/104
+* MeshPackets for sending should be reference counted so that API clients would have the option of checking sent status (would allow removing the nasty 30 sec timer in gpio watch sending)
+
+For high speed/lots of devices/short range tasks:
+
+- When guessing numhops for sending: if I've heard from many local (0 hop neighbors) decrease hopcount by 2 rather than 1. 
+This should nicely help 'router' nodes do the right thing when long range, or if there are many local nodes for short range.
+- fix timeouts/delays to be based on packet length at current radio settings
+
+* update faq with antennas https://meshtastic.discourse.group/t/range-test-ideas-requested/738/2
+* update faq on recommended android version and phones
+* add help link inside the app, reference a page on the wiki
+* turn on amazon reviews support
+* add a tablet layout (with map next to messages) in the android app
+
+# Old docs to merge
+
+MESH RADIO PROTOCOL
+
+Old TODO notes on the mesh radio protocol, merge into real docs someday...
+
+for each named group we have a pre-shared key known by all group members and
+wrapped around the device. you can only be in one group at a time (FIXME?!) To
+join the group we read a qr code with the preshared key and ParamsCodeEnum. that
+gets sent via bluetooth to the device.  ParamsCodeEnum maps to a set of various
+radio params (regulatory region, center freq, SF, bandwidth, bitrate, power
+etc...) so all members of the mesh can have their radios set the same way.
+
+once in that group, we can talk between 254 node numbers.
+to get our node number (and announce our presence in the channel) we pick a
+random node number and broadcast as that node with WANT-NODENUM(my globally
+unique name).  If anyone on the channel has seen someone _else_ using that name
+within the last 24 hrs(?) they reply with DENY-NODENUM. Note: we might receive
+multiple denies.  Note: this allows others to speak up for some other node that
+might be saving battery right now. Any time we hear from another node (for any
+message type), we add that node number to the unpickable list.  To dramatically
+decrease the odds a node number we request is already used by someone. If no one
+denies within TBD seconds, we assume that we have that node number.  As long as
+we keep talking to folks at least once every 24 hrs, others should remember we
+have it.
+
+Once we have a node number we can broadcast POSITION-UPDATE(my globally unique
+name, lat, lon, alt, amt battery remaining).  All receivers will use this to a)
+update the mapping of who is at what node nums, b) the time of last rx, c)
+position.  If we haven't heard from that node in a while we reply to that node
+(only) with our current POSITION_UPDATE state - so that node (presumably just
+rejoined the network) can build a map of all participants.
+
+We will periodically broadcast POSITION-UPDATE as needed based on distance moved
+or a periodic minimum heartbeat.
+
+If user wants to send a text they can SEND_TEXT(dest user, short text message).
+Dest user is a node number, or 0xff for broadcast.
 
 # Medium priority
 
-Items to complete before the first beta release.
+Items to complete before 1.0.
 
-- turn on watchdog timer (because lib code seems buggy)
-- show battery level as % full
-- rx signal measurements -3 marginal, -9 bad, 10 great, -10 means almost unusable. So scale this into % signal strength. preferably as a graph, with an X indicating loss of comms.
-
-# Pre-beta priority
-
-During the beta timeframe the following improvements 'would be nice'
+# Post 1.0 ideas
 
 - finish DSR for unicast
 - check fcc rules on duty cycle. we might not need to freq hop. https://www.sunfiretesting.com/LoRa-FCC-Certification-Guide/ . Might need to add enforcement for europe though.
-- pick channel center frequency based on channel name? "dolphin" would hash to 900Mhz, "cat" to 905MHz etc? allows us to hide the concept of channel # from hte user.
 - make a no bluetooth configured yet screen - include this screen in the loop if the user hasn't yet paired
 - if radio params change fundamentally, discard the nodedb
 - re-enable the bluetooth battery level service on the T-BEAM
-- implement first cut of router mode: preferentially handle flooding, and change sleep and GPS behaviors
 - provide generalized (but slow) internet message forwarding service if one of our nodes has internet connectivity (MQTT) [ Not a requirement but a personal interest ]
 
-# Low priority
+# Low priority ideas
 
 Items after the first final candidate release.
 
+- implement nimble battery level service
+- Nimble implement device info service remaining fields (hw version etc)
+- Turn on RPA addresses for the device side in Nimble
+- Try to teardown less of the Nimble protocol stack across sleep
 - dynamic frequency scaling could save a lot of power on ESP32, but it seems to corrupt uart (even with ref_tick set correctly)
 - Change back to using a fixed sized MemoryPool rather than MemoryDynamic (see bug #149)
 - scan to find channels with low background noise? (Use CAD mode of the RF95 to automatically find low noise channels)
@@ -62,11 +264,17 @@ Items after the first final candidate release.
 - split out the software update utility so other projects can use it. Have the appload specify the URL for downloads.
 - read the PMU battery fault indicators and blink/led/warn user on screen
 - discard very old nodedb records (> 1wk)
-- add a watchdog timer
 - handle millis() rollover in GPS.getTime - otherwise we will break after 50 days
 - report esp32 device code bugs back to the mothership via android
 - change BLE bonding to something more secure. see comment by pSecurity->setAuthenticationMode(ESP_LE_AUTH_BOND)
 
+Changes related to wifi support on ESP32:
+
+- iram space: https://esp32.com/viewtopic.php?t=8460
+- set https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-guides/external-ram.html spi ram bss
+- figure out if iram or bluetooth classic caused ble problems
+- post bug on esp32-arduino with BLE bug findings
+
 # Spinoff project ideas
 
 - an open source version of https://www.burnair.ch/skynet/

docs/software/android-too-old.md

@@ -0,0 +1,13 @@
+# Your android application needs updating
+
+Hi.
+
+If you've landed here that means your android application is too old for the running device firmware.  Usually our updates are backwards compatible, but in this special circumstance it is not.  Sorry.
+
+Probably, what this means is that you installed the **alpha test** version of the firmware from github.  We really love people helping with development by running the alpha test binaries.  But if you aren't ready to sign up for that right now, please go back to [github](https://github.com/meshtastic/Meshtastic-device/releases) and install the latest **not alpha** 1.1.x firmware.
+
+If you **do** intend to run the alpha test please [opt-in](https://play.google.com/apps/testing/com.geeksville.mesh) to receive the alpha test version of the android application.
+
+If you are willing to be an alpha tester, please keep an eye on our forum where we post frequent release notes.  We also will actively help you with any bugs you might encounter (along our shared journey of new feature goodness).
+
+If you have problems/questions please post in our [forum](https://meshtastic.discourse.group) and some nice person will probably help.

docs/software/ant.md

@@ -0,0 +1,14 @@
+# ANT protocol notes
+
+SD340 terms are reasonable for NRF52
+https://www.thisisant.com/developer/components/nrf52832#tab_protocol_stacks_tab
+
+Profiles to implement:
+
+tracker
+https://www.thisisant.com/developer/ant-plus/device-profiles/#4365_tab
+
+ebike
+https://www.thisisant.com/developer/ant-plus/device-profiles/#527_tab
+
+no profile for messaging?

docs/software/build-instructions.md

@@ -1,22 +1,58 @@
 # Build instructions
 
-This project uses the simple PlatformIO build system. You can use the IDE, but for brevity
-in these instructions I describe use of their command line tool.
+This project uses the simple PlatformIO build system. PlatformIO is an extension to Microsoft VSCode.  Workflows from building from the GUI or from the commandline are listed below.  
 
-1. Purchase a suitable radio (see above)
+If you encounter any problems, please post a question in [our forum](meshtastic.discourse.group).  And when you learn a fix, update these instructions for the next person (i.e. edit this file and send in a [pull-request](https://opensource.com/article/19/7/create-pull-request-github) which we will eagerly merge).
+
+## GUI
+
+1. Purchase a suitable [radio](https://github.com/meshtastic/Meshtastic-device/wiki/Hardware-Information).
+2. Install [Python](https://www.python.org/downloads/).
+3. Install [Git](https://git-scm.com/downloads).
+4. Reboot your computer.
+5. Install [PlatformIO](https://platformio.org/platformio-ide).
+6. Click the PlatformIO icon on the side bar. ![platformio icon](https://user-images.githubusercontent.com/47490997/89482668-77c7ea00-d7ee-11ea-8785-5faf8ff99800.png)
+7. Under `Quick Access, Miscellaneous, Clone Git Project` enter the URL of the Meshtastic repo found [here](https://github.com/meshtastic/Meshtastic-device). ![image](https://user-images.githubusercontent.com/47490997/89483047-4c91ca80-d7ef-11ea-91f4-1d53d4e8acd9.png) 
+8. Select a file location to save the repo.
+9. Once loaded, open the `platformio.ini` file. 
+10. At the line `default_envs` you can change it to the board type you are building for ie. `tlora-v2, tlora-v1, tlora-v2-1-1.6, tbeam, heltec, tbeam0.7` (boards are listed further down in the file).
+11. The hardware can be configured for different countries by adding a definition to the `configuration.h` file. `#define HW_VERSION_US` or `HW_VERSION_EU433, HW_VERSION_EU865, HW_VERSION_CN, HW_VERSION_JP`. Other country settings can be found in `MeshRadio.h`. The default is `HW_VERSION_US`.
+12. Click the PlatformIO icon on the side bar. Under `Project Tasks` you can now build or upload.
+
+Note - To get a clean build you may have to delete the auto-generated file `./.vscode/c_cpp_properties.json`, close and re-open Visual Studio and WAIT until the file is auto-generated before compiling again.
+
+## Command Line
+
+1. Purchase a suitable [radio](https://github.com/meshtastic/Meshtastic-device/wiki/Hardware-Information).
 2. Install [PlatformIO](https://platformio.org/platformio-ide)
-3. Download this git repo and cd into it
+3. Download this git repo and cd into it:
+
+```
+git clone https://github.com/meshtastic/Meshtastic-device.git
+cd Meshtastic-device
+```
 4. Run `git submodule update --init --recursive` to pull in dependencies this project needs.
-5. If you are outside the USA, edit [platformio.ini](/platformio.ini) to set the correct frequency range for your country. The line you need to change starts with `hw_version` and instructions are provided above that line. Options are provided for `EU433`, `EU835`, `CN`, `JP` and `US` (default). Pull-requests eagerly accepted for other countries.
+5. If you are outside the USA, run "export COUNTRY=EU865" (or whatever) to set the correct frequency range for your country. Options are provided for `EU433`, `EU865`, `CN`, `JP` and `US` (default). Pull-requests eagerly accepted for other countries.
 6. Plug the radio into your USB port
-7. Type `pio run --environment XXX -t upload` (This command will fetch dependencies, build the project and install it on the board via USB). For XXX, use the board type you have (either `tbeam`, `heltec`, `ttgo-lora32-v1`, `ttgo-lora32-v2`).
+7. Type `pio run --environment XXX -t upload` (This command will fetch dependencies, build the project and install it on the board via USB). For XXX, use the board type you have (either `tlora-v2, tlora-v1, tlora-v2-1-1.6, tbeam, heltec, tbeam0.7`).
 8. Platform IO also installs a very nice VisualStudio Code based IDE, see their [tutorial](https://docs.platformio.org/en/latest/tutorials/espressif32/arduino_debugging_unit_testing.html) if you'd like to use it.
 
 ## Decoding stack traces
 
+### Option 1
+
 If you get a crash, you can decode the addresses from the `Backtrace:` line:
 
 1. Save the `Backtrace: 0x....` line to a file, e.g., `backtrace.txt`.
 2. Run `bin/exception_decoder.py backtrace.txt` (this uses symbols from the
    last `firmware.elf`, so you must be running the same binary that's still in
    your `.pio/build` directory).
+
+### Option 2
+
+You can run the exception decoder to monitor the serial output and decode backtraces in real time.
+
+1. From within PlatformIO, open a new terminal.
+2. At the the terminal, enter:
+   `pio device monitor --port /dev/cu.SLAB_USBtoUART -f esp32_exception_decoder`
+   Replace the value of port with the location of your serial port.

docs/software/crypto.md

@@ -5,33 +5,47 @@ the project developers are not cryptography experts. Therefore we ask two things
 
 - If you are a cryptography expert, please review these notes and our questions below. Can you help us by reviewing our
   notes below and offering advice? We will happily give as much or as little credit as you wish ;-).
-- Consider our existing solution 'alpha' and probably fairly secure against a not particularly aggressive adversary. But until
-  it is reviewed by someone smarter than us, assume it might have flaws.
+- Consider our existing solution 'alpha' and probably fairly secure against a not particularly aggressive adversary
+(but we can't yet make a more confident statement).
 
-## Notes on implementation
+## Summary of strengths/weaknesses of our current implementation
+
+Based on comments from reviewers (see below), here's some tips for usage of these radios.  So you can know the level of protection offered:
+
+* It is pretty likely that the AES256 security is implemented 'correctly' and an observer will not be able to decode your messages.
+* Warning: If an attacker is able to get one of the radios in their posession, they could either a) extract the channel key from that device or b) use that radio to listen to new communications.
+* Warning: If an attacker is able to get the "Channel QR code/URL" that you share with others - that attacker could then be able to read any messages sent on the channel (either tomorrow or in the past - if they kept a raw copy of those broadcast packets)
+
+Possible future areas of work (if there is enough interest - post in our [forum](https://meshtastic.discourse.group) if you want this):
+
+1. Optionally requiring users to provide a PIN to regain access to the mesh.  This could be based on: intentionally locking the device, time since last use, or any member could force all members to reauthenticate,
+2. Until a device reauthenticates, any other access via BLE or USB would be blocked (this would protect against attackers who are not prepared to write custom software to extract and reverse engineer meshtastic flash memory)
+3. Turning on read-back protection in the device fuse-bits (this would extend protection in #2 to block all but **extremely** advanced attacks involving chip disassembly)
+4. Time limiting keys used for message transmission and automatically cycling them on a schedule.  This would protect past messages from being decoded even if an attacker learns the current key.
+
+### Notes for reviewers
+
+If you are reviewing our implementation, this is a brief statement of our method.
 
 - We do all crypto at the SubPacket (payload) level only, so that all meshtastic nodes will route for others - even those channels which are encrypted with a different key.
 - Mostly based on reading [Wikipedia](<https://en.wikipedia.org/wiki/Block_cipher_mode_of_operation#Counter_(CTR)>) and using the modes the ESP32 provides support for in hardware.
 - We use AES256-CTR as a stream cypher (with zero padding on the last BLOCK) because it is well supported with hardware acceleration.
-
-Parameters for our CTR implementation:
-
 - Our AES key is 128 or 256 bits, shared as part of the 'Channel' specification.
-- Each SubPacket will be sent as a series of 16 byte BLOCKS.
-- The node number concatenated with the packet number is used as the NONCE. This counter will be stored in flash in the device and should essentially never repeat. If the user makes a new 'Channel' (i.e. picking a new random 256 bit key), the packet number will start at zero. The packet number is sent
-  in cleartext with each packet. The node number can be derived from the "from" field of each packet.
-- Each BLOCK for a packet has an incrementing COUNTER. COUNTER starts at zero for the first block of each packet.
-- The IV for each block is constructed by concatenating the NONCE as the upper 96 bits of the IV and the COUNTER as the bottom 32 bits. Note: since our packets are small counter will really never be higher than 32 (five bits).
+- The node number concatenated with the packet number is used as the NONCE. This nonce will be stored in flash in the device and should essentially never repeat. If the user makes a new 'Channel' (i.e. picking a new random 256 bit key), the packet number will start at zero. 
+- The packet number is sent in cleartext with each packet. The node number can be derived from the "from" field of each packet. (Cleartext is acceptable because it merely provides IV for each encryption run)
+- Each 16 byte BLOCK for a packet has an incrementing COUNTER. COUNTER starts at zero for the first block of each packet.
+- The IV for each block is constructed by concatenating the NONCE as the upper 96 bits of the IV and the COUNTER as the bottom 32 bits. Since our packets are small counter portion will really never be higher than 32 (five bits).
 
-```
-You can encrypt separate messages by dividing the nonce_counter buffer in two areas: the first one used for a per-message nonce, handled by yourself, and the second one updated by this function internally.
-For example, you might reserve the first 12 bytes for the per-message nonce, and the last 4 bytes for internal use. In that case, before calling this function on a new message you need to set the first 12 bytes of nonce_counter to your chosen nonce value, the last 4 to 0, and nc_off to 0 (which will cause stream_block to be ignored). That way, you can encrypt at most 2**96 messages of up to 2**32 blocks each with the same key.
+## Comments from reviewer #1
 
-The per-message nonce (or information sufficient to reconstruct it) needs to be communicated with the ciphertext and must be unique. The recommended way to ensure uniqueness is to use a message counter. An alternative is to generate random nonces, but this limits the number of messages that can be securely encrypted: for example, with 96-bit random nonces, you should not encrypt more than 2**32 messages with the same key.
+This reviewer is a cryptography professional, but would like to remain anonymous. We thank them for their comments ;-):
 
-Note that for both stategies, sizes are measured in blocks and that an AES block is 16 bytes.
-```
+I'm assuming that meshtastic is being used to hike in places where someone capable is trying to break it - like you were going to walk around DefCon using these. I spent about an hour reviewing the encryption, and have the following notes:
 
-## Remaining todo
+* The write-up isn't quite as clear as the code.
+* The code is using AES-CTR mode correctly to ensure confidentiality.
+* The comment for initNonce really covers the necessary information.
+* I think the bigger encryption question is "what does the encryption need to do"? As it stands, an attacker who has yet to capture any of the devices cannot reasonably capture text or location data. An attacker who captures any device in the channel/mesh can read everything going to that device, everything stored on that device, and any other communication within the channel that they captured in encrypted form. If that capability basically matches your expectations, it is suitable for whatever adventures this was intended for, then, based on information publicly available or widely disclosed, the encryption is good. If those properties are distressing (like, device history is deliberately limited and you don't want a device captured today to endanger the information sent over the channel yesterday) we could talk about ways to achieve that (most likely synchronizing time and replacing the key with its own SHA256 every X hours, and ensuring the old key is not retained unnecessarily).
+* Two other things to keep in mind are that AES-CTR does not itself provide authenticity (e.g. an attacker can flip bits in replaying data and scramble the resulting plaintext), and that the current scheme gives some hints about transmission in the size. So, if you worry about an adversary deliberately messing-up messages or knowing the length of a text message, it looks like those might be possible.
 
-- Have the app change the crypto key when the user generates a new channel
+I'm guessing that the network behaves somewhat like a store-and-forward network - or, at least, that the goal is to avoid establishing a two-way connection to transmit data. I'm afraid I haven't worked with mesh networks much, but remember studying them briefly in school about ten years ago.

docs/software/device-api.md

@@ -1,12 +1,28 @@
-# Bluetooth API
+# Bluetooth/serial/TCP protocol API
 
-The Bluetooth API is design to have only a few characteristics and most polymorphism comes from the flexible set of Google Protocol Buffers which are sent over the wire. We use protocol buffers extensively both for the bluetooth API and for packets inside the mesh or when providing packets to other applications on the phone.
+(This document describes the protocol for external API clients using our devices.  If you are interested in running your own code on the device itself, see the [on-device](plugin-api.md) documentation instead)
 
-## A note on MTU sizes
+The Device API is design to have only a simple stream of ToRadio and FromRadio packets and all polymorphism comes from the flexible set of Google Protocol Buffers which are sent over the wire. We use protocol buffers extensively both for the bluetooth API and for packets inside the mesh or when providing packets to other applications on the phone.
 
-This device will work with any MTU size, but it is highly recommended that you call your phone's "setMTU function to increase MTU to 512 bytes" as soon as you connect to a service. This will dramatically improve performance when reading/writing packets.
+## Streaming version
 
-## MeshBluetoothService
+This protocol is **almost** identical when it is deployed over BLE, Serial/USB or TCP (our three currently supported transports for connecting to phone/PC). Most of this document is in terms of the original BLE version, but this section describes the small changes when this API is exposed over a Streaming (non datagram) transport. The streaming version has the following changes:
+
+- We assume the stream is reliable (though the protocol will resynchronize if bytes are lost or corrupted). i.e. we do not include CRCs or error correction codes.
+- Packets always have a four byte header (described below) prefixed before each packet. This header provides framing characters and length.
+- The stream going towards the radio is only a series of ToRadio packets (with the extra 4 byte headers)
+- The stream going towards the PC is a stream of FromRadio packets (with the 4 byte headers), or if the receiver state machine does not see valid header bytes it can (optionally) print those bytes as the debug console from the radio. This allows the device to emit regular serial debugging messages (which can be understood by a terminal program) but also switch to a more structured set of protobufs once it sees that the PC client has sent a protobuf towards it.
+
+The 4 byte header is constructed to both provide framing and to not look line 'normal' 7 bit ASCII.
+
+- Byte 0: START1 (0x94)
+- Byte 1: START2 (0xc3)
+- Byte 2: MSB of protobuf length
+- Byte 3: LSB of protobuf length
+
+The receiver will validate length and if >512 it will assume the packet is corrupted and return to looking for START1. While looking for START1 any other characters are printed as "debug output". For small example implementation of this reader see the meshtastic-python implementation.
+
+## MeshBluetoothService (the BLE API)
 
 This is the main bluetooth service for the device and provides the API your app should use to get information about the mesh, send packets or provision the radio.
 
@@ -28,7 +44,6 @@ Expected sequence for initial download:
 - Read a RadioConfig from "radio" - used to get the channel and radio settings
 - Read a User from "user" - to get the username for this node
 - Read a MyNodeInfo from "mynode" to get information about this local device
-- Write an empty record to "nodeinfo" to restart the nodeinfo reading state machine
 - Read a series of NodeInfo packets to build the phone's copy of the current NodeDB for the mesh
 - Read a endConfig packet that indicates that the entire state you need has been sent.
 - Read a series of MeshPackets until it returns empty to get any messages that arrived for this node while the phone was away
@@ -71,16 +86,20 @@ Not all messages are kept in the fromradio queue (filtered based on SubPacket):
 - No WantNodeNum / DenyNodeNum messages are kept
   A variable keepAllPackets, if set to true will suppress this behavior and instead keep everything for forwarding to the phone (for debugging)
 
-## Protobuf API
+### A note on MTU sizes
+
+This device will work with any MTU size, but it is highly recommended that you call your phone's "setMTU function to increase MTU to 512 bytes" as soon as you connect to a service. This will dramatically improve performance when reading/writing packets.
+
+### Protobuf API
 
 On connect, you should send a want_config_id protobuf to the device. This will cause the device to send its node DB and radio config via the fromradio endpoint. After sending the full DB, the radio will send a want_config_id to indicate it is done sending the configuration.
 
-## Other bluetooth services
+### Other bluetooth services
 
-This document focuses on the core mesh service, but it is worth noting that the following other Bluetooth services are also
+This document focuses on the core device protocol, but it is worth noting that the following other Bluetooth services are also
 provided by the device.
 
-### BluetoothSoftwareUpdate
+#### BluetoothSoftwareUpdate
 
 The software update service. For a sample function that performs a software update using this API see [startUpdate](https://github.com/meshtastic/Meshtastic-Android/blob/master/app/src/main/java/com/geeksville/mesh/service/SoftwareUpdateService.kt).
 
@@ -94,14 +113,15 @@ Characteristics
 | e272ebac-d463-4b98-bc84-5cc1a39ee517 | write       | data, variable sized, recommended 512 bytes, write one for each block of file                                     |
 | 4826129c-c22a-43a3-b066-ce8f0d5bacc6 | write       | crc32, write last - writing this will complete the OTA operation, now you can read result                         |
 | 5e134862-7411-4424-ac4a-210937432c77 | read,notify | result code, readable but will notify when the OTA operation completes                                            |
+| 5e134862-7411-4424-ac4a-210937432c67 | write | sets the region for programming, currently only 0 (app) or 100 (spiffs) are defined, if not set app is assumed |
 | GATT_UUID_SW_VERSION_STR/0x2a28      | read        | We also implement these standard GATT entries because SW update probably needs them:                              |
 | GATT_UUID_MANU_NAME/0x2a29           | read        |                                                                                                                   |
 | GATT_UUID_HW_VERSION_STR/0x2a27      | read        |                                                                                                                   |
 
-### DeviceInformationService
+#### DeviceInformationService
 
 Implements the standard BLE contract for this service (has software version, hardware model, serial number, etc...)
 
-### BatteryLevelService
+#### BatteryLevelService
 
 Implements the standard BLE contract service, provides battery level in a way that most client devices should automatically understand (i.e. it should show in the bluetooth devices screen automatically)

docs/software/esp32-arduino-build-notes.md

@@ -2,7 +2,7 @@
 
 We build our own custom version of esp32-arduino, in order to get some fixes we've made but haven't yet been merged in master.
 
-These are a set of currently unformatted notes on how to build and install them.  Most developers should not care about this, because
+These are a set of currently unformatted notes on how to build and install them. Most developers should not care about this, because
 you'll automatically get our fixed libraries.
 
 ```
@@ -10,7 +10,20 @@ you'll automatically get our fixed libraries.
   IDF release/v3.3 46b12a560
   IDF release/v3.3 367c3c09c
   https://docs.espressif.com/projects/esp-idf/en/release-v3.3/get-started/linux-setup.html
-  kevinh@kevin-server:~/development/meshtastic/esp32-arduino-lib-builder\$ python /home/kevinh/development/meshtastic/esp32-arduino-lib-builder/esp-idf/components/esptool*py/esptool/esptool.py --chip esp32 --port /dev/ttyUSB0 --baud 921600 --before default_reset --after hard_reset write_flash -z --flash_mode dout --flash_freq 40m --flash_size detect 0x1000 /home/kevinh/development/meshtastic/esp32-arduino-lib-builder/build/bootloader/bootloader.bin
+  kevinh@kevin-server:~/development/meshtastic/esp32-arduino-lib-builder\$ python /home/kevinh/development/meshtastic/
   cp -a out/tools/sdk/* components/arduino/tools/sdk
-  cp -ar components/arduino/ ~/.platformio/packages/framework-arduinoespressif32@src-fba9d33740f719f712e9f8b07da6ea13/
+  cp -ar components/arduino/* ~/.platformio/packages/framework-arduinoespressif32
+  
+  /// @src-fba9d33740f719f712e9f8b07da6ea13/
+
+  or
+
+  cp -ar out/tools/sdk/* ~/.platformio/packages/framework-arduinoespressif32/tools/sdk
+
+```
+
+How to flash new bootloader
+
+```
+esp32-arduino-lib-builder/esp-idf/components/esptool*py/esptool/esptool.py --chip esp32 --port /dev/ttyUSB0 --baud 921600 --before default_reset --after hard_reset write_flash -z --flash_mode dout --flash_freq 40m --flash_size detect 0x1000 /home/kevinh/development/meshtastic/esp32-arduino-lib-builder/build/bootloader/bootloader.bin
 ```

docs/software/gps-todo.txt

@@ -0,0 +1,17 @@
+You probably don't care about this ugly file of personal notes ;-)
+
+for taiwan region:
+bin/run.sh --set region 8
+
+time only mode
+./bin/run.sh --set gps_operation 3
+
+ublox parsing failure
+
+record power measurements and update spreadsheet
+
+have loop methods return allowable sleep time (from their perspective)
+increase main cpu sleep time
+
+warn people about crummy gps antennas - add to faq
+

docs/software/mesh-alg.md

@@ -1,5 +1,75 @@
 # Mesh broadcast algorithm
 
+## Current algorithm
+
+The routing protocol for Meshtastic is really quite simple (and suboptimal). It is heavily influenced by the mesh routing algorithm used in [Radiohead](https://www.airspayce.com/mikem/arduino/RadioHead/) (which was used in very early versions of this project). It has four conceptual layers.
+
+### A note about protocol buffers
+
+Because we want our devices to work across various vendors and implementations, we use [Protocol Buffers](https://github.com/meshtastic/Meshtastic-protobufs) pervasively. For information on how the protocol buffers are used wrt API clients see [sw-design](sw-design.md), for purposes of this document you mostly only
+need to consider the MeshPacket and Subpacket message types.
+
+### Layer 1: Non reliable zero hop messaging
+
+This layer is conventional non-reliable lora packet transmission. The transmitted packet has the following representation on the ether:
+
+- A 32 bit LORA preamble (to allow receiving radios to synchronize clocks and start framing). We use a longer than minimum (8 bit) preamble to maximize the amount of time the LORA receivers can stay asleep, which dramatically lowers power consumption.
+
+After the preamble the 16 byte packet header is transmitted. This header is described directly by the PacketHeader class in the C++ source code. But indirectly it matches the first portion of the "MeshPacket" protobuf definition. But notably: this portion of the packet is sent directly as the following 16 bytes (rather than using the protobuf encoding). We do this to both save airtime and to allow receiving radio hardware the option of filtering packets before even waking the main CPU.
+
+- to (4 bytes): the unique NodeId of the destination (or 0xffffffff for NodeNum_BROADCAST)
+- from (4 bytes): the unique NodeId of the sender)
+- id (4 bytes): the unique (wrt the sending node only) packet ID number for this packet. We use a large (32 bit) packet ID to ensure there is enough unique state to protect any encrypted payload from attack.
+- flags (4 bytes): Only a few bits are are currently used - 3 bits for for the "HopLimit" (see below) and 1 bit for "WantAck"
+
+After the packet header the actual packet is placed onto the the wire. These bytes are merely the encrypted packed protobuf encoding of the SubPacket protobuf. A full description of our encryption is available in [crypto](crypto.md). It is worth noting that only this SubPacket is encrypted, headers are not. Which leaves open the option of eventually allowing nodes to route packets without knowing the keys used to encrypt.
+
+NodeIds are constructed from the bottom four bytes of the macaddr of the bluetooth address. Because the OUI is assigned by the IEEE and we currently only support a few CPU manufacturers, the upper byte is defacto guaranteed unique for each vendor. The bottom 3 bytes are guaranteed unique by that vendor.
+
+To prevent collisions all transmitters will listen before attempting to send. If they hear some other node transmitting, they will reattempt transmission in x milliseconds. This retransmission delay is random between FIXME and FIXME (these two numbers are currently hardwired, but really should be scaled based on expected packet transmission time at current channel settings).
+
+### Layer 2: Reliable zero hop messaging
+
+This layer adds reliable messaging between the node and its immediate neighbors (only).
+
+The default messaging provided by layer-1 is extended by setting the "want-ack" flag in the MeshPacket protobuf. If want-ack is set the following documentation from mesh.proto applies:
+
+"""This packet is being sent as a reliable message, we would prefer it to arrive
+at the destination. We would like to receive a ack packet in response.
+
+Broadcasts messages treat this flag specially: Since acks for broadcasts would
+rapidly flood the channel, the normal ack behavior is suppressed. Instead,
+the original sender listens to see if at least one node is rebroadcasting this
+packet (because naive flooding algorithm). If it hears that the odds (given
+typical LoRa topologies) the odds are very high that every node should
+eventually receive the message. So FloodingRouter.cpp generates an implicit
+ack which is delivered to the original sender. If after some time we don't
+hear anyone rebroadcast our packet, we will timeout and retransmit, using the
+regular resend logic."""
+
+If a transmitting node does not receive an ACK (or a NAK) packet within FIXME milliseconds, it will use layer-1 to attempt a retransmission of the sent packet. A reliable packet (at this 'zero hop' level) will be resent a maximum of three times. If no ack or nak has been received by then the local node will internally generate a nak (either for local consumption or use by higher layers of the protocol).
+
+### Layer 3: (Naive) flooding for multi-hop messaging
+
+Given our use-case for the initial release, most of our protocol is built around [flooding](<https://en.wikipedia.org/wiki/Flooding_(computer_networking)>). The implementation is currently 'naive' - i.e. it doesn't try to optimize flooding other than abandoning retransmission once we've seen a nearby receiver has acked the packet. Therefore, for each source packet up to N retransmissions might occur (if there are N nodes in the mesh).
+
+Each node in the mesh, if it sees a packet on the ether with HopLimit set to a value other than zero, it will decrement that HopLimit and attempt retransmission on behalf of the original sending node.
+
+### Layer 4: DSR for multi-hop unicast messaging
+
+This layer is not yet fully implemented (and not yet used). But eventually (if we stay with our own transport rather than switching to QMesh or Reticulum)
+we will use conventional DSR for unicast messaging. Currently (even when not requiring 'broadcasts') we send any multi-hop unicasts as 'broadcasts' so that we can
+leverage our (functional) flooding implementation. This is suboptimal but it is a very rare use-case, because the odds are high that most nodes (given our small networks and 'hiking' use case) are within a very small number of hops. When any node witnesses an ack for a packet, it will realize that it can abandon its own
+broadcast attempt for that packet.
+
+## Misc notes on remaining tasks
+
+This section is currently poorly formatted, it is mostly a mere set of todo lists and notes for @geeksville during his initial development. After release 1.0 ideas for future optimization include:
+
+- Make flood-routing less naive (because we have GPS and radio signal strength as heuristics to avoid redundant retransmissions)
+- If nodes have been user marked as 'routers', preferentially do flooding via those nodes
+- Fully implement DSR to improve unicast efficiency (or switch to QMesh/Reticulum as these projects mature)
+
 great source of papers and class notes: http://www.cs.jhu.edu/~cs647/
 
 flood routing improvements
@@ -19,20 +89,21 @@ reliable messaging tasks (stage one for DSR):
 - DONE once an ack comes in, remove the packet from the retry list and deliver the ack to the original sender
 - DONE after three retries, deliver a no-ack packet to the original sender (i.e. the phone app or mesh router service)
 - DONE test one hop ack/nak with the python framework
-- Do stress test with acks
+- DONE Do stress test with acks
 
 dsr tasks
 
-- oops I might have broken message reception
+- DONE oops I might have broken message reception
 - DONE Don't use broadcasts for the network pings (close open github issue)
 - DONE add ignoreSenders to radioconfig to allow testing different mesh topologies by refusing to see certain senders
-- test multihop delivery with the python framework
+- DONE test multihop delivery with the python framework
 
 optimizations / low priority:
 
+- read this [this](http://pages.cs.wisc.edu/~suman/pubs/nadv-mobihoc05.pdf) paper and others and make our naive flood routing less naive
 - read @cyclomies long email with good ideas on optimizations and reply
-- Remove NodeNum assignment algorithm (now that we use 4 byte node nums)
-- make android app warn if firmware is too old or too new to talk to
+- DONE Remove NodeNum assignment algorithm (now that we use 4 byte node nums)
+- DONE make android app warn if firmware is too old or too new to talk to
 - change nodenums and packetids in protobuf to be fixed32
 - low priority: think more careful about reliable retransmit intervals
 - make ReliableRouter.pending threadsafe
@@ -145,23 +216,3 @@ look into the literature for this idea specifically.
   build the most recent version of reality, and if some nodes are too far, then nodes closer in will eventually forward their changes to the distributed db.
 - construct non ambigious rules for who broadcasts to request db updates. ideally the algorithm should nicely realize node X can see most other nodes, so they should just listen to all those nodes and minimize the # of broadcasts. the distributed picture of nodes rssi could be useful here?
 - possibly view the BLE protocol to the radio the same way - just a process of reconverging the node/msgdb database.
-
-# Old notes
-
-FIXME, merge into the above:
-
-good description of batman protocol: https://www.open-mesh.org/projects/open-mesh/wiki/BATMANConcept
-
-interesting paper on lora mesh: https://portal.research.lu.se/portal/files/45735775/paper.pdf
-It seems like DSR might be the algorithm used by RadioheadMesh. DSR is described in https://tools.ietf.org/html/rfc4728
-https://en.wikipedia.org/wiki/Dynamic_Source_Routing
-
-broadcast solution:
-Use naive flooding at first (FIXME - do some math for a 20 node, 3 hop mesh. A single flood will require a max of 20 messages sent)
-Then move to MPR later (http://www.olsr.org/docs/report_html/node28.html). Use altitude and location as heursitics in selecting the MPR set
-
-compare to db sync algorithm?
-
-what about never flooding gps broadcasts. instead only have them go one hop in the common case, but if any node X is looking at the position of Y on their gui, then send a unicast to Y asking for position update. Y replies.
-
-If Y were to die, at least the neighbor nodes of Y would have their last known position of Y.

docs/software/mqtt.md

@@ -0,0 +1,192 @@
+
+# Table of Contents
+- [Table of Contents](#table-of-contents)
+  - [Abstract](#abstract)
+  - [Short term goals](#short-term-goals)
+  - [Long term goals](#long-term-goals)
+  - [Multiple Channel support / Security](#multiple-channel-support--security)
+  - [On device API](#on-device-api)
+  - [MQTT transport](#mqtt-transport)
+    - [Topics](#topics)
+      - [Service Envelope](#service-envelope)
+      - [NODEID](#nodeid)
+      - [USERID](#userid)
+      - [CHANNELID](#channelid)
+    - [Gateway nodes](#gateway-nodes)
+    - [Optional web services](#optional-web-services)
+      - [Public MQTT broker service](#public-mqtt-broker-service)
+      - [Riot.im messaging bridge](#riotim-messaging-bridge)
+    - [Deprecated concepts](#deprecated-concepts)
+      - [MESHID (deprecated)](#meshid-deprecated)
+      - [DESTCLASS (deprecated)](#destclass-deprecated)
+      - [DESTID (deprecated)](#destid-deprecated)
+  - [Rejected idea: RAW UDP](#rejected-idea-raw-udp)
+  - [Development plan](#development-plan)
+    - [Work items](#work-items)
+    - [Enhancements in following releases](#enhancements-in-following-releases)
+
+## Abstract
+
+This is a mini-doc/RFC sketching out a development plan to satisfy a number of 1.1 goals.
+
+- [MQTT](https://opensource.com/article/18/6/mqtt) internet accessible API.  Issue #[369](https://github.com/meshtastic/Meshtastic-device/issues/169)
+- An open API to easily run custom mini-apps on the devices
+- A text messaging bridge when a node in the mesh can gateway to the internet. Issue #[353](https://github.com/meshtastic/Meshtastic-device/issues/353) and this nicely documented [android issue](https://github.com/meshtastic/Meshtastic-Android/issues/2).
+- An easy way to let desktop app developers remotely control GPIOs. Issue #[182](https://github.com/meshtastic/Meshtastic-device/issues/182)
+- Remote attribute access (to change settings of distant nodes). Issue #182
+
+## Short term goals
+
+- We want a clean API for novice developers to write mini "apps" that run **on the device** with the existing messaging/location "apps".
+- We want the ability to have a gateway web service, so that if any node in the mesh can connect to the internet (via its connected phone app or directly) then that node will provide bidirectional messaging between nodes and the internet.
+- We want an easy way for novice developers to remotely read and control GPIOs (because this is an often requested use case), without those developers having to write any device code.
+- We want a way to gateway text messaging between our current private meshes and the broader internet (when that mesh is able to connect to the internet)
+- We want a way to remotely set any device/channel parameter on a node. This is particularly important for administering physically inaccessible router nodes. Ideally this mechanism would also be used for administering the local node (so one common mechanism for both cases).
+- This work should be independent of our current (semi-custom) LoRa transport, so that in the future we can swap out that transport if we wish (to QMesh or Reticulum?)
+- Our networks are (usually) very slow and low bandwidth, so the messaging must be very airtime efficient.
+
+## Long term goals
+
+- Store and forward messaging should be supported, so apps can send messages that might be delivered to their destination in **hours** or **days** if a node/mesh was partitioned.
+
+## Multiple Channel support / Security
+
+Mini-apps API can bind to particular channels. They will only see messages sent on that channel.
+
+During the 1.0 timeframe only one channel was supported per node.  Starting in the 1.1 tree we will do things like "remote admin operations / channel settings etc..." are on the "Control" channel and only especially trusted users should have the keys to access that channel.
+
+FIXME - explain this more, talk about how useful for users and security domains.
+- add channels as security
+
+## On device API
+
+For information on the related on-device API see [here](device-api.md).
+
+## MQTT transport
+
+Any gateway-device will contact the MQTT broker. 
+
+### Topics
+
+The "mesh/crypt/CHANNELID/NODEID/PORTID" [topic](https://www.hivemq.com/blog/mqtt-essentials-part-5-mqtt-topics-best-practices/) will be used for messages sent from/to a mesh.
+
+Gateway nodes will foward any MeshPacket from a local mesh channel with uplink_enabled.  The packet (encapsulated in a ServiceEnvelope) will remain encrypted with the key for the specified channel.  
+
+For any channels in the local node with downlink_enabled, the gateway node will forward packets from MQTT to the local mesh.  It will do this by subscribing to mesh/crypt/CHANNELID/# and forwarding relevant packets.
+
+If the channelid 'well known'/public it could be decrypted by a web service (if the web service was provided with the associated channel key), in which case it will be decrypted by a web service and appear at "mesh/clear/CHANNELID/NODEID/PORTID". Note: This is not in the initial deliverable. 
+
+FIXME, discuss how text message global mirroring could scale (or not)
+FIXME, possibly don't global mirror text messages - instead rely on matrix/riot? 
+FIXME, discuss possible attacks by griefers and how they can be prvented
+
+#### Service Envelope
+
+The payload published on mesh/... will always be wrapped in a [ServiceEnvelope protobuf](https://github.com/meshtastic/Meshtastic-protobufs/blob/master/docs/docs.md#.ServiceEnvelope).
+
+ServiceEnvelope will include the message, and full information about arrival time, who forwarded it, source channel, source mesh id, etc... 
+
+#### NODEID
+
+The unique ID for a node. A hex string that starts with a ! symbol.
+
+#### USERID
+
+A user ID string. This string is either a user ID if known or a nodeid to simply deliver the message to whoever the local user is of a particular device (i.e. person who might see the screen). FIXME, see what riot.im uses and perhaps use that convention? Or use the signal +phone number convention? Or the email addr?
+
+#### CHANNELID
+
+FIXME, figure out how channelids work
+
+### Gateway nodes
+
+Any meshtastic node that has a direct connection to the internet (either via a helper app or installed wifi/4G/satellite hardware) can function as a "Gateway node". 
+
+Gateway nodes (via code running in the phone) will contain two tables to whitelist particular traffic to either be delivered toward the internet, or down toward the mesh. Users that are developing custom apps will be able to customize these filters/subscriptions.
+
+Since multiple gateway nodes might be connected to a single mesh, it is possible that duplicate messages will be published on any particular topic.  Therefore subscribers to these topics should
+deduplicate if needed by using the packet ID of each message.
+
+### Optional web services
+
+#### Public MQTT broker service
+
+An existing public [MQTT broker](https://mosquitto.org/) will be the default for this service, but clients can use any MQTT broker they choose. 
+
+FIXME - figure out how to avoid impersonation (because we are initially using a public mqtt server with no special security options).  FIXME, include some ideas on this in the ServiceEnvelope documentation.
+
+#### Riot.im messaging bridge
+
+@Geeksville will run a riot.im bridge that talks to the public MQTT broker and sends/receives into the riot.im network.
+
+There is apparently [already](https://github.com/derEisele/tuple) a riot.im [bridge](https://matrix.org/bridges/) for MQTT. That will possibly need to be customized a bit. But by doing this, we should be able to let random riot.im users send/receive messages to/from any meshtastic device. (FIXME ponder security).  See this [issue](https://github.com/meshtastic/Meshtastic-Android/issues/2#issuecomment-645660990) with discussion with the dev.
+
+### Deprecated concepts
+
+You can ignore these for now...
+
+#### MESHID (deprecated)
+
+Earlier drafts of this document included the concept of a MESHID.  That concept has been removed for now, but might be useful in the future.  The old idea is listed below:
+
+A unique ID for this mesh. There will be some sort of key exchange process so that the mesh ID can not be impersonated by other meshes. 
+
+#### DESTCLASS (deprecated)
+
+Earlier drafts of this document included the concept of a DESTCLASS.  That concept has been removed for now, but might be useful in the future.  The old idea is listed below:
+
+The type of DESTID this message should be delivered to. A short one letter sequence:
+
+| Symbol | Description                                                   |
+| ------ | ------------------------------------------------------------- |
+| R      | riot.im                                                       |
+| L      | local mesh node ID or ^all                                    |
+| A      | an application specific message, ID will be an APP ID         |
+| S      | SMS gateway, DESTID is a phone number to reach via Twilio.com |
+| E      | Emergency message, see bug #fixme for more context            |
+
+#### DESTID (deprecated)
+
+Earlier drafts of this document included the concept of a DESTCLASS.  That concept has been removed for now, but might be useful in the future.  The old idea is listed below:
+
+Can be...
+
+- an internet username: kevinh@geeksville.com
+- ^ALL for anyone
+- An app ID (to allow apps out in the web to receive arbitrary binary data from nodes or simply other apps using meshtastic as a transport). They would connect to the MQTT broker and subscribe to their topic
+
+## Rejected idea: RAW UDP
+
+A number of commenters have requested/proposed using UDP for the transport.  We've considered this option and decided to use MQTT instead for the following reasons:
+
+  - Most UDP uses cases would need to have a server anyways so that nodes can reach each other from anywhere (i.e. if most gateways will be behind some form of NAT which would need to be tunnelled)
+  - Raw UDP is dropped **very** agressively by many cellular providers.  MQTT from the gateway to a broker can be done over a TCP connection for this reason.
+  - MQTT provides a nice/documented/standard security model to build upon
+  - MQTT is fairly wire efficient with multiple broker implementations/providers and numerous client libraries for any language.  The actual implementation of MQTT is quite simple.
+  
+## Development plan
+
+Given the previous problem/goals statement, here's the initial thoughts on the work items required.  As this idea becomes a bit more fully baked we should add details
+on how this will be implemented and guesses at approximate work items.
+
+### Work items
+
+- Change nodeIDs to be base64 instead of eight hex digits.
+- DONE Refactor the position features into a position "mini-app". Use only the new public on-device API to implement this app.
+- DONE Refactor the on device texting features into a messaging "mini-app". (Similar to the position mini-app)
+- Add new multi channel concept
+- Send new channels to python client
+- Let python client add channels
+- Add portion of channelid to the raw lora packet header
+- Confirm that we can now forward encrypted packets without decrypting at each node
+- Use a channel named "remotehw" to secure the GPIO service.  If that channel is not found, don't even start the service.  Document this as the standard method for securing services.
+- Add first cut of the "gateway node" code (i.e. MQTT broker client) to the python API (very little code needed for this component)
+- Confirm that texting works to/from the internet
+- Confirm that positions are optionally sent to the internet
+- Add the first cut of the "gateway node" code to the android app (very little code needed for this component)
+
+### Enhancements in following releases
+
+The initial gateway will be added to the python tool.  But the gateway implementation is designed to be fairly trivial/dumb.  After the initial release the actual gateway code can be ported to also run inside of the android app.  In fact, we could have ESP32 based nodes include a built-in "gateway node" implementation.
+
+Store and forward could be added so that nodes on the mesh could deliver messages (i.e. text messages) on an "as possible" basis.  This would allow things like "hiker sends a message to friend - mesh can not currently reach friend - eventually (days later) mesh can somehow reach friend, message gets delivered"

docs/software/nrf52-TODO.md

@@ -1,19 +1,113 @@
 # NRF52 TODO
 
+- Possibly switch from softdevice to Apachy Newt: https://github.com/espressif/esp-nimble
+  https://github.com/apache/mynewt-core - use nimble BLE on both ESP32 and NRF52
+
+## RAK815
+
+### PPR1 TODO
+
+* V_BK for the GPS should probably be supplied from something always on
+
+* use S113 soft device 7.2.0
+* properly test charge controller config and read battery/charge status
+* fix bluetooth
+* fix LCD max contrast (currently too high, needs to be about 40?)
+* save brightness settings in flash
+* make ST7567Wire driver less ugly, move OLED stuff into a common class treee
+* add LCD power save mode for lcd per page 31 of datasheet
+* add LCD power off sequence per datasheet to lcd driver
+* leave LCD screen on most of the time (because it needs little power)
+
+### general nrf52 TODO:
+
+- turn off transitions on eink screens
+- change update interval on eink from 1/sec frames to one frame every 5 mins
+- enter SDS state at correct time (to protect battery or loss of phone contact)
+- show screen on eink when we enter SDS state (with app info and say sleeping)
+- require button press to pair
+
+- shrink soft device RAM usage
+- get nrf52832 working again (currently OOM)
+- i2c gps comms not quite right
+- ble: AdafruitBluefruit::begin - adafruit_ble_task was assigned an invalid stack pointer. out of memory?
+- measure power draw
+
+### Bootloader
+
+Install our (temporarily hacked up) adafruit bootloader
+
+```
+kevinh@kevin-server:~/development/meshtastic/Adafruit_nRF52_Bootloader$ make BOARD=rak815 sd flash
+LD rak815_bootloader-0.3.2-111-g9478eb7-dirty.out
+   text	   data	    bss	    dec	    hex	filename
+  20888	   1124	  15006	  37018	   909a	_build/build-rak815/rak815_bootloader-0.3.2-111-g9478eb7-dirty.out
+Create rak815_bootloader-0.3.2-111-g9478eb7-dirty.hex
+Create rak815_bootloader-0.3.2-111-g9478eb7-dirty-nosd.hex
+Flashing: rak815_bootloader-0.3.2-111-g9478eb7-dirty-nosd.hex
+nrfjprog --program _build/build-rak815/rak815_bootloader-0.3.2-111-g9478eb7-dirty-nosd.hex --sectoranduicrerase -f nrf52 --reset
+Parsing hex file.
+Erasing page at address 0x0.
+Erasing page at address 0x74000.
+Erasing page at address 0x75000.
+Erasing page at address 0x76000.
+Erasing page at address 0x77000.
+Erasing page at address 0x78000.
+Erasing page at address 0x79000.
+Erasing UICR flash area.
+Applying system reset.
+Checking that the area to write is not protected.
+Programming device.
+Applying system reset.
+Run.
+```
+
+### Appload
+
+tips on installing https://github.com/platformio/platform-nordicnrf52/issues/8#issuecomment-374017768
+
+to see console output over jlink:
+
+```
+12:17
+in one tab run "bin/nrf52832-gdbserver.sh" - leave this running the whole time while developing/debugging
+12:17
+~/development/meshtastic/meshtastic-esp32$ bin/nrf52-console.sh
+###RTT Client: ************************************************************
+###RTT Client: *               SEGGER Microcontroller GmbH                *
+###RTT Client: *   Solutions for real time microcontroller applications   *
+###RTT Client: ************************************************************
+###RTT Client: *                                                          *
+###RTT Client: *       (c) 2012 - 2016  SEGGER Microcontroller GmbH       *
+###RTT Client: *                                                          *
+###RTT Client: *     www.segger.com     Support: support@segger.com       *
+###RTT Client: *                                                          *
+###RTT Client: ************************************************************
+###RTT Client: *                                                          *
+###RTT Client: * SEGGER J-Link RTT Client   Compiled Apr  7 2020 15:01:22 *
+###RTT Client: *                                                          *
+###RTT Client: ************************************************************
+###RTT Client: -----------------------------------------------
+###RTT Client: Connecting to J-Link RTT Server via localhost:19021 ..............
+###RTT Client: Connected.
+SEGGER J-Link V6.70c - Real time terminal output
+SEGGER J-Link ARM V9.6, SN=69663845
+Process: JLinkGDBServerCLExein another tab run:
+12:18
+On NRF52 I've been using the jlink fake serial console.  But since the rak815 has the serial port hooked up we can switch back to that once the basics are working.
+```
+
 ## Misc work items
 
 RAM investigation.
 nRF52832-QFAA 64KB ram, 512KB flash vs
 nrf52832-QFAB 32KB ram, 512kb flash
+nrf52833 128KB RAM
 nrf52840 256KB RAM, 1MB flash
 
-platform.json
+Manual hacks needed to build (for now):
 
-    "framework-arduinoadafruitnrf52": {
-      "type": "framework",
-      "optional": true,
-      "version": "https://github.com/meshtastic/Adafruit_nRF52_Arduino.git"
-    },
+kevinh@kevin-server:~/.platformio/packages/framework-arduinoadafruitnrf52/variants\$ ln -s ~/development/meshtastic/meshtastic-esp32/variants/\* .
 
 ## Initial work items
 
@@ -122,7 +216,7 @@ Nice ideas worth considering someday...
 - DONE neg 7 error code from receive
 - DONE remove unused sx1262 lib from github
 - at boot we are starting our message IDs at 1, rather we should start them at a random number. also, seed random based on timer. this could be the cause of our first message not seen bug.
-- add a NEMA based GPS driver to test GPS
+- add a NMEA based GPS driver to test GPS
 - DONE use "variants" to get all gpio bindings
 - DONE plug in correct variants for the real board
 - turn on DFU assistance in the appload using the nordic DFU helper lib call

docs/software/pinetab.md

@@ -0,0 +1,34 @@
+# Pinetab
+
+These are **preliminary** notes on support for Meshtastic in the Pinetab.
+
+A RF95 is connected via a CH341 USB-SPI chip.
+
+Pin assignments:
+CS0 from RF95 goes to CS0 on CH341
+DIO0 from RF95 goes to INT on CH341
+RST from RF95 goes to RST on CH341
+
+This linux driver claims to provide USB-SPI support: https://github.com/gschorcht/spi-ch341-usb
+Notes here on using that driver: https://www.linuxquestions.org/questions/linux-hardware-18/ch341-usb-to-spi-adaptor-driver-doesn%27t-work-4175622736/
+
+Or if **absolutely** necessary could bitbang: https://www.cnx-software.com/2018/02/16/wch-ch341-usb-to-serial-chip-gets-linux-driver-to-control-gpios-over-usb/
+
+## Portduino tasks
+
+* How to access spi devices via ioctl (spidev): https://www.raspberrypi.org/documentation/hardware/raspberrypi/spi/README.md#:~:text=Troubleshooting-,Overview,bus)%2C%20UARTs%2C%20etc.
+* access gpio via libgpiod?
+* Use dkms to distribute driver? 
+* echo 100 > /sys/module/spi_ch341_usb/parameters/poll_period
+
+## Task list
+
+* Port meshtastic to build (under platformio) for a poxix target.  spec: no screen, no gpios, sim network interface, posix threads, posix semaphores & queues, IO to the console only
+Use ARM linux: https://platformio.org/platforms/linux_arm
+And  linux native: https://platformio.org/platforms/native
+
+* Test cs341 driver - just test reading/writing a register and detecting interrupts, confirm can see rf95
+* Make a radiolib spi module that targets the cs341 (and builds on linux)
+* use new radiolib module to hook pinebook lora to meshtastic, confirm mesh discovery works
+* Make a subclass of StreamAPI that works as a posix TCP server
+* Use new TCP endpoint from meshtastic-python

docs/software/plugin-api.md

@@ -0,0 +1,81 @@
+# Plugin-API
+
+This is a tutorial on how to write small plugins which run on the device.  Plugins are bits regular 'arduino' code that can send and receive packets to other nodes/apps/PCs using our mesh.
+
+## Key concepts
+
+All plugins should be subclasses of MeshPlugin.  By inheriting from this class and creating an instance of your new plugin your plugin will be automatically registered to receive packets.
+
+Messages are sent to particular port numbers (similar to UDP networking).  Your new plugin should eventually pick its own port number (see below), but at first you can simply use PRIVATE_APP (which is the default).
+
+Packets can be sent/received either as raw binary structures or as [Protobufs](https://developers.google.com/protocol-buffers).
+
+### Class heirarchy
+
+The relevant bits of the class heirarchy are as follows
+
+* [MeshPlugin](/src/mesh/MeshPlugin.h) (in src/mesh/MeshPlugin.h) - you probably don't want to use this baseclass directly
+  * [SinglePortPlugin](/src/mesh/SinglePortPlugin.h) (in src/mesh/SinglePortPlugin.h) - for plugins that send/receive from a single port number (the normal case)
+    * [ProtobufPlugin](/src/mesh/ProtobufPlugin.h) (in src/mesh/ProtobufPlugin.h) - for plugins that send/receive a single particular Protobuf type.  Inherit from this if you are using protocol buffers in your plugin.
+
+You will typically want to inherit from either SinglePortPlugin (if you are just sending/receiving raw bytes) or ProtobufPlugin (if you are sending/receiving protobufs).  You'll implement your own handleReceived/handleReceivedProtobuf - probably based on the example code.
+
+If your plugin needs to perform any operations at startup you can override and implement the setup() method to run your code.
+
+If you need to send a packet you can call service.sendToMesh with code like this (from the examples):
+
+```
+    MeshPacket *p = allocReply();
+    p->to = dest;
+
+    service.sendToMesh(p);
+```
+
+## Example plugins
+
+A number of [key services](/src/plugins) are implemented using the plugin API, these plugins are as follows:
+
+* [TextMessagePlugin](/src/plugins/TextMessagePlugin.h) - receives text messages and displays them on the LCD screen/stores them in the local DB
+* [NodeInfoPlugin](/src/plugins/NodeInfoPlugin.h) - receives/sends User information to other nodes so that usernames are available in the databases
+* [RemoteHardwarePlugin](/src/plugins/RemoteHardwarePlugin.h) - a plugin that provides easy remote access to device hardware (for things like turning GPIOs on or off).  Intended to be a more extensive example and provide a useful feature of its own.  See [remote-hardware](remote-hardware.md) for details.
+* [ReplyPlugin](/src/plugins/ReplyPlugin.h) - a simple plugin that just replies to any packet it receives (provides a 'ping' service).
+
+## Getting started
+
+The easiest way to get started is:
+
+* [Build and install](build-instructions.md) the standard codebase from github.
+* Copy [src/plugins/ReplyPlugin.*](/src/plugins/ReplyPlugin.cpp) into src/plugins/YourPlugin.*.  Then change the port number from *PortNum_REPLY_APP* to *PortNum_PRIVATE_APP*.
+* Edit plugins/Plugins.cpp:setupPlugins() to add a call to create an instance of your plugin (see comment at head of that function)
+* Rebuild with your new messaging goodness and install on the device
+* Use the [meshtastic commandline tool](https://github.com/meshtastic/Meshtastic-python) to send a packet to your board, for example "*meshtastic --dest 1234 --sendping*", where *1234* is another mesh node to send the ping to.
+
+## Threading
+
+It is very common that you would like your plugin to be invoked periodically.
+We use a crude/basic cooperative threading system to allow this on any of our supported platforms.  Simply inherit from OSThread and implement runOnce().  See the OSThread [documentation](/src/concurrency/OSThread.h) for more details.  For an example consumer of this API see RemoteHardwarePlugin::runOnce.
+
+## Sending messages
+
+If you would like to proactively send messages (rather than just responding to them), just call service.sendToMesh().  For an example of this see [NodeInfoPlugin::sendOurNodeInfo(...)](/src/plugins/NodeInfoPlugin.cpp).
+
+## Picking a port number
+
+For any new 'apps' that run on the device or via sister apps on phones/PCs they should pick and use a unique 'portnum' for their application.
+
+If you are making a new app using meshtastic, please send in a pull request to add your 'portnum' to [the master list](https://github.com/meshtastic/Meshtastic-protobufs/blob/master/portnums.proto).  PortNums should be assigned in the following range:
+
+* **0-63** Core Meshtastic use; do not use for third party apps
+* **64-127** Registered 3rd party apps.  Send in a pull request that adds a new entry to portnums.proto to register your application
+* **256-511** Use one of these portnums for your private applications that you don't want to register publically
+* **1024-66559** Are reserved for use by IP tunneling (see *FIXME* for more information)
+
+All other values are reserved.
+
+## How to add custom protocol buffers
+
+If you would like to use protocol buffers to define the structures you send over the mesh (recommended), here's how to do that.
+
+* Create a new .proto file in the protos directory.  You can use the existing [remote_hardware.proto](https://github.com/meshtastic/Meshtastic-protobufs/blob/master/remote_hardware.proto) file as an example.
+* Run "bin/regen-protos.sh" to regenerate the C code for accessing the protocol buffers.  If you don't have the required nanopb tool, follow the instructions printed by the script to get it.
+* Done!  You can now use your new protobuf just like any of the existing protobufs in meshtastic.

docs/software/plugins/ExternalNotificationPlugin.md

@@ -0,0 +1,89 @@
+# About
+
+The ExternalNotification Plugin will allow you to connect a speaker, LED or other device to notify you when a message has been received from the mesh network.
+
+# Configuration
+
+These are the settings that can be configured.
+
+    ext_notification_plugin_enabled
+        Is the plugin enabled?
+        
+        0 = Disabled (Default)
+        1 = Enabled
+
+    ext_notification_plugin_active
+        Is your external circuit triggered when our GPIO is low or high?
+
+        0 = Active Low (Default)
+        1 = Active High
+
+    ext_notification_plugin_alert_message
+        Do you want to be notified on an incoming message?
+
+        0 = Disabled (Default)
+        1 = Alert when a text message comes
+
+    ext_notification_plugin_alert_bell
+        Do you want to be notified on an incoming bell?
+
+        0 = Disabled (Default)
+        1 = Alert when the bell character is received
+
+    ext_notification_plugin_output
+        What GPIO is your external circuit attached?
+
+        GPIO of the output. (Default = 13)
+
+    ext_notification_plugin_output_ms
+        How long do you want us to trigger your external circuit?
+    
+        Amount of time in ms for the alert. Default is 1000.
+
+
+# Usage Notes
+
+For basic usage, start with:
+
+	ext_notification_plugin_enabled = 1
+	ext_notification_plugin_alert_message = 1
+    
+Depending on how your external cirtcuit configured is configured, you may need to set the active state to true.
+
+	ext_notification_plugin_active = 1
+	
+## Alert Types
+
+We support being alerted on two events:
+
+1) Incoming Text Message
+
+2) Incoming Text Message that contains the ascii bell character. At present, only the Python API can send an ascii bell character, but more support may be added in the future.
+
+### Bell Character
+
+The bell character is ASCII 0x07. Include 0x07 anywhere in the text message and with ext_notification_plugin_alert_bell enabled, we will issue an external notification.
+    
+# External Hardware
+
+Be mindful of the max current sink and source of the esp32 GPIO. The easiest devices to interface with would be either an LED or Active Buzzer.
+
+Ideas for external hardware:
+
+* LED
+* Active Buzzer
+* Flame thrower
+* Strobe Light
+* Siren
+    
+# Known Problems
+
+* This won't directly support an passive (normal) speaker as it does not generate any audio wave forms.
+* This currently only supports the esp32. Other targets may be possible, I just don't have to test with.
+* This plugin only monitors text messages. We won't trigger on any other packet types.
+
+# Need more help?
+
+Go to the Meshtastic Discourse Group if you have any questions or to share how you have used this.
+
+https://meshtastic.discourse.group

docs/software/plugins/RangeTestPlugin.md

@@ -0,0 +1,105 @@
+# About
+
+The RangeTest Plugin will help you perform range and coverage tests.
+
+# Configuration
+
+These are the settings that can be configured.
+
+    range_test_plugin_enabled
+        Is the plugin enabled?
+        
+        0 = Disabled (Default)
+        1 = Enabled
+
+    range_test_plugin_save
+        If enabled, we will save a log of all received messages to /static/rangetest.csv which you can access from the webserver. We will abort
+        writing if there is less than 50k of space on the filesystem to prevent filling up the storage.
+
+        0 = Disabled (Default)
+        1 = Enabled
+
+    range_test_plugin_sender
+        Number of seconds to wait between sending packets. Using the long_slow channel configuration, it's best not to go more frequent than once every 60 seconds. You can be more agressive with faster settings. 0 is default which disables sending messages.
+
+# Usage Notes
+
+For basic usage, you will need two devices both with a GPS. A device with a paired phone with GPS may work, I have not tried it. 
+
+The first thing to do is to turn on the plugin. With the plugin turned on, the other settings will be available:
+
+	range_test_plugin_enabled = 1
+
+If you want to send a message every 60 seconds:
+    
+	range_test_plugin_sender = 60
+
+To save a log of the messages:
+
+    range_test_plugin_save = 1
+
+Recommended settings for a sender at different radio settings:
+
+    Long Slow  ... range_test_plugin_sender = 60
+    Long Alt   ... range_test_plugin_sender = 30
+    Medium     ... range_test_plugin_sender = 15
+    Short Fast ... range_test_plugin_sender = 15
+
+## Python API Examples
+
+### Sender 
+
+    meshtastic --set range_test_plugin_enabled 1
+    meshtastic --set range_test_plugin_sender 60
+
+### Receiver
+
+    meshtastic --set range_test_plugin_enabled 1
+    meshtastic --set range_test_plugin_save 1
+
+## Other things to keep in mind
+
+Be sure to turn off either the plugin configured as a sender or the device where the plugin setup as sender when not in use. This will use a lot of time on air and will spam your channel.
+
+Also be mindful of your space usage on the file system. It has protections from filling up the space but it's best to delete old range test results.
+
+
+# Known Problems
+
+* If turned on, using mesh network will become unwieldly because messages are sent over the same channel as the other messages. See TODO below.
+
+# TODO
+
+* Right now range test messages go over the TEXT_MESSAGE_APP port. We need a toggle to switch to optionally send over RANGE_TEST_APP.
+
+# FAQ
+
+Q: Where is rangetest.csv saved?
+A: Turn on the WiFi on your device as either a WiFi client or a WiFi AP. Once you can connect to your device, go to /static and you will see rangetest.csv.
+
+Q: Do I need to have WiFi turned on for the file to be saved?
+A: Nope, it'll just work.
+
+Q: Do I need a phone for this plugin?
+A: There's no need for a phone.
+
+Q: Can I use this as a message logger?
+A: While it's not the intended purpose, sure, why not. Do it!
+
+Q: What will happen if I run out of space on my device?
+A: We have a protection in place to keep you from completly filling up your device. This will make sure that other device critical functions will continue to work. We will reserve at least 50k of free space.
+
+Q: What do I do with the rangetest.csv file when I'm done?
+A: Go to /static and delete the file.
+
+Q: Can I use this as a sender while on battery power?
+A: Yes, but your battery will run down quicker than normal. While sending, we tell the device not to go into low-power mode since it needs to keep to a fairly strict timer.
+
+Q: Why is this operating on incoming messages instead of the existing location discovery protocol?
+A: This plugin is still young and currently supports monitoring just one port at a time. I decided to use the existing message port because that is easy to test with. A future version will listen to multiple ports to be more promiscuous. 
+
+# Need more help?
+
+Go to the Meshtastic Discourse Group if you have any questions or to share how you have used this.
+
+https://meshtastic.discourse.group

docs/software/plugins/SerialPlugin.md

@@ -0,0 +1,40 @@
+# About
+
+A simple interface to send messages over the mesh network by sending strings
+over a serial port.
+
+Default is to use RX GPIO 16 and TX GPIO 17.
+
+
+# Basic Usage:
+
+        1) Enable the plugin by setting serialplugin_enabled to 1.
+        2) Set the pins (serialplugin_rxd / serialplugin_rxd) for your preferred RX and TX GPIO pins.
+           On tbeam, recommend to use:
+                RXD 35
+                TXD 15
+        3) Set serialplugin_timeout to the amount of time to wait before we consider
+           your packet as "done".
+        4) (Optional) In SerialPlugin.h set the port to PortNum_TEXT_MESSAGE_APP if you want to
+           send messages to/from the general text message channel.
+        5) Connect to your device over the serial interface at 38400 8N1.
+        6) Send a packet up to 240 bytes in length. This will get relayed over the mesh network.
+        7) (Optional) Set serialplugin_echo to 1 and any message you send out will be echoed back
+           to your device.
+
+# TODO (in this order):
+
+* Define a verbose RX mode to report on mesh and packet infomration.
+        - This won't happen any time soon.
+
+# Known Problems
+
+    * Until the plugin is initilized by the startup sequence, the TX pin is in a floating
+      state. Device connected to that pin may see this as "noise".
+    * Will not work on NRF and the Linux device targets.
+    
+# Need help?
+
+Need help with this plugin? Post your question on the Meshtastic Discourse:
+
+https://meshtastic.discourse.group

docs/software/plugins/StoreForwardPlugin.md

@@ -0,0 +1,105 @@
+# About
+
+This is a work in progress and is not yet available.
+
+The Store Forward Plugin is an implementation of a Store and Forward system to enable resilient messaging in the event that a client device is disconnected from the main network.
+
+Because of the increased network traffic for this overhead, it's not adviced to use this if you are duty cycle limited for your airtime usage nor is it adviced to use this for SF12 (Long range but Slow).
+
+# Requirements
+
+Initial Requirements:
+
+* Must be installed on a router node.
+* * This is an artificial limitation, but is in place to enforce best practices.
+* * Router nodes are intended to be always online. If this plugin misses any messages, the reliability of the stored messages will be reduced
+* Esp32 Processor based device with external PSRAM. (tbeam v1.0 and tbeamv1.1, maybe others)
+
+# Implementation timeline
+
+Not necessarily in this order:
+
+UC 1) MVP - automagically forward packets to a client that may have missed packets.
+
+UC 2) Client Interface (Web, Android, Python or iOS when that happens) to optionally request packets be resent. This is to support the case where Router has not detected that the client was away. This is because the router will only know you're away if you've been gone for a period of time but will have no way of knowing if you were offline for a short number of minutes. This will cover the case where you have ducked into a cave or you're swapping out your battery.
+
+UC 3) router sends a periodic “heartbeat” to let the clients know they’re part of the main mesh
+
+UC 4) support for a mesh to have multiple routers that have the store & forward functionality (for redundancy)
+
+UC 5) Support for "long term" delayed messages and "short term" delayed messages. Handle the cases slightly different to improve user expierence. A short term delayed message would be a message that was resent becaue a node was not heard from for <5 minutes. A long term delayed message is a message that has not been delivered in >5 minutes.
+
+UC 6) Eventually we could add a "want_store_and_forward" bit to MeshPacket and that could be nicer than whitelists in this plugin. Initially we'd only set that bit in text messages (and any other plugin messages that can cope with this). This change would be backward wire compatible so can add easily later.
+
+UC 7) Currently the way we allocate messages in the device code is super inefficient. It always allocates the worst case message size. Really we should dynamically allocate just the # of bytes we need. This would allow many more MeshPackets to be kept in RAM.
+
+UC 8) We'll want a "delayed" bit in MeshPacket. This will indicate that the message was not received in real time.
+
+# Things to consider
+
+Not all these cases will be initially implemented. It's just a running stream of thoughts to be considered.
+
+## Main Mesh Network with Router
+
+The store and forward plugin is intended to be enabled on a router that designates your "main" mesh network.
+
+## Store and Forward on Multiple Routers
+
+If multiple routers with the plugin are enabled, they should be able to share their stored database amongst each other. This enable resilliancy from one router going offline.
+
+## Fragmented networks - No router
+
+In this case, the mesh network has been fragmented by two client devices leaving the main network.
+
+If two Meshtastic devices walk away from the main mesh, they will be able to message each other but not message the main network. When they return to the main network, they will receive the messages they have missed from the main mesh network.
+
+## Fragmented network - With routers
+
+In this case, we have two routers separate by a great distance, each serving multiple devices. One of the routers have gone offline. This has now created two physically seaprated mesh networks using the same channel configuration.
+
+Q: How do we rejoin both fragmented networks? Do we care about messages that were unrouted between fagments?
+
+## Identifing Delayed Messages
+
+When a message is replayed for a node, identify the packet as "Delayed". This will indicate that the message was not received in real time.
+
+# Router Data Structures
+
+Structure of received messages:
+
+    receivedMessages
+      Port_No
+      packetID
+      to
+      from
+      rxTimeMsec
+      data
+
+Structure of nodes and last time we heard from them. This is a record of any packet type.
+
+    receivedRecord
+      From
+      rxTimeMillis
+
+# General Operation for UC1 - automagically forward packets to a client that may have missed packets
+
+On every handled packet
+* Record the sender from and the time we heard from that sender into senderRecord.
+
+On every handled packet
+
+* If the packet is a message, save the messsage into receivedMessages
+
+On every handled packet, if we have not heard from that sender in a period of time greater than timeAway, let's assume that they have been away from the network.
+
+* In this case, we will resend them all the messages they have missed since they were gone
+
+## Expected problems this implementation
+
+* If the client has been away for less than 5 minutes and has received the previously sent message, the client will gracefully ignore it. This is thanks to PacketHistory::wasSeenRecently in PacketHistory.cpp.
+* * If the client has been away for more than 5 minutes and we resend packets that they have already received, it's possible they will see duplicate messages. This should be unlikely but is still possible. 
+
+
+# Designed limitations
+
+The Store and Forward plugin will subscribe to specific packet types and channels and only save those. This will both reduce the amount of data we will need to store and reduce the overhead on the network. Eg: There's no need to replay ACK packets nor is there's no need to replay old location packets.

docs/software/power.md

@@ -32,21 +32,25 @@ From lower to higher power consumption.
   onEntry: setBluetoothOn(true)
   onExit:
 
-- full on (ON) - Everything is on
-  onEntry: setBluetoothOn(true), screen.setOn(true)
-  onExit: screen.setOn(false)
-
 - serial API usage (SERIAL) - Screen is on, device doesn't sleep, bluetooth off
   onEntry: setBluetooth off, screen on
   onExit:
 
+- full on (ON) - Everything is on, can eventually timeout and lower to a lower power state
+  onEntry: setBluetoothOn(true), screen.setOn(true)
+  onExit: screen->setOn(false)
+
+- has power (POWER) - Screen is on, device doesn't sleep, bluetooth on, will stay in this state as long as we have power
+  onEntry: setBluetooth off, screen on
+  onExit:
+
 ## Behavior
 
 ### events that increase CPU activity
 
 - At cold boot: The initial state (after setup() has run) is DARK
 - While in DARK: if we receive EVENT_BOOT, transition to ON (and show the bootscreen). This event will be sent if we detect we woke due to reset (as opposed to deep sleep)
-- While in LS: Once every position_broadcast_secs (default 15 mins) - the unit will wake into DARK mode and broadcast a "networkPing" (our position) and stay alive for wait_bluetooth_secs (default 30 seconds). This allows other nodes to have a record of our last known position if we go away and allows a paired phone to hear from us and download messages.
+- While in LS: Once every position_broadcast_secs (default 15 mins) - the unit will wake into DARK mode and broadcast a "networkPing" (our position) and stay alive for wait_bluetooth_secs (default 60 seconds). This allows other nodes to have a record of our last known position if we go away and allows a paired phone to hear from us and download messages.
 - While in LS: Every send*owner_interval (defaults to 4, i.e. one hour), when we wake to send our position we \_also* broadcast our owner. This lets new nodes on the network find out about us or correct duplicate node number assignments.
 - While in LS/NB/DARK: If the user presses a button (EVENT_PRESS) we go to full ON mode for screen_on_secs (default 30 seconds). Multiple presses keeps resetting this timeout
 - While in LS/NB/DARK: If we receive new text messages (EVENT_RECEIVED_TEXT_MSG), we go to full ON mode for screen_on_secs (same as if user pressed a button)
@@ -56,9 +60,11 @@ From lower to higher power consumption.
 - While in NB/DARK/ON: If we receive EVENT_NODEDB_UPDATED we transition to ON (so the new screen can be shown)
 - While in DARK: While the phone talks to us over BLE (EVENT_CONTACT_FROM_PHONE) reset any sleep timers and stay in DARK (needed for bluetooth sw update and nice user experience if the user is reading/replying to texts)
 - while in LS/NB/DARK: if SERIAL_CONNECTED, go to serial
+- while in any state: if we have AC power, go to POWER
 
 ### events that decrease cpu activity
 
+- While in POWER: if lose AC go to ON
 - While in SERIAL: if SERIAL_DISCONNECTED, go to NB
 - While in ON: If PRESS event occurs, reset screen_on_secs timer and tell the screen to handle the pess
 - While in ON: If it has been more than screen_on_secs since a press, lower to DARK

docs/software/rak815.md

@@ -0,0 +1,6 @@
+# RAK815
+
+Notes on trying to get the RAK815 working with meshtastic.
+
+good tutorial: https://www.hackster.io/naresh-krish/getting-started-with-rak815-tracker-module-and-arduino-1c7bc9
+(includes software serial link - possibly useful for GPS)

docs/software/read-error.txt

@@ -0,0 +1,148 @@
+nimble stress test error (private notes for @geeksville)
+
+findings:
+only happens when stress testing multiple sleepwake cycles?
+failed packets all have initial mbuflen of zero (should be 1)
+restarting the connection on phone sometimes (but not always) fixes it (is the larger config nonce pushing packet size up too large?)
+- packets >= 79 bytes (FromRadio) cause INVALID_OFFSET (7) gatt errors to be sent to the app
+- some packets are missing
+
+theory:
+some sort of leak in mbuf storage during unfortunately timed sleep shutdowns
+
+
+device side
+
+
+connection updated; status=0 handle=0 our_ota_addr_type=0 our_ota_addr=00:24:62:ab:dd:df
+ our_id_addr_type=0 our_id_addr=00:24:62:ab:dd:df
+ peer_ota_addr_type=0 peer_ota_addr=00:7c:d9:5c:ba:2e
+ peer_id_addr_type=0 peer_id_addr=00:7c:d9:5c:ba:2e
+ conn_itvl=36 conn_latency=0 supervision_timeout=500 encrypted=1 authenticated=1 bonded=1
+
+BLE fromRadio called
+getFromRadio, !available
+toRadioWriteCb data 0x3ffc3d72, len 4
+Trigger powerFSM 9
+Transition powerFSM transition=Contact from phone, from=DARK to=DARK
+Client wants config, nonce=6864
+Reset nodeinfo read pointer
+toRadioWriteCb data 0x3ffc3d72, len 4
+Trigger powerFSM 9
+Transition powerFSM transition=Contact from phone, from=DARK to=DARK
+Client wants config, nonce=6863
+Reset nodeinfo read pointer
+BLE fromRadio called
+getFromRadio, state=2
+encoding toPhone packet to phone variant=3, 50 bytes
+BLE fromRadio called
+getFromRadio, state=3
+encoding toPhone packet to phone variant=6, 83 bytes
+BLE fromRadio called
+getFromRadio, state=4
+Sending nodeinfo: num=0xabdddf38, lastseen=1595606850, id=!2462abdddf38, name=Bob b
+encoding toPhone packet to phone variant=4, 67 bytes
+BLE fromRadio called
+getFromRadio, state=4
+Sending nodeinfo: num=0x28b200b4, lastseen=1595606804, id=!246f28b200b4, name=Unknown 00b4
+encoding toPhone packet to phone variant=4, 80 bytes
+BLE fromRadio called
+getFromRadio, state=4
+Sending nodeinfo: num=0xabf84098, lastseen=1593680756, id=!2462abf84098, name=bx n
+encoding toPhone packet to phone variant=4, 72 bytes
+BLE fromRadio called
+getFromRadio, state=4
+Sending nodeinfo: num=0x83f0d8e5, lastseen=1594686931, id=!e8e383f0d8e5, name=Unknown d8e5
+encoding toPhone packet to phone variant=4, 64 bytes
+BLE fromRadio called
+getFromRadio, state=4
+Sending nodeinfo: num=0xd1dc7764, lastseen=1595602082, id=!f008d1dc7764, name=dg
+encoding toPhone packet to phone variant=4, 52 bytes
+BLE fromRadio called
+getFromRadio, state=4
+Sending nodeinfo: num=0xd1dc7828, lastseen=1595598298, id=!f008d1dc7828, name=ryan 
+encoding toPhone packet to phone variant=4, 54 bytes
+BLE fromRadio called
+getFromRadio, state=4
+Done sending nodeinfos
+getFromRadio, state=5
+
+
+
+phone side
+
+2020-07-24 09:11:00.642 6478-6545/com.geeksville.mesh W/com.geeksville.mesh.service.BluetoothInterface: Attempting reconnect
+2020-07-24 09:11:00.642 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: connect
+2020-07-24 09:11:00.642 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: connect
+2020-07-24 09:11:00.643 6478-6545/com.geeksville.mesh D/BluetoothGatt: connect() - device: 24:62:AB:DD:DF:3A, auto: false
+2020-07-24 09:11:00.643 6478-6545/com.geeksville.mesh D/BluetoothGatt: registerApp()
+2020-07-24 09:11:00.643 6478-6545/com.geeksville.mesh D/BluetoothGatt: registerApp() - UUID=026baf7f-d2de-43f1-961f-4e00e04c6fbb
+2020-07-24 09:11:00.645 6478-27868/com.geeksville.mesh D/BluetoothGatt: onClientRegistered() - status=0 clientIf=4
+2020-07-24 09:11:01.022 6478-27868/com.geeksville.mesh D/BluetoothGatt: onClientConnectionState() - status=0 clientIf=4 device=24:62:AB:DD:DF:3A
+2020-07-24 09:11:01.022 6478-27868/com.geeksville.mesh I/com.geeksville.mesh.service.SafeBluetooth: new bluetooth connection state 2, status 0
+2020-07-24 09:11:01.023 6478-27868/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work connect is completed, resuming status=0, res=kotlin.Unit
+2020-07-24 09:11:01.023 6478-6545/com.geeksville.mesh I/com.geeksville.mesh.service.BluetoothInterface: Connected to radio!
+2020-07-24 09:11:01.023 6478-6545/com.geeksville.mesh D/BluetoothGatt: refresh() - device: 24:62:AB:DD:DF:3A
+2020-07-24 09:11:01.526 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: discover
+2020-07-24 09:11:01.526 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: discover
+2020-07-24 09:11:01.526 6478-6545/com.geeksville.mesh D/BluetoothGatt: discoverServices() - device: 24:62:AB:DD:DF:3A
+2020-07-24 09:11:01.829 6478-27868/com.geeksville.mesh D/BluetoothGatt: onConnectionUpdated() - Device=24:62:AB:DD:DF:3A interval=6 latency=0 timeout=500 status=0
+2020-07-24 09:11:02.008 6478-27868/com.geeksville.mesh D/BluetoothGatt: onSearchComplete() = Device=24:62:AB:DD:DF:3A Status=0
+2020-07-24 09:11:02.009 6478-27868/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work discover is completed, resuming status=0, res=kotlin.Unit
+2020-07-24 09:11:02.009 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: Discovered services!
+2020-07-24 09:11:02.095 6478-27868/com.geeksville.mesh D/BluetoothGatt: onConnectionUpdated() - Device=24:62:AB:DD:DF:3A interval=36 latency=0 timeout=500 status=0
+2020-07-24 09:11:03.010 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.RadioInterfaceService: Broadcasting connection=true
+2020-07-24 09:11:03.012 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:03.012 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:03.012 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Received broadcast com.geeksville.mesh.CONNECT_CHANGED
+2020-07-24 09:11:03.012 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: onConnectionChanged=CONNECTED
+2020-07-24 09:11:03.012 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Starting config nonce=6878
+2020-07-24 09:11:03.013 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: queuing 4 bytes to f75c76d2-129e-4dad-a1dd-7866124401e7 *** sending start config
+2020-07-24 09:11:03.013 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: writeC f75c76d2-129e-4dad-a1dd-7866124401e7
+2020-07-24 09:11:03.015 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Received broadcast com.geeksville.mesh.CONNECT_CHANGED
+2020-07-24 09:11:03.015 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: onConnectionChanged=CONNECTED
+2020-07-24 09:11:03.015 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Starting config nonce=6877
+2020-07-24 09:11:03.016 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: device sleep timeout cancelled
+2020-07-24 09:11:03.016 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: queuing 4 bytes to f75c76d2-129e-4dad-a1dd-7866124401e7
+2020-07-24 09:11:03.016 6478-6545/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: writeC f75c76d2-129e-4dad-a1dd-7866124401e7
+2020-07-24 09:11:03.130 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: writeC f75c76d2-129e-4dad-a1dd-7866124401e7
+2020-07-24 09:11:03.132 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5 is completed, resuming status=0, res=android.bluetooth.BluetoothGattCharacteristic@556a315
+2020-07-24 09:11:03.132 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: Done reading from radio, fromradio is empty
+2020-07-24 09:11:03.132 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: starting setNotify(ed9da18c-a800-4f66-a670-aa7547e34453, true) *** start notify
+2020-07-24 09:11:03.132 6478-19966/com.geeksville.mesh D/BluetoothGatt: setCharacteristicNotification() - uuid: ed9da18c-a800-4f66-a670-aa7547e34453 enable: true
+2020-07-24 09:11:03.133 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: writeD  
+2020-07-24 09:11:03.220 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: writeC f75c76d2-129e-4dad-a1dd-7866124401e7
+2020-07-24 09:11:03.221 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work writeC f75c76d2-129e-4dad-a1dd-7866124401e7 is completed, resuming status=0, res=android.bluetooth.BluetoothGattCharacteristic@2d2062a
+2020-07-24 09:11:03.221 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: write of 4 bytes completed
+2020-07-24 09:11:03.221 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:03.310 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: writeD
+2020-07-24 09:11:03.311 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work writeC f75c76d2-129e-4dad-a1dd-7866124401e7 is completed, resuming status=0, res=android.bluetooth.BluetoothGattCharacteristic@2d2062a
+2020-07-24 09:11:03.311 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: write of 4 bytes completed
+2020-07-24 09:11:03.400 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:03.402 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work writeD is completed, resuming status=0, res=android.bluetooth.BluetoothGattDescriptor@fc99c1b
+2020-07-24 09:11:03.402 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Notify enable=true completed
+2020-07-24 09:11:03.769 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5 is completed, resuming status=0, res=android.bluetooth.BluetoothGattCharacteristic@556a315
+2020-07-24 09:11:03.769 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: Received 80 bytes from radio **** received an 80 byte fromradio.  Why did we miss three previous reads?
+2020-07-24 09:11:03.774 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:03.774 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:03.774 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Received broadcast com.geeksville.mesh.RECEIVE_FROMRADIO
+2020-07-24 09:11:03.776 6478-6478/com.geeksville.mesh E/com.geeksville.mesh.service.MeshService: Invalid Protobuf from radio, len=80 (exception Protocol message had invalid UTF-8.)
+2020-07-24 09:11:03.776 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Received broadcast com.geeksville.mesh.RECEIVE_FROMRADIO
+2020-07-24 09:11:03.776 6478-6478/com.geeksville.mesh E/com.geeksville.mesh.service.MeshService: Invalid Protobuf from radio, len=80 (exception Protocol message had invalid UTF-8.)
+2020-07-24 09:11:04.031 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5 is completed, resuming status=0, res=android.bluetooth.BluetoothGattCharacteristic@556a315
+2020-07-24 09:11:04.031 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.BluetoothInterface: Received 52 bytes from radio *** received 52 bytes - where did the previous two read results go?
+2020-07-24 09:11:04.033 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: Enqueuing work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:04.033 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth$BluetoothContinuation: Starting work: readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:04.034 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Received broadcast com.geeksville.mesh.RECEIVE_FROMRADIO
+2020-07-24 09:11:04.035 6478-6478/com.geeksville.mesh E/com.geeksville.mesh.service.MeshService: Invalid Protobuf from radio, len=52 (exception While parsing a protocol message, the input ended unexpectedly in the middle of a field.  This could mean either that the input has been truncated or that an embedded message misreported its own length.)
+2020-07-24 09:11:04.036 6478-6478/com.geeksville.mesh D/com.geeksville.mesh.service.MeshService: Received broadcast com.geeksville.mesh.RECEIVE_FROMRADIO
+2020-07-24 09:11:04.036 6478-6478/com.geeksville.mesh E/com.geeksville.mesh.service.MeshService: Invalid Protobuf from radio, len=52 (exception While parsing a protocol message, the input ended unexpectedly in the middle of a field.  This could mean either that the input has been truncated or that an embedded message misreported its own length.)
+2020-07-24 09:11:04.210 6478-19966/com.geeksville.mesh D/com.geeksville.mesh.service.SafeBluetooth: work readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5 is completed, resuming status=7, res=android.bluetooth.BluetoothGattCharacteristic@556a315 *** read failed
+2020-07-24 09:11:04.211 6478-19966/com.geeksville.mesh W/com.geeksville.mesh.service.BluetoothInterface: Scheduling reconnect because error during doReadFromRadio - disconnecting, Bluetooth status=7 while doing readC 8ba2bcc2-ee02-4a55-a531-c525c5e454d5
+2020-07-24 09:11:04.211 6478-6545/com.geeksville.mesh W/com.geeksville.mesh.service.BluetoothInterface: Forcing disconnect and hopefully device will comeback (disabling forced refresh)
+2020-07-24 09:11:04.211 6478-6545/com.geeksville.mesh I/com.geeksville.mesh.service.SafeBluetooth: Closing our GATT connection
+2020-07-24 09:11:04.211 6478-6545/com.geeksville.mesh D/BluetoothGatt: cancelOpen() - device: 24:62:AB:DD:DF:3A
+2020-07-24 09:11:04.214 6478-19966/com.geeksville.mesh D/BluetoothGatt: onClientConnectionState() - status=0 clientIf=4 device=24:62:AB:DD:DF:3A
+2020-07-24 09:11:04.215 6478-19966/com.geeksville.mesh I/com.geeksville.mesh.service.SafeBluetooth: new bluetooth connection state 0, status 0
+2020-07-24 09:11:04.215 6478-19966/com.geeksville.mesh I/com.geeksville.mesh.service.SafeBluetooth: Got disconnect because we are shutting down, closing gatt
+2020-07-24 09:11:04.215 6478-19966/com.geeksville.mesh D/BluetoothGatt: close()

docs/software/remote-hardware-service.md

@@ -0,0 +1,23 @@
+# Remote Hardware Service
+
+FIXME - the following are a collection of notes moved from elsewhere.  We need to refactor these notes into actual documentation on the remote-hardware/gpio service.
+
+### 1.7.2. New 'no-code-IOT' mini-app
+
+Add a new 'remote GPIO/serial port/SPI/I2C access' mini-app. This new standard app would use the MQTT messaging layer to let users (developers that don't need to write device code) do basic (potentially dangerous) operations remotely.
+
+#### 1.7.2.1. Supported operations in the initial release
+
+Initially supported features for no-code-IOT.
+
+- Set any GPIO
+- Read any GPIO
+
+#### 1.7.2.2. Supported operations eventually
+
+General ideas for no-code IOT.
+
+- Subscribe for notification of GPIO input status change (i.e. when pin goes low, send my app a message)
+- Write/read N bytes over I2C/SPI bus Y (as one atomic I2C/SPI transaction)
+- Send N bytes out serial port Z
+- Subscribe for notification for when regex X matches the bytes that were received on serial port Z

docs/software/sw-design.md

@@ -1,9 +1,10 @@
 This is a mini design doc for developing the meshtastic software.
 
 * [Build instructions](build-instructions.md)
+* [On device plugin API](plugin-api.md) - a tutorial on how to write small Plugins which run on the device and can message other nodes.
 * [TODO](TODO.md) - read this if you are looking for things to do (or curious about currently missing features)
 * Our [project board](https://github.com/orgs/meshtastic/projects/1) - shows what things we are currently working on and remaining work items for the current release.
 * [Power Management](power.md)
 * [Mesh algorithm](mesh-alg.md)
-* [Bluetooth API](bluetooth-api.md) and porting guide for new clients (iOS, python, etc...)
+* [External client API](device-api.md) and porting guide for new clients (iOS, python, etc...)
 * TODO: how to port the device code to a new device.

images/genfiles.sh

@@ -1,11 +0,0 @@
-# using height of 50 to have 14 pixels beneath icon for text
-inkscape -z -e icon.png -w 50 -h 50 icon-24px.svg
-convert icon.png -background white -alpha Background ../src/icon.xbm
-
-inkscape -z -e compass.png -w 48 -h 48 location_searching-24px.svg
-convert compass.png -background white -alpha Background ../src/compass.xbm
-
-inkscape -z -e face.png -w 13 -h 13 face-24px.svg
-
-inkscape -z -e pin.png -w 13 -h 13 room-24px.svg
-convert pin.png -background white -alpha Background ../src/pin.xbm

lib/nanopb/include/pb.h

@@ -11,7 +11,7 @@
  *****************************************************************/
 
 /* Enable support for dynamically allocated fields */
-#define PB_ENABLE_MALLOC 1
+/* #define PB_ENABLE_MALLOC 1 */
 
 /* Define this if your CPU / compiler combination does not support
  * unaligned memory access to packed structures. */
@@ -55,7 +55,7 @@
 
 /* Version of the nanopb library. Just in case you want to check it in
  * your own program. */
-#define NANOPB_VERSION nanopb-0.4.1
+#define NANOPB_VERSION nanopb-0.4.4
 
 /* Include all the system headers needed by nanopb. You will need the
  * definitions of the following:
@@ -276,17 +276,18 @@ typedef struct pb_field_iter_s pb_field_iter_t;
 /* This structure is used in auto-generated constants
  * to specify struct fields.
  */
-PB_PACKED_STRUCT_START
 typedef struct pb_msgdesc_s pb_msgdesc_t;
 struct pb_msgdesc_s {
-    pb_size_t field_count;
     const uint32_t *field_info;
     const pb_msgdesc_t * const * submsg_info;
     const pb_byte_t *default_value;
 
     bool (*field_callback)(pb_istream_t *istream, pb_ostream_t *ostream, const pb_field_iter_t *field);
-} pb_packed;
-PB_PACKED_STRUCT_END
+
+    pb_size_t field_count;
+    pb_size_t required_field_count;
+    pb_size_t largest_tag;
+};
 
 /* Iterator for message descriptor */
 struct pb_field_iter_s {
@@ -469,137 +470,181 @@ struct pb_extension_s {
     }; \
     const pb_msgdesc_t structname ## _msg = \
     { \
-       0 msgname ## _FIELDLIST(PB_GEN_FIELD_COUNT, structname), \
        structname ## _field_info, \
        structname ## _submsg_info, \
        msgname ## _DEFAULT, \
        msgname ## _CALLBACK, \
+       0 msgname ## _FIELDLIST(PB_GEN_FIELD_COUNT, structname), \
+       0 msgname ## _FIELDLIST(PB_GEN_REQ_FIELD_COUNT, structname), \
+       0 msgname ## _FIELDLIST(PB_GEN_LARGEST_TAG, structname), \
     }; \
     msgname ## _FIELDLIST(PB_GEN_FIELD_INFO_ASSERT_ ## width, structname)
 
 #define PB_GEN_FIELD_COUNT(structname, atype, htype, ltype, fieldname, tag) +1
+#define PB_GEN_REQ_FIELD_COUNT(structname, atype, htype, ltype, fieldname, tag) \
+    + (PB_HTYPE_ ## htype == PB_HTYPE_REQUIRED)
+#define PB_GEN_LARGEST_TAG(structname, atype, htype, ltype, fieldname, tag) \
+    * 0 + tag
 
+/* X-macro for generating the entries in struct_field_info[] array. */
 #define PB_GEN_FIELD_INFO_1(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO(1, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_1(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_2(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO(2, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_2(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_4(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO(4, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_4(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_8(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO(8, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_8(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_AUTO(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO_AUTO2(PB_FIELDINFO_WIDTH_AUTO(atype, htype, ltype), structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_AUTO2(PB_FIELDINFO_WIDTH_AUTO(_PB_ATYPE_ ## atype, _PB_HTYPE_ ## htype, _PB_LTYPE_ ## ltype), \
+                   tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
-#define PB_GEN_FIELD_INFO_AUTO2(width, structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO(width, structname, atype, htype, ltype, fieldname, tag)
+#define PB_FIELDINFO_AUTO2(width, tag, type, data_offset, data_size, size_offset, array_size) \
+    PB_FIELDINFO_AUTO3(width, tag, type, data_offset, data_size, size_offset, array_size)
 
-#define PB_GEN_FIELD_INFO(width, structname, atype, htype, ltype, fieldname, tag) \
-    PB_FIELDINFO_ ## width(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
-                   PB_DATA_OFFSET_ ## atype(htype, structname, fieldname), \
-                   PB_DATA_SIZE_ ## atype(htype, structname, fieldname), \
-                   PB_SIZE_OFFSET_ ## atype(htype, structname, fieldname), \
-                   PB_ARRAY_SIZE_ ## atype(htype, structname, fieldname))
+#define PB_FIELDINFO_AUTO3(width, tag, type, data_offset, data_size, size_offset, array_size) \
+    PB_FIELDINFO_ ## width(tag, type, data_offset, data_size, size_offset, array_size)
 
+/* X-macro for generating asserts that entries fit in struct_field_info[] array.
+ * The structure of macros here must match the structure above in PB_GEN_FIELD_INFO_x(),
+ * but it is not easily reused because of how macro substitutions work. */
 #define PB_GEN_FIELD_INFO_ASSERT_1(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO_ASSERT(1, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_ASSERT_1(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_ASSERT_2(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO_ASSERT(2, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_ASSERT_2(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_ASSERT_4(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO_ASSERT(4, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_ASSERT_4(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_ASSERT_8(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO_ASSERT(8, structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_ASSERT_8(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
 #define PB_GEN_FIELD_INFO_ASSERT_AUTO(structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO_ASSERT_AUTO2(PB_FIELDINFO_WIDTH_AUTO(atype, htype, ltype), structname, atype, htype, ltype, fieldname, tag)
+    PB_FIELDINFO_ASSERT_AUTO2(PB_FIELDINFO_WIDTH_AUTO(_PB_ATYPE_ ## atype, _PB_HTYPE_ ## htype, _PB_LTYPE_ ## ltype), \
+                   tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
+                   PB_DATA_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_DATA_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_SIZE_OFFSET_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname), \
+                   PB_ARRAY_SIZE_ ## atype(_PB_HTYPE_ ## htype, structname, fieldname))
 
-#define PB_GEN_FIELD_INFO_ASSERT_AUTO2(width, structname, atype, htype, ltype, fieldname, tag) \
-    PB_GEN_FIELD_INFO_ASSERT(width, structname, atype, htype, ltype, fieldname, tag)
+#define PB_FIELDINFO_ASSERT_AUTO2(width, tag, type, data_offset, data_size, size_offset, array_size) \
+    PB_FIELDINFO_ASSERT_AUTO3(width, tag, type, data_offset, data_size, size_offset, array_size)
 
-#define PB_GEN_FIELD_INFO_ASSERT(width, structname, atype, htype, ltype, fieldname, tag) \
-    PB_FIELDINFO_ASSERT_ ## width(tag, PB_ATYPE_ ## atype | PB_HTYPE_ ## htype | PB_LTYPE_MAP_ ## ltype, \
-                   PB_DATA_OFFSET_ ## atype(htype, structname, fieldname), \
-                   PB_DATA_SIZE_ ## atype(htype, structname, fieldname), \
-                   PB_SIZE_OFFSET_ ## atype(htype, structname, fieldname), \
-                   PB_ARRAY_SIZE_ ## atype(htype, structname, fieldname))
+#define PB_FIELDINFO_ASSERT_AUTO3(width, tag, type, data_offset, data_size, size_offset, array_size) \
+    PB_FIELDINFO_ASSERT_ ## width(tag, type, data_offset, data_size, size_offset, array_size)
 
-#define PB_DATA_OFFSET_STATIC(htype, structname, fieldname) PB_DATA_OFFSET_ ## htype(structname, fieldname)
-#define PB_DATA_OFFSET_POINTER(htype, structname, fieldname) PB_DATA_OFFSET_ ## htype(structname, fieldname)
-#define PB_DATA_OFFSET_CALLBACK(htype, structname, fieldname) PB_DATA_OFFSET_ ## htype(structname, fieldname)
-#define PB_DATA_OFFSET_REQUIRED(structname, fieldname) offsetof(structname, fieldname)
-#define PB_DATA_OFFSET_SINGULAR(structname, fieldname) offsetof(structname, fieldname)
-#define PB_DATA_OFFSET_ONEOF(structname, fieldname) offsetof(structname, PB_ONEOF_NAME(FULL, fieldname))
-#define PB_DATA_OFFSET_OPTIONAL(structname, fieldname) offsetof(structname, fieldname)
-#define PB_DATA_OFFSET_REPEATED(structname, fieldname) offsetof(structname, fieldname)
-#define PB_DATA_OFFSET_FIXARRAY(structname, fieldname) offsetof(structname, fieldname)
+#define PB_DATA_OFFSET_STATIC(htype, structname, fieldname) PB_DO ## htype(structname, fieldname)
+#define PB_DATA_OFFSET_POINTER(htype, structname, fieldname) PB_DO ## htype(structname, fieldname)
+#define PB_DATA_OFFSET_CALLBACK(htype, structname, fieldname) PB_DO ## htype(structname, fieldname)
+#define PB_DO_PB_HTYPE_REQUIRED(structname, fieldname) offsetof(structname, fieldname)
+#define PB_DO_PB_HTYPE_SINGULAR(structname, fieldname) offsetof(structname, fieldname)
+#define PB_DO_PB_HTYPE_ONEOF(structname, fieldname) offsetof(structname, PB_ONEOF_NAME(FULL, fieldname))
+#define PB_DO_PB_HTYPE_OPTIONAL(structname, fieldname) offsetof(structname, fieldname)
+#define PB_DO_PB_HTYPE_REPEATED(structname, fieldname) offsetof(structname, fieldname)
+#define PB_DO_PB_HTYPE_FIXARRAY(structname, fieldname) offsetof(structname, fieldname)
 
-#define PB_SIZE_OFFSET_STATIC(htype, structname, fieldname) PB_SIZE_OFFSET_ ## htype(structname, fieldname)
-#define PB_SIZE_OFFSET_POINTER(htype, structname, fieldname) PB_SIZE_OFFSET_PTR_ ## htype(structname, fieldname)
-#define PB_SIZE_OFFSET_CALLBACK(htype, structname, fieldname) PB_SIZE_OFFSET_CB_ ## htype(structname, fieldname)
-#define PB_SIZE_OFFSET_REQUIRED(structname, fieldname) 0
-#define PB_SIZE_OFFSET_SINGULAR(structname, fieldname) 0
-#define PB_SIZE_OFFSET_ONEOF(structname, fieldname) PB_SIZE_OFFSET_ONEOF2(structname, PB_ONEOF_NAME(FULL, fieldname), PB_ONEOF_NAME(UNION, fieldname))
-#define PB_SIZE_OFFSET_ONEOF2(structname, fullname, unionname) PB_SIZE_OFFSET_ONEOF3(structname, fullname, unionname)
-#define PB_SIZE_OFFSET_ONEOF3(structname, fullname, unionname) pb_delta(structname, fullname, which_ ## unionname)
-#define PB_SIZE_OFFSET_OPTIONAL(structname, fieldname) pb_delta(structname, fieldname, has_ ## fieldname)
-#define PB_SIZE_OFFSET_REPEATED(structname, fieldname) pb_delta(structname, fieldname, fieldname ## _count)
-#define PB_SIZE_OFFSET_FIXARRAY(structname, fieldname) 0
-#define PB_SIZE_OFFSET_PTR_REQUIRED(structname, fieldname) 0
-#define PB_SIZE_OFFSET_PTR_SINGULAR(structname, fieldname) 0
-#define PB_SIZE_OFFSET_PTR_ONEOF(structname, fieldname) PB_SIZE_OFFSET_ONEOF(structname, fieldname)
-#define PB_SIZE_OFFSET_PTR_OPTIONAL(structname, fieldname) 0
-#define PB_SIZE_OFFSET_PTR_REPEATED(structname, fieldname) PB_SIZE_OFFSET_REPEATED(structname, fieldname)
-#define PB_SIZE_OFFSET_PTR_FIXARRAY(structname, fieldname) 0
-#define PB_SIZE_OFFSET_CB_REQUIRED(structname, fieldname) 0
-#define PB_SIZE_OFFSET_CB_SINGULAR(structname, fieldname) 0
-#define PB_SIZE_OFFSET_CB_ONEOF(structname, fieldname) PB_SIZE_OFFSET_ONEOF(structname, fieldname)
-#define PB_SIZE_OFFSET_CB_OPTIONAL(structname, fieldname) 0
-#define PB_SIZE_OFFSET_CB_REPEATED(structname, fieldname) 0
-#define PB_SIZE_OFFSET_CB_FIXARRAY(structname, fieldname) 0
+#define PB_SIZE_OFFSET_STATIC(htype, structname, fieldname) PB_SO ## htype(structname, fieldname)
+#define PB_SIZE_OFFSET_POINTER(htype, structname, fieldname) PB_SO_PTR ## htype(structname, fieldname)
+#define PB_SIZE_OFFSET_CALLBACK(htype, structname, fieldname) PB_SO_CB ## htype(structname, fieldname)
+#define PB_SO_PB_HTYPE_REQUIRED(structname, fieldname) 0
+#define PB_SO_PB_HTYPE_SINGULAR(structname, fieldname) 0
+#define PB_SO_PB_HTYPE_ONEOF(structname, fieldname) PB_SO_PB_HTYPE_ONEOF2(structname, PB_ONEOF_NAME(FULL, fieldname), PB_ONEOF_NAME(UNION, fieldname))
+#define PB_SO_PB_HTYPE_ONEOF2(structname, fullname, unionname) PB_SO_PB_HTYPE_ONEOF3(structname, fullname, unionname)
+#define PB_SO_PB_HTYPE_ONEOF3(structname, fullname, unionname) pb_delta(structname, fullname, which_ ## unionname)
+#define PB_SO_PB_HTYPE_OPTIONAL(structname, fieldname) pb_delta(structname, fieldname, has_ ## fieldname)
+#define PB_SO_PB_HTYPE_REPEATED(structname, fieldname) pb_delta(structname, fieldname, fieldname ## _count)
+#define PB_SO_PB_HTYPE_FIXARRAY(structname, fieldname) 0
+#define PB_SO_PTR_PB_HTYPE_REQUIRED(structname, fieldname) 0
+#define PB_SO_PTR_PB_HTYPE_SINGULAR(structname, fieldname) 0
+#define PB_SO_PTR_PB_HTYPE_ONEOF(structname, fieldname) PB_SO_PB_HTYPE_ONEOF(structname, fieldname)
+#define PB_SO_PTR_PB_HTYPE_OPTIONAL(structname, fieldname) 0
+#define PB_SO_PTR_PB_HTYPE_REPEATED(structname, fieldname) PB_SO_PB_HTYPE_REPEATED(structname, fieldname)
+#define PB_SO_PTR_PB_HTYPE_FIXARRAY(structname, fieldname) 0
+#define PB_SO_CB_PB_HTYPE_REQUIRED(structname, fieldname) 0
+#define PB_SO_CB_PB_HTYPE_SINGULAR(structname, fieldname) 0
+#define PB_SO_CB_PB_HTYPE_ONEOF(structname, fieldname) PB_SO_PB_HTYPE_ONEOF(structname, fieldname)
+#define PB_SO_CB_PB_HTYPE_OPTIONAL(structname, fieldname) 0
+#define PB_SO_CB_PB_HTYPE_REPEATED(structname, fieldname) 0
+#define PB_SO_CB_PB_HTYPE_FIXARRAY(structname, fieldname) 0
 
-#define PB_ARRAY_SIZE_STATIC(htype, structname, fieldname) PB_ARRAY_SIZE_ ## htype(structname, fieldname)
-#define PB_ARRAY_SIZE_POINTER(htype, structname, fieldname) PB_ARRAY_SIZE_PTR_ ## htype(structname, fieldname)
+#define PB_ARRAY_SIZE_STATIC(htype, structname, fieldname) PB_AS ## htype(structname, fieldname)
+#define PB_ARRAY_SIZE_POINTER(htype, structname, fieldname) PB_AS_PTR ## htype(structname, fieldname)
 #define PB_ARRAY_SIZE_CALLBACK(htype, structname, fieldname) 1
-#define PB_ARRAY_SIZE_REQUIRED(structname, fieldname) 1
-#define PB_ARRAY_SIZE_SINGULAR(structname, fieldname) 1
-#define PB_ARRAY_SIZE_OPTIONAL(structname, fieldname) 1
-#define PB_ARRAY_SIZE_ONEOF(structname, fieldname) 1
-#define PB_ARRAY_SIZE_REPEATED(structname, fieldname) pb_arraysize(structname, fieldname)
-#define PB_ARRAY_SIZE_FIXARRAY(structname, fieldname) pb_arraysize(structname, fieldname)
-#define PB_ARRAY_SIZE_PTR_REQUIRED(structname, fieldname) 1
-#define PB_ARRAY_SIZE_PTR_SINGULAR(structname, fieldname) 1
-#define PB_ARRAY_SIZE_PTR_OPTIONAL(structname, fieldname) 1
-#define PB_ARRAY_SIZE_PTR_ONEOF(structname, fieldname) 1
-#define PB_ARRAY_SIZE_PTR_REPEATED(structname, fieldname) 1
-#define PB_ARRAY_SIZE_PTR_FIXARRAY(structname, fieldname) pb_arraysize(structname, fieldname[0])
+#define PB_AS_PB_HTYPE_REQUIRED(structname, fieldname) 1
+#define PB_AS_PB_HTYPE_SINGULAR(structname, fieldname) 1
+#define PB_AS_PB_HTYPE_OPTIONAL(structname, fieldname) 1
+#define PB_AS_PB_HTYPE_ONEOF(structname, fieldname) 1
+#define PB_AS_PB_HTYPE_REPEATED(structname, fieldname) pb_arraysize(structname, fieldname)
+#define PB_AS_PB_HTYPE_FIXARRAY(structname, fieldname) pb_arraysize(structname, fieldname)
+#define PB_AS_PTR_PB_HTYPE_REQUIRED(structname, fieldname) 1
+#define PB_AS_PTR_PB_HTYPE_SINGULAR(structname, fieldname) 1
+#define PB_AS_PTR_PB_HTYPE_OPTIONAL(structname, fieldname) 1
+#define PB_AS_PTR_PB_HTYPE_ONEOF(structname, fieldname) 1
+#define PB_AS_PTR_PB_HTYPE_REPEATED(structname, fieldname) 1
+#define PB_AS_PTR_PB_HTYPE_FIXARRAY(structname, fieldname) pb_arraysize(structname, fieldname[0])
 
-#define PB_DATA_SIZE_STATIC(htype, structname, fieldname) PB_DATA_SIZE_ ## htype(structname, fieldname)
-#define PB_DATA_SIZE_POINTER(htype, structname, fieldname) PB_DATA_SIZE_PTR_ ## htype(structname, fieldname)
-#define PB_DATA_SIZE_CALLBACK(htype, structname, fieldname) PB_DATA_SIZE_CB_ ## htype(structname, fieldname)
-#define PB_DATA_SIZE_REQUIRED(structname, fieldname) pb_membersize(structname, fieldname)
-#define PB_DATA_SIZE_SINGULAR(structname, fieldname) pb_membersize(structname, fieldname)
-#define PB_DATA_SIZE_OPTIONAL(structname, fieldname) pb_membersize(structname, fieldname)
-#define PB_DATA_SIZE_ONEOF(structname, fieldname) pb_membersize(structname, PB_ONEOF_NAME(FULL, fieldname))
-#define PB_DATA_SIZE_REPEATED(structname, fieldname) pb_membersize(structname, fieldname[0])
-#define PB_DATA_SIZE_FIXARRAY(structname, fieldname) pb_membersize(structname, fieldname[0])
-#define PB_DATA_SIZE_PTR_REQUIRED(structname, fieldname) pb_membersize(structname, fieldname[0])
-#define PB_DATA_SIZE_PTR_SINGULAR(structname, fieldname) pb_membersize(structname, fieldname[0])
-#define PB_DATA_SIZE_PTR_OPTIONAL(structname, fieldname) pb_membersize(structname, fieldname[0])
-#define PB_DATA_SIZE_PTR_ONEOF(structname, fieldname) pb_membersize(structname, PB_ONEOF_NAME(FULL, fieldname)[0])
-#define PB_DATA_SIZE_PTR_REPEATED(structname, fieldname) pb_membersize(structname, fieldname[0])
-#define PB_DATA_SIZE_PTR_FIXARRAY(structname, fieldname) pb_membersize(structname, fieldname[0][0])
-#define PB_DATA_SIZE_CB_REQUIRED(structname, fieldname) pb_membersize(structname, fieldname)
-#define PB_DATA_SIZE_CB_SINGULAR(structname, fieldname) pb_membersize(structname, fieldname)
-#define PB_DATA_SIZE_CB_OPTIONAL(structname, fieldname) pb_membersize(structname, fieldname)
-#define PB_DATA_SIZE_CB_ONEOF(structname, fieldname) pb_membersize(structname, PB_ONEOF_NAME(FULL, fieldname))
-#define PB_DATA_SIZE_CB_REPEATED(structname, fieldname) pb_membersize(structname, fieldname)
-#define PB_DATA_SIZE_CB_FIXARRAY(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DATA_SIZE_STATIC(htype, structname, fieldname) PB_DS ## htype(structname, fieldname)
+#define PB_DATA_SIZE_POINTER(htype, structname, fieldname) PB_DS_PTR ## htype(structname, fieldname)
+#define PB_DATA_SIZE_CALLBACK(htype, structname, fieldname) PB_DS_CB ## htype(structname, fieldname)
+#define PB_DS_PB_HTYPE_REQUIRED(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DS_PB_HTYPE_SINGULAR(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DS_PB_HTYPE_OPTIONAL(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DS_PB_HTYPE_ONEOF(structname, fieldname) pb_membersize(structname, PB_ONEOF_NAME(FULL, fieldname))
+#define PB_DS_PB_HTYPE_REPEATED(structname, fieldname) pb_membersize(structname, fieldname[0])
+#define PB_DS_PB_HTYPE_FIXARRAY(structname, fieldname) pb_membersize(structname, fieldname[0])
+#define PB_DS_PTR_PB_HTYPE_REQUIRED(structname, fieldname) pb_membersize(structname, fieldname[0])
+#define PB_DS_PTR_PB_HTYPE_SINGULAR(structname, fieldname) pb_membersize(structname, fieldname[0])
+#define PB_DS_PTR_PB_HTYPE_OPTIONAL(structname, fieldname) pb_membersize(structname, fieldname[0])
+#define PB_DS_PTR_PB_HTYPE_ONEOF(structname, fieldname) pb_membersize(structname, PB_ONEOF_NAME(FULL, fieldname)[0])
+#define PB_DS_PTR_PB_HTYPE_REPEATED(structname, fieldname) pb_membersize(structname, fieldname[0])
+#define PB_DS_PTR_PB_HTYPE_FIXARRAY(structname, fieldname) pb_membersize(structname, fieldname[0][0])
+#define PB_DS_CB_PB_HTYPE_REQUIRED(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DS_CB_PB_HTYPE_SINGULAR(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DS_CB_PB_HTYPE_OPTIONAL(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DS_CB_PB_HTYPE_ONEOF(structname, fieldname) pb_membersize(structname, PB_ONEOF_NAME(FULL, fieldname))
+#define PB_DS_CB_PB_HTYPE_REPEATED(structname, fieldname) pb_membersize(structname, fieldname)
+#define PB_DS_CB_PB_HTYPE_FIXARRAY(structname, fieldname) pb_membersize(structname, fieldname)
 
 #define PB_ONEOF_NAME(type, tuple) PB_EXPAND(PB_ONEOF_NAME_ ## type tuple)
 #define PB_ONEOF_NAME_UNION(unionname,membername,fullname) unionname
@@ -607,37 +652,37 @@ struct pb_extension_s {
 #define PB_ONEOF_NAME_FULL(unionname,membername,fullname) fullname
 
 #define PB_GEN_SUBMSG_INFO(structname, atype, htype, ltype, fieldname, tag) \
-    PB_SUBMSG_INFO_ ## htype(ltype, structname, fieldname)
+    PB_SUBMSG_INFO_ ## htype(_PB_LTYPE_ ## ltype, structname, fieldname)
 
-#define PB_SUBMSG_INFO_REQUIRED(ltype, structname, fieldname) PB_SUBMSG_INFO_ ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
-#define PB_SUBMSG_INFO_SINGULAR(ltype, structname, fieldname) PB_SUBMSG_INFO_ ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
-#define PB_SUBMSG_INFO_OPTIONAL(ltype, structname, fieldname) PB_SUBMSG_INFO_ ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
+#define PB_SUBMSG_INFO_REQUIRED(ltype, structname, fieldname) PB_SI ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
+#define PB_SUBMSG_INFO_SINGULAR(ltype, structname, fieldname) PB_SI ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
+#define PB_SUBMSG_INFO_OPTIONAL(ltype, structname, fieldname) PB_SI ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
 #define PB_SUBMSG_INFO_ONEOF(ltype, structname, fieldname) PB_SUBMSG_INFO_ONEOF2(ltype, structname, PB_ONEOF_NAME(UNION, fieldname), PB_ONEOF_NAME(MEMBER, fieldname))
 #define PB_SUBMSG_INFO_ONEOF2(ltype, structname, unionname, membername) PB_SUBMSG_INFO_ONEOF3(ltype, structname, unionname, membername)
-#define PB_SUBMSG_INFO_ONEOF3(ltype, structname, unionname, membername) PB_SUBMSG_INFO_ ## ltype(structname ## _ ## unionname ## _ ## membername ## _MSGTYPE)
-#define PB_SUBMSG_INFO_REPEATED(ltype, structname, fieldname) PB_SUBMSG_INFO_ ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
-#define PB_SUBMSG_INFO_FIXARRAY(ltype, structname, fieldname) PB_SUBMSG_INFO_ ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
-#define PB_SUBMSG_INFO_BOOL(t)
-#define PB_SUBMSG_INFO_BYTES(t)
-#define PB_SUBMSG_INFO_DOUBLE(t)
-#define PB_SUBMSG_INFO_ENUM(t)
-#define PB_SUBMSG_INFO_UENUM(t)
-#define PB_SUBMSG_INFO_FIXED32(t)
-#define PB_SUBMSG_INFO_FIXED64(t)
-#define PB_SUBMSG_INFO_FLOAT(t)
-#define PB_SUBMSG_INFO_INT32(t)
-#define PB_SUBMSG_INFO_INT64(t)
-#define PB_SUBMSG_INFO_MESSAGE(t)  PB_SUBMSG_DESCRIPTOR(t)
-#define PB_SUBMSG_INFO_MSG_W_CB(t) PB_SUBMSG_DESCRIPTOR(t)
-#define PB_SUBMSG_INFO_SFIXED32(t)
-#define PB_SUBMSG_INFO_SFIXED64(t)
-#define PB_SUBMSG_INFO_SINT32(t)
-#define PB_SUBMSG_INFO_SINT64(t)
-#define PB_SUBMSG_INFO_STRING(t)
-#define PB_SUBMSG_INFO_UINT32(t)
-#define PB_SUBMSG_INFO_UINT64(t)
-#define PB_SUBMSG_INFO_EXTENSION(t)
-#define PB_SUBMSG_INFO_FIXED_LENGTH_BYTES(t)
+#define PB_SUBMSG_INFO_ONEOF3(ltype, structname, unionname, membername) PB_SI ## ltype(structname ## _ ## unionname ## _ ## membername ## _MSGTYPE)
+#define PB_SUBMSG_INFO_REPEATED(ltype, structname, fieldname) PB_SI ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
+#define PB_SUBMSG_INFO_FIXARRAY(ltype, structname, fieldname) PB_SI ## ltype(structname ## _ ## fieldname ## _MSGTYPE)
+#define PB_SI_PB_LTYPE_BOOL(t)
+#define PB_SI_PB_LTYPE_BYTES(t)
+#define PB_SI_PB_LTYPE_DOUBLE(t)
+#define PB_SI_PB_LTYPE_ENUM(t)
+#define PB_SI_PB_LTYPE_UENUM(t)
+#define PB_SI_PB_LTYPE_FIXED32(t)
+#define PB_SI_PB_LTYPE_FIXED64(t)
+#define PB_SI_PB_LTYPE_FLOAT(t)
+#define PB_SI_PB_LTYPE_INT32(t)
+#define PB_SI_PB_LTYPE_INT64(t)
+#define PB_SI_PB_LTYPE_MESSAGE(t)  PB_SUBMSG_DESCRIPTOR(t)
+#define PB_SI_PB_LTYPE_MSG_W_CB(t) PB_SUBMSG_DESCRIPTOR(t)
+#define PB_SI_PB_LTYPE_SFIXED32(t)
+#define PB_SI_PB_LTYPE_SFIXED64(t)
+#define PB_SI_PB_LTYPE_SINT32(t)
+#define PB_SI_PB_LTYPE_SINT64(t)
+#define PB_SI_PB_LTYPE_STRING(t)
+#define PB_SI_PB_LTYPE_UINT32(t)
+#define PB_SI_PB_LTYPE_UINT64(t)
+#define PB_SI_PB_LTYPE_EXTENSION(t)
+#define PB_SI_PB_LTYPE_FIXED_LENGTH_BYTES(t)
 #define PB_SUBMSG_DESCRIPTOR(t)    &(t ## _msg),
 
 /* The field descriptors use a variable width format, with width of either
@@ -726,37 +771,37 @@ struct pb_extension_s {
  * The generator will give explicit size argument when it knows that a message
  * structure grows beyond 1-word format limits.
  */
-#define PB_FIELDINFO_WIDTH_AUTO(atype, htype, ltype) PB_FIELDINFO_WIDTH_ ## atype(htype, ltype)
-#define PB_FIELDINFO_WIDTH_STATIC(htype, ltype) PB_FIELDINFO_WIDTH_ ## htype(ltype)
-#define PB_FIELDINFO_WIDTH_POINTER(htype, ltype) PB_FIELDINFO_WIDTH_ ## htype(ltype)
-#define PB_FIELDINFO_WIDTH_CALLBACK(htype, ltype) 2
-#define PB_FIELDINFO_WIDTH_REQUIRED(ltype) PB_FIELDINFO_WIDTH_ ## ltype
-#define PB_FIELDINFO_WIDTH_SINGULAR(ltype) PB_FIELDINFO_WIDTH_ ## ltype
-#define PB_FIELDINFO_WIDTH_OPTIONAL(ltype) PB_FIELDINFO_WIDTH_ ## ltype
-#define PB_FIELDINFO_WIDTH_ONEOF(ltype) PB_FIELDINFO_WIDTH_ ## ltype
-#define PB_FIELDINFO_WIDTH_REPEATED(ltype) 2
-#define PB_FIELDINFO_WIDTH_FIXARRAY(ltype) 2
-#define PB_FIELDINFO_WIDTH_BOOL      1
-#define PB_FIELDINFO_WIDTH_BYTES     2
-#define PB_FIELDINFO_WIDTH_DOUBLE    1
-#define PB_FIELDINFO_WIDTH_ENUM      1
-#define PB_FIELDINFO_WIDTH_UENUM     1
-#define PB_FIELDINFO_WIDTH_FIXED32   1
-#define PB_FIELDINFO_WIDTH_FIXED64   1
-#define PB_FIELDINFO_WIDTH_FLOAT     1
-#define PB_FIELDINFO_WIDTH_INT32     1
-#define PB_FIELDINFO_WIDTH_INT64     1
-#define PB_FIELDINFO_WIDTH_MESSAGE   2
-#define PB_FIELDINFO_WIDTH_MSG_W_CB  2
-#define PB_FIELDINFO_WIDTH_SFIXED32  1
-#define PB_FIELDINFO_WIDTH_SFIXED64  1
-#define PB_FIELDINFO_WIDTH_SINT32    1
-#define PB_FIELDINFO_WIDTH_SINT64    1
-#define PB_FIELDINFO_WIDTH_STRING    2
-#define PB_FIELDINFO_WIDTH_UINT32    1
-#define PB_FIELDINFO_WIDTH_UINT64    1
-#define PB_FIELDINFO_WIDTH_EXTENSION 1
-#define PB_FIELDINFO_WIDTH_FIXED_LENGTH_BYTES 2
+#define PB_FIELDINFO_WIDTH_AUTO(atype, htype, ltype) PB_FI_WIDTH ## atype(htype, ltype)
+#define PB_FI_WIDTH_PB_ATYPE_STATIC(htype, ltype) PB_FI_WIDTH ## htype(ltype)
+#define PB_FI_WIDTH_PB_ATYPE_POINTER(htype, ltype) PB_FI_WIDTH ## htype(ltype)
+#define PB_FI_WIDTH_PB_ATYPE_CALLBACK(htype, ltype) 2
+#define PB_FI_WIDTH_PB_HTYPE_REQUIRED(ltype) PB_FI_WIDTH ## ltype
+#define PB_FI_WIDTH_PB_HTYPE_SINGULAR(ltype) PB_FI_WIDTH ## ltype
+#define PB_FI_WIDTH_PB_HTYPE_OPTIONAL(ltype) PB_FI_WIDTH ## ltype
+#define PB_FI_WIDTH_PB_HTYPE_ONEOF(ltype) PB_FI_WIDTH ## ltype
+#define PB_FI_WIDTH_PB_HTYPE_REPEATED(ltype) 2
+#define PB_FI_WIDTH_PB_HTYPE_FIXARRAY(ltype) 2
+#define PB_FI_WIDTH_PB_LTYPE_BOOL      1
+#define PB_FI_WIDTH_PB_LTYPE_BYTES     2
+#define PB_FI_WIDTH_PB_LTYPE_DOUBLE    1
+#define PB_FI_WIDTH_PB_LTYPE_ENUM      1
+#define PB_FI_WIDTH_PB_LTYPE_UENUM     1
+#define PB_FI_WIDTH_PB_LTYPE_FIXED32   1
+#define PB_FI_WIDTH_PB_LTYPE_FIXED64   1
+#define PB_FI_WIDTH_PB_LTYPE_FLOAT     1
+#define PB_FI_WIDTH_PB_LTYPE_INT32     1
+#define PB_FI_WIDTH_PB_LTYPE_INT64     1
+#define PB_FI_WIDTH_PB_LTYPE_MESSAGE   2
+#define PB_FI_WIDTH_PB_LTYPE_MSG_W_CB  2
+#define PB_FI_WIDTH_PB_LTYPE_SFIXED32  1
+#define PB_FI_WIDTH_PB_LTYPE_SFIXED64  1
+#define PB_FI_WIDTH_PB_LTYPE_SINT32    1
+#define PB_FI_WIDTH_PB_LTYPE_SINT64    1
+#define PB_FI_WIDTH_PB_LTYPE_STRING    2
+#define PB_FI_WIDTH_PB_LTYPE_UINT32    1
+#define PB_FI_WIDTH_PB_LTYPE_UINT64    1
+#define PB_FI_WIDTH_PB_LTYPE_EXTENSION 1
+#define PB_FI_WIDTH_PB_LTYPE_FIXED_LENGTH_BYTES 2
 
 /* The mapping from protobuf types to LTYPEs is done using these macros. */
 #define PB_LTYPE_MAP_BOOL               PB_LTYPE_BOOL

lib/nanopb/include/pb_common.h

@@ -32,6 +32,10 @@ bool pb_field_iter_next(pb_field_iter_t *iter);
  * Returns false if no such field exists. */
 bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag);
 
+/* Find a field with type PB_LTYPE_EXTENSION, or return false if not found.
+ * There can be only one extension range field per message. */
+bool pb_field_iter_find_extension(pb_field_iter_t *iter);
+
 #ifdef PB_VALIDATE_UTF8
 /* Validate UTF-8 text string */
 bool pb_validate_utf8(const char *s);

lib/nanopb/include/pb_decode.h

@@ -113,6 +113,9 @@ bool pb_decode_ex(pb_istream_t *stream, const pb_msgdesc_t *fields, void *dest_s
  * pb_decode() returns with an error, the message is already released.
  */
 void pb_release(const pb_msgdesc_t *fields, void *dest_struct);
+#else
+/* Allocation is not supported, so release is no-op */
+#define pb_release(fields, dest_struct) PB_UNUSED(fields); PB_UNUSED(dest_struct);
 #endif
 
 
@@ -121,11 +124,14 @@ void pb_release(const pb_msgdesc_t *fields, void *dest_struct);
  **************************************/
 
 /* Create an input stream for reading from a memory buffer.
+ *
+ * msglen should be the actual length of the message, not the full size of
+ * allocated buffer.
  *
  * Alternatively, you can use a custom stream that reads directly from e.g.
  * a file or a network socket.
  */
-pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize);
+pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t msglen);
 
 /* Function to read from a pb_istream_t. You can use this if you need to
  * read some custom header data, or to read data in field callbacks.