From 6e9cf82b6817fafe6aa13565b87c924c61fdaaa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Wed, 6 Apr 2022 14:11:39 +0200 Subject: [PATCH 01/40] fixes #1355 --- src/mqtt/MQTT.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mqtt/MQTT.cpp b/src/mqtt/MQTT.cpp index 5ecbd56c1..6f2841149 100644 --- a/src/mqtt/MQTT.cpp +++ b/src/mqtt/MQTT.cpp @@ -290,8 +290,8 @@ String MQTT::downstreamPacketToJson(MeshPacket *mp) memset(&scratch, 0, sizeof(scratch)); if (pb_decode_from_bytes(mp->decoded.payload.bytes, mp->decoded.payload.size, &Telemetry_msg, &scratch)) { + decoded = &scratch; if (decoded->which_variant == Telemetry_environment_metrics_tag) { - decoded = &scratch; msgPayload = Json::object{ {"temperature", decoded->variant.environment_metrics.temperature}, {"relative_humidity", decoded->variant.environment_metrics.relative_humidity}, From 10a7071300941055f3b413d51dca4ae300554c35 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Wed, 6 Apr 2022 13:03:44 +0000 Subject: [PATCH 02/40] Remove send owner interval (#1361) * NodeInfo slimfast * Removed send_owner_interval --- proto | 2 +- src/mesh/NodeDB.h | 2 -- src/mesh/generated/admin.pb.h | 2 +- src/mesh/generated/radioconfig.pb.h | 11 ++++------- src/mesh/generated/telemetry.pb.h | 8 ++++---- 5 files changed, 10 insertions(+), 15 deletions(-) diff --git a/proto b/proto index e1ef519d0..c6abcf8d6 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit e1ef519d07b180a05298857374fd60a6165b7a75 +Subproject commit c6abcf8d6b9c9dec630381c50198bed92258d4e8 diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index 551d30932..c9a3a46bc 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -146,7 +146,6 @@ extern NodeDB nodeDB; prefs.ls_secs = oneday - prefs.send_owner_interval = 2 # Send an owner packet every other network ping prefs.position_broadcast_secs = 12 hours # send either position or owner every 12hrs # get a new GPS position once per day @@ -166,7 +165,6 @@ extern NodeDB nodeDB; #define PREF_GET(name, defaultVal) \ inline uint32_t getPref_##name() { return radioConfig.preferences.name ? radioConfig.preferences.name : (defaultVal); } -PREF_GET(send_owner_interval, IF_ROUTER(2, 4)) PREF_GET(position_broadcast_secs, IF_ROUTER(12 * 60 * 60, 15 * 60)) // Defaulting Telemetry to the same as position interval for now PREF_GET(telemetry_module_device_update_interval, IF_ROUTER(12 * 60 * 60, 15 * 60)) diff --git a/src/mesh/generated/admin.pb.h b/src/mesh/generated/admin.pb.h index 1f93abd56..ef9db43cb 100644 --- a/src/mesh/generated/admin.pb.h +++ b/src/mesh/generated/admin.pb.h @@ -129,7 +129,7 @@ extern const pb_msgdesc_t AdminMessage_msg; #define AdminMessage_fields &AdminMessage_msg /* Maximum encoded size of messages (where known) */ -#define AdminMessage_size 610 +#define AdminMessage_size 604 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/generated/radioconfig.pb.h b/src/mesh/generated/radioconfig.pb.h index 5e8dc1d60..7a764e6fc 100644 --- a/src/mesh/generated/radioconfig.pb.h +++ b/src/mesh/generated/radioconfig.pb.h @@ -260,7 +260,6 @@ typedef enum _RadioConfig_UserPreferences_TelemetrySensorType { /* Struct definitions */ typedef struct _RadioConfig_UserPreferences { uint32_t position_broadcast_secs; - uint32_t send_owner_interval; uint32_t wait_bluetooth_secs; uint32_t screen_on_secs; uint32_t phone_timeout_secs; @@ -401,13 +400,12 @@ extern "C" { /* Initializer values for message structs */ #define RadioConfig_init_default {false, RadioConfig_UserPreferences_init_default} -#define RadioConfig_UserPreferences_init_default {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} +#define RadioConfig_UserPreferences_init_default {0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} #define RadioConfig_init_zero {false, RadioConfig_UserPreferences_init_zero} -#define RadioConfig_UserPreferences_init_zero {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} +#define RadioConfig_UserPreferences_init_zero {0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} /* Field tags (for use in manual encoding/decoding) */ #define RadioConfig_UserPreferences_position_broadcast_secs_tag 1 -#define RadioConfig_UserPreferences_send_owner_interval_tag 2 #define RadioConfig_UserPreferences_wait_bluetooth_secs_tag 4 #define RadioConfig_UserPreferences_screen_on_secs_tag 5 #define RadioConfig_UserPreferences_phone_timeout_secs_tag 6 @@ -502,7 +500,6 @@ X(a, STATIC, OPTIONAL, MESSAGE, preferences, 1) #define RadioConfig_UserPreferences_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, UINT32, position_broadcast_secs, 1) \ -X(a, STATIC, SINGULAR, UINT32, send_owner_interval, 2) \ X(a, STATIC, SINGULAR, UINT32, wait_bluetooth_secs, 4) \ X(a, STATIC, SINGULAR, UINT32, screen_on_secs, 5) \ X(a, STATIC, SINGULAR, UINT32, phone_timeout_secs, 6) \ @@ -597,8 +594,8 @@ extern const pb_msgdesc_t RadioConfig_UserPreferences_msg; #define RadioConfig_UserPreferences_fields &RadioConfig_UserPreferences_msg /* Maximum encoded size of messages (where known) */ -#define RadioConfig_UserPreferences_size 604 -#define RadioConfig_size 607 +#define RadioConfig_UserPreferences_size 598 +#define RadioConfig_size 601 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/generated/telemetry.pb.h b/src/mesh/generated/telemetry.pb.h index b3bf3df1d..1fb5ea0d7 100644 --- a/src/mesh/generated/telemetry.pb.h +++ b/src/mesh/generated/telemetry.pb.h @@ -40,10 +40,10 @@ typedef struct _EnvironmentMetrics { /* Types of Measurements the telemetry module is equipped to handle */ typedef struct _Telemetry { - /* This is usually not sent over the mesh (to save space), but it is sent - from the phone so that the local device can set its RTC If it is sent over - the mesh (because there are devices on the mesh without GPS), it will only - be sent by devices which has a hardware GPS clock (IE Mobile Phone). + /* This is usually not sent over the mesh (to save space), but it is sent + from the phone so that the local device can set its RTC If it is sent over + the mesh (because there are devices on the mesh without GPS), it will only + be sent by devices which has a hardware GPS clock (IE Mobile Phone). seconds since 1970 */ uint32_t time; /* Key native device metrics such as battery level */ From a20ba7e686eedd7318aa46bb88ddfb2648ce205e Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Wed, 6 Apr 2022 11:04:11 -0500 Subject: [PATCH 03/40] Add pca10059_diy_eink to boards matrix --- .github/workflows/main_matrix.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main_matrix.yml b/.github/workflows/main_matrix.yml index 2f2e7b25c..8d76cfdac 100644 --- a/.github/workflows/main_matrix.yml +++ b/.github/workflows/main_matrix.yml @@ -162,6 +162,7 @@ jobs: - board: rak4631_19003 - board: rak4631_5005_eink - board: t-echo + - board: pca10059_diy_eink runs-on: ubuntu-latest steps: From 11323acb23cb7c3a4662fe484f8006d99b7edbba Mon Sep 17 00:00:00 2001 From: Sacha Weatherstone Date: Fri, 8 Apr 2022 13:40:41 +1000 Subject: [PATCH 04/40] Update link to site --- src/graphics/Screen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index e30367f5b..04d66ba87 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -256,7 +256,7 @@ static void drawCriticalFaultFrame(OLEDDisplay *display, OLEDDisplayUiState *sta display->drawString(0 + x, 0 + y, tempBuf); display->setTextAlignment(TEXT_ALIGN_LEFT); display->setFont(FONT_SMALL); - display->drawString(0 + x, FONT_HEIGHT_MEDIUM + y, "For help, please post on\nmeshtastic.discourse.group"); + display->drawString(0 + x, FONT_HEIGHT_MEDIUM + y, "For help, please visit \nmeshtastic.org"); } // Ignore messages orginating from phone (from the current node 0x0) unless range test or store and forward module are enabled From 0f5b0b5f005b77a3a6974bee443a9d40b86ee37a Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Fri, 8 Apr 2022 19:51:13 -0700 Subject: [PATCH 05/40] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 65aed143b..2abb45c0f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ [![Open in Visual Studio Code](https://open.vscode.dev/badges/open-in-vscode.svg)](https://open.vscode.dev/meshtastic/Meshtastic-device) [![Continuous Integration](https://github.com/meshtastic/Meshtastic-device/actions/workflows/main.yml/badge.svg)](https://github.com/meshtastic/Meshtastic-device/actions/workflows/main.yml) ![GitHub all releases](https://img.shields.io/github/downloads/meshtastic/meshtastic-device/total) +[![CLA assistant](https://cla-assistant.io/readme/badge/meshtastic/Meshtastic-device)](https://cla-assistant.io/meshtastic/Meshtastic-device) ## This repository contains the device firmware used in the [Meshtastic](https://meshtastic.org) project. From 69ed47704058ff2787f83bd52d1fe3db6f69900d Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sat, 9 Apr 2022 22:42:43 -0700 Subject: [PATCH 06/40] Disable TX if region is unset --- src/mesh/DSRRouter.cpp | 250 --------------------------------- src/mesh/DSRRouter.h | 80 ----------- src/mesh/RadioLibInterface.cpp | 24 ++-- 3 files changed, 15 insertions(+), 339 deletions(-) delete mode 100644 src/mesh/DSRRouter.cpp delete mode 100644 src/mesh/DSRRouter.h diff --git a/src/mesh/DSRRouter.cpp b/src/mesh/DSRRouter.cpp deleted file mode 100644 index c2a170ec7..000000000 --- a/src/mesh/DSRRouter.cpp +++ /dev/null @@ -1,250 +0,0 @@ -#include "configuration.h" -#include "DSRRouter.h" - -/* when we receive any packet - -- sniff and update tables (especially useful to find adjacent nodes). Update user, network and position info. -- if we need to route() that packet, resend it to the next_hop based on our nodedb. -- if it is broadcast or destined for our node, deliver locally -- handle routereply/routeerror/routediscovery messages as described below -- then free it - -routeDiscovery - -- if we've already passed through us (or is from us), then it ignore it -- use the nodes already mentioned in the request to update our routing table -- if they were looking for us, send back a routereply -- if max_hops is zero and they weren't looking for us, drop (FIXME, send back error - I think not though?) -- if we receive a discovery packet, we use it to populate next_hop (if needed) towards the requester (after decrementing max_hops) -- if we receive a discovery packet, and we have a next_hop in our nodedb for that destination we send a (reliable) we send a route -reply towards the requester - -when sending any reliable packet - -- if timeout doing retries, send a routeError (nak) message back towards the original requester. all nodes eavesdrop on that -packet and update their route caches. - -when we receive a routereply packet - -- update next_hop on the node, if the new reply needs fewer hops than the existing one (we prefer shorter paths). fixme, someday -use a better heuristic - -when we receive a routeError packet - -- delete the route for that failed recipient, restartRouteDiscovery() -- if we receive routeerror in response to a discovery, -- fixme, eventually keep caches of possible other routes. -*/ - -ErrorCode DSRRouter::send(MeshPacket *p) -{ - // We only consider multihop routing packets (i.e. those with dest set) - if (p->decoded.dest) { - // add an entry for this pending message - auto pending = startRetransmission(p); - // FIXME - when acks come in for this packet, we should _not_ delete the record unless the ack was from - // the final dest. We need to keep that record around until FIXME - // Also we should not retransmit multihop entries in that table at all - - // If we have an entry in our routing tables, just send it, otherwise start a route discovery - NodeNum nextHop = getNextHop(p->decoded.dest); - if (nextHop) { - sendNextHop(nextHop, p); // start a reliable single hop send - } else { - pending->wantRoute = true; - - // start discovery, but only if we don't already a discovery in progress for that node number - startDiscovery(p->decoded.dest); - } - - return ERRNO_OK; - } else - return ReliableRouter::send(p); -} - -void DSRRouter::sniffReceived(const MeshPacket *p, const Routing *c) -{ - // Learn 0 hop routes by just hearing any adjacent nodes - // But treat broadcasts carefully, because when flood broadcasts go out they keep the same original "from". So we want to - // ignore rebroadcasts. - // this will also add records for any ACKs we receive for our messages - if (p->to != NODENUM_BROADCAST || p->hop_limit != HOP_RELIABLE) { - addRoute(getFrom(p), getFrom(p), 0); // We are adjacent with zero hops - } - - if (c) - switch (c->which_variant) { - case Routing_route_request_tag: - // Handle route discovery packets (will be a broadcast message) - // FIXME - always start request with the senders nodenum - if (weAreInRoute(c->route_request)) { - DEBUG_MSG("Ignoring a route request that contains us\n"); - } else { - updateRoutes(c->route_request, - true); // Update our routing tables based on the route that came in so far on this request - - if (p->decoded.dest == getNodeNum()) { - // They were looking for us, send back a route reply (the sender address will be first in the list) - sendRouteReply(c->route_request); - } else { - // They were looking for someone else, forward it along (as a zero hop broadcast) - NodeNum nextHop = getNextHop(p->decoded.dest); - if (nextHop) { - // in our route cache, reply to the requester (the sender address will be first in the list) - sendRouteReply(c->route_request, nextHop); - } else { - // Not in our route cache, rebroadcast on their behalf (after adding ourselves to the request route) - resendRouteRequest(p); - } - } - } - break; - case Routing_route_reply_tag: - updateRoutes(c->route_reply, false); - - // FIXME, if any of our current pending packets were waiting for this route, send them (and leave them as regular - // pending packets until ack arrives) - // FIXME, if we don't get a route reply at all (or a route error), timeout and generate a routeerror TIMEOUT on our - // own... - break; - case Routing_error_reason_tag: - removeRoute(p->decoded.dest); - - // FIXME: if any pending packets were waiting on this route, delete them - break; - default: - break; - } - - // We simply ignore ACKs - because ReliableRouter will delete the pending packet for us - - // Handle regular packets - if (p->to == getNodeNum()) { // Destined for us (at least for this hop) - - // We need to route this packet to some other node - if (p->decoded.dest && p->decoded.dest != p->to) { - // if we have a route out, resend the packet to the next hop, otherwise return RouteError no-route available - - NodeNum nextHop = getNextHop(p->decoded.dest); - if (nextHop) { - sendNextHop(nextHop, p); // start a reliable single hop send - } else { - // We don't have a route out - assert(p->decoded.source); // I think this is guaranteed by now - - // FIXME - what if the current packet _is_ a route error packet? - sendRouteError(p, Routing_Error_NO_ROUTE); - } - - // FIXME, stop local processing of this packet - } - - if (c) { - // handle naks - convert them to route error packets - // All naks are generated locally, because we failed resending the packet too many times - PacketId nakId = c->error_reason ? p->decoded.request_id : 0; - if (nakId) { - auto pending = findPendingPacket(p->to, nakId); - if (pending && - pending->packet->decoded.source) { // if source not set, this was not a multihop packet, just ignore - removeRoute(pending->packet->decoded.dest); // We no longer have a route to the specified node - - sendRouteError(p, Routing_Error_GOT_NAK); - } - } - } - } - - ReliableRouter::sniffReceived(p, c); -} - -/** - * Does our node appear in the specified route - */ -bool DSRRouter::weAreInRoute(const RouteDiscovery &route) -{ - return true; // FIXME -} - -/** - * Given a DSR route, use that route to update our DB of possible routes - * - * Note: routes are always listed in the same order - from sender to receipient (i.e. route_replies also use this some order) - * - * @param isRequest is true if we are looking at a route request, else we are looking at a reply - **/ -void DSRRouter::updateRoutes(const RouteDiscovery &route, bool isRequest) -{ - DEBUG_MSG("FIXME not implemented updateRoutes\n"); -} - -/** - * send back a route reply (the sender address will be first in the list) - */ -void DSRRouter::sendRouteReply(const RouteDiscovery &route, NodeNum toAppend) -{ - DEBUG_MSG("FIXME not implemented sendRoute\n"); -} - -/** - * Given a nodenum return the next node we should forward to if we want to reach that node. - * - * @return 0 if no route found - */ -NodeNum DSRRouter::getNextHop(NodeNum dest) -{ - DEBUG_MSG("FIXME not implemented getNextHop\n"); - return 0; -} - -/** Not in our route cache, rebroadcast on their behalf (after adding ourselves to the request route) - * - * We will bump down hop_limit in this call. - */ -void DSRRouter::resendRouteRequest(const MeshPacket *p) -{ - DEBUG_MSG("FIXME not implemented resendRoute\n"); -} - -/** - * Record that forwarder can reach dest for us, but they will need numHops to get there. - * If our routing tables already have something that can reach that node in fewer hops we will keep the existing route - * instead. - */ -void DSRRouter::addRoute(NodeNum dest, NodeNum forwarder, uint8_t numHops) -{ - DEBUG_MSG("FIXME not implemented addRoute\n"); -} - -/** - * Record that we no longer have a route to the dest - */ -void DSRRouter::removeRoute(NodeNum dest) -{ - DEBUG_MSG("FIXME not implemented removeRoute\n"); -} - -/** - * Forward the specified packet to the specified node - */ -void DSRRouter::sendNextHop(NodeNum n, const MeshPacket *p) -{ - DEBUG_MSG("FIXME not implemented sendNextHop\n"); -} - -/** - * Send a route error packet towards whoever originally sent this message - */ -void DSRRouter::sendRouteError(const MeshPacket *p, Routing_Error err) -{ - DEBUG_MSG("FIXME not implemented sendRouteError\n"); -} - -/** make a copy of p, start discovery, but only if we don't - * already a discovery in progress for that node number. Caller has already scheduled this message for retransmission - * when the discovery is complete. - */ -void DSRRouter::startDiscovery(NodeNum dest) -{ - DEBUG_MSG("FIXME not implemented startDiscovery\n"); -} \ No newline at end of file diff --git a/src/mesh/DSRRouter.h b/src/mesh/DSRRouter.h deleted file mode 100644 index 0caa9310a..000000000 --- a/src/mesh/DSRRouter.h +++ /dev/null @@ -1,80 +0,0 @@ -#include "ReliableRouter.h" - -class DSRRouter : public ReliableRouter -{ - - protected: - /** - * Every (non duplicate) packet this node receives will be passed through this method. This allows subclasses to - * update routing tables etc... based on what we overhear (even for messages not destined to our node) - */ - virtual void sniffReceived(const MeshPacket *p, const Routing *c) override; - - /** - * Send a packet on a suitable interface. This routine will - * later free() the packet to pool. This routine is not allowed to stall. - * If the txmit queue is full it might return an error - */ - virtual ErrorCode send(MeshPacket *p) override; - - private: - /** - * Does our node appear in the specified route - */ - bool weAreInRoute(const RouteDiscovery &route); - - /** - * Given a DSR route, use that route to update our DB of possible routes - * - * Note: routes are always listed in the same order - from sender to receipient (i.e. route_replies also use this some order) - * - * @param isRequest is true if we are looking at a route request, else we are looking at a reply - **/ - void updateRoutes(const RouteDiscovery &route, bool isRequest); - - /** - * send back a route reply (the sender address will be first in the list) - */ - void sendRouteReply(const RouteDiscovery &route, NodeNum toAppend = 0); - - /** - * Given a nodenum return the next node we should forward to if we want to reach that node. - * - * @return 0 if no route found - */ - NodeNum getNextHop(NodeNum dest); - - /** Not in our route cache, rebroadcast on their behalf (after adding ourselves to the request route) - * - * We will bump down hop_limit in this call. - */ - void resendRouteRequest(const MeshPacket *p); - - /** - * Record that forwarder can reach dest for us, but they will need numHops to get there. - * If our routing tables already have something that can reach that node in fewer hops we will keep the existing route - * instead. - */ - void addRoute(NodeNum dest, NodeNum forwarder, uint8_t numHops); - - /** - * Record that we no longer have a route to the dest - */ - void removeRoute(NodeNum dest); - - /** - * Forward the specified packet to the specified node - */ - void sendNextHop(NodeNum n, const MeshPacket *p); - - /** - * Send a route error packet towards whoever originally sent this message - */ - void sendRouteError(const MeshPacket *p, Routing_Error err); - - /** make a copy of p, start discovery, but only if we don't - * already a discovery in progress for that node number. Caller has already scheduled this message for retransmission - * when the discovery is complete. - */ - void startDiscovery(NodeNum dest); -}; diff --git a/src/mesh/RadioLibInterface.cpp b/src/mesh/RadioLibInterface.cpp index 0e55e23b0..5f0e57210 100644 --- a/src/mesh/RadioLibInterface.cpp +++ b/src/mesh/RadioLibInterface.cpp @@ -1,8 +1,8 @@ -#include "configuration.h" #include "RadioLibInterface.h" #include "MeshTypes.h" #include "NodeDB.h" #include "SPILock.h" +#include "configuration.h" #include "error.h" #include "mesh-pb-constants.h" #include @@ -93,8 +93,15 @@ bool RadioLibInterface::canSendImmediately() /// bluetooth comms code. If the txmit queue is empty it might return an error ErrorCode RadioLibInterface::send(MeshPacket *p) { - if (disabled || radioConfig.preferences.is_lora_tx_disabled) { - DEBUG_MSG("send - lora_tx_disabled\n"); + if (radioConfig.preferences.region != RegionCode_Unset) { + if (disabled || radioConfig.preferences.is_lora_tx_disabled) { + DEBUG_MSG("send - lora_tx_disabled\n"); + packetPool.release(p); + return ERRNO_DISABLED; + } + + } else { + DEBUG_MSG("send - lora_tx_disabled because RegionCode_Unset\n"); packetPool.release(p); return ERRNO_DISABLED; } @@ -115,9 +122,9 @@ ErrorCode RadioLibInterface::send(MeshPacket *p) // in response to a packet we just received. So we want to make sure the other side has had a chance to reconfigure its radio /* We assume if rx_snr = 0 and rx_rssi = 0, the packet was not generated locally. - * This assumption is valid because of the offset generated by the radio to account for the noise - * floor. - */ + * This assumption is valid because of the offset generated by the radio to account for the noise + * floor. + */ if (p->rx_snr == 0 && p->rx_rssi == 0) { startTransmitTimer(true); } else { @@ -125,7 +132,6 @@ ErrorCode RadioLibInterface::send(MeshPacket *p) DEBUG_MSG("rx_snr found. hop_limit:%d rx_snr:%f\n", p->hop_limit, p->rx_snr); startTransmitTimerSNR(p->rx_snr); } - return res; #else @@ -311,7 +317,7 @@ void RadioLibInterface::handleReceiveInterrupt() printPacket("Lora RX", mp); - //xmitMsec = getPacketTime(mp); + // xmitMsec = getPacketTime(mp); airTime->logAirtime(RX_LOG, xmitMsec); deliverToReceiver(mp); @@ -334,7 +340,7 @@ void RadioLibInterface::startSend(MeshPacket *txp) size_t numbytes = beginSending(txp); int res = iface->startTransmit(radiobuf, numbytes); - if(res != ERR_NONE) { + if (res != ERR_NONE) { RECORD_CRITICALERROR(CriticalErrorCode_RadioSpiBug); // This send failed, but make sure to 'complete' it properly From 16ae867c2d0fb30927bbdc6a25b6fc3e1e4f92e2 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 10 Apr 2022 19:15:10 -0700 Subject: [PATCH 07/40] Show welcome screen if region unset --- platformio.ini | 2 +- src/graphics/Screen.cpp | 56 +++++++++++++++++++++++++++++++++-------- src/graphics/Screen.h | 2 ++ 3 files changed, 48 insertions(+), 12 deletions(-) diff --git a/platformio.ini b/platformio.ini index b2c375901..e692888c2 100644 --- a/platformio.ini +++ b/platformio.ini @@ -42,7 +42,7 @@ build_flags = -Wno-missing-field-initializers monitor_speed = 921600 lib_deps = - https://github.com/meshtastic/esp8266-oled-ssd1306.git ; ESP8266_SSD1306 + https://github.com/meshtastic/esp8266-oled-ssd1306.git#53580644255b48ebb7a737343c6b4e71c7e11cf2 ; ESP8266_SSD1306 mathertel/OneButton@^2.0.3 ; OneButton library for non-blocking button debounce 1202 ; CRC32, explicitly needed because dependency is missing in the ble ota update lib https://github.com/meshtastic/arduino-fsm.git diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index 04d66ba87..d00106f0a 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -168,6 +168,28 @@ static void drawSSLScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int16 } } +// Used when booting without a region set +static void drawWelcomeScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int16_t x, int16_t y) +{ + + display->setFont(FONT_SMALL); + + display->setTextAlignment(TEXT_ALIGN_CENTER); + display->drawString(64 + x, y, "Welcome to Meshtastic!"); + + display->setTextAlignment(TEXT_ALIGN_LEFT); + + display->drawString(x, y + FONT_HEIGHT_SMALL * 2 - 3, "Set the region using the"); + display->drawString(x, y + FONT_HEIGHT_SMALL * 3 - 3, "Android, iOS, Flasher or"); + display->drawString(x, y + FONT_HEIGHT_SMALL * 4 - 3, "CLI."); + +#ifndef NO_ESP32 + yield(); + esp_task_wdt_reset(); +#endif + +} + #ifdef HAS_EINK /// Used on eink displays while in deep sleep static void drawSleepScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int16_t x, int16_t y) @@ -766,7 +788,7 @@ void Screen::setup() useDisplay = true; #ifdef AutoOLEDWire_h - dispdev.setDetected(screen_model); + dispdev.setDetected(screen_model); #endif // I think this is not needed - redundant with ui.init @@ -862,6 +884,10 @@ int32_t Screen::runOnce() showingBootScreen = false; } + if (radioConfig.preferences.region == RegionCode_Unset) { + setWelcomeFrames(); + } + // Process incoming commands. for (;;) { ScreenCmd cmd; @@ -975,6 +1001,18 @@ void Screen::setSSLFrames() } } +/* show a message that the SSL cert is being built + * it is expected that this will be used during the boot phase */ +void Screen::setWelcomeFrames() +{ + if (address_found) { + // DEBUG_MSG("showing Welcome frames\n"); + static FrameCallback welcomeFrames[] = {drawWelcomeScreen}; + ui.setFrames(welcomeFrames, 1); + ui.update(); + } +} + // restore our regular frame list void Screen::setFrames() { @@ -1366,7 +1404,7 @@ void DebugInfo::drawFrameSettings(OLEDDisplay *display, OLEDDisplayUiState *stat } auto mode = ""; - + if (channels.getPrimary().modem_config == 0) { mode = "VLongSlow"; } else if (channels.getPrimary().modem_config == 1) { @@ -1380,7 +1418,7 @@ void DebugInfo::drawFrameSettings(OLEDDisplay *display, OLEDDisplayUiState *stat } else if (channels.getPrimary().modem_config == 5) { mode = "ShortSlow"; } else if (channels.getPrimary().modem_config == 6) { - mode = "ShortFast"; + mode = "ShortFast"; } else { mode = "Custom"; } @@ -1432,8 +1470,7 @@ void DebugInfo::drawFrameSettings(OLEDDisplay *display, OLEDDisplayUiState *stat // Display Channel Utilization char chUtil[13]; sprintf(chUtil, "ChUtil %2.0f%%", airTime->channelUtilizationPercent()); - display->drawString(x + SCREEN_WIDTH - display->getStringWidth(chUtil), - y + FONT_HEIGHT_SMALL * 1, chUtil); + display->drawString(x + SCREEN_WIDTH - display->getStringWidth(chUtil), y + FONT_HEIGHT_SMALL * 1, chUtil); // Line 3 if (radioConfig.preferences.gps_format != GpsCoordinateFormat_GpsFormatDMS) // if DMS then don't draw altitude @@ -1491,16 +1528,13 @@ int Screen::handleTextMessage(const MeshPacket *packet) int Screen::handleUIFrameEvent(const UIFrameEvent *event) { if (showingNormalScreen) { - if (event->frameChanged) - { + if (event->frameChanged) { setFrames(); // Regen the list of screens (will show new text message) - } - else if (event->needRedraw) - { + } else if (event->needRedraw) { setFastFramerate(); // TODO: We might also want switch to corresponding frame, // but we don't know the exact frame number. - //ui.switchToFrame(0); + // ui.switchToFrame(0); } } diff --git a/src/graphics/Screen.h b/src/graphics/Screen.h index 2242532d7..59d68c902 100644 --- a/src/graphics/Screen.h +++ b/src/graphics/Screen.h @@ -235,6 +235,8 @@ class Screen : public concurrency::OSThread /// Draws our SSL cert screen during boot (called from WebServer) void setSSLFrames(); + void setWelcomeFrames(); + protected: /// Updates the UI. // From 14eef4276252a173c898de394165d9a3e987d3ba Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 10 Apr 2022 19:27:11 -0700 Subject: [PATCH 08/40] Disable indicator on welcome screen. --- src/graphics/Screen.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index d00106f0a..7bc7ea170 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -1007,6 +1007,8 @@ void Screen::setWelcomeFrames() { if (address_found) { // DEBUG_MSG("showing Welcome frames\n"); + ui.disableAllIndicators(); + static FrameCallback welcomeFrames[] = {drawWelcomeScreen}; ui.setFrames(welcomeFrames, 1); ui.update(); From 6eab8f84d10b0cee917082651793c70c5712190e Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Sun, 10 Apr 2022 19:34:13 -0700 Subject: [PATCH 09/40] Updated text for better fit --- src/graphics/Screen.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index 7bc7ea170..0ac19cbce 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -175,13 +175,13 @@ static void drawWelcomeScreen(OLEDDisplay *display, OLEDDisplayUiState *state, i display->setFont(FONT_SMALL); display->setTextAlignment(TEXT_ALIGN_CENTER); - display->drawString(64 + x, y, "Welcome to Meshtastic!"); + display->drawString(64 + x, y, "Meshtastic :)"); display->setTextAlignment(TEXT_ALIGN_LEFT); display->drawString(x, y + FONT_HEIGHT_SMALL * 2 - 3, "Set the region using the"); - display->drawString(x, y + FONT_HEIGHT_SMALL * 3 - 3, "Android, iOS, Flasher or"); - display->drawString(x, y + FONT_HEIGHT_SMALL * 4 - 3, "CLI."); + display->drawString(x, y + FONT_HEIGHT_SMALL * 3 - 3, "Meshtastic Android, iOS,"); + display->drawString(x, y + FONT_HEIGHT_SMALL * 4 - 3, "Flasher or CLI client."); #ifndef NO_ESP32 yield(); From 340737f2a8f7fcc17326cc9caa072adeea271f3c Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Mon, 11 Apr 2022 18:53:29 -0700 Subject: [PATCH 10/40] Work around for issue with unknown age due to not having time Work around for issue with unknown age due to not having time --- src/graphics/Screen.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index 0ac19cbce..0ccc46b22 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -668,8 +668,18 @@ static void drawNodeInfo(OLEDDisplay *display, OLEDDisplayUiState *state, int16_ snprintf(lastStr, sizeof(lastStr), "%u seconds ago", agoSecs); else if (agoSecs < 120 * 60) // last 2 hrs snprintf(lastStr, sizeof(lastStr), "%u minutes ago", agoSecs / 60); - else - snprintf(lastStr, sizeof(lastStr), "%u hours ago", agoSecs / 60 / 60); + else { + + uint32_t hours_in_month = 730; + + // Only show hours ago if it's been less than 6 months. Otherwise, we may have bad + // data. + if ((agoSecs / 60 / 60) < (hours_in_month * 6)) { + snprintf(lastStr, sizeof(lastStr), "%u hours ago", agoSecs / 60 / 60); + } else { + snprintf(lastStr, sizeof(lastStr), "unknown age"); + } + } static char distStr[20]; strcpy(distStr, "? km"); // might not have location data From 478274aff13249065523bffb59094c9c9e893a50 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Mon, 11 Apr 2022 20:09:48 -0700 Subject: [PATCH 11/40] Beginning of compression --- src/mesh/Router.cpp | 49 +- src/mesh/compression/unishox2.c | 1345 +++++++++++++++++++++++++++++++ src/mesh/compression/unishox2.h | 278 +++++++ 3 files changed, 1660 insertions(+), 12 deletions(-) create mode 100644 src/mesh/compression/unishox2.c create mode 100644 src/mesh/compression/unishox2.h diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index 08de01ccd..34ab85a26 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -1,12 +1,15 @@ -#include "configuration.h" #include "Router.h" #include "Channels.h" #include "CryptoEngine.h" #include "NodeDB.h" #include "RTC.h" +#include "configuration.h" #include "main.h" #include "mesh-pb-constants.h" #include "modules/RoutingModule.h" +extern "C" { +#include "mesh/compression/unishox2.h" +} #if defined(HAS_WIFI) || defined(PORTDUINO) #include "mqtt/MQTT.h" @@ -210,29 +213,27 @@ ErrorCode Router::send(MeshPacket *p) if (p->which_payloadVariant == MeshPacket_decoded_tag) { ChannelIndex chIndex = p->channel; // keep as a local because we are about to change it - - #if defined(HAS_WIFI) || defined(PORTDUINO) - //check if we should send decrypted packets to mqtt + // check if we should send decrypted packets to mqtt - //truth table: + // truth table: /* mqtt_server mqtt_encryption_enabled should_encrypt * not set 0 1 * not set 1 1 * set 0 0 * set 1 1 - * + * * => so we only decrypt mqtt if they have a custom mqtt server AND mqtt_encryption_enabled is FALSE - */ + */ bool shouldActuallyEncrypt = true; if (*radioConfig.preferences.mqtt_server && !radioConfig.preferences.mqtt_encryption_enabled) { shouldActuallyEncrypt = false; } - + DEBUG_MSG("Should encrypt MQTT?: %d\n", shouldActuallyEncrypt); - //the packet is currently in a decrypted state. send it now if they want decrypted packets + // the packet is currently in a decrypted state. send it now if they want decrypted packets if (mqtt && !shouldActuallyEncrypt) mqtt->onSend(*p, chIndex); #endif @@ -244,8 +245,8 @@ ErrorCode Router::send(MeshPacket *p) } #if defined(HAS_WIFI) || defined(PORTDUINO) - //the packet is now encrypted. - //check if we should send encrypted packets to mqtt + // the packet is now encrypted. + // check if we should send encrypted packets to mqtt if (mqtt && shouldActuallyEncrypt) mqtt->onSend(*p, chIndex); #endif @@ -276,7 +277,7 @@ bool perhapsDecode(MeshPacket *p) if (p->which_payloadVariant == MeshPacket_decoded_tag) return true; // If packet was already decoded just return - //assert(p->which_payloadVariant == MeshPacket_encrypted_tag); + // assert(p->which_payloadVariant == MeshPacket_encrypted_tag); // Try to find a channel that works with this hash for (ChannelIndex chIndex = 0; chIndex < channels.getNumChannels(); chIndex++) { @@ -324,6 +325,30 @@ Routing_Error perhapsEncode(MeshPacket *p) size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), Data_fields, &p->decoded); + if (1) { + // int orig_len, compressed_len; + int compressed_len; + char compressed_out[100] = {0}; + // char *orig = (char *)"Hiiiiiiii! :) How are you doing? I am doing well."; + // char *orig = (char *)"Meshtastic"; + + // orig_len = strlen(orig); + // compressed_len = unishox2_compress_simple(orig, orig_len, compressed_out); + compressed_len = unishox2_compress_simple((char *)bytes, numbytes, compressed_out); + + Serial.println(compressed_len); + Serial.println(compressed_out); + //&p->decoded.portnum; + + char decompressed_out[100] = {}; + int decompressed_len; + + decompressed_len = unishox2_decompress_simple(compressed_out, compressed_len, decompressed_out); + + Serial.println(decompressed_len); + Serial.println(decompressed_out); + } + if (numbytes > MAX_RHPACKETLEN) return Routing_Error_TOO_LARGE; diff --git a/src/mesh/compression/unishox2.c b/src/mesh/compression/unishox2.c new file mode 100644 index 000000000..e9d02e8a5 --- /dev/null +++ b/src/mesh/compression/unishox2.c @@ -0,0 +1,1345 @@ +/* + * Copyright (C) 2020 Siara Logics (cc) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @author Arundale Ramanathan + * + */ +/** + * @file unishox2.c + * @author Arundale Ramanathan, James Z. M. Gao + * @brief Main code of Unishox2 Compression and Decompression library + * + * This file implements the code for the Unishox API function \n + * defined in unishox2.h + */ + +#include +#include +#include +#include +#include + +#include "unishox2.h" + +/// byte is unsigned char +typedef unsigned char byte; + +/// possible horizontal sets and states +enum {USX_ALPHA = 0, USX_SYM, USX_NUM, USX_DICT, USX_DELTA, USX_NUM_TEMP}; + +/// This 2D array has the characters for the sets USX_ALPHA, USX_SYM and USX_NUM. Where a character cannot fit into a byte, 0 is used and handled in code. +byte usx_sets[][28] = {{ 0, ' ', 'e', 't', 'a', 'o', 'i', 'n', + 's', 'r', 'l', 'c', 'd', 'h', 'u', 'p', 'm', 'b', + 'g', 'w', 'f', 'y', 'v', 'k', 'q', 'j', 'x', 'z'}, + {'"', '{', '}', '_', '<', '>', ':', '\n', + 0, '[', ']', '\\', ';', '\'', '\t', '@', '*', '&', + '?', '!', '^', '|', '\r', '~', '`', 0, 0, 0}, + { 0, ',', '.', '0', '1', '9', '2', '5', '-', + '/', '3', '4', '6', '7', '8', '(', ')', ' ', + '=', '+', '$', '%', '#', 0, 0, 0, 0, 0}}; + +/// Stores position of letter in usx_sets. +/// First 3 bits - position in usx_hcodes +/// Next 5 bits - position in usx_vcodes +byte usx_code_94[94]; + +/// Vertical codes starting from the MSB +byte usx_vcodes[] = { 0x00, 0x40, 0x60, 0x80, 0x90, 0xA0, 0xB0, + 0xC0, 0xD0, 0xD8, 0xE0, 0xE4, 0xE8, 0xEC, + 0xEE, 0xF0, 0xF2, 0xF4, 0xF6, 0xF7, 0xF8, + 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF }; + +/// Length of each veritical code +byte usx_vcode_lens[] = { 2, 3, 3, 4, 4, 4, 4, + 4, 5, 5, 6, 6, 6, 7, + 7, 7, 7, 7, 8, 8, 8, + 8, 8, 8, 8, 8, 8, 8 }; + +/// Vertical Codes and Set number for frequent sequences in sets USX_SYM and USX_NUM. First 3 bits indicate set (USX_SYM/USX_NUM) and rest are vcode positions +byte usx_freq_codes[] = {(1 << 5) + 25, (1 << 5) + 26, (1 << 5) + 27, (2 << 5) + 23, (2 << 5) + 24, (2 << 5) + 25}; + +/// Not used +const int UTF8_MASK[] = {0xE0, 0xF0, 0xF8}; +/// Not used +const int UTF8_PREFIX[] = {0xC0, 0xE0, 0xF0}; + +/// Minimum length to consider as repeating sequence +#define NICE_LEN 5 + +/// Set (USX_NUM - 2) and vertical code (26) for encoding repeating letters +#define RPT_CODE ((2 << 5) + 26) +/// Set (USX_NUM - 2) and vertical code (27) for encoding terminator +#define TERM_CODE ((2 << 5) + 27) +/// Set (USX_SYM - 1) and vertical code (7) for encoding Line feed \\n +#define LF_CODE ((1 << 5) + 7) +/// Set (USX_NUM - 1) and vertical code (8) for encoding \\r\\n +#define CRLF_CODE ((1 << 5) + 8) +/// Set (USX_NUM - 1) and vertical code (22) for encoding \\r +#define CR_CODE ((1 << 5) + 22) +/// Set (USX_NUM - 1) and vertical code (14) for encoding \\t +#define TAB_CODE ((1 << 5) + 14) +/// Set (USX_NUM - 2) and vertical code (17) for space character when it appears in USX_NUM state \\r +#define NUM_SPC_CODE ((2 << 5) + 17) + +/// Code for special code (11111) when state=USX_DELTA +#define UNI_STATE_SPL_CODE 0xF8 +/// Length of Code for special code when state=USX_DELTA +#define UNI_STATE_SPL_CODE_LEN 5 +/// Code for switch code when state=USX_DELTA +#define UNI_STATE_SW_CODE 0x80 +/// Length of Code for Switch code when state=USX_DELTA +#define UNI_STATE_SW_CODE_LEN 2 + +/// Switch code in USX_ALPHA and USX_NUM 00 +#define SW_CODE 0 +/// Length of Switch code +#define SW_CODE_LEN 2 +/// Terminator bit sequence for Preset 1. Length varies depending on state as per following macros +#define TERM_BYTE_PRESET_1 0 +/// Length of Terminator bit sequence when state is lower +#define TERM_BYTE_PRESET_1_LEN_LOWER 6 +/// Length of Terminator bit sequence when state is upper +#define TERM_BYTE_PRESET_1_LEN_UPPER 4 + +/// Offset at which usx_code_94 starts +#define USX_OFFSET_94 33 + +/// global to indicate whether initialization is complete or not +byte is_inited = 0; + +/// Fills the usx_code_94 94 letter array based on sets of characters at usx_sets \n +/// For each element in usx_code_94, first 3 msb bits is set (USX_ALPHA / USX_SYM / USX_NUM) \n +/// and the rest 5 bits indicate the vertical position in the corresponding set +void init_coder() { + if (is_inited) + return; + memset(usx_code_94, '\0', sizeof(usx_code_94)); + for (int i = 0; i < 3; i++) { + for (int j = 0; j < 28; j++) { + byte c = usx_sets[i][j]; + if (c != 0 && c > 32) { + usx_code_94[c - USX_OFFSET_94] = (i << 5) + j; + if (c >= 'a' && c <= 'z') + usx_code_94[c - USX_OFFSET_94 - ('a' - 'A')] = (i << 5) + j; + } + } + } + is_inited = 1; +} + +/// Mask for retrieving each code to be encoded according to its length +unsigned int usx_mask[] = {0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE, 0xFF}; + +/// Appends specified number of bits to the output (out) \n +/// If maximum limit (olen) is reached, -1 is returned \n +/// Otherwise clen bits in code are appended to out starting with MSB +int append_bits(char *out, int olen, int ol, byte code, int clen) { + + byte cur_bit; + byte blen; + unsigned char a_byte; + int oidx; + + //printf("%d,%x,%d,%d\n", ol, code, clen, state); + + while (clen > 0) { + cur_bit = ol % 8; + blen = clen; + a_byte = code & usx_mask[blen - 1]; + a_byte >>= cur_bit; + if (blen + cur_bit > 8) + blen = (8 - cur_bit); + oidx = ol / 8; + if (oidx < 0 || olen <= oidx) + return -1; + if (cur_bit == 0) + out[oidx] = a_byte; + else + out[oidx] |= a_byte; + code <<= blen; + ol += blen; + clen -= blen; + } + return ol; +} + +/// This is a safe call to append_bits() making sure it does not write past olen +#define SAFE_APPEND_BITS(exp) do { \ + const int newidx = (exp); \ + if (newidx < 0) return newidx; \ +} while (0) + +/// Appends switch code to out depending on the state (USX_DELTA or other) +int append_switch_code(char *out, int olen, int ol, byte state) { + if (state == USX_DELTA) { + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, UNI_STATE_SPL_CODE, UNI_STATE_SPL_CODE_LEN)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, UNI_STATE_SW_CODE, UNI_STATE_SW_CODE_LEN)); + } else + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, SW_CODE, SW_CODE_LEN)); + return ol; +} + +/// Appends given horizontal and veritical code bits to out +int append_code(char *out, int olen, int ol, byte code, byte *state, const byte usx_hcodes[], const byte usx_hcode_lens[]) { + byte hcode = code >> 5; + byte vcode = code & 0x1F; + if (!usx_hcode_lens[hcode] && hcode != USX_ALPHA) + return ol; + switch (hcode) { + case USX_ALPHA: + if (*state != USX_ALPHA) { + SAFE_APPEND_BITS(ol = append_switch_code(out, olen, ol, *state)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + *state = USX_ALPHA; + } + break; + case USX_SYM: + SAFE_APPEND_BITS(ol = append_switch_code(out, olen, ol, *state)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, usx_hcodes[USX_SYM], usx_hcode_lens[USX_SYM])); + break; + case USX_NUM: + if (*state != USX_NUM) { + SAFE_APPEND_BITS(ol = append_switch_code(out, olen, ol, *state)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, usx_hcodes[USX_NUM], usx_hcode_lens[USX_NUM])); + if (usx_sets[hcode][vcode] >= '0' && usx_sets[hcode][vcode] <= '9') + *state = USX_NUM; + } + } + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, usx_vcodes[vcode], usx_vcode_lens[vcode])); + return ol; +} + +/// Length of bits used to represent count for each level +const byte count_bit_lens[5] = {2, 4, 7, 11, 16}; +/// Cumulative counts represented at each level +const int32_t count_adder[5] = {4, 20, 148, 2196, 67732}; +/// Codes used to specify the level that the count belongs to +const byte count_codes[] = {0x01, 0x82, 0xC3, 0xE4, 0xF4}; +/// Encodes given count to out +int encodeCount(char *out, int olen, int ol, int count) { + // First five bits are code and Last three bits of codes represent length + for (int i = 0; i < 5; i++) { + if (count < count_adder[i]) { + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, (count_codes[i] & 0xF8), count_codes[i] & 0x07)); + uint16_t count16 = (count - (i ? count_adder[i - 1] : 0)) << (16 - count_bit_lens[i]); + if (count_bit_lens[i] > 8) { + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, count16 >> 8, 8)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, count16 & 0xFF, count_bit_lens[i] - 8)); + } else + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, count16 >> 8, count_bit_lens[i])); + return ol; + } + } + return ol; +} + +/// Length of bits used to represent delta code for each level +const byte uni_bit_len[5] = {6, 12, 14, 16, 21}; +/// Cumulative delta codes represented at each level +const int32_t uni_adder[5] = {0, 64, 4160, 20544, 86080}; + +/// Encodes the unicode code point given by code to out. prev_code is used to calculate the delta +int encodeUnicode(char *out, int olen, int ol, int32_t code, int32_t prev_code) { + // First five bits are code and Last three bits of codes represent length + //const byte codes[8] = {0x00, 0x42, 0x83, 0xA3, 0xC3, 0xE4, 0xF5, 0xFD}; + const byte codes[6] = {0x01, 0x82, 0xC3, 0xE4, 0xF5, 0xFD}; + int32_t till = 0; + int32_t diff = code - prev_code; + if (diff < 0) + diff = -diff; + //printf("%ld, ", code); + //printf("Diff: %d\n", diff); + for (int i = 0; i < 5; i++) { + till += (1 << uni_bit_len[i]); + if (diff < till) { + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, (codes[i] & 0xF8), codes[i] & 0x07)); + //if (diff) { + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, prev_code > code ? 0x80 : 0, 1)); + int32_t val = diff - uni_adder[i]; + //printf("Val: %d\n", val); + if (uni_bit_len[i] > 16) { + val <<= (24 - uni_bit_len[i]); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, val >> 16, 8)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, (val >> 8) & 0xFF, 8)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, val & 0xFF, uni_bit_len[i] - 16)); + } else + if (uni_bit_len[i] > 8) { + val <<= (16 - uni_bit_len[i]); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, val >> 8, 8)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, val & 0xFF, uni_bit_len[i] - 8)); + } else { + val <<= (8 - uni_bit_len[i]); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, val & 0xFF, uni_bit_len[i])); + } + return ol; + } + } + return ol; +} + +/// Reads UTF-8 character from in. Also returns the number of bytes occupied by the UTF-8 character in utf8len +int32_t readUTF8(const char *in, int len, int l, int *utf8len) { + int32_t ret = 0; + if (l < (len - 1) && (in[l] & 0xE0) == 0xC0 && (in[l + 1] & 0xC0) == 0x80) { + *utf8len = 2; + ret = (in[l] & 0x1F); + ret <<= 6; + ret += (in[l + 1] & 0x3F); + if (ret < 0x80) + ret = 0; + } else + if (l < (len - 2) && (in[l] & 0xF0) == 0xE0 && (in[l + 1] & 0xC0) == 0x80 + && (in[l + 2] & 0xC0) == 0x80) { + *utf8len = 3; + ret = (in[l] & 0x0F); + ret <<= 6; + ret += (in[l + 1] & 0x3F); + ret <<= 6; + ret += (in[l + 2] & 0x3F); + if (ret < 0x0800) + ret = 0; + } else + if (l < (len - 3) && (in[l] & 0xF8) == 0xF0 && (in[l + 1] & 0xC0) == 0x80 + && (in[l + 2] & 0xC0) == 0x80 && (in[l + 3] & 0xC0) == 0x80) { + *utf8len = 4; + ret = (in[l] & 0x07); + ret <<= 6; + ret += (in[l + 1] & 0x3F); + ret <<= 6; + ret += (in[l + 2] & 0x3F); + ret <<= 6; + ret += (in[l + 3] & 0x3F); + if (ret < 0x10000) + ret = 0; + } + return ret; +} + +/// Finds the longest matching sequence from the beginning of the string. \n +/// If a match is found and it is longer than NICE_LEN, it is encoded as a repeating sequence to out \n +/// This is also used for Unicode strings \n +/// This is a crude implementation that is not optimized. Assuming only short strings \n +/// are encoded, this is not much of an issue. +int matchOccurance(const char *in, int len, int l, char *out, int olen, int *ol, byte *state, const byte usx_hcodes[], const byte usx_hcode_lens[]) { + int j, k; + int longest_dist = 0; + int longest_len = 0; + for (j = l - NICE_LEN; j >= 0; j--) { + for (k = l; k < len && j + k - l < l; k++) { + if (in[k] != in[j + k - l]) + break; + } + while ((((unsigned char) in[k]) >> 6) == 2) + k--; // Skip partial UTF-8 matches + //if ((in[k - 1] >> 3) == 0x1E || (in[k - 1] >> 4) == 0x0E || (in[k - 1] >> 5) == 0x06) + // k--; + if ((k - l) > (NICE_LEN - 1)) { + int match_len = k - l - NICE_LEN; + int match_dist = l - j - NICE_LEN + 1; + if (match_len > longest_len) { + longest_len = match_len; + longest_dist = match_dist; + } + } + } + if (longest_len) { + SAFE_APPEND_BITS(*ol = append_switch_code(out, olen, *ol, *state)); + SAFE_APPEND_BITS(*ol = append_bits(out, olen, *ol, usx_hcodes[USX_DICT], usx_hcode_lens[USX_DICT])); + //printf("Len:%d / Dist:%d/%.*s\n", longest_len, longest_dist, longest_len + NICE_LEN, in + l - longest_dist - NICE_LEN + 1); + SAFE_APPEND_BITS(*ol = encodeCount(out, olen, *ol, longest_len)); + SAFE_APPEND_BITS(*ol = encodeCount(out, olen, *ol, longest_dist)); + l += (longest_len + NICE_LEN); + l--; + return l; + } + return -l; +} + +/// This is used only when encoding a string array +/// Finds the longest matching sequence from the previous array element to the beginning of the string array. \n +/// If a match is found and it is longer than NICE_LEN, it is encoded as a repeating sequence to out \n +/// This is also used for Unicode strings \n +/// This is a crude implementation that is not optimized. Assuming only short strings \n +/// are encoded, this is not much of an issue. +int matchLine(const char *in, int len, int l, char *out, int olen, int *ol, struct us_lnk_lst *prev_lines, byte *state, const byte usx_hcodes[], const byte usx_hcode_lens[]) { + int last_ol = *ol; + int last_len = 0; + int last_dist = 0; + int last_ctx = 0; + int line_ctr = 0; + int j = 0; + do { + int i, k; + int line_len = (int)strlen(prev_lines->data); + int limit = (line_ctr == 0 ? l : line_len); + for (; j < limit; j++) { + for (i = l, k = j; k < line_len && i < len; k++, i++) { + if (prev_lines->data[k] != in[i]) + break; + } + while ((((unsigned char) prev_lines->data[k]) >> 6) == 2) + k--; // Skip partial UTF-8 matches + if ((k - j) >= NICE_LEN) { + if (last_len) { + if (j > last_dist) + continue; + //int saving = ((k - j) - last_len) + (last_dist - j) + (last_ctx - line_ctr); + //if (saving < 0) { + // //printf("No savng: %d\n", saving); + // continue; + //} + *ol = last_ol; + } + last_len = (k - j); + last_dist = j; + last_ctx = line_ctr; + SAFE_APPEND_BITS(*ol = append_switch_code(out, olen, *ol, *state)); + SAFE_APPEND_BITS(*ol = append_bits(out, olen, *ol, usx_hcodes[USX_DICT], usx_hcode_lens[USX_DICT])); + SAFE_APPEND_BITS(*ol = encodeCount(out, olen, *ol, last_len - NICE_LEN)); + SAFE_APPEND_BITS(*ol = encodeCount(out, olen, *ol, last_dist)); + SAFE_APPEND_BITS(*ol = encodeCount(out, olen, *ol, last_ctx)); + /* + if ((*ol - last_ol) > (last_len * 4)) { + last_len = 0; + *ol = last_ol; + }*/ + //printf("Len: %d, Dist: %d, Line: %d\n", last_len, last_dist, last_ctx); + j += last_len; + } + } + line_ctr++; + prev_lines = prev_lines->previous; + } while (prev_lines && prev_lines->data != NULL); + if (last_len) { + l += last_len; + l--; + return l; + } + return -l; +} + +/// Returns 4 bit code assuming ch falls between '0' to '9', \n +/// 'A' to 'F' or 'a' to 'f' +byte getBaseCode(char ch) { + if (ch >= '0' && ch <= '9') + return (ch - '0') << 4; + else if (ch >= 'A' && ch <= 'F') + return (ch - 'A' + 10) << 4; + else if (ch >= 'a' && ch <= 'f') + return (ch - 'a' + 10) << 4; + return 0; +} + +/// Enum indicating nibble type - USX_NIB_NUM means ch is a number '0' to '9', \n +/// USX_NIB_HEX_LOWER means ch is between 'a' to 'f', \n +/// USX_NIB_HEX_UPPER means ch is between 'A' to 'F' +enum {USX_NIB_NUM = 0, USX_NIB_HEX_LOWER, USX_NIB_HEX_UPPER, USX_NIB_NOT}; +/// Gets 4 bit code assuming ch falls between '0' to '9', \n +/// 'A' to 'F' or 'a' to 'f' +char getNibbleType(char ch) { + if (ch >= '0' && ch <= '9') + return USX_NIB_NUM; + else if (ch >= 'a' && ch <= 'f') + return USX_NIB_HEX_LOWER; + else if (ch >= 'A' && ch <= 'F') + return USX_NIB_HEX_UPPER; + return USX_NIB_NOT; +} + +/// Starts coding of nibble sets +int append_nibble_escape(char *out, int olen, int ol, byte state, const byte usx_hcodes[], const byte usx_hcode_lens[]) { + SAFE_APPEND_BITS(ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, usx_hcodes[USX_NUM], usx_hcode_lens[USX_NUM])); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, 0, 2)); + return ol; +} + +/// Returns minimum value of two longs +long min_of(long c, long i) { + return c > i ? i : c; +} + +/// Appends the terminator code depending on the state, preset and whether full terminator needs to be encoded to out or not \n +int append_final_bits(char *const out, const int olen, int ol, const byte state, const byte is_all_upper, const byte usx_hcodes[], const byte usx_hcode_lens[]) { + if (usx_hcode_lens[USX_ALPHA]) { + if (USX_NUM != state) { + // for num state, append TERM_CODE directly + // for other state, switch to Num Set first + SAFE_APPEND_BITS(ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, usx_hcodes[USX_NUM], usx_hcode_lens[USX_NUM])); + } + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, usx_vcodes[TERM_CODE & 0x1F], usx_vcode_lens[TERM_CODE & 0x1F])); + } else { + // preset 1, terminate at 2 or 3 SW_CODE, i.e., 4 or 6 continuous 0 bits + // see discussion: https://github.com/siara-cc/Unishox/issues/19#issuecomment-922435580 + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, TERM_BYTE_PRESET_1, is_all_upper ? TERM_BYTE_PRESET_1_LEN_UPPER : TERM_BYTE_PRESET_1_LEN_LOWER)); + } + + // fill byte with the last bit + SAFE_APPEND_BITS(ol = append_bits(out, olen, ol, (ol == 0 || out[(ol-1)/8] << ((ol-1)&7) >= 0) ? 0 : 0xFF, (8 - ol % 8) & 7)); + + return ol; +} + +/// Macro used in the main compress function so that if the output len exceeds given maximum length (olen) it can exit +#define SAFE_APPEND_BITS2(olen, exp) do { \ + const int newidx = (exp); \ + const int __olen = (olen); \ + if (newidx < 0) return __olen >= 0 ? __olen + 1 : (1 - __olen) * 4; \ +} while (0) + +// Main API function. See unishox2.h for documentation +int unishox2_compress_lines(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen), const byte usx_hcodes[], const byte usx_hcode_lens[], const char *usx_freq_seq[], const char *usx_templates[], struct us_lnk_lst *prev_lines) { + + byte state; + + int l, ll, ol; + char c_in, c_next; + int prev_uni; + byte is_upper, is_all_upper; +#if (UNISHOX_API_OUT_AND_LEN(0,1)) == 0 + const int olen = INT_MAX - 1; + const int rawolen = olen; + const byte need_full_term_codes = 0; +#else + const int rawolen = olen; + byte need_full_term_codes = 0; + if (olen < 0) { + need_full_term_codes = 1; + olen *= -1; + } +#endif + + init_coder(); + ol = 0; + prev_uni = 0; + state = USX_ALPHA; + is_all_upper = 0; + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, UNISHOX_MAGIC_BITS, UNISHOX_MAGIC_BIT_LEN)); // magic bit(s) + for (l=0; l 0) { + continue; + } else if (l < 0 && ol < 0) { + return olen + 1; + } + l = -l; + } else { + l = matchOccurance(in, len, l, out, olen, &ol, &state, usx_hcodes, usx_hcode_lens); + if (l > 0) { + continue; + } else if (l < 0 && ol < 0) { + return olen + 1; + } + l = -l; + } + } + + c_in = in[l]; + if (l && len > 4 && l < (len - 4) && usx_hcode_lens[USX_NUM]) { + if (c_in == in[l - 1] && c_in == in[l + 1] && c_in == in[l + 2] && c_in == in[l + 3]) { + int rpt_count = l + 4; + while (rpt_count < len && in[rpt_count] == c_in) + rpt_count++; + rpt_count -= l; + SAFE_APPEND_BITS2(rawolen, ol = append_code(out, olen, ol, RPT_CODE, &state, usx_hcodes, usx_hcode_lens)); + SAFE_APPEND_BITS2(rawolen, ol = encodeCount(out, olen, ol, rpt_count - 4)); + l += rpt_count; + l--; + continue; + } + } + + if (l <= (len - 36) && usx_hcode_lens[USX_NUM]) { + if (in[l + 8] == '-' && in[l + 13] == '-' && in[l + 18] == '-' && in[l + 23] == '-') { + char hex_type = USX_NIB_NUM; + int uid_pos = l; + for (; uid_pos < l + 36; uid_pos++) { + char c_uid = in[uid_pos]; + if (c_uid == '-' && (uid_pos == 8 || uid_pos == 13 || uid_pos == 18 || uid_pos == 23)) + continue; + char nib_type = getNibbleType(c_uid); + if (nib_type == USX_NIB_NOT) + break; + if (nib_type != USX_NIB_NUM) { + if (hex_type != USX_NIB_NUM && hex_type != nib_type) + break; + hex_type = nib_type; + } + } + if (uid_pos == l + 36) { + SAFE_APPEND_BITS2(rawolen, ol = append_nibble_escape(out, olen, ol, state, usx_hcodes, usx_hcode_lens)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, (hex_type == USX_NIB_HEX_LOWER ? 0xC0 : 0xF0), + (hex_type == USX_NIB_HEX_LOWER ? 3 : 5))); + for (uid_pos = l; uid_pos < l + 36; uid_pos++) { + char c_uid = in[uid_pos]; + if (c_uid != '-') + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, getBaseCode(c_uid), 4)); + } + //printf("GUID:\n"); + l += 35; + continue; + } + } + } + + if (l < (len - 5) && usx_hcode_lens[USX_NUM]) { + char hex_type = USX_NIB_NUM; + int hex_len = 0; + do { + char nib_type = getNibbleType(in[l + hex_len]); + if (nib_type == USX_NIB_NOT) + break; + if (nib_type != USX_NIB_NUM) { + if (hex_type != USX_NIB_NUM && hex_type != nib_type) + break; + hex_type = nib_type; + } + hex_len++; + } while (l + hex_len < len); + if (hex_len > 10 && hex_type == USX_NIB_NUM) + hex_type = USX_NIB_HEX_LOWER; + if ((hex_type == USX_NIB_HEX_LOWER || hex_type == USX_NIB_HEX_UPPER) && hex_len > 3) { + SAFE_APPEND_BITS2(rawolen, ol = append_nibble_escape(out, olen, ol, state, usx_hcodes, usx_hcode_lens)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, (hex_type == USX_NIB_HEX_LOWER ? 0x80 : 0xE0), (hex_type == USX_NIB_HEX_LOWER ? 2 : 4))); + SAFE_APPEND_BITS2(rawolen, ol = encodeCount(out, olen, ol, hex_len)); + do { + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, getBaseCode(in[l++]), 4)); + } while (--hex_len); + l--; + continue; + } + } + + if (usx_templates != NULL) { + int i; + for (i = 0; i < 5; i++) { + if (usx_templates[i]) { + int rem = (int)strlen(usx_templates[i]); + int j = 0; + for (; j < rem && l + j < len; j++) { + char c_t = usx_templates[i][j]; + c_in = in[l + j]; + if (c_t == 'f' || c_t == 'F') { + if (getNibbleType(c_in) != (c_t == 'f' ? USX_NIB_HEX_LOWER : USX_NIB_HEX_UPPER) + && getNibbleType(c_in) != USX_NIB_NUM) { + break; + } + } else + if (c_t == 'r' || c_t == 't' || c_t == 'o') { + if (c_in < '0' || c_in > (c_t == 'r' ? '7' : (c_t == 't' ? '3' : '1'))) + break; + } else + if (c_t != c_in) + break; + } + if (((float)j / rem) > 0.66) { + //printf("%s\n", usx_templates[i]); + rem = rem - j; + SAFE_APPEND_BITS2(rawolen, ol = append_nibble_escape(out, olen, ol, state, usx_hcodes, usx_hcode_lens)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, 0, 1)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, (count_codes[i] & 0xF8), count_codes[i] & 0x07)); + SAFE_APPEND_BITS2(rawolen, ol = encodeCount(out, olen, ol, rem)); + for (int k = 0; k < j; k++) { + char c_t = usx_templates[i][k]; + if (c_t == 'f' || c_t == 'F') + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, getBaseCode(in[l + k]), 4)); + else if (c_t == 'r' || c_t == 't' || c_t == 'o') { + c_t = (c_t == 'r' ? 3 : (c_t == 't' ? 2 : 1)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, (in[l + k] - '0') << (8 - c_t), c_t)); + } + } + l += j; + l--; + break; + } + } + } + if (i < 5) + continue; + } + + if (usx_freq_seq != NULL) { + int i; + for (i = 0; i < 6; i++) { + int seq_len = (int)strlen(usx_freq_seq[i]); + if (len - seq_len >= 0 && l <= len - seq_len) { + if (memcmp(usx_freq_seq[i], in + l, seq_len) == 0 && usx_hcode_lens[usx_freq_codes[i] >> 5]) { + SAFE_APPEND_BITS2(rawolen, ol = append_code(out, olen, ol, usx_freq_codes[i], &state, usx_hcodes, usx_hcode_lens)); + l += seq_len; + l--; + break; + } + } + } + if (i < 6) + continue; + } + + c_in = in[l]; + + is_upper = 0; + if (c_in >= 'A' && c_in <= 'Z') + is_upper = 1; + else { + if (is_all_upper) { + is_all_upper = 0; + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + state = USX_ALPHA; + } + } + if (is_upper && !is_all_upper) { + if (state == USX_NUM) { + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + state = USX_ALPHA; + } + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + if (state == USX_DELTA) { + state = USX_ALPHA; + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + } + } + c_next = 0; + if (l+1 < len) + c_next = in[l+1]; + + if (c_in >= 32 && c_in <= 126) { + if (is_upper && !is_all_upper) { + for (ll=l+4; ll>=l && ll 'Z') + break; + } + if (ll == l-1) { + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + state = USX_ALPHA; + is_all_upper = 1; + } + } + if (state == USX_DELTA && (c_in == ' ' || c_in == '.' || c_in == ',')) { + byte spl_code = (c_in == ',' ? 0xC0 : (c_in == '.' ? 0xE0 : (c_in == ' ' ? 0 : 0xFF))); + if (spl_code != 0xFF) { + byte spl_code_len = (c_in == ',' ? 3 : (c_in == '.' ? 4 : (c_in == ' ' ? 1 : 4))); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, UNI_STATE_SPL_CODE, UNI_STATE_SPL_CODE_LEN)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, spl_code, spl_code_len)); + continue; + } + } + c_in -= 32; + if (is_all_upper && is_upper) + c_in += 32; + if (c_in == 0) { + if (state == USX_NUM) + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_vcodes[NUM_SPC_CODE & 0x1F], usx_vcode_lens[NUM_SPC_CODE & 0x1F])); + else + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_vcodes[1], usx_vcode_lens[1])); + } else { + c_in--; + SAFE_APPEND_BITS2(rawolen, ol = append_code(out, olen, ol, usx_code_94[(int)c_in], &state, usx_hcodes, usx_hcode_lens)); + } + } else + if (c_in == 13 && c_next == 10) { + SAFE_APPEND_BITS2(rawolen, ol = append_code(out, olen, ol, CRLF_CODE, &state, usx_hcodes, usx_hcode_lens)); + l++; + } else + if (c_in == 10) { + if (state == USX_DELTA) { + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, UNI_STATE_SPL_CODE, UNI_STATE_SPL_CODE_LEN)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, 0xF0, 4)); + } else + SAFE_APPEND_BITS2(rawolen, ol = append_code(out, olen, ol, LF_CODE, &state, usx_hcodes, usx_hcode_lens)); + } else + if (c_in == 13) { + SAFE_APPEND_BITS2(rawolen, ol = append_code(out, olen, ol, CR_CODE, &state, usx_hcodes, usx_hcode_lens)); + } else + if (c_in == '\t') { + SAFE_APPEND_BITS2(rawolen, ol = append_code(out, olen, ol, TAB_CODE, &state, usx_hcodes, usx_hcode_lens)); + } else { + int utf8len; + int32_t uni = readUTF8(in, len, l, &utf8len); + if (uni) { + l += utf8len; + if (state != USX_DELTA) { + int32_t uni2 = readUTF8(in, len, l, &utf8len); + if (uni2) { + if (state != USX_ALPHA) { + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + } + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_ALPHA], usx_hcode_lens[USX_ALPHA])); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_vcodes[1], usx_vcode_lens[1])); // code for space (' ') + state = USX_DELTA; + } else { + SAFE_APPEND_BITS2(rawolen, ol = append_switch_code(out, olen, ol, state)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, usx_hcodes[USX_DELTA], usx_hcode_lens[USX_DELTA])); + } + } + SAFE_APPEND_BITS2(rawolen, ol = encodeUnicode(out, olen, ol, uni, prev_uni)); + //printf("%d:%d:%d\n", l, utf8len, uni); + prev_uni = uni; + l--; + } else { + int bin_count = 1; + for (int bi = l + 1; bi < len; bi++) { + char c_bi = in[bi]; + //if (c_bi > 0x1F && c_bi != 0x7F) + // break; + if (readUTF8(in, len, bi, &utf8len)) + break; + if (bi < (len - 4) && c_bi == in[bi - 1] && c_bi == in[bi + 1] && c_bi == in[bi + 2] && c_bi == in[bi + 3]) + break; + bin_count++; + } + //printf("Bin:%d:%d:%x:%d\n", l, (unsigned char) c_in, (unsigned char) c_in, bin_count); + SAFE_APPEND_BITS2(rawolen, ol = append_nibble_escape(out, olen, ol, state, usx_hcodes, usx_hcode_lens)); + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, 0xF8, 5)); + SAFE_APPEND_BITS2(rawolen, ol = encodeCount(out, olen, ol, bin_count)); + do { + SAFE_APPEND_BITS2(rawolen, ol = append_bits(out, olen, ol, in[l++], 8)); + } while (--bin_count); + l--; + } + } + } + + if (need_full_term_codes) { + const int orig_ol = ol; + SAFE_APPEND_BITS2(rawolen, ol = append_final_bits(out, olen, ol, state, is_all_upper, usx_hcodes, usx_hcode_lens)); + return (ol / 8) * 4 + (((ol-orig_ol)/8) & 3); + } else { + const int rst = (ol + 7) / 8; + append_final_bits(out, rst, ol, state, is_all_upper, usx_hcodes, usx_hcode_lens); + return rst; + } +} + +// Main API function. See unishox2.h for documentation +int unishox2_compress(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen), const byte usx_hcodes[], const byte usx_hcode_lens[], const char *usx_freq_seq[], const char *usx_templates[]) { + return unishox2_compress_lines(in, len, UNISHOX_API_OUT_AND_LEN(out, olen), usx_hcodes, usx_hcode_lens, usx_freq_seq, usx_templates, NULL); +} + +// Main API function. See unishox2.h for documentation +int unishox2_compress_simple(const char *in, int len, char *out) { + return unishox2_compress_lines(in, len, UNISHOX_API_OUT_AND_LEN(out, INT_MAX - 1), USX_HCODES_DFLT, USX_HCODE_LENS_DFLT, USX_FREQ_SEQ_DFLT, USX_TEMPLATES, NULL); +} + +// Reads one bit from in +int readBit(const char *in, int bit_no) { + return in[bit_no >> 3] & (0x80 >> (bit_no % 8)); +} + +// Reads next 8 bits, if available +int read8bitCode(const char *in, int len, int bit_no) { + int bit_pos = bit_no & 0x07; + int char_pos = bit_no >> 3; + byte code = (((byte)in[char_pos]) << bit_pos); + if (bit_no + bit_pos < len) { + code |= ((byte)in[++char_pos]) >> (8 - bit_pos); + } else + code |= (0xFF >> (8 - bit_pos)); + return code; +} + +/// The list of veritical codes is split into 5 sections. Used by readVCodeIdx() +#define SECTION_COUNT 5 +/// Used by readVCodeIdx() for finding the section under which the code read using read8bitCode() falls +byte usx_vsections[] = {0x7F, 0xBF, 0xDF, 0xEF, 0xFF}; +/// Used by readVCodeIdx() for finding the section vertical position offset +byte usx_vsection_pos[] = {0, 4, 8, 12, 20}; +/// Used by readVCodeIdx() for masking the code read by read8bitCode() +byte usx_vsection_mask[] = {0x7F, 0x3F, 0x1F, 0x0F, 0x0F}; +/// Used by readVCodeIdx() for shifting the code read by read8bitCode() to obtain the vpos +byte usx_vsection_shift[] = {5, 4, 3, 1, 0}; + +/// Vertical decoder lookup table - 3 bits code len, 5 bytes vertical pos +/// code len is one less as 8 cannot be accommodated in 3 bits +byte usx_vcode_lookup[36] = { + (1 << 5) + 0, (1 << 5) + 0, (2 << 5) + 1, (2 << 5) + 2, // Section 1 + (3 << 5) + 3, (3 << 5) + 4, (3 << 5) + 5, (3 << 5) + 6, // Section 2 + (3 << 5) + 7, (3 << 5) + 7, (4 << 5) + 8, (4 << 5) + 9, // Section 3 + (5 << 5) + 10, (5 << 5) + 10, (5 << 5) + 11, (5 << 5) + 11, // Section 4 + (5 << 5) + 12, (5 << 5) + 12, (6 << 5) + 13, (6 << 5) + 14, + (6 << 5) + 15, (6 << 5) + 15, (6 << 5) + 16, (6 << 5) + 16, // Section 5 + (6 << 5) + 17, (6 << 5) + 17, (7 << 5) + 18, (7 << 5) + 19, + (7 << 5) + 20, (7 << 5) + 21, (7 << 5) + 22, (7 << 5) + 23, + (7 << 5) + 24, (7 << 5) + 25, (7 << 5) + 26, (7 << 5) + 27 +}; + +/// Decodes the vertical code from the given bitstream at in \n +/// This is designed to use less memory using a 36 byte buffer \n +/// compared to using a 256 byte buffer to decode the next 8 bits read by read8bitCode() \n +/// by splitting the list of vertical codes. \n +/// Decoder is designed for using less memory, not speed. \n +/// Returns the veritical code index or 99 if match could not be found. \n +/// Also updates bit_no_p with how many ever bits used by the vertical code. +int readVCodeIdx(const char *in, int len, int *bit_no_p) { + if (*bit_no_p < len) { + byte code = read8bitCode(in, len, *bit_no_p); + int i = 0; + do { + if (code <= usx_vsections[i]) { + byte vcode = usx_vcode_lookup[usx_vsection_pos[i] + ((code & usx_vsection_mask[i]) >> usx_vsection_shift[i])]; + (*bit_no_p) += ((vcode >> 5) + 1); + if (*bit_no_p > len) + return 99; + return vcode & 0x1F; + } + } while (++i < SECTION_COUNT); + } + return 99; +} + +/// Mask for retrieving each code to be decoded according to its length \n +/// Same as usx_mask so redundant +byte len_masks[] = {0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE, 0xFF}; +/// Decodes the horizontal code from the given bitstream at in \n +/// depending on the hcodes defined using usx_hcodes and usx_hcode_lens \n +/// Returns the horizontal code index or 99 if match could not be found. \n +/// Also updates bit_no_p with how many ever bits used by the horizontal code. +int readHCodeIdx(const char *in, int len, int *bit_no_p, const byte usx_hcodes[], const byte usx_hcode_lens[]) { + if (!usx_hcode_lens[USX_ALPHA]) + return USX_ALPHA; + if (*bit_no_p < len) { + byte code = read8bitCode(in, len, *bit_no_p); + for (int code_pos = 0; code_pos < 5; code_pos++) { + if (usx_hcode_lens[code_pos] && (code & len_masks[usx_hcode_lens[code_pos] - 1]) == usx_hcodes[code_pos]) { + *bit_no_p += usx_hcode_lens[code_pos]; + return code_pos; + } + } + } + return 99; +} + +// TODO: Last value check.. Also len check in readBit +/// Returns the position of step code (0, 10, 110, etc.) encountered in the stream +int getStepCodeIdx(const char *in, int len, int *bit_no_p, int limit) { + int idx = 0; + while (*bit_no_p < len && readBit(in, *bit_no_p)) { + idx++; + (*bit_no_p)++; + if (idx == limit) + return idx; + } + if (*bit_no_p >= len) + return 99; + (*bit_no_p)++; + return idx; +} + +/// Reads specified number of bits and builds the corresponding integer +int32_t getNumFromBits(const char *in, int len, int bit_no, int count) { + int32_t ret = 0; + while (count-- && bit_no < len) { + ret += (readBit(in, bit_no) ? 1 << count : 0); + bit_no++; + } + return count < 0 ? ret : -1; +} + +/// Decodes the count from the given bit stream at in. Also updates bit_no_p +int32_t readCount(const char *in, int *bit_no_p, int len) { + int idx = getStepCodeIdx(in, len, bit_no_p, 4); + if (idx == 99) + return -1; + if (*bit_no_p + count_bit_lens[idx] - 1 >= len) + return -1; + int32_t count = getNumFromBits(in, len, *bit_no_p, count_bit_lens[idx]) + (idx ? count_adder[idx - 1] : 0); + (*bit_no_p) += count_bit_lens[idx]; + return count; +} + +/// Decodes the Unicode codepoint from the given bit stream at in. Also updates bit_no_p \n +/// When the step code is 5, reads the next step code to find out the special code. +int32_t readUnicode(const char *in, int *bit_no_p, int len) { + int idx = getStepCodeIdx(in, len, bit_no_p, 5); + if (idx == 99) + return 0x7FFFFF00 + 99; + if (idx == 5) { + int idx = getStepCodeIdx(in, len, bit_no_p, 4); + return 0x7FFFFF00 + idx; + } + if (idx >= 0) { + int sign = (*bit_no_p < len ? readBit(in, *bit_no_p) : 0); + (*bit_no_p)++; + if (*bit_no_p + uni_bit_len[idx] - 1 >= len) + return 0x7FFFFF00 + 99; + int32_t count = getNumFromBits(in, len, *bit_no_p, uni_bit_len[idx]); + count += uni_adder[idx]; + (*bit_no_p) += uni_bit_len[idx]; + //printf("Sign: %d, Val:%d", sign, count); + return sign ? -count : count; + } + return 0; +} + +/// Macro to ensure that the decoder does not append more than olen bytes to out +#define DEC_OUTPUT_CHAR(out, olen, ol, c) do { \ + char *const obuf = (out); \ + const int oidx = (ol); \ + const int limit = (olen); \ + if (limit <= oidx) return limit + 1; \ + else if (oidx < 0) return 0; \ + else obuf[oidx] = (c); \ +} while (0) + +/// Macro to ensure that the decoder does not append more than olen bytes to out +#define DEC_OUTPUT_CHARS(olen, exp) do { \ + const int newidx = (exp); \ + const int limit = (olen); \ + if (newidx > limit) return limit + 1; \ +} while (0) + +/// Write given unicode code point to out as a UTF-8 sequence +int writeUTF8(char *out, int olen, int ol, int uni) { + if (uni < (1 << 11)) { + DEC_OUTPUT_CHAR(out, olen, ol++, 0xC0 + (uni >> 6)); + DEC_OUTPUT_CHAR(out, olen, ol++, 0x80 + (uni & 0x3F)); + } else + if (uni < (1 << 16)) { + DEC_OUTPUT_CHAR(out, olen, ol++, 0xE0 + (uni >> 12)); + DEC_OUTPUT_CHAR(out, olen, ol++, 0x80 + ((uni >> 6) & 0x3F)); + DEC_OUTPUT_CHAR(out, olen, ol++, 0x80 + (uni & 0x3F)); + } else { + DEC_OUTPUT_CHAR(out, olen, ol++, 0xF0 + (uni >> 18)); + DEC_OUTPUT_CHAR(out, olen, ol++, 0x80 + ((uni >> 12) & 0x3F)); + DEC_OUTPUT_CHAR(out, olen, ol++, 0x80 + ((uni >> 6) & 0x3F)); + DEC_OUTPUT_CHAR(out, olen, ol++, 0x80 + (uni & 0x3F)); + } + return ol; +} + +/// Decode repeating sequence and appends to out +int decodeRepeat(const char *in, int len, char *out, int olen, int ol, int *bit_no, struct us_lnk_lst *prev_lines) { + if (prev_lines) { + int32_t dict_len = readCount(in, bit_no, len) + NICE_LEN; + if (dict_len < NICE_LEN) + return ol; + int32_t dist = readCount(in, bit_no, len); + if (dist < 0) + return ol; + int32_t ctx = readCount(in, bit_no, len); + if (ctx < 0) + return ol; + struct us_lnk_lst *cur_line = prev_lines; + const int left = olen - ol; + while (ctx--) + cur_line = cur_line->previous; + if (left <= 0) return olen + 1; + memmove(out + ol, cur_line->data + dist, min_of(left, dict_len)); + if (left < dict_len) return olen + 1; + ol += dict_len; + } else { + int32_t dict_len = readCount(in, bit_no, len) + NICE_LEN; + if (dict_len < NICE_LEN) + return ol; + int32_t dist = readCount(in, bit_no, len) + NICE_LEN - 1; + if (dist < NICE_LEN - 1) + return ol; + const int32_t left = olen - ol; + //printf("Decode len: %d, dist: %d\n", dict_len - NICE_LEN, dist - NICE_LEN + 1); + if (left <= 0) return olen + 1; + memmove(out + ol, out + ol - dist, min_of(left, dict_len)); + if (left < dict_len) return olen + 1; + ol += dict_len; + } + return ol; +} + +/// Returns hex character corresponding to the 4 bit nibble +char getHexChar(int32_t nibble, int hex_type) { + if (nibble >= 0 && nibble <= 9) + return '0' + nibble; + else if (hex_type < USX_NIB_HEX_UPPER) + return 'a' + nibble - 10; + return 'A' + nibble - 10; +} + +// Main API function. See unishox2.h for documentation +int unishox2_decompress_lines(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen), const byte usx_hcodes[], const byte usx_hcode_lens[], const char *usx_freq_seq[], const char *usx_templates[], struct us_lnk_lst *prev_lines) { + + int dstate; + int bit_no; + int h, v; + byte is_all_upper; +#if (UNISHOX_API_OUT_AND_LEN(0,1)) == 0 + const int olen = INT_MAX - 1; +#endif + + init_coder(); + int ol = 0; + bit_no = UNISHOX_MAGIC_BIT_LEN; // ignore the magic bit + dstate = h = USX_ALPHA; + is_all_upper = 0; + + int prev_uni = 0; + + len <<= 3; + while (bit_no < len) { + int orig_bit_no = bit_no; + if (dstate == USX_DELTA || h == USX_DELTA) { + if (dstate != USX_DELTA) + h = dstate; + int32_t delta = readUnicode(in, &bit_no, len); + if ((delta >> 8) == 0x7FFFFF) { + int spl_code_idx = delta & 0x000000FF; + if (spl_code_idx == 99) + break; + switch (spl_code_idx) { + case 0: + DEC_OUTPUT_CHAR(out, olen, ol++, ' '); + continue; + case 1: + h = readHCodeIdx(in, len, &bit_no, usx_hcodes, usx_hcode_lens); + if (h == 99) { + bit_no = len; + continue; + } + if (h == USX_DELTA || h == USX_ALPHA) { + dstate = h; + continue; + } + if (h == USX_DICT) { + DEC_OUTPUT_CHARS(olen, ol = decodeRepeat(in, len, out, olen, ol, &bit_no, prev_lines)); + h = dstate; + continue; + } + break; + case 2: + DEC_OUTPUT_CHAR(out, olen, ol++, ','); + continue; + case 3: + DEC_OUTPUT_CHAR(out, olen, ol++, '.'); + continue; + case 4: + DEC_OUTPUT_CHAR(out, olen, ol++, 10); + continue; + } + } else { + prev_uni += delta; + DEC_OUTPUT_CHARS(olen, ol = writeUTF8(out, olen, ol, prev_uni)); + //printf("%ld, ", prev_uni); + } + if (dstate == USX_DELTA && h == USX_DELTA) + continue; + } else + h = dstate; + char c = 0; + byte is_upper = is_all_upper; + v = readVCodeIdx(in, len, &bit_no); + if (v == 99 || h == 99) { + bit_no = orig_bit_no; + break; + } + if (v == 0 && h != USX_SYM) { + if (bit_no >= len) + break; + if (h != USX_NUM || dstate != USX_DELTA) { + h = readHCodeIdx(in, len, &bit_no, usx_hcodes, usx_hcode_lens); + if (h == 99 || bit_no >= len) { + bit_no = orig_bit_no; + break; + } + } + if (h == USX_ALPHA) { + if (dstate == USX_ALPHA) { + if (!usx_hcode_lens[USX_ALPHA] && TERM_BYTE_PRESET_1 == (read8bitCode(in, len, bit_no - SW_CODE_LEN) & (0xFF << (8 - (is_all_upper ? TERM_BYTE_PRESET_1_LEN_UPPER : TERM_BYTE_PRESET_1_LEN_LOWER))))) + break; // Terminator for preset 1 + if (is_all_upper) { + is_upper = is_all_upper = 0; + continue; + } + v = readVCodeIdx(in, len, &bit_no); + if (v == 99) { + bit_no = orig_bit_no; + break; + } + if (v == 0) { + h = readHCodeIdx(in, len, &bit_no, usx_hcodes, usx_hcode_lens); + if (h == 99) { + bit_no = orig_bit_no; + break; + } + if (h == USX_ALPHA) { + is_all_upper = 1; + continue; + } + } + is_upper = 1; + } else { + dstate = USX_ALPHA; + continue; + } + } else + if (h == USX_DICT) { + DEC_OUTPUT_CHARS(olen, ol = decodeRepeat(in, len, out, olen, ol, &bit_no, prev_lines)); + continue; + } else + if (h == USX_DELTA) { + //printf("Sign: %d, bitno: %d\n", sign, bit_no); + //printf("Code: %d\n", prev_uni); + //printf("BitNo: %d\n", bit_no); + continue; + } else { + if (h != USX_NUM || dstate != USX_DELTA) + v = readVCodeIdx(in, len, &bit_no); + if (v == 99) { + bit_no = orig_bit_no; + break; + } + if (h == USX_NUM && v == 0) { + int idx = getStepCodeIdx(in, len, &bit_no, 5); + if (idx == 0) { + idx = getStepCodeIdx(in, len, &bit_no, 4); + int32_t rem = readCount(in, &bit_no, len); + if (rem < 0) + break; + rem = (int)strlen(usx_templates[idx]) - rem; + int eof = 0; + for (int j = 0; j < rem; j++) { + char c_t = usx_templates[idx][j]; + if (c_t == 'f' || c_t == 'r' || c_t == 't' || c_t == 'o' || c_t == 'F') { + char nibble_len = (c_t == 'f' || c_t == 'F' ? 4 : (c_t == 'r' ? 3 : (c_t == 't' ? 2 : 1))); + const int32_t raw_char = getNumFromBits(in, len, bit_no, nibble_len); + if (raw_char < 0) { + eof = 1; + break; + } + DEC_OUTPUT_CHAR(out, olen, ol++, getHexChar((char)raw_char, + c_t == 'f' ? USX_NIB_HEX_LOWER : USX_NIB_HEX_UPPER)); + bit_no += nibble_len; + } else + DEC_OUTPUT_CHAR(out, olen, ol++, c_t); + } + if (eof) break; // reach input eof + } else + if (idx == 5) { + int32_t bin_count = readCount(in, &bit_no, len); + if (bin_count < 0) + break; + if (bin_count == 0) // invalid encoding + break; + do { + const int32_t raw_char = getNumFromBits(in, len, bit_no, 8); + if (raw_char < 0) + break; + DEC_OUTPUT_CHAR(out, olen, ol++, (char)raw_char); + bit_no += 8; + } while (--bin_count); + if (bin_count > 0) break; // reach input eof + } else { + int32_t nibble_count = 0; + if (idx == 2 || idx == 4) + nibble_count = 32; + else { + nibble_count = readCount(in, &bit_no, len); + if (nibble_count < 0) + break; + if (nibble_count == 0) // invalid encoding + break; + } + do { + int32_t nibble = getNumFromBits(in, len, bit_no, 4); + if (nibble < 0) + break; + DEC_OUTPUT_CHAR(out, olen, ol++, getHexChar(nibble, idx < 3 ? USX_NIB_HEX_LOWER : USX_NIB_HEX_UPPER)); + if ((idx == 2 || idx == 4) && (nibble_count == 25 || nibble_count == 21 || nibble_count == 17 || nibble_count == 13)) + DEC_OUTPUT_CHAR(out, olen, ol++, '-'); + bit_no += 4; + } while (--nibble_count); + if (nibble_count > 0) break; // reach input eof + } + if (dstate == USX_DELTA) + h = USX_DELTA; + continue; + } + } + } + if (is_upper && v == 1) { + h = dstate = USX_DELTA; // continuous delta coding + continue; + } + if (h < 3 && v < 28) + c = usx_sets[h][v]; + if (c >= 'a' && c <= 'z') { + dstate = USX_ALPHA; + if (is_upper) + c -= 32; + } else { + if (c >= '0' && c <= '9') { + dstate = USX_NUM; + } else if (c == 0) { + if (v == 8) { + DEC_OUTPUT_CHAR(out, olen, ol++, '\r'); + DEC_OUTPUT_CHAR(out, olen, ol++, '\n'); + } else if (h == USX_NUM && v == 26) { + int32_t count = readCount(in, &bit_no, len); + if (count < 0) + break; + count += 4; + if (ol <= 0) + return 0; // invalid encoding + char rpt_c = out[ol - 1]; + while (count--) + DEC_OUTPUT_CHAR(out, olen, ol++, rpt_c); + } else if (h == USX_SYM && v > 24) { + v -= 25; + const int freqlen = (int)strlen(usx_freq_seq[v]); + const int left = olen - ol; + if (left <= 0) return olen + 1; + memcpy(out + ol, usx_freq_seq[v], min_of(left, freqlen)); + if (left < freqlen) return olen + 1; + ol += freqlen; + } else if (h == USX_NUM && v > 22 && v < 26) { + v -= (23 - 3); + const int freqlen = (int)strlen(usx_freq_seq[v]); + const int left = olen - ol; + if (left <= 0) return olen + 1; + memcpy(out + ol, usx_freq_seq[v], min_of(left, freqlen)); + if (left < freqlen) return olen + 1; + ol += freqlen; + } else + break; // Terminator + if (dstate == USX_DELTA) + h = USX_DELTA; + continue; + } + } + if (dstate == USX_DELTA) + h = USX_DELTA; + DEC_OUTPUT_CHAR(out, olen, ol++, c); + } + + return ol; + +} + +// Main API function. See unishox2.h for documentation +int unishox2_decompress(const char *in, int len, UNISHOX_API_OUT_AND_LEN(char *out, int olen), const byte usx_hcodes[], const byte usx_hcode_lens[], const char *usx_freq_seq[], const char *usx_templates[]) { + return unishox2_decompress_lines(in, len, UNISHOX_API_OUT_AND_LEN(out, olen), usx_hcodes, usx_hcode_lens, usx_freq_seq, usx_templates, NULL); +} + +// Main API function. See unishox2.h for documentation +int unishox2_decompress_simple(const char *in, int len, char *out) { + return unishox2_decompress(in, len, UNISHOX_API_OUT_AND_LEN(out, INT_MAX - 1), USX_PSET_DFLT); +} diff --git a/src/mesh/compression/unishox2.h b/src/mesh/compression/unishox2.h new file mode 100644 index 000000000..bbbd7a759 --- /dev/null +++ b/src/mesh/compression/unishox2.h @@ -0,0 +1,278 @@ +/* + * Copyright (C) 2020 Siara Logics (cc) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * @author Arundale Ramanathan + * + */ + +/** + * @file unishox2.h + * @author Arundale Ramanathan, James Z. M. Gao + * @brief API for Unishox2 Compression and Decompression + * + * This file describes each function of the Unishox2 API \n + * For finding out how this API can be used in your program, \n + * please see test_unishox2.c. + */ + +#ifndef unishox2 +#define unishox2 + +#define UNISHOX_VERSION "2.0" ///< Unicode spec version + +/** + * Macro switch to enable/disable output buffer length parameter in low level api \n + * Disabled by default \n + * When this macro is defined, the all the API functions \n + * except the simple API functions accept an additional parameter olen \n + * that enables the developer to pass the size of the output buffer provided \n + * so that the api function may not write beyond that length. \n + * This can be disabled if the developer knows that the buffer provided is sufficient enough \n + * so no additional parameter is passed and the program is faster since additional check \n + * for output length is not performed at each step \n + * The simple api, i.e. unishox2_(de)compress_simple will always omit the buffer length + */ +#ifndef UNISHOX_API_WITH_OUTPUT_LEN +# define UNISHOX_API_WITH_OUTPUT_LEN 0 +#endif + +/// Upto 8 bits of initial magic bit sequence can be included. Bit count can be specified with UNISHOX_MAGIC_BIT_LEN +#ifndef UNISHOX_MAGIC_BITS +# define UNISHOX_MAGIC_BITS 0xFF +#endif + +/// Desired length of Magic bits defined by UNISHOX_MAGIC_BITS +#ifdef UNISHOX_MAGIC_BIT_LEN +# if UNISHOX_MAGIC_BIT_LEN < 0 || 9 <= UNISHOX_MAGIC_BIT_LEN +# error "UNISHOX_MAGIC_BIT_LEN need between [0, 8)" +# endif +#else +# define UNISHOX_MAGIC_BIT_LEN 1 +#endif + +//enum {USX_ALPHA = 0, USX_SYM, USX_NUM, USX_DICT, USX_DELTA}; + +/// Default Horizontal codes. When composition of text is know beforehand, the other hcodes in this section can be used to achieve more compression. +#define USX_HCODES_DFLT (const unsigned char[]) {0x00, 0x40, 0x80, 0xC0, 0xE0} +/// Length of each default hcode +#define USX_HCODE_LENS_DFLT (const unsigned char[]) {2, 2, 2, 3, 3} + +/// Horizontal codes preset for English Alphabet content only +#define USX_HCODES_ALPHA_ONLY (const unsigned char[]) {0x00, 0x00, 0x00, 0x00, 0x00} +/// Length of each Alpha only hcode +#define USX_HCODE_LENS_ALPHA_ONLY (const unsigned char[]) {0, 0, 0, 0, 0} + +/// Horizontal codes preset for Alpha Numeric content only +#define USX_HCODES_ALPHA_NUM_ONLY (const unsigned char[]) {0x00, 0x00, 0x80, 0x00, 0x00} +/// Length of each Alpha numeric hcode +#define USX_HCODE_LENS_ALPHA_NUM_ONLY (const unsigned char[]) {1, 0, 1, 0, 0} + +/// Horizontal codes preset for Alpha Numeric and Symbol content only +#define USX_HCODES_ALPHA_NUM_SYM_ONLY (const unsigned char[]) {0x00, 0x80, 0xC0, 0x00, 0x00} +/// Length of each Alpha numeric and symbol hcodes +#define USX_HCODE_LENS_ALPHA_NUM_SYM_ONLY (const unsigned char[]) {1, 2, 2, 0, 0} + +/// Horizontal codes preset favouring Alphabet content +#define USX_HCODES_FAVOR_ALPHA (const unsigned char[]) {0x00, 0x80, 0xA0, 0xC0, 0xE0} +/// Length of each hcode favouring Alpha content +#define USX_HCODE_LENS_FAVOR_ALPHA (const unsigned char[]) {1, 3, 3, 3, 3} + +/// Horizontal codes preset favouring repeating sequences +#define USX_HCODES_FAVOR_DICT (const unsigned char[]) {0x00, 0x40, 0xC0, 0x80, 0xE0} +/// Length of each hcode favouring repeating sequences +#define USX_HCODE_LENS_FAVOR_DICT (const unsigned char[]) {2, 2, 3, 2, 3} + +/// Horizontal codes preset favouring symbols +#define USX_HCODES_FAVOR_SYM (const unsigned char[]) {0x80, 0x00, 0xA0, 0xC0, 0xE0} +/// Length of each hcode favouring symbols +#define USX_HCODE_LENS_FAVOR_SYM (const unsigned char[]) {3, 1, 3, 3, 3} + +//#define USX_HCODES_FAVOR_UMLAUT {0x00, 0x40, 0xE0, 0xC0, 0x80} +//#define USX_HCODE_LENS_FAVOR_UMLAUT {2, 2, 3, 3, 2} + +/// Horizontal codes preset favouring umlaut letters +#define USX_HCODES_FAVOR_UMLAUT (const unsigned char[]) {0x80, 0xA0, 0xC0, 0xE0, 0x00} +/// Length of each hcode favouring umlaut letters +#define USX_HCODE_LENS_FAVOR_UMLAUT (const unsigned char[]) {3, 3, 3, 3, 1} + +/// Horizontal codes preset for no repeating sequences +#define USX_HCODES_NO_DICT (const unsigned char[]) {0x00, 0x40, 0x80, 0x00, 0xC0} +/// Length of each hcode for no repeating sequences +#define USX_HCODE_LENS_NO_DICT (const unsigned char[]) {2, 2, 2, 0, 2} + +/// Horizontal codes preset for no Unicode characters +#define USX_HCODES_NO_UNI (const unsigned char[]) {0x00, 0x40, 0x80, 0xC0, 0x00} +/// Length of each hcode for no Unicode characters +#define USX_HCODE_LENS_NO_UNI (const unsigned char[]) {2, 2, 2, 2, 0} + +/// Default frequently occuring sequences. When composition of text is know beforehand, the other sequences in this section can be used to achieve more compression. +#define USX_FREQ_SEQ_DFLT (const char *[]) {"\": \"", "\": ", ""} +/// Frequently occuring sequences in XML content +#define USX_FREQ_SEQ_XML (const char *[]) {"", " Date: Mon, 11 Apr 2022 22:12:04 -0700 Subject: [PATCH 12/40] compression WIP compression works. next is to store it in the proto as a oneof and then decompress it on use. --- platformio.ini | 5 ++++- src/mesh/Router.cpp | 21 ++++++++++++--------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/platformio.ini b/platformio.ini index e692888c2..ba5c37489 100644 --- a/platformio.ini +++ b/platformio.ini @@ -110,11 +110,14 @@ lib_ignore = ESP32 BLE Arduino platform_packages = framework-arduinoespressif32@https://github.com/meshtastic/arduino-esp32.git#4cde0f5d412d2695184f32e8a47e9bea57b45276 -; leave this commented out to avoid breaking Windows +; leave this commented out to avoid breaking Windows ;upload_port = /dev/ttyUSB0 ;monitor_port = /dev/ttyUSB0 +upload_port = /dev/cu.SLAB_USBtoUART +monitor_port = /dev/cu.SLAB_USBtoUART + ; customize the partition table ; http://docs.platformio.org/en/latest/platforms/espressif32.html#partition-tables board_build.partitions = partition-table.csv diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index 34ab85a26..c81359604 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -325,26 +325,29 @@ Routing_Error perhapsEncode(MeshPacket *p) size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), Data_fields, &p->decoded); - if (1) { - // int orig_len, compressed_len; + if (p->decoded.portnum == PortNum_TEXT_MESSAGE_APP) { + + char original_payload[Constants_DATA_PAYLOAD_LEN]; + memcpy(original_payload, p->decoded.payload.bytes, p->decoded.payload.size); + int compressed_len; char compressed_out[100] = {0}; - // char *orig = (char *)"Hiiiiiiii! :) How are you doing? I am doing well."; - // char *orig = (char *)"Meshtastic"; - // orig_len = strlen(orig); - // compressed_len = unishox2_compress_simple(orig, orig_len, compressed_out); - compressed_len = unishox2_compress_simple((char *)bytes, numbytes, compressed_out); + compressed_len = unishox2_compress_simple(original_payload, p->decoded.payload.size, compressed_out); + Serial.print("Original length - "); + Serial.println(p->decoded.payload.size); + + Serial.print("Compressed length - "); Serial.println(compressed_len); - Serial.println(compressed_out); - //&p->decoded.portnum; + //Serial.println(compressed_out); char decompressed_out[100] = {}; int decompressed_len; decompressed_len = unishox2_decompress_simple(compressed_out, compressed_len, decompressed_out); + Serial.print("Decompressed length - "); Serial.println(decompressed_len); Serial.println(decompressed_out); } From b59368ca3c7104d2fe4e5bd59aa35e01e9fcbf2b Mon Sep 17 00:00:00 2001 From: amerinoj <60507916+amerinoj@users.noreply.github.com> Date: Tue, 12 Apr 2022 14:13:25 +0200 Subject: [PATCH 13/40] Updated pinut in variant tlorav2_1_16 (#1365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Added the last mac address bytes in bluetooth device name ESP32Bluetooth.cpp Add default notification pinout to tlora_v2_1_16/variant.h * updated ADC_MULTIPLIER value for more accuracy * Update variant.h Updated LORA_RESET GPIO in this board is 23 Removed LORA_DIO1 35 , gpio 35 is a battery pin Removed LORA_DIO2 34, gpio 34 is unuse in this board Co-authored-by: Ben Meadors Co-authored-by: Thomas Göttgens --- variants/tlora_v2_1_16/variant.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/variants/tlora_v2_1_16/variant.h b/variants/tlora_v2_1_16/variant.h index 958127b76..16d02e150 100644 --- a/variants/tlora_v2_1_16/variant.h +++ b/variants/tlora_v2_1_16/variant.h @@ -22,7 +22,5 @@ #define USE_RF95 #define LORA_DIO0 26 // a No connect on the SX1262 module -#define LORA_RESET 14 -#define LORA_DIO1 35 // Not really used -#define LORA_DIO2 34 // Not really used +#define LORA_RESET 23 From 794167c701ca7853d769c0ff1f65fb84454f4443 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Tue, 12 Apr 2022 13:04:24 +0000 Subject: [PATCH 14/40] Removed phone sds timeout (#1372) --- proto | 2 +- src/PowerFSM.cpp | 3 --- src/mesh/NodeDB.h | 2 -- src/mesh/generated/admin.pb.h | 2 +- src/mesh/generated/radioconfig.pb.h | 11 ++++------- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/proto b/proto index c6abcf8d6..6ff115232 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit c6abcf8d6b9c9dec630381c50198bed92258d4e8 +Subproject commit 6ff115232f052c01f479cc8288beb67fcde83504 diff --git a/src/PowerFSM.cpp b/src/PowerFSM.cpp index 0106cff47..a6fd758ac 100644 --- a/src/PowerFSM.cpp +++ b/src/PowerFSM.cpp @@ -363,9 +363,6 @@ void PowerFSM_setup() if (meshSds != UINT32_MAX) powerFSM.add_timed_transition(lowPowerState, &stateSDS, meshSds * 1000, NULL, "mesh timeout"); - // removing for now, because some users don't even have phones - // powerFSM.add_timed_transition(lowPowerState, &stateSDS, getPref_phone_sds_timeout_sec() * 1000, NULL, "phone - // timeout"); powerFSM.run_machine(); // run one interation of the state machine, so we run our on enter tasks for the initial DARK state } diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index c9a3a46bc..4cea56068 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -141,7 +141,6 @@ extern NodeDB nodeDB; # prefs.position_broadcast_secs = FIXME possibly broadcast only once an hr prefs.wait_bluetooth_secs = 1 # Don't stay in bluetooth mode prefs.mesh_sds_timeout_secs = never - prefs.phone_sds_timeout_sec = never # try to stay in light sleep one full day, then briefly wake and sleep again prefs.ls_secs = oneday @@ -176,7 +175,6 @@ PREF_GET(wait_bluetooth_secs, IF_ROUTER(1, 60)) PREF_GET(screen_on_secs, 60) PREF_GET(mesh_sds_timeout_secs, IF_ROUTER(NODE_DELAY_FOREVER, 2 * 60 * 60)) -PREF_GET(phone_sds_timeout_sec, IF_ROUTER(NODE_DELAY_FOREVER, 2 * 60 * 60)) PREF_GET(sds_secs, 365 * 24 * 60 * 60) // We default to sleeping (with bluetooth off for 5 minutes at a time). This seems to be a good tradeoff between diff --git a/src/mesh/generated/admin.pb.h b/src/mesh/generated/admin.pb.h index ef9db43cb..bd33f7b90 100644 --- a/src/mesh/generated/admin.pb.h +++ b/src/mesh/generated/admin.pb.h @@ -129,7 +129,7 @@ extern const pb_msgdesc_t AdminMessage_msg; #define AdminMessage_fields &AdminMessage_msg /* Maximum encoded size of messages (where known) */ -#define AdminMessage_size 604 +#define AdminMessage_size 598 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/generated/radioconfig.pb.h b/src/mesh/generated/radioconfig.pb.h index 7a764e6fc..db1a82837 100644 --- a/src/mesh/generated/radioconfig.pb.h +++ b/src/mesh/generated/radioconfig.pb.h @@ -263,7 +263,6 @@ typedef struct _RadioConfig_UserPreferences { uint32_t wait_bluetooth_secs; uint32_t screen_on_secs; uint32_t phone_timeout_secs; - uint32_t phone_sds_timeout_sec; uint32_t mesh_sds_timeout_secs; uint32_t sds_secs; uint32_t ls_secs; @@ -400,16 +399,15 @@ extern "C" { /* Initializer values for message structs */ #define RadioConfig_init_default {false, RadioConfig_UserPreferences_init_default} -#define RadioConfig_UserPreferences_init_default {0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} +#define RadioConfig_UserPreferences_init_default {0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} #define RadioConfig_init_zero {false, RadioConfig_UserPreferences_init_zero} -#define RadioConfig_UserPreferences_init_zero {0, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} +#define RadioConfig_UserPreferences_init_zero {0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, _RegionCode_MIN, _ChargeCurrent_MIN, 0, _Role_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", 0, _GpsCoordinateFormat_MIN, 0, 0, 0, 0, 0, {0, 0, 0}, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_Serial_Mode_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, _RadioConfig_UserPreferences_TelemetrySensorType_MIN, 0, 0, 0, 0, 0, 0, 0, 0, "", "", 0, 0, 0, 0, 0, 0, _InputEventChar_MIN, _InputEventChar_MIN, _InputEventChar_MIN, 0, 0, "", 0, 0, 0, _RadioConfig_UserPreferences_Serial_Baud_MIN, 0} /* Field tags (for use in manual encoding/decoding) */ #define RadioConfig_UserPreferences_position_broadcast_secs_tag 1 #define RadioConfig_UserPreferences_wait_bluetooth_secs_tag 4 #define RadioConfig_UserPreferences_screen_on_secs_tag 5 #define RadioConfig_UserPreferences_phone_timeout_secs_tag 6 -#define RadioConfig_UserPreferences_phone_sds_timeout_sec_tag 7 #define RadioConfig_UserPreferences_mesh_sds_timeout_secs_tag 8 #define RadioConfig_UserPreferences_sds_secs_tag 9 #define RadioConfig_UserPreferences_ls_secs_tag 10 @@ -503,7 +501,6 @@ X(a, STATIC, SINGULAR, UINT32, position_broadcast_secs, 1) \ X(a, STATIC, SINGULAR, UINT32, wait_bluetooth_secs, 4) \ X(a, STATIC, SINGULAR, UINT32, screen_on_secs, 5) \ X(a, STATIC, SINGULAR, UINT32, phone_timeout_secs, 6) \ -X(a, STATIC, SINGULAR, UINT32, phone_sds_timeout_sec, 7) \ X(a, STATIC, SINGULAR, UINT32, mesh_sds_timeout_secs, 8) \ X(a, STATIC, SINGULAR, UINT32, sds_secs, 9) \ X(a, STATIC, SINGULAR, UINT32, ls_secs, 10) \ @@ -594,8 +591,8 @@ extern const pb_msgdesc_t RadioConfig_UserPreferences_msg; #define RadioConfig_UserPreferences_fields &RadioConfig_UserPreferences_msg /* Maximum encoded size of messages (where known) */ -#define RadioConfig_UserPreferences_size 598 -#define RadioConfig_size 601 +#define RadioConfig_UserPreferences_size 592 +#define RadioConfig_size 595 #ifdef __cplusplus } /* extern "C" */ From 0e4699d8b67c74b07e737bf050d41fd018ca64c8 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Tue, 12 Apr 2022 21:13:35 -0700 Subject: [PATCH 15/40] updating proto submodule to latest --- proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto b/proto index 6ff115232..870a62b27 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 6ff115232f052c01f479cc8288beb67fcde83504 +Subproject commit 870a62b27e6d28dd1427e14996aed22cffd8dd01 From f511baba9ac8885cd47a98ac6503a0b0c245ab42 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Tue, 12 Apr 2022 21:25:33 -0700 Subject: [PATCH 16/40] Bump to 1.3.6 --- version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.properties b/version.properties index df418fa9b..13cd63471 100644 --- a/version.properties +++ b/version.properties @@ -1,4 +1,4 @@ [VERSION] major = 1 minor = 3 -build = 5 +build = 6 From b76424db50003d1fa90388e88508c84c64689d0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bal=C3=A1zs=20Kelemen?= <10376327+prampec@users.noreply.github.com> Date: Thu, 14 Apr 2022 00:43:06 +0200 Subject: [PATCH 17/40] Make Observer to be able to observe multiple Observables. (#1234) * Make Observer to be able to observe multiple Observables. * Fix Observer destructor cleanup. Co-authored-by: Sacha Weatherstone --- src/Observer.h | 24 ++++++++++++------------ src/gps/GPS.cpp | 4 ++-- src/mesh/PhoneAPI.cpp | 2 +- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/Observer.h b/src/Observer.h index fb024dc82..ccc7f37b6 100644 --- a/src/Observer.h +++ b/src/Observer.h @@ -11,13 +11,13 @@ template class Observable; */ template class Observer { - Observable *observed = NULL; + std::list *> observed; public: virtual ~Observer(); - /// Stop watching our current obserable - void unobserve(); + /// Stop watching the obserable + void unobserve(Observable *o); /// Start watching a specified observable void observe(Observable *o); @@ -87,21 +87,21 @@ template class Observable template Observer::~Observer() { - unobserve(); + for (typename std::list *>::const_iterator iterator = observed.begin(); iterator != observed.end(); + ++iterator) { + (*iterator)->removeObserver(this); + } + observed.clear(); } -template void Observer::unobserve() +template void Observer::unobserve(Observable *o) { - if (observed) - observed->removeObserver(this); - observed = NULL; + o->removeObserver(this); + observed.remove(o); } template void Observer::observe(Observable *o) { - // We can only watch one thing at a time - assert(!observed); - - observed = o; + observed.push_back(o); o->addObserver(this); } diff --git a/src/gps/GPS.cpp b/src/gps/GPS.cpp index 8562f0ebe..aa92d96c8 100644 --- a/src/gps/GPS.cpp +++ b/src/gps/GPS.cpp @@ -95,8 +95,8 @@ bool GPS::setup() GPS::~GPS() { // we really should unregister our sleep observer - notifySleepObserver.unobserve(); - notifyDeepSleepObserver.unobserve(); + notifySleepObserver.unobserve(¬ifySleep); + notifyDeepSleepObserver.unobserve(¬ifyDeepSleep); } bool GPS::hasLock() { return hasValidLocation; } diff --git a/src/mesh/PhoneAPI.cpp b/src/mesh/PhoneAPI.cpp index ca794036f..033ef4906 100644 --- a/src/mesh/PhoneAPI.cpp +++ b/src/mesh/PhoneAPI.cpp @@ -48,7 +48,7 @@ void PhoneAPI::close() if (state != STATE_SEND_NOTHING) { state = STATE_SEND_NOTHING; - unobserve(); + unobserve(&service.fromNumChanged); releasePhonePacket(); // Don't leak phone packets on shutdown onConnectionChanged(false); From ecc114f1cdbe0c04bbd342157f2adcd91afbc550 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Wed, 13 Apr 2022 19:23:35 -0700 Subject: [PATCH 18/40] temp work on compression --- src/configuration.h | 6 ++++ src/mesh/Router.cpp | 50 +++++++++++++++++++++++-------- src/mesh/generated/mesh.pb.h | 31 +++++++++++-------- src/mesh/generated/telemetry.pb.h | 8 ++--- src/mesh/http/WiFiAPClient.cpp | 17 ++++++++--- 5 files changed, 80 insertions(+), 32 deletions(-) diff --git a/src/configuration.h b/src/configuration.h index 742d9e75e..59d0cf712 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -125,6 +125,12 @@ along with this program. If not, see . #define RTC_DATA_ATTR #endif +// ----------------------------------------------------------------------------- +// Feature toggles +// ----------------------------------------------------------------------------- + +//#define DISABLE_NTP + // ----------------------------------------------------------------------------- // OLED & Input // ----------------------------------------------------------------------------- diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index c81359604..9487231fd 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -303,6 +303,14 @@ bool perhapsDecode(MeshPacket *p) // parsing was successful p->which_payloadVariant = MeshPacket_decoded_tag; // change type to decoded p->channel = chIndex; // change to store the index instead of the hash + + + // Decompress if needed. jm + if (p->decoded.which_payloadVariant == Data_payload_compressed_tag) { + // Decompress the file + } + + printPacket("decoded message", p); return true; } @@ -321,35 +329,53 @@ Routing_Error perhapsEncode(MeshPacket *p) if (p->which_payloadVariant == MeshPacket_decoded_tag) { static uint8_t bytes[MAX_RHPACKETLEN]; // we have to use a scratch buffer because a union - // printPacket("pre encrypt", p); // portnum valid here - size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), Data_fields, &p->decoded); + // Only allow encryption on the text message app. + // TODO: Allow modules to opt into compression. if (p->decoded.portnum == PortNum_TEXT_MESSAGE_APP) { char original_payload[Constants_DATA_PAYLOAD_LEN]; memcpy(original_payload, p->decoded.payload.bytes, p->decoded.payload.size); - int compressed_len; - char compressed_out[100] = {0}; + char compressed_out[Constants_DATA_PAYLOAD_LEN] = {0}; - compressed_len = unishox2_compress_simple(original_payload, p->decoded.payload.size, compressed_out); + int compressed_len = unishox2_compress_simple(original_payload, p->decoded.payload.size, compressed_out); Serial.print("Original length - "); Serial.println(p->decoded.payload.size); Serial.print("Compressed length - "); Serial.println(compressed_len); - //Serial.println(compressed_out); + // Serial.println(compressed_out); - char decompressed_out[100] = {}; - int decompressed_len; + // If the compressed length is greater than or equal to the original size, don't use the compressed form + if (compressed_len >= p->decoded.payload.size) { - decompressed_len = unishox2_decompress_simple(compressed_out, compressed_len, decompressed_out); + DEBUG_MSG("Not compressing message. Not enough benefit from doing so.\n"); + // Set the uncompressed payload varient anyway. Shouldn't hurt? + p->decoded.which_payloadVariant = Data_payload_tag; - Serial.print("Decompressed length - "); - Serial.println(decompressed_len); - Serial.println(decompressed_out); + // Otherwise we use the compressor + } else { + DEBUG_MSG("Compressing message.\n"); + // Copy the compressed data into the meshpacket + p->decoded.payload_compressed.size = compressed_len; + memcpy(p->decoded.payload_compressed.bytes, compressed_out, compressed_len); + + p->decoded.which_payloadVariant = Data_payload_compressed_tag; + } + + if (0) { + char decompressed_out[Constants_DATA_PAYLOAD_LEN] = {}; + int decompressed_len; + + decompressed_len = unishox2_decompress_simple(compressed_out, compressed_len, decompressed_out); + + Serial.print("Decompressed length - "); + Serial.println(decompressed_len); + Serial.println(decompressed_out); + } } if (numbytes > MAX_RHPACKETLEN) diff --git a/src/mesh/generated/mesh.pb.h b/src/mesh/generated/mesh.pb.h index b4a9dbf90..8ce6fc803 100644 --- a/src/mesh/generated/mesh.pb.h +++ b/src/mesh/generated/mesh.pb.h @@ -500,6 +500,7 @@ typedef struct _User { } User; typedef PB_BYTES_ARRAY_T(237) Data_payload_t; +typedef PB_BYTES_ARRAY_T(237) Data_payload_compressed_t; /* (Formerly called SubPacket) The payload portion fo a packet, this is the actual bytes that are sent inside a radio packet (because from/to are broken out by the comms library) */ @@ -507,30 +508,34 @@ typedef struct _Data { /* Formerly named typ and of type Type */ PortNum portnum; /* TODO: REPLACE */ - Data_payload_t payload; + pb_size_t which_payloadVariant; + union { + Data_payload_t payload; + Data_payload_compressed_t payload_compressed; + }; + /* TODO: REPLACE */ + bool want_response; /* Not normally used, but for testing a sender can request that recipient responds in kind (i.e. if it received a position, it should unicast back it's position). Note: that if you set this on a broadcast you will receive many replies. */ - bool want_response; + uint32_t dest; /* The address of the destination node. This field is is filled in by the mesh radio device software, application layer software should never need it. RouteDiscovery messages _must_ populate this. Other message types might need to if they are doing multihop routing. */ - uint32_t dest; + uint32_t source; /* The address of the original sender for this message. This field should _only_ be populated for reliable multihop packets (to keep packets small). */ - uint32_t source; + uint32_t request_id; /* Only used in routing or response messages. Indicates the original message ID that this message is reporting failure on. (formerly called original_id) */ - uint32_t request_id; - /* If set, this message is intened to be a reply to a previously sent message with the defined id. */ uint32_t reply_id; + /* If set, this message is intened to be a reply to a previously sent message with the defined id. */ + uint32_t emoji; /* Defaults to false. If true, then what is in the payload should be treated as an emoji like giving a message a heart or poop emoji. */ - uint32_t emoji; - /* Location structure */ bool has_location; Location location; } Data; @@ -738,7 +743,7 @@ extern "C" { #define User_init_default {"", "", "", {0}, _HardwareModel_MIN, 0, _Team_MIN, 0, 0, 0} #define RouteDiscovery_init_default {0, {0, 0, 0, 0, 0, 0, 0, 0}} #define Routing_init_default {0, {RouteDiscovery_init_default}} -#define Data_init_default {_PortNum_MIN, {0, {0}}, 0, 0, 0, 0, 0, 0, false, Location_init_default} +#define Data_init_default {_PortNum_MIN, 0, {{0, {0}}}, 0, 0, 0, 0, 0, 0, false, Location_init_default} #define Location_init_default {0, 0, 0, 0, 0} #define MeshPacket_init_default {0, 0, 0, 0, {Data_init_default}, 0, 0, 0, 0, 0, _MeshPacket_Priority_MIN, 0, _MeshPacket_Delayed_MIN} #define NodeInfo_init_default {0, false, User_init_default, false, Position_init_default, 0, 0, false, DeviceMetrics_init_default} @@ -751,7 +756,7 @@ extern "C" { #define User_init_zero {"", "", "", {0}, _HardwareModel_MIN, 0, _Team_MIN, 0, 0, 0} #define RouteDiscovery_init_zero {0, {0, 0, 0, 0, 0, 0, 0, 0}} #define Routing_init_zero {0, {RouteDiscovery_init_zero}} -#define Data_init_zero {_PortNum_MIN, {0, {0}}, 0, 0, 0, 0, 0, 0, false, Location_init_zero} +#define Data_init_zero {_PortNum_MIN, 0, {{0, {0}}}, 0, 0, 0, 0, 0, 0, false, Location_init_zero} #define Location_init_zero {0, 0, 0, 0, 0} #define MeshPacket_init_zero {0, 0, 0, 0, {Data_init_zero}, 0, 0, 0, 0, 0, _MeshPacket_Priority_MIN, 0, _MeshPacket_Delayed_MIN} #define NodeInfo_init_zero {0, false, User_init_zero, false, Position_init_zero, 0, 0, false, DeviceMetrics_init_zero} @@ -825,6 +830,7 @@ extern "C" { #define User_ant_azimuth_tag 12 #define Data_portnum_tag 1 #define Data_payload_tag 2 +#define Data_payload_compressed_tag 10 #define Data_want_response_tag 3 #define Data_dest_tag 4 #define Data_source_tag 5 @@ -923,14 +929,15 @@ X(a, STATIC, ONEOF, UENUM, (variant,error_reason,error_reason), 3) #define Data_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, UENUM, portnum, 1) \ -X(a, STATIC, SINGULAR, BYTES, payload, 2) \ +X(a, STATIC, ONEOF, BYTES, (payloadVariant,payload,payload), 2) \ X(a, STATIC, SINGULAR, BOOL, want_response, 3) \ X(a, STATIC, SINGULAR, FIXED32, dest, 4) \ X(a, STATIC, SINGULAR, FIXED32, source, 5) \ X(a, STATIC, SINGULAR, FIXED32, request_id, 6) \ X(a, STATIC, SINGULAR, FIXED32, reply_id, 7) \ X(a, STATIC, SINGULAR, FIXED32, emoji, 8) \ -X(a, STATIC, OPTIONAL, MESSAGE, location, 9) +X(a, STATIC, OPTIONAL, MESSAGE, location, 9) \ +X(a, STATIC, ONEOF, BYTES, (payloadVariant,payload_compressed,payload_compressed), 10) #define Data_CALLBACK NULL #define Data_DEFAULT NULL #define Data_location_MSGTYPE Location diff --git a/src/mesh/generated/telemetry.pb.h b/src/mesh/generated/telemetry.pb.h index 1fb5ea0d7..b3bf3df1d 100644 --- a/src/mesh/generated/telemetry.pb.h +++ b/src/mesh/generated/telemetry.pb.h @@ -40,10 +40,10 @@ typedef struct _EnvironmentMetrics { /* Types of Measurements the telemetry module is equipped to handle */ typedef struct _Telemetry { - /* This is usually not sent over the mesh (to save space), but it is sent - from the phone so that the local device can set its RTC If it is sent over - the mesh (because there are devices on the mesh without GPS), it will only - be sent by devices which has a hardware GPS clock (IE Mobile Phone). + /* This is usually not sent over the mesh (to save space), but it is sent + from the phone so that the local device can set its RTC If it is sent over + the mesh (because there are devices on the mesh without GPS), it will only + be sent by devices which has a hardware GPS clock (IE Mobile Phone). seconds since 1970 */ uint32_t time; /* Key native device metrics such as battery level */ diff --git a/src/mesh/http/WiFiAPClient.cpp b/src/mesh/http/WiFiAPClient.cpp index afda19702..be07acc53 100644 --- a/src/mesh/http/WiFiAPClient.cpp +++ b/src/mesh/http/WiFiAPClient.cpp @@ -10,10 +10,13 @@ #include "target_specific.h" #include #include -#include #include #include +#ifndef DISABLE_NTP +#include +#endif + using namespace concurrency; static void WiFiEvent(WiFiEvent_t event); @@ -23,7 +26,10 @@ DNSServer dnsServer; // NTP WiFiUDP ntpUDP; + +#ifndef DISABLE_NTP NTPClient timeClient(ntpUDP, "0.pool.ntp.org"); +#endif uint8_t wifiDisconnectReason = 0; @@ -67,13 +73,13 @@ static int32_t reconnectWiFi() DEBUG_MSG("... Reconnecting to WiFi access point\n"); WiFi.mode(WIFI_MODE_STA); WiFi.begin(wifiName, wifiPsw); - // Starting timeClient; } } - //if (*wifiName) { +#ifndef DISABLE_NTP + // if (*wifiName) { if (WiFi.isConnected()) { DEBUG_MSG("Updating NTP time\n"); if (timeClient.update()) { @@ -89,6 +95,7 @@ static int32_t reconnectWiFi() DEBUG_MSG("NTP Update failed\n"); } } +#endif return 30 * 1000; // every 30 seconds } @@ -155,9 +162,11 @@ static void onNetworkConnected() MDNS.addService("https", "tcp", 443); } +#ifndef DISABLE_NTP DEBUG_MSG("Starting NTP time client\n"); timeClient.begin(); - timeClient.setUpdateInterval(60*60); // Update once an hour + timeClient.setUpdateInterval(60 * 60); // Update once an hour +#endif initWebServer(); initApiServer(); From 7e977aea005aebc8e128822a0548b07bb005343d Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Wed, 13 Apr 2022 21:59:25 -0700 Subject: [PATCH 19/40] Add welcome screen feature toggle --- platformio.ini | 5 +++-- src/configuration.h | 2 ++ src/graphics/Screen.cpp | 2 ++ src/mesh/Router.cpp | 8 ++++---- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/platformio.ini b/platformio.ini index ba5c37489..7bb2f339e 100644 --- a/platformio.ini +++ b/platformio.ini @@ -115,8 +115,9 @@ platform_packages = ;upload_port = /dev/ttyUSB0 ;monitor_port = /dev/ttyUSB0 -upload_port = /dev/cu.SLAB_USBtoUART -monitor_port = /dev/cu.SLAB_USBtoUART +; Please don't delete these lines. JM uses them. +;upload_port = /dev/cu.SLAB_USBtoUART +;monitor_port = /dev/cu.SLAB_USBtoUART ; customize the partition table ; http://docs.platformio.org/en/latest/platforms/espressif32.html#partition-tables diff --git a/src/configuration.h b/src/configuration.h index 59d0cf712..5de429c6b 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -131,6 +131,8 @@ along with this program. If not, see . //#define DISABLE_NTP +#define DISABLE_WELCOME_UNSET + // ----------------------------------------------------------------------------- // OLED & Input // ----------------------------------------------------------------------------- diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index 0ccc46b22..2b2cb7789 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -894,9 +894,11 @@ int32_t Screen::runOnce() showingBootScreen = false; } +#ifndef DISABLE_WELCOME_UNSET if (radioConfig.preferences.region == RegionCode_Unset) { setWelcomeFrames(); } +#endif // Process incoming commands. for (;;) { diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index 9487231fd..cb5bea03e 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -360,13 +360,13 @@ Routing_Error perhapsEncode(MeshPacket *p) } else { DEBUG_MSG("Compressing message.\n"); // Copy the compressed data into the meshpacket - p->decoded.payload_compressed.size = compressed_len; - memcpy(p->decoded.payload_compressed.bytes, compressed_out, compressed_len); + //p->decoded.payload_compressed.size = compressed_len; + //memcpy(p->decoded.payload_compressed.bytes, compressed_out, compressed_len); - p->decoded.which_payloadVariant = Data_payload_compressed_tag; + //p->decoded.which_payloadVariant = Data_payload_compressed_tag; } - if (0) { + if (1) { char decompressed_out[Constants_DATA_PAYLOAD_LEN] = {}; int decompressed_len; From a10ea604afce41bbaabf318deeb44acadb46164f Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Thu, 14 Apr 2022 15:51:48 -0700 Subject: [PATCH 20/40] Fixes for cppcheck errors --- src/configuration.h | 2 ++ src/mesh/compression/unishox2.c | 51 ++++++++++++++++----------------- 2 files changed, 26 insertions(+), 27 deletions(-) diff --git a/src/configuration.h b/src/configuration.h index 5de429c6b..31c7f0422 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -129,8 +129,10 @@ along with this program. If not, see . // Feature toggles // ----------------------------------------------------------------------------- +// Disable use of the NTP library and related features //#define DISABLE_NTP +// Disable the welcome screen and allow #define DISABLE_WELCOME_UNSET // ----------------------------------------------------------------------------- diff --git a/src/mesh/compression/unishox2.c b/src/mesh/compression/unishox2.c index e9d02e8a5..0ca650bb1 100644 --- a/src/mesh/compression/unishox2.c +++ b/src/mesh/compression/unishox2.c @@ -129,11 +129,9 @@ void init_coder() { for (int i = 0; i < 3; i++) { for (int j = 0; j < 28; j++) { byte c = usx_sets[i][j]; - if (c != 0 && c > 32) { - usx_code_94[c - USX_OFFSET_94] = (i << 5) + j; - if (c >= 'a' && c <= 'z') - usx_code_94[c - USX_OFFSET_94 - ('a' - 'A')] = (i << 5) + j; - } + usx_code_94[c - USX_OFFSET_94] = (i << 5) + j; + if (c >= 'a' && c <= 'z') + usx_code_94[c - USX_OFFSET_94 - ('a' - 'A')] = (i << 5) + j; } } is_inited = 1; @@ -147,32 +145,31 @@ unsigned int usx_mask[] = {0x80, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC, 0xFE, 0xFF}; /// Otherwise clen bits in code are appended to out starting with MSB int append_bits(char *out, int olen, int ol, byte code, int clen) { - byte cur_bit; - byte blen; - unsigned char a_byte; - int oidx; //printf("%d,%x,%d,%d\n", ol, code, clen, state); while (clen > 0) { - cur_bit = ol % 8; - blen = clen; - a_byte = code & usx_mask[blen - 1]; - a_byte >>= cur_bit; - if (blen + cur_bit > 8) - blen = (8 - cur_bit); - oidx = ol / 8; - if (oidx < 0 || olen <= oidx) - return -1; - if (cur_bit == 0) - out[oidx] = a_byte; - else - out[oidx] |= a_byte; - code <<= blen; - ol += blen; - clen -= blen; - } - return ol; + int oidx; + unsigned char a_byte; + + byte cur_bit = ol % 8; + byte blen = clen; + a_byte = code & usx_mask[blen - 1]; + a_byte >>= cur_bit; + if (blen + cur_bit > 8) + blen = (8 - cur_bit); + oidx = ol / 8; + if (oidx < 0 || olen <= oidx) + return -1; + if (cur_bit == 0) + out[oidx] = a_byte; + else + out[oidx] |= a_byte; + code <<= blen; + ol += blen; + clen -= blen; + } + return ol; } /// This is a safe call to append_bits() making sure it does not write past olen From dc20cbb6725e3e1a1e38381c245e163f5aec6714 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Thu, 14 Apr 2022 15:57:31 -0700 Subject: [PATCH 21/40] one more fix for cppcheck --- src/mesh/compression/unishox2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mesh/compression/unishox2.c b/src/mesh/compression/unishox2.c index 0ca650bb1..8e91185a9 100644 --- a/src/mesh/compression/unishox2.c +++ b/src/mesh/compression/unishox2.c @@ -973,7 +973,7 @@ int32_t readUnicode(const char *in, int *bit_no_p, int len) { if (idx == 99) return 0x7FFFFF00 + 99; if (idx == 5) { - int idx = getStepCodeIdx(in, len, bit_no_p, 4); + idx = getStepCodeIdx(in, len, bit_no_p, 4); return 0x7FFFFF00 + idx; } if (idx >= 0) { From bb22b6ec584df9eb607433f382a1370e7de2a4f0 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Thu, 14 Apr 2022 16:15:19 -0700 Subject: [PATCH 22/40] bump to 1.3.7 --- version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.properties b/version.properties index 13cd63471..73fb57178 100644 --- a/version.properties +++ b/version.properties @@ -1,4 +1,4 @@ [VERSION] major = 1 minor = 3 -build = 6 +build = 7 From fc0508f254118d43cff99735ae936428db2e5a7e Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Thu, 14 Apr 2022 21:02:52 -0700 Subject: [PATCH 23/40] Fix for welcome screen going over ble pairing --- src/graphics/Screen.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index 2b2cb7789..73d77e77a 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -187,7 +187,6 @@ static void drawWelcomeScreen(OLEDDisplay *display, OLEDDisplayUiState *state, i yield(); esp_task_wdt_reset(); #endif - } #ifdef HAS_EINK @@ -895,7 +894,7 @@ int32_t Screen::runOnce() } #ifndef DISABLE_WELCOME_UNSET - if (radioConfig.preferences.region == RegionCode_Unset) { + if (showingNormalScreen && radioConfig.preferences.region == RegionCode_Unset) { setWelcomeFrames(); } #endif From 87da7794785bf5a692d4029a9748a66ea0c8a7df Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Thu, 14 Apr 2022 21:31:31 -0700 Subject: [PATCH 24/40] Updated welcome screen w/ text, pages and logo Updated welcome screen w/ text, pages and logo --- src/graphics/Screen.cpp | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index 73d77e77a..6c6be7580 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -172,16 +172,29 @@ static void drawSSLScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int16 static void drawWelcomeScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int16_t x, int16_t y) { - display->setFont(FONT_SMALL); + if ((millis() / 10000) % 2) { + display->setFont(FONT_SMALL); - display->setTextAlignment(TEXT_ALIGN_CENTER); - display->drawString(64 + x, y, "Meshtastic :)"); + display->setTextAlignment(TEXT_ALIGN_CENTER); + display->drawString(64 + x, y, "//\\ E S H T /\\ S T / C"); - display->setTextAlignment(TEXT_ALIGN_LEFT); + display->setTextAlignment(TEXT_ALIGN_LEFT); - display->drawString(x, y + FONT_HEIGHT_SMALL * 2 - 3, "Set the region using the"); - display->drawString(x, y + FONT_HEIGHT_SMALL * 3 - 3, "Meshtastic Android, iOS,"); - display->drawString(x, y + FONT_HEIGHT_SMALL * 4 - 3, "Flasher or CLI client."); + display->drawString(x, y + FONT_HEIGHT_SMALL * 2 - 3, "Set the region using the"); + display->drawString(x, y + FONT_HEIGHT_SMALL * 3 - 3, "Meshtastic Android, iOS,"); + display->drawString(x, y + FONT_HEIGHT_SMALL * 4 - 3, "Flasher or CLI client."); + } else { + display->setFont(FONT_SMALL); + + display->setTextAlignment(TEXT_ALIGN_CENTER); + display->drawString(64 + x, y, "//\\ E S H T /\\ S T / C"); + + display->setTextAlignment(TEXT_ALIGN_LEFT); + + display->drawString(x, y + FONT_HEIGHT_SMALL * 2 - 3, "Visit meshtastic.org"); + display->drawString(x, y + FONT_HEIGHT_SMALL * 3 - 3, "for more information."); + display->drawString(x, y + FONT_HEIGHT_SMALL * 4 - 3, ""); + } #ifndef NO_ESP32 yield(); From 0c285aac6e27f63e20811d421db8d3d31a55c4d7 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Thu, 14 Apr 2022 21:32:00 -0700 Subject: [PATCH 25/40] enable welcome text --- src/configuration.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/configuration.h b/src/configuration.h index 31c7f0422..8f0d3cfa5 100644 --- a/src/configuration.h +++ b/src/configuration.h @@ -133,7 +133,7 @@ along with this program. If not, see . //#define DISABLE_NTP // Disable the welcome screen and allow -#define DISABLE_WELCOME_UNSET +// #define DISABLE_WELCOME_UNSET // ----------------------------------------------------------------------------- // OLED & Input From 8124ecbfd85a3b70911f42acc6f3af9aaca8d79c Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Fri, 15 Apr 2022 18:11:17 -0700 Subject: [PATCH 26/40] Change to unishox library --- src/mesh/compression/unishox2.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/mesh/compression/unishox2.c b/src/mesh/compression/unishox2.c index 8e91185a9..ef3828852 100644 --- a/src/mesh/compression/unishox2.c +++ b/src/mesh/compression/unishox2.c @@ -129,9 +129,11 @@ void init_coder() { for (int i = 0; i < 3; i++) { for (int j = 0; j < 28; j++) { byte c = usx_sets[i][j]; - usx_code_94[c - USX_OFFSET_94] = (i << 5) + j; - if (c >= 'a' && c <= 'z') - usx_code_94[c - USX_OFFSET_94 - ('a' - 'A')] = (i << 5) + j; + if (c > 32) { + usx_code_94[c - USX_OFFSET_94] = (i << 5) + j; + if (c >= 'a' && c <= 'z') + usx_code_94[c - USX_OFFSET_94 - ('a' - 'A')] = (i << 5) + j; + } } } is_inited = 1; @@ -168,8 +170,8 @@ int append_bits(char *out, int olen, int ol, byte code, int clen) { code <<= blen; ol += blen; clen -= blen; - } - return ol; + } + return ol; } /// This is a safe call to append_bits() making sure it does not write past olen From 79a41bd81c90fbfe9064fedbc635667946085628 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Fri, 15 Apr 2022 22:30:21 -0700 Subject: [PATCH 27/40] updating proto submodule to latest --- proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto b/proto index 870a62b27..e342be425 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 870a62b27e6d28dd1427e14996aed22cffd8dd01 +Subproject commit e342be425503cec75e0d0fc775d6cdbc04456200 From 5d7990667d9afd4fcb51cd63d6d6ab754a350b8d Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Fri, 15 Apr 2022 23:16:40 -0700 Subject: [PATCH 28/40] Max nodes to 64 and remove targz --- src/mesh/generated/deviceonly.pb.h | 8 +- src/mesh/http/ContentHandler.cpp | 150 +++-------------------------- 2 files changed, 16 insertions(+), 142 deletions(-) diff --git a/src/mesh/generated/deviceonly.pb.h b/src/mesh/generated/deviceonly.pb.h index 8250cf568..b65c60aae 100644 --- a/src/mesh/generated/deviceonly.pb.h +++ b/src/mesh/generated/deviceonly.pb.h @@ -33,7 +33,7 @@ typedef struct _DeviceState { User owner; /* TODO: REPLACE */ pb_size_t node_db_count; - NodeInfo node_db[80]; + NodeInfo node_db[64]; /* Received packets saved for delivery to the phone */ pb_size_t receive_queue_count; MeshPacket receive_queue[1]; @@ -59,9 +59,9 @@ extern "C" { #endif /* Initializer values for message structs */ -#define DeviceState_init_default {false, MyNodeInfo_init_default, false, User_init_default, 0, {NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default}, 0, {MeshPacket_init_default}, false, MeshPacket_init_default, 0, 0, 0} +#define DeviceState_init_default {false, MyNodeInfo_init_default, false, User_init_default, 0, {NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default}, 0, {MeshPacket_init_default}, false, MeshPacket_init_default, 0, 0, 0} #define ChannelFile_init_default {0, {Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default}} -#define DeviceState_init_zero {false, MyNodeInfo_init_zero, false, User_init_zero, 0, {NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero}, 0, {MeshPacket_init_zero}, false, MeshPacket_init_zero, 0, 0, 0} +#define DeviceState_init_zero {false, MyNodeInfo_init_zero, false, User_init_zero, 0, {NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero}, 0, {MeshPacket_init_zero}, false, MeshPacket_init_zero, 0, 0, 0} #define ChannelFile_init_zero {0, {Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero}} /* Field tags (for use in manual encoding/decoding) */ @@ -108,7 +108,7 @@ extern const pb_msgdesc_t ChannelFile_msg; /* Maximum encoded size of messages (where known) */ #define ChannelFile_size 832 -#define DeviceState_size 23903 +#define DeviceState_size 19327 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/http/ContentHandler.cpp b/src/mesh/http/ContentHandler.cpp index 754360d56..ba78b4ea5 100644 --- a/src/mesh/http/ContentHandler.cpp +++ b/src/mesh/http/ContentHandler.cpp @@ -47,11 +47,9 @@ using namespace httpsserver; #include HTTPClient httpClient; -// needed for ESP32-targz #define DEST_FS_USES_LITTLEFS #define ESP_ARDUINO_VERSION_VAL(major, minor, patch) ((major << 16) | (minor << 8) | (patch)) #define ESP_ARDUINO_VERSION ESP_ARDUINO_VERSION_VAL(1, 0, 4) -#include // We need to specify some content-type mapping, so the resources get delivered with the // right content type and are displayed correctly in the browser @@ -63,57 +61,13 @@ char contentTypes[][2][32] = {{".txt", "text/plain"}, {".html", "text/html"} {".svg", "image/svg+xml"}, {"", ""}}; // const char *tarURL = "https://www.casler.org/temp/meshtastic-web.tar"; -const char *tarURL = "https://api-production-871d.up.railway.app/mirror/webui"; -const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security) +// const char *tarURL = "https://api-production-871d.up.railway.app/mirror/webui"; +// const char *certificate = NULL; // change this as needed, leave as is for no TLS check (yolo security) // Our API to handle messages to and from the radio. HttpAPI webAPI; -WiFiClient *getTarHTTPClientPtr(WiFiClientSecure *client, const char *url, const char *cert = NULL) -{ - if (cert == NULL) { - // New versions don't have setInsecure - // client->setInsecure(); - } else { - client->setCACert(cert); - } - const char *UserAgent = "ESP32-HTTP-GzUpdater-Client"; - httpClient.setReuse(true); // handle 301 redirects gracefully - httpClient.setUserAgent(UserAgent); - httpClient.setConnectTimeout(10000); // 10s timeout = 10000 - if (!httpClient.begin(*client, url)) { - log_e("Can't open url %s", url); - return nullptr; - } - const char *headerKeys[] = {"location", "redirect", "Content-Type", "Content-Length", "Content-Disposition"}; - const size_t numberOfHeaders = 5; - httpClient.collectHeaders(headerKeys, numberOfHeaders); - int httpCode = httpClient.GET(); - // file found at server - if (httpCode == HTTP_CODE_FOUND || httpCode == HTTP_CODE_MOVED_PERMANENTLY) { - String newlocation = ""; - String headerLocation = httpClient.header("location"); - String headerRedirect = httpClient.header("redirect"); - if (headerLocation != "") { - newlocation = headerLocation; - Serial.printf("302 (location): %s => %s\n", url, headerLocation.c_str()); - } else if (headerRedirect != "") { - Serial.printf("301 (redirect): %s => %s\n", url, headerLocation.c_str()); - newlocation = headerRedirect; - } - httpClient.end(); - if (newlocation != "") { - log_w("Found 302/301 location header: %s", newlocation.c_str()); - return getTarHTTPClientPtr(client, newlocation.c_str(), cert); - } else { - log_e("Empty redirect !!"); - return nullptr; - } - } - if (httpCode != 200) - return nullptr; - return httpClient.getStreamPtr(); -} + void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) { @@ -131,9 +85,9 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) ResourceNode *nodeAdmin = new ResourceNode("/admin", "GET", &handleAdmin); ResourceNode *nodeAdminSettings = new ResourceNode("/admin/settings", "GET", &handleAdminSettings); ResourceNode *nodeAdminSettingsApply = new ResourceNode("/admin/settings/apply", "POST", &handleAdminSettingsApply); - ResourceNode *nodeAdminFs = new ResourceNode("/admin/fs", "GET", &handleFs); - ResourceNode *nodeUpdateFs = new ResourceNode("/admin/fs/update", "POST", &handleUpdateFs); - ResourceNode *nodeDeleteFs = new ResourceNode("/admin/fs/delete", "GET", &handleDeleteFsContent); +// ResourceNode *nodeAdminFs = new ResourceNode("/admin/fs", "GET", &handleFs); +// ResourceNode *nodeUpdateFs = new ResourceNode("/admin/fs/update", "POST", &handleUpdateFs); +// ResourceNode *nodeDeleteFs = new ResourceNode("/admin/fs/delete", "GET", &handleDeleteFsContent); ResourceNode *nodeRestart = new ResourceNode("/restart", "POST", &handleRestart); ResourceNode *nodeFormUpload = new ResourceNode("/upload", "POST", &handleFormUpload); @@ -160,10 +114,10 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) secureServer->registerNode(nodeJsonFsBrowseStatic); secureServer->registerNode(nodeJsonDelete); secureServer->registerNode(nodeJsonReport); - secureServer->registerNode(nodeUpdateFs); - secureServer->registerNode(nodeDeleteFs); +// secureServer->registerNode(nodeUpdateFs); +// secureServer->registerNode(nodeDeleteFs); secureServer->registerNode(nodeAdmin); - secureServer->registerNode(nodeAdminFs); +// secureServer->registerNode(nodeAdminFs); secureServer->registerNode(nodeAdminSettings); secureServer->registerNode(nodeAdminSettingsApply); secureServer->registerNode(nodeRoot); // This has to be last @@ -181,10 +135,10 @@ void registerHandlers(HTTPServer *insecureServer, HTTPSServer *secureServer) insecureServer->registerNode(nodeJsonFsBrowseStatic); insecureServer->registerNode(nodeJsonDelete); insecureServer->registerNode(nodeJsonReport); - insecureServer->registerNode(nodeUpdateFs); - insecureServer->registerNode(nodeDeleteFs); +// insecureServer->registerNode(nodeUpdateFs); +// insecureServer->registerNode(nodeDeleteFs); insecureServer->registerNode(nodeAdmin); - insecureServer->registerNode(nodeAdminFs); +// insecureServer->registerNode(nodeAdminFs); insecureServer->registerNode(nodeAdminSettings); insecureServer->registerNode(nodeAdminSettingsApply); insecureServer->registerNode(nodeRoot); // This has to be last @@ -727,86 +681,6 @@ void handleHotspot(HTTPRequest *req, HTTPResponse *res) res->println("\n"); } -void handleUpdateFs(HTTPRequest *req, HTTPResponse *res) -{ - res->setHeader("Content-Type", "text/html"); - res->setHeader("Access-Control-Allow-Origin", "*"); - // res->setHeader("Access-Control-Allow-Methods", "POST"); - - res->println("

Meshtastic

\n"); - res->println("Downloading Meshtastic Web Content..."); - - WiFiClientSecure *client = new WiFiClientSecure; - Stream *streamptr = getTarHTTPClientPtr(client, tarURL, certificate); - - delay(5); // Let other network operations run - - if (streamptr != nullptr) { - DEBUG_MSG("Connection to content server ... success!\n"); - - DEBUG_MSG("Deleting files from /static : \n"); - - htmlDeleteDir("/static"); - - delay(5); // Let other network operations run - - TarUnpacker *TARUnpacker = new TarUnpacker(); - TARUnpacker->haltOnError(false); // stop on fail (manual restart/reset required) - TARUnpacker->setTarVerify(false); // true = enables health checks but slows down the overall process - TARUnpacker->setupFSCallbacks(targzTotalBytesFn, targzFreeBytesFn); // prevent the partition from exploding, recommended - TARUnpacker->setLoggerCallback(BaseUnpacker::targzPrintLoggerCallback); // gz log verbosity - TARUnpacker->setTarProgressCallback( - BaseUnpacker::defaultProgressCallback); // prints the untarring progress for each individual file - TARUnpacker->setTarStatusProgressCallback( - BaseUnpacker::defaultTarStatusProgressCallback); // print the filenames as they're expanded - TARUnpacker->setTarMessageCallback(BaseUnpacker::targzPrintLoggerCallback); // tar log verbosity - - String contentLengthStr = httpClient.header("Content-Length"); - contentLengthStr.trim(); - int64_t streamSize = -1; - if (contentLengthStr != "") { - streamSize = atoi(contentLengthStr.c_str()); - Serial.printf("Stream size %d\n", streamSize); - res->printf("Stream size %d

\n", streamSize); - } - - if (!TARUnpacker->tarStreamExpander(streamptr, streamSize, FSCom, "/static")) { - res->printf("tarStreamExpander failed with return code #%d\n", TARUnpacker->tarGzGetError()); - Serial.printf("tarStreamExpander failed with return code #%d\n", TARUnpacker->tarGzGetError()); - // Close the connection on error and free up memory - client->stop(); - - return; - } else { - /* - // print leftover bytes if any (probably zero-fill from the server) - while (httpClient.connected()) { - size_t streamSize = streamptr->available(); - if (streamSize) { - Serial.printf("%02x ", streamptr->read()); - } else - break; - } - */ - } - - } else { - res->printf("Failed to establish http connection\n"); - Serial.println("Failed to establish http connection"); - return; - // Close the connection on error and free up memory - client->stop(); - } - - res->println("Done! Restarting the device. Click this in 10 seconds"); - - /* - * This is a work around for a bug where we run out of memory. - * TODO: Fixme! - */ - // ESP.restart(); - webServerThread->requestRestart = (millis() / 1000) + 5; -} void handleDeleteFsContent(HTTPRequest *req, HTTPResponse *res) { From 5327d6162a9f587fc5df941f4f06c57e126d1913 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Fri, 15 Apr 2022 23:17:30 -0700 Subject: [PATCH 29/40] remove targz library --- platformio.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio.ini b/platformio.ini index 7bb2f339e..67c6ae9b3 100644 --- a/platformio.ini +++ b/platformio.ini @@ -102,7 +102,6 @@ lib_deps = ${environmental.lib_deps} https://github.com/meshtastic/esp32_https_server.git h2zero/NimBLE-Arduino@1.3.7 - tobozo/ESP32-targz@^1.1.4 arduino-libraries/NTPClient@^3.1.0 lorol/LittleFS_esp32@^1.0.6 lib_ignore = From 90974751498aecc3923f36bf293df838aa3aa16e Mon Sep 17 00:00:00 2001 From: Sacha Weatherstone Date: Mon, 18 Apr 2022 23:56:16 +1000 Subject: [PATCH 30/40] Add Repobeats stats image to readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 2abb45c0f..9ba829bf5 100644 --- a/README.md +++ b/README.md @@ -17,3 +17,7 @@ Manual Method [For nRF52 devices click here](https://meshtastic.org/docs/getting-started/flashing-nrf52) For developer information and specific building instructions, please see the [developer documentation](https://meshtastic.org/docs/developers) + +## Stats + +![Alt](https://repobeats.axiom.co/api/embed/99a2cf5622bb4807f9e8c3b86589f1133cce58a2.svg 'Repobeats analytics image') From f279f9614eaa6255689339ffff46daddfad4711b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 18 Apr 2022 18:11:17 +0200 Subject: [PATCH 31/40] Enable OEM Bootlogo, needs protobuf update --- src/graphics/Screen.cpp | 77 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 74 insertions(+), 3 deletions(-) diff --git a/src/graphics/Screen.cpp b/src/graphics/Screen.cpp index 6c6be7580..a4bddaf17 100644 --- a/src/graphics/Screen.cpp +++ b/src/graphics/Screen.cpp @@ -31,6 +31,7 @@ along with this program. If not, see . #include "graphics/images.h" #include "main.h" #include "mesh-pb-constants.h" +#include "mesh/generated/deviceonly.pb.h" #include "mesh/Channels.h" #include "modules/TextMessageModule.h" #include "sleep.h" @@ -44,6 +45,8 @@ along with this program. If not, see . using namespace meshtastic; /** @todo remove */ +extern bool loadProto(const char *filename, size_t protoSize, size_t objSize, const pb_msgdesc_t *fields, void *dest_struct); + namespace graphics { @@ -77,6 +80,10 @@ static char ourId[5]; // GeoCoord object for the screen GeoCoord geoCoord; +// OEM Config File +static const char *oemConfigFile = "/prefs/oem.proto"; +OEMStore oemStore; + #ifdef SHOW_REDRAWS static bool heartbeat = false; #endif @@ -148,6 +155,54 @@ static void drawBootScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int1 drawIconScreen(region, display, state, x, y); } +static void drawOEMIconScreen(const char *upperMsg, OLEDDisplay *display, OLEDDisplayUiState *state, int16_t x, int16_t y) +{ + // draw an xbm image. + // Please note that everything that should be transitioned + // needs to be drawn relative to x and y + + // draw centered icon left to right and centered above the one line of app text + display->drawXbm(x + (SCREEN_WIDTH - oemStore.oem_icon_width) / 2, y + (SCREEN_HEIGHT - FONT_HEIGHT_MEDIUM - oemStore.oem_icon_height) / 2 + 2, + oemStore.oem_icon_width, oemStore.oem_icon_height, (const uint8_t *)oemStore.oem_icon_bits.bytes); + + switch(oemStore.oem_font){ + case 0: + display->setFont(FONT_SMALL); + break; + case 2: + display->setFont(FONT_LARGE); + break; + default: + display->setFont(FONT_MEDIUM); + break; + } + + display->setTextAlignment(TEXT_ALIGN_LEFT); + const char *title = oemStore.oem_text; + display->drawString(x + getStringCenteredX(title), y + SCREEN_HEIGHT - FONT_HEIGHT_MEDIUM, title); + display->setFont(FONT_SMALL); + + // Draw region in upper left + if (upperMsg) + display->drawString(x + 0, y + 0, upperMsg); + + // Draw version in upper right + char buf[16]; + snprintf(buf, sizeof(buf), "%s", + xstr(APP_VERSION_SHORT)); // Note: we don't bother printing region or now, it makes the string too long + display->drawString(x + SCREEN_WIDTH - display->getStringWidth(buf), y + 0, buf); + screen->forceDisplay(); + + // FIXME - draw serial # somewhere? +} + +static void drawOEMBootScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int16_t x, int16_t y) +{ + // Draw region in upper left + const char *region = myRegion ? myRegion->name : NULL; + drawOEMIconScreen(region, display, state, x, y); +} + // Used on boot when a certificate is being created static void drawSSLScreen(OLEDDisplay *display, OLEDDisplayUiState *state, int16_t x, int16_t y) { @@ -813,8 +868,8 @@ void Screen::setup() dispdev.setDetected(screen_model); #endif - // I think this is not needed - redundant with ui.init - // dispdev.resetOrientation(); + // Load OEM config from Proto file if existent + loadProto(oemConfigFile, OEMStore_size, sizeof(oemConfigFile), OEMStore_fields, &oemStore); // Initialising the UI will init the display too. ui.init(); @@ -867,6 +922,7 @@ void Screen::setup() // twice initially. ui.update(); ui.update(); + serialSinceMsec = millis(); // Subscribe to status updates powerStatusObserver.observe(&powerStatus->onNewStatus); @@ -897,7 +953,7 @@ int32_t Screen::runOnce() return RUN_SAME; } - // Show boot screen for first 3 seconds, then switch to normal operation. + // Show boot screen for first 5 seconds, then switch to normal operation. // serialSinceMsec adjusts for additional serial wait time during nRF52 bootup static bool showingBootScreen = true; if (showingBootScreen && (millis() > (5000 + serialSinceMsec))) { @@ -906,6 +962,21 @@ int32_t Screen::runOnce() showingBootScreen = false; } + // If we have an OEM Boot screen, toggle after 2,5 seconds + if(strlen(oemStore.oem_text) > 0){ + static bool showingOEMBootScreen = true; + if (showingOEMBootScreen && (millis() > (2500 + serialSinceMsec))) { + DEBUG_MSG("Switch to OEM screen...\n"); + // Change frames. + static FrameCallback bootOEMFrames[] = {drawOEMBootScreen}; + static const int bootOEMFrameCount = sizeof(bootOEMFrames) / sizeof(bootOEMFrames[0]); + ui.setFrames(bootOEMFrames, bootOEMFrameCount); + ui.update(); + ui.update(); + showingOEMBootScreen = false; + } + } + #ifndef DISABLE_WELCOME_UNSET if (showingNormalScreen && radioConfig.preferences.region == RegionCode_Unset) { setWelcomeFrames(); From 748416d9e3006d0ebedd4741cf17bba81926c968 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 18 Apr 2022 18:27:17 +0200 Subject: [PATCH 32/40] protobuf update --- src/mesh/generated/deviceonly.pb.c | 4 ++ src/mesh/generated/deviceonly.pb.h | 60 ++++++++++++++++++++++++++++-- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/src/mesh/generated/deviceonly.pb.c b/src/mesh/generated/deviceonly.pb.c index a91649052..76aba1aef 100644 --- a/src/mesh/generated/deviceonly.pb.c +++ b/src/mesh/generated/deviceonly.pb.c @@ -12,4 +12,8 @@ PB_BIND(DeviceState, DeviceState, 4) PB_BIND(ChannelFile, ChannelFile, 2) +PB_BIND(OEMStore, OEMStore, 2) + + + diff --git a/src/mesh/generated/deviceonly.pb.h b/src/mesh/generated/deviceonly.pb.h index 8250cf568..a8df04eae 100644 --- a/src/mesh/generated/deviceonly.pb.h +++ b/src/mesh/generated/deviceonly.pb.h @@ -11,6 +11,17 @@ #error Regenerate this file with the current version of nanopb generator. #endif +/* Enum definitions */ +/* TODO: REPLACE */ +typedef enum _ScreenFonts { + /* TODO: REPLACE */ + ScreenFonts_FONT_SMALL = 0, + /* TODO: REPLACE */ + ScreenFonts_FONT_MEDIUM = 1, + /* TODO: REPLACE */ + ScreenFonts_FONT_LARGE = 2 +} ScreenFonts; + /* Struct definitions */ /* The on-disk saved channels */ typedef struct _ChannelFile { @@ -33,7 +44,7 @@ typedef struct _DeviceState { User owner; /* TODO: REPLACE */ pb_size_t node_db_count; - NodeInfo node_db[80]; + NodeInfo node_db[64]; /* Received packets saved for delivery to the phone */ pb_size_t receive_queue_count; MeshPacket receive_queue[1]; @@ -53,16 +64,40 @@ typedef struct _DeviceState { bool did_gps_reset; } DeviceState; +typedef PB_BYTES_ARRAY_T(2048) OEMStore_oem_icon_bits_t; +/* This can be used for customizing the firmware distribution. If populated, + show a secondary bootup screen with cuatom logo and text for 2.5 seconds. */ +typedef struct _OEMStore { + /* The Logo width in Px */ + uint32_t oem_icon_width; + /* The Logo height in Px */ + uint32_t oem_icon_height; + /* The Logo in xbm bytechar format */ + OEMStore_oem_icon_bits_t oem_icon_bits; + /* Use this font for the OEM text. */ + ScreenFonts oem_font; + /* Use this font for the OEM text. */ + char oem_text[40]; +} OEMStore; + + +/* Helper constants for enums */ +#define _ScreenFonts_MIN ScreenFonts_FONT_SMALL +#define _ScreenFonts_MAX ScreenFonts_FONT_LARGE +#define _ScreenFonts_ARRAYSIZE ((ScreenFonts)(ScreenFonts_FONT_LARGE+1)) + #ifdef __cplusplus extern "C" { #endif /* Initializer values for message structs */ -#define DeviceState_init_default {false, MyNodeInfo_init_default, false, User_init_default, 0, {NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default}, 0, {MeshPacket_init_default}, false, MeshPacket_init_default, 0, 0, 0} +#define DeviceState_init_default {false, MyNodeInfo_init_default, false, User_init_default, 0, {NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default}, 0, {MeshPacket_init_default}, false, MeshPacket_init_default, 0, 0, 0} #define ChannelFile_init_default {0, {Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default}} -#define DeviceState_init_zero {false, MyNodeInfo_init_zero, false, User_init_zero, 0, {NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero}, 0, {MeshPacket_init_zero}, false, MeshPacket_init_zero, 0, 0, 0} +#define OEMStore_init_default {0, 0, {0, {0}}, _ScreenFonts_MIN, ""} +#define DeviceState_init_zero {false, MyNodeInfo_init_zero, false, User_init_zero, 0, {NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero}, 0, {MeshPacket_init_zero}, false, MeshPacket_init_zero, 0, 0, 0} #define ChannelFile_init_zero {0, {Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero}} +#define OEMStore_init_zero {0, 0, {0, {0}}, _ScreenFonts_MIN, ""} /* Field tags (for use in manual encoding/decoding) */ #define ChannelFile_channels_tag 1 @@ -74,6 +109,11 @@ extern "C" { #define DeviceState_version_tag 8 #define DeviceState_no_save_tag 9 #define DeviceState_did_gps_reset_tag 11 +#define OEMStore_oem_icon_width_tag 1 +#define OEMStore_oem_icon_height_tag 2 +#define OEMStore_oem_icon_bits_tag 3 +#define OEMStore_oem_font_tag 4 +#define OEMStore_oem_text_tag 5 /* Struct field encoding specification for nanopb */ #define DeviceState_FIELDLIST(X, a) \ @@ -99,16 +139,28 @@ X(a, STATIC, REPEATED, MESSAGE, channels, 1) #define ChannelFile_DEFAULT NULL #define ChannelFile_channels_MSGTYPE Channel +#define OEMStore_FIELDLIST(X, a) \ +X(a, STATIC, SINGULAR, UINT32, oem_icon_width, 1) \ +X(a, STATIC, SINGULAR, UINT32, oem_icon_height, 2) \ +X(a, STATIC, SINGULAR, BYTES, oem_icon_bits, 3) \ +X(a, STATIC, SINGULAR, UENUM, oem_font, 4) \ +X(a, STATIC, SINGULAR, STRING, oem_text, 5) +#define OEMStore_CALLBACK NULL +#define OEMStore_DEFAULT NULL + extern const pb_msgdesc_t DeviceState_msg; extern const pb_msgdesc_t ChannelFile_msg; +extern const pb_msgdesc_t OEMStore_msg; /* Defines for backwards compatibility with code written before nanopb-0.4.0 */ #define DeviceState_fields &DeviceState_msg #define ChannelFile_fields &ChannelFile_msg +#define OEMStore_fields &OEMStore_msg /* Maximum encoded size of messages (where known) */ #define ChannelFile_size 832 -#define DeviceState_size 23903 +#define DeviceState_size 19327 +#define OEMStore_size 2106 #ifdef __cplusplus } /* extern "C" */ From 7c071e2361866892d5bbb1744b15feaa16327e81 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Mon, 18 Apr 2022 12:29:20 -0700 Subject: [PATCH 33/40] updating proto submodule to latest --- proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proto b/proto index e342be425..8bac81b63 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit e342be425503cec75e0d0fc775d6cdbc04456200 +Subproject commit 8bac81b6317552ca873e324fbd520c77c4a937a3 From 136e2e96f747f744afeb4df38638f97ada234e94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 18 Apr 2022 21:55:58 +0200 Subject: [PATCH 34/40] Remove References to Pre-1.2 Preffile 1.3+ need a clean install anyway, so no point migrating these any more. --- src/mesh/NodeDB.cpp | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index a394db13e..ad113e8ab 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -276,11 +276,9 @@ void NodeDB::pickNewNodeNum() myNodeInfo.my_node_num = r; } -static const char *preffileOld = "/db.proto"; static const char *preffile = "/prefs/db.proto"; static const char *radiofile = "/prefs/radio.proto"; static const char *channelfile = "/prefs/channels.proto"; -// const char *preftmp = "/db.proto.tmp"; /** Load a protobuf from a file, return true for success */ bool loadProto(const char *filename, size_t protoSize, size_t objSize, const pb_msgdesc_t *fields, void *dest_struct) @@ -290,13 +288,6 @@ bool loadProto(const char *filename, size_t protoSize, size_t objSize, const pb_ auto f = FSCom.open(filename); - // FIXME, temporary hack until every node in the universe is 1.2 or later - look for prefs in the old location (so we can - // preserve region) - if (!f && filename == preffile) { - filename = preffileOld; - f = FSCom.open(filename); - } - bool okay = false; if (f) { DEBUG_MSG("Loading %s\n", filename); @@ -399,8 +390,6 @@ void NodeDB::saveToDisk() saveProto(radiofile, RadioConfig_size, sizeof(RadioConfig), RadioConfig_fields, &radioConfig); saveChannelsToDisk(); - // remove any pre 1.2 pref files, turn on after 1.2 is in beta - // if(okay) FSCom.remove(preffileOld); } else { DEBUG_MSG("***** DEVELOPMENT MODE - DO NOT RELEASE - not saving to flash *****\n"); } From ed62b6916c2960972bf5e0119c782ee5e8fb0dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 18 Apr 2022 22:46:45 +0200 Subject: [PATCH 35/40] tryfix #1363 Ask for the register several times and only go on if the answer is the same in 2 consecutive tries. --- src/debug/i2cScan.h | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/debug/i2cScan.h b/src/debug/i2cScan.h index 7bb5eadd5..9e7a49a62 100644 --- a/src/debug/i2cScan.h +++ b/src/debug/i2cScan.h @@ -6,21 +6,28 @@ uint8_t oled_probe(byte addr) { uint8_t r = 0; + uint8_t r_prev = 0; + uint8_t c = 0; uint8_t o_probe = 0; - Wire.beginTransmission(addr); - Wire.write(0x00); - Wire.endTransmission(); - Wire.requestFrom((int)addr, 1); - if (Wire.available()) { - r = Wire.read(); - } - r &= 0x0f; - if (r == 0x08 || r == 0x00) { - o_probe = 2; // SH1106 - } else if ( r == 0x03 || r == 0x06 || r == 0x07) { - o_probe = 1; // SSD1306 - } - DEBUG_MSG("0x%x subtype probed\n", r); + do { + r_prev = r; + Wire.beginTransmission(addr); + Wire.write(0x00); + Wire.endTransmission(); + Wire.requestFrom((int)addr, 1); + if (Wire.available()) { + r = Wire.read(); + } + r &= 0x0f; + + if (r == 0x08 || r == 0x00) { + o_probe = 2; // SH1106 + } else if ( r == 0x03 || r == 0x06 || r == 0x07) { + o_probe = 1; // SSD1306 + } + c++; + } while ((r != r_prev) && (c < 4)); + DEBUG_MSG("0x%x subtype probed in %i tries \n", r, c); return o_probe; } From 6fe9f0b42fe3eaf037ddbc865dd35e33f933500a Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Mon, 18 Apr 2022 17:00:36 -0700 Subject: [PATCH 36/40] Disable compression / decompression (for testing) --- src/mesh/Router.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index cb5bea03e..68e53a507 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -340,7 +340,8 @@ Routing_Error perhapsEncode(MeshPacket *p) char compressed_out[Constants_DATA_PAYLOAD_LEN] = {0}; - int compressed_len = unishox2_compress_simple(original_payload, p->decoded.payload.size, compressed_out); + int compressed_len; + // compressed_len = unishox2_compress_simple(original_payload, p->decoded.payload.size, compressed_out); Serial.print("Original length - "); Serial.println(p->decoded.payload.size); @@ -366,11 +367,11 @@ Routing_Error perhapsEncode(MeshPacket *p) //p->decoded.which_payloadVariant = Data_payload_compressed_tag; } - if (1) { + if (0) { char decompressed_out[Constants_DATA_PAYLOAD_LEN] = {}; int decompressed_len; - decompressed_len = unishox2_decompress_simple(compressed_out, compressed_len, decompressed_out); + // decompressed_len = unishox2_decompress_simple(compressed_out, compressed_len, decompressed_out); Serial.print("Decompressed length - "); Serial.println(decompressed_len); From 90df7c2488ccdc3291913ace5a3c7f1d6e829573 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Mon, 18 Apr 2022 17:20:03 -0700 Subject: [PATCH 37/40] bump to 1.3.8 --- version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.properties b/version.properties index 73fb57178..d46454914 100644 --- a/version.properties +++ b/version.properties @@ -1,4 +1,4 @@ [VERSION] major = 1 minor = 3 -build = 7 +build = 8 From b82bf5c72999a122eeaca33e8b20e27866193144 Mon Sep 17 00:00:00 2001 From: caveman99 Date: Tue, 19 Apr 2022 20:26:22 +0000 Subject: [PATCH 38/40] [create-pull-request] automated change --- proto | 2 +- src/mesh/generated/deviceonly.pb.h | 2 ++ src/mesh/generated/radioconfig.pb.h | 11 ++++++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/proto b/proto index 8bac81b63..a578453b3 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 8bac81b6317552ca873e324fbd520c77c4a937a3 +Subproject commit a578453b3c17794b61fb6cf4470ecaac8287d6d2 diff --git a/src/mesh/generated/deviceonly.pb.h b/src/mesh/generated/deviceonly.pb.h index 4aa637f03..a8df04eae 100644 --- a/src/mesh/generated/deviceonly.pb.h +++ b/src/mesh/generated/deviceonly.pb.h @@ -93,6 +93,8 @@ extern "C" { /* Initializer values for message structs */ #define DeviceState_init_default {false, MyNodeInfo_init_default, false, User_init_default, 0, {NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default}, 0, {MeshPacket_init_default}, false, MeshPacket_init_default, 0, 0, 0} +#define ChannelFile_init_default {0, {Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default}} +#define OEMStore_init_default {0, 0, {0, {0}}, _ScreenFonts_MIN, ""} #define DeviceState_init_zero {false, MyNodeInfo_init_zero, false, User_init_zero, 0, {NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero}, 0, {MeshPacket_init_zero}, false, MeshPacket_init_zero, 0, 0, 0} #define ChannelFile_init_zero {0, {Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero}} #define OEMStore_init_zero {0, 0, {0, {0}}, _ScreenFonts_MIN, ""} diff --git a/src/mesh/generated/radioconfig.pb.h b/src/mesh/generated/radioconfig.pb.h index db1a82837..b6a50bb90 100644 --- a/src/mesh/generated/radioconfig.pb.h +++ b/src/mesh/generated/radioconfig.pb.h @@ -210,7 +210,12 @@ typedef enum _RadioConfig_UserPreferences_Serial_Baud { /* TODO: REPLACE */ RadioConfig_UserPreferences_Serial_Baud_BAUD_576000 = 10, /* TODO: REPLACE */ - RadioConfig_UserPreferences_Serial_Baud_BAUD_921600 = 11 + RadioConfig_UserPreferences_Serial_Baud_BAUD_921600 = 11, + /* TODO: REPLACE */ + RadioConfig_UserPreferences_Serial_Baud_BAUD_110 = 12, + RadioConfig_UserPreferences_Serial_Baud_BAUD_300 = 13, + RadioConfig_UserPreferences_Serial_Baud_BAUD_600 = 14, + RadioConfig_UserPreferences_Serial_Baud_BAUD_1200 = 15 } RadioConfig_UserPreferences_Serial_Baud; /* Defines the device's role on the Mesh network @@ -381,8 +386,8 @@ typedef struct _RadioConfig { #define _InputEventChar_ARRAYSIZE ((InputEventChar)(InputEventChar_KEY_BACK+1)) #define _RadioConfig_UserPreferences_Serial_Baud_MIN RadioConfig_UserPreferences_Serial_Baud_BAUD_Default -#define _RadioConfig_UserPreferences_Serial_Baud_MAX RadioConfig_UserPreferences_Serial_Baud_BAUD_921600 -#define _RadioConfig_UserPreferences_Serial_Baud_ARRAYSIZE ((RadioConfig_UserPreferences_Serial_Baud)(RadioConfig_UserPreferences_Serial_Baud_BAUD_921600+1)) +#define _RadioConfig_UserPreferences_Serial_Baud_MAX RadioConfig_UserPreferences_Serial_Baud_BAUD_1200 +#define _RadioConfig_UserPreferences_Serial_Baud_ARRAYSIZE ((RadioConfig_UserPreferences_Serial_Baud)(RadioConfig_UserPreferences_Serial_Baud_BAUD_1200+1)) #define _RadioConfig_UserPreferences_Serial_Mode_MIN RadioConfig_UserPreferences_Serial_Mode_MODE_Default #define _RadioConfig_UserPreferences_Serial_Mode_MAX RadioConfig_UserPreferences_Serial_Mode_MODE_PROTO From 293921e95a99247789bd792790912264513a4f61 Mon Sep 17 00:00:00 2001 From: Dmitry Galenko Date: Tue, 19 Apr 2022 22:37:04 +0200 Subject: [PATCH 39/40] Brother EP-44 support --- src/modules/esp32/SerialModule.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/modules/esp32/SerialModule.cpp b/src/modules/esp32/SerialModule.cpp index 20b41c418..46f778f87 100644 --- a/src/modules/esp32/SerialModule.cpp +++ b/src/modules/esp32/SerialModule.cpp @@ -93,6 +93,18 @@ int32_t SerialModule::runOnce() if (radioConfig.preferences.serial_module_baud == RadioConfig_UserPreferences_Serial_Baud_BAUD_Default) { baud = 38400; + + } else if (radioConfig.preferences.serial_module_baud == RadioConfig_UserPreferences_Serial_Baud_BAUD_110) { + baud = 110; + + } else if (radioConfig.preferences.serial_module_baud == RadioConfig_UserPreferences_Serial_Baud_BAUD_300) { + baud = 300; + + } else if (radioConfig.preferences.serial_module_baud == RadioConfig_UserPreferences_Serial_Baud_BAUD_600) { + baud = 600; + + } else if (radioConfig.preferences.serial_module_baud == RadioConfig_UserPreferences_Serial_Baud_BAUD_1200) { + baud = 1200; } else if (radioConfig.preferences.serial_module_baud == RadioConfig_UserPreferences_Serial_Baud_BAUD_2400) { baud = 2400; From 6b012ca5b022ad5ebc384eb164aaedb3bc8a2585 Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Tue, 19 Apr 2022 22:41:00 -0700 Subject: [PATCH 40/40] Add fiscal contributors badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9ba829bf5..5eb417171 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ [![Continuous Integration](https://github.com/meshtastic/Meshtastic-device/actions/workflows/main.yml/badge.svg)](https://github.com/meshtastic/Meshtastic-device/actions/workflows/main.yml) ![GitHub all releases](https://img.shields.io/github/downloads/meshtastic/meshtastic-device/total) [![CLA assistant](https://cla-assistant.io/readme/badge/meshtastic/Meshtastic-device)](https://cla-assistant.io/meshtastic/Meshtastic-device) +[![Fiscal Contributors](https://opencollective.com/meshtastic/tiers/badge.svg)](https://opencollective.com/meshtastic/tiers/badge.svg) ## This repository contains the device firmware used in the [Meshtastic](https://meshtastic.org) project.