diff --git a/arch/nrf52/nrf52.ini b/arch/nrf52/nrf52.ini index 57b276978..0c6154c83 100644 --- a/arch/nrf52/nrf52.ini +++ b/arch/nrf52/nrf52.ini @@ -17,7 +17,7 @@ build_flags = -DLFS_NO_ASSERT ; Disable LFS assertions , see https://github.com/meshtastic/firmware/pull/3818 -DMESHTASTIC_EXCLUDE_AUDIO=1 -DMESHTASTIC_EXCLUDE_PAXCOUNTER=1 - -DMAX_NUM_NODES=80 + -DMAX_NUM_NODES=70 build_src_filter = ${arduino_base.build_src_filter} - - - - - - - - - - diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 45679e3f6..2a0d66a33 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -57,6 +57,7 @@ NodeDB *nodeDB = nullptr; // we have plenty of ram so statically alloc this tempbuf (for now) EXT_RAM_BSS_ATTR meshtastic_DeviceState devicestate; meshtastic_MyNodeInfo &myNodeInfo = devicestate.my_node; +meshtastic_NodeDatabase nodeDatabase; meshtastic_LocalConfig config; meshtastic_DeviceUIConfig uiconfig{.screen_brightness = 153, .screen_timeout = 30}; meshtastic_LocalModuleConfig moduleConfig; @@ -144,7 +145,7 @@ uint32_t get_st7789_id(uint8_t cs, uint8_t sck, uint8_t mosi, uint8_t dc, uint8_ #endif -bool meshtastic_DeviceState_callback(pb_istream_t *istream, pb_ostream_t *ostream, const pb_field_iter_t *field) +bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostream, const pb_field_iter_t *field) { if (ostream) { std::vector const *vec = (std::vector *)field->pData; @@ -193,6 +194,7 @@ NodeDB::NodeDB() cleanupMeshDB(); uint32_t devicestateCRC = crc32Buffer(&devicestate, sizeof(devicestate)); + uint32_t nodeDatabaseCRC = crc32Buffer(&nodeDatabase, sizeof(nodeDatabase)); uint32_t configCRC = crc32Buffer(&config, sizeof(config)); uint32_t channelFileCRC = crc32Buffer(&channelFile, sizeof(channelFile)); @@ -249,15 +251,15 @@ NodeDB::NodeDB() // Ensure macaddr is set to our macaddr as it will be copied in our info below memcpy(owner.macaddr, ourMacAddr, sizeof(owner.macaddr)); - // Include our owner in the node db under our nodenum - meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); if (!config.has_security) { config.has_security = true; + config.security = meshtastic_Config_SecurityConfig_init_default; config.security.serial_enabled = config.device.serial_enabled; config.security.is_managed = config.device.is_managed; } #if !(MESHTASTIC_EXCLUDE_PKI_KEYGEN || MESHTASTIC_EXCLUDE_PKI) + if (!owner.is_licensed) { bool keygenSuccess = false; if (config.security.private_key.size == 32) { @@ -284,7 +286,8 @@ NodeDB::NodeDB() crypto->setDHPrivateKey(config.security.private_key.bytes); } #endif - + // Include our owner in the node db under our nodenum + meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); info->user = TypeConversions::ConvertToUserLite(owner); info->has_user = true; @@ -321,8 +324,12 @@ NodeDB::NodeDB() moduleConfig.neighbor_info.update_interval = Default::getConfiguredOrMinimumValue(moduleConfig.neighbor_info.update_interval, min_neighbor_info_broadcast_secs); + LOG_DEBUG("nodeDatabaseCRC: %u, crc32Buffer(&nodeDatabase, sizeof(nodeDatabase): %u", nodeDatabaseCRC, + crc32Buffer(&nodeDatabase, sizeof(nodeDatabase))); if (devicestateCRC != crc32Buffer(&devicestate, sizeof(devicestate))) saveWhat |= SEGMENT_DEVICESTATE; + if (nodeDatabaseCRC != crc32Buffer(&nodeDatabase, sizeof(nodeDatabase))) + saveWhat |= SEGMENT_NODEDATABASE; if (configCRC != crc32Buffer(&config, sizeof(config))) saveWhat |= SEGMENT_CONFIG; if (channelFileCRC != crc32Buffer(&channelFile, sizeof(channelFile))) @@ -437,6 +444,7 @@ bool NodeDB::factoryReset(bool eraseBleBonds) #endif spiLock->unlock(); // second, install default state (this will deal with the duplicate mac address issue) + installDefaultNodeDatabase(); installDefaultDeviceState(); installDefaultConfig(!eraseBleBonds); // Also preserve the private key if we're not erasing BLE bonds installDefaultModuleConfig(); @@ -461,6 +469,15 @@ bool NodeDB::factoryReset(bool eraseBleBonds) return true; } +void NodeDB::installDefaultNodeDatabase() +{ + LOG_DEBUG("Install default NodeDatabase"); + nodeDatabase.version = DEVICESTATE_CUR_VER; + nodeDatabase.nodes = std::vector(MAX_NUM_NODES); + numMeshNodes = 0; + meshNodes = &nodeDatabase.nodes; +} + void NodeDB::installDefaultConfig(bool preserveKey = false) { uint8_t private_key_temp[32]; @@ -788,9 +805,10 @@ void NodeDB::resetNodes() if (!config.position.fixed_position) clearLocalPosition(); numMeshNodes = 1; - std::fill(devicestate.node_db_lite.begin() + 1, devicestate.node_db_lite.end(), meshtastic_NodeInfoLite()); + std::fill(nodeDatabase.nodes.begin() + 1, nodeDatabase.nodes.end(), meshtastic_NodeInfoLite()); devicestate.has_rx_text_message = false; devicestate.has_rx_waypoint = false; + saveNodeDatabaseToDisk(); saveDeviceStateToDisk(); if (neighborInfoModule && moduleConfig.neighbor_info.enabled) neighborInfoModule->resetNeighbors(); @@ -806,10 +824,10 @@ void NodeDB::removeNodeByNum(NodeNum nodeNum) removed++; } numMeshNodes -= removed; - std::fill(devicestate.node_db_lite.begin() + numMeshNodes, devicestate.node_db_lite.begin() + numMeshNodes + 1, + std::fill(nodeDatabase.nodes.begin() + numMeshNodes, nodeDatabase.nodes.begin() + numMeshNodes + 1, meshtastic_NodeInfoLite()); LOG_DEBUG("NodeDB::removeNodeByNum purged %d entries. Save changes", removed); - saveDeviceStateToDisk(); + saveNodeDatabaseToDisk(); } void NodeDB::clearLocalPosition() @@ -838,7 +856,7 @@ void NodeDB::cleanupMeshDB() } } numMeshNodes -= removed; - std::fill(devicestate.node_db_lite.begin() + numMeshNodes, devicestate.node_db_lite.begin() + numMeshNodes + removed, + std::fill(nodeDatabase.nodes.begin() + numMeshNodes, nodeDatabase.nodes.begin() + numMeshNodes + removed, meshtastic_NodeInfoLite()); LOG_DEBUG("cleanupMeshDB purged %d entries", removed); } @@ -848,13 +866,9 @@ void NodeDB::installDefaultDeviceState() LOG_INFO("Install default DeviceState"); // memset(&devicestate, 0, sizeof(meshtastic_DeviceState)); - numMeshNodes = 0; - meshNodes = &devicestate.node_db_lite; - // init our devicestate with valid flags so protobuf writing/reading will work devicestate.has_my_node = true; devicestate.has_owner = true; - // devicestate.node_db_lite_count = 0; devicestate.version = DEVICESTATE_CUR_VER; devicestate.receive_queue_count = 0; // Not yet implemented FIXME devicestate.has_rx_waypoint = false; @@ -908,7 +922,9 @@ void NodeDB::pickNewNodeNum() myNodeInfo.my_node_num = nodeNum; } -static const char *prefFileName = "/prefs/db.proto"; +static const char *deviceStateFileName = "/prefs/device.proto"; +static const char *legacyPrefFileName = "/prefs/db.proto"; +static const char *nodeDatabaseFileName = "/prefs/nodes.proto"; static const char *configFileName = "/prefs/config.proto"; static const char *uiconfigFileName = "/prefs/uiconfig.proto"; static const char *moduleConfigFileName = "/prefs/module.proto"; @@ -949,20 +965,41 @@ LoadFileResult NodeDB::loadProto(const char *filename, size_t protoSize, size_t void NodeDB::loadFromDisk() { - devicestate.version = - 0; // Mark the current device state as completely unusable, so that if we fail reading the entire file from + // Mark the current device state as completely unusable, so that if we fail reading the entire file from // disk we will still factoryReset to restore things. + devicestate.version = 0; #ifdef ARCH_ESP32 spiLock->lock(); + // If the legacy deviceState exists, start over with a factory reset + if (FSCom.exists(legacyPrefFileName)) { + rmDir("/prefs"); + } if (FSCom.exists("/static/static")) rmDir("/static/static"); // Remove bad static web files bundle from initial 2.5.13 release spiLock->unlock(); #endif + auto state = loadProto(nodeDatabaseFileName, getMaxNodesAllocatedSize(), sizeof(meshtastic_NodeDatabase), + &meshtastic_NodeDatabase_msg, &nodeDatabase); + if (nodeDatabase.version < DEVICESTATE_MIN_VER) { + LOG_WARN("NodeDatabase %d is old, discard", nodeDatabase.version); + installDefaultNodeDatabase(); + } else { + meshNodes = &nodeDatabase.nodes; + numMeshNodes = nodeDatabase.nodes.size(); + LOG_INFO("Loaded saved nodedatabase version %d, with nodes count: %d", nodeDatabase.version, nodeDatabase.nodes.size()); + } + + if (numMeshNodes > MAX_NUM_NODES) { + LOG_WARN("Node count %d exceeds MAX_NUM_NODES %d, truncating", numMeshNodes, MAX_NUM_NODES); + numMeshNodes = MAX_NUM_NODES; + } + meshNodes->resize(MAX_NUM_NODES); + // static DeviceState scratch; We no longer read into a tempbuf because this structure is 15KB of valuable RAM - auto state = loadProto(prefFileName, sizeof(meshtastic_DeviceState) + MAX_NUM_NODES_FS * sizeof(meshtastic_NodeInfo), - sizeof(meshtastic_DeviceState), &meshtastic_DeviceState_msg, &devicestate); + state = loadProto(deviceStateFileName, meshtastic_DeviceState_size, sizeof(meshtastic_DeviceState), + &meshtastic_DeviceState_msg, &devicestate); // See https://github.com/meshtastic/firmware/issues/4184#issuecomment-2269390786 // It is very important to try and use the saved prefs even if we fail to read meshtastic_DeviceState. Because most of our @@ -976,15 +1013,8 @@ void NodeDB::loadFromDisk() LOG_WARN("Devicestate %d is old, discard", devicestate.version); installDefaultDeviceState(); } else { - LOG_INFO("Loaded saved devicestate version %d, with nodecount: %d", devicestate.version, devicestate.node_db_lite.size()); - meshNodes = &devicestate.node_db_lite; - numMeshNodes = devicestate.node_db_lite.size(); + LOG_INFO("Loaded saved devicestate version %d", devicestate.version); } - if (numMeshNodes > MAX_NUM_NODES) { - LOG_WARN("Node count %d exceeds MAX_NUM_NODES %d, truncating", numMeshNodes, MAX_NUM_NODES); - numMeshNodes = MAX_NUM_NODES; - } - meshNodes->resize(MAX_NUM_NODES); state = loadProto(configFileName, meshtastic_LocalConfig_size, sizeof(meshtastic_LocalConfig), &meshtastic_LocalConfig_msg, &config); @@ -1150,14 +1180,24 @@ bool NodeDB::saveDeviceStateToDisk() #endif // Note: if MAX_NUM_NODES=100 and meshtastic_NodeInfoLite_size=166, so will be approximately 17KB // Because so huge we _must_ not use fullAtomic, because the filesystem is probably too small to hold two copies of this - return saveProto(prefFileName, sizeof(devicestate) + numMeshNodes * meshtastic_NodeInfoLite_size, &meshtastic_DeviceState_msg, - &devicestate, false); + return saveProto(deviceStateFileName, meshtastic_DeviceState_size, &meshtastic_DeviceState_msg, &devicestate, true); +} + +bool NodeDB::saveNodeDatabaseToDisk() +{ +#ifdef FSCom + spiLock->lock(); + FSCom.mkdir("/prefs"); + spiLock->unlock(); +#endif + size_t nodeDatabaseSize; + pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &nodeDatabase); + return saveProto(nodeDatabaseFileName, nodeDatabaseSize, &meshtastic_NodeDatabase_msg, &nodeDatabase, false); } bool NodeDB::saveToDiskNoRetry(int saveWhat) { bool success = true; - #ifdef FSCom spiLock->lock(); FSCom.mkdir("/prefs"); @@ -1202,11 +1242,16 @@ bool NodeDB::saveToDiskNoRetry(int saveWhat) success &= saveDeviceStateToDisk(); } + if (saveWhat & SEGMENT_NODEDATABASE) { + success &= saveNodeDatabaseToDisk(); + } + return success; } bool NodeDB::saveToDisk(int saveWhat) { + LOG_DEBUG("Save to disk %d", saveWhat); bool success = saveToDiskNoRetry(saveWhat); if (!success) { @@ -1388,7 +1433,7 @@ bool NodeDB::updateUser(uint32_t nodeId, meshtastic_User &p, uint8_t channelInde // We just changed something about a User, // store our DB unless we just did so less than a minute ago if (!Throttle::isWithinTimespanMs(lastNodeDbSave, ONE_MINUTE_MS)) { - saveToDisk(SEGMENT_DEVICESTATE); + saveToDisk(SEGMENT_NODEDATABASE); lastNodeDbSave = millis(); } else { LOG_DEBUG("Defer NodeDB saveToDisk for now"); @@ -1402,6 +1447,10 @@ bool NodeDB::updateUser(uint32_t nodeId, meshtastic_User &p, uint8_t channelInde /// we updateGUI and updateGUIforNode if we think our this change is big enough for a redraw void NodeDB::updateFrom(const meshtastic_MeshPacket &mp) { + // if (mp.from == getNodeNum()) { + // LOG_DEBUG("Ignore update from self"); + // return; + // } if (mp.which_payload_variant == meshtastic_MeshPacket_decoded_tag && mp.from) { LOG_DEBUG("Update DB node 0x%x, rx_time=%u", mp.from, mp.rx_time); diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index d244a94ba..ee2b5f516 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include "MeshTypes.h" @@ -21,11 +22,13 @@ DeviceState versions used to be defined in the .proto file but really only this #define SEGMENT_MODULECONFIG 2 #define SEGMENT_DEVICESTATE 4 #define SEGMENT_CHANNELS 8 +#define SEGMENT_NODEDATABASE 16 -#define DEVICESTATE_CUR_VER 23 -#define DEVICESTATE_MIN_VER 22 +#define DEVICESTATE_CUR_VER 24 +#define DEVICESTATE_MIN_VER 24 extern meshtastic_DeviceState devicestate; +extern meshtastic_NodeDatabase nodeDatabase; extern meshtastic_ChannelFile channelFile; extern meshtastic_MyNodeInfo &myNodeInfo; extern meshtastic_LocalConfig config; @@ -75,7 +78,8 @@ class NodeDB /// write to flash /// @return true if the save was successful - bool saveToDisk(int saveWhat = SEGMENT_CONFIG | SEGMENT_MODULECONFIG | SEGMENT_DEVICESTATE | SEGMENT_CHANNELS); + bool saveToDisk(int saveWhat = SEGMENT_CONFIG | SEGMENT_MODULECONFIG | SEGMENT_DEVICESTATE | SEGMENT_CHANNELS | + SEGMENT_NODEDATABASE); /** Reinit radio config if needed, because either: * a) sometimes a buggy android app might send us bogus settings or @@ -148,6 +152,15 @@ class NodeDB virtual meshtastic_NodeInfoLite *getMeshNode(NodeNum n); size_t getNumMeshNodes() { return numMeshNodes; } + size_t getMaxNodesAllocatedSize() + { + meshtastic_NodeDatabase emptyNodeDatabase; + emptyNodeDatabase.version = DEVICESTATE_CUR_VER; + size_t nodeDatabaseSize; + pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &emptyNodeDatabase); + return nodeDatabaseSize + (MAX_NUM_NODES * meshtastic_NodeInfoLite_size); + } + // returns true if the maximum number of nodes is reached or we are running low on memory bool isFull(); @@ -188,8 +201,8 @@ class NodeDB void cleanupMeshDB(); /// Reinit device state from scratch (not loading from disk) - void installDefaultDeviceState(), installDefaultChannels(), installDefaultConfig(bool preserveKey), - installDefaultModuleConfig(); + void installDefaultDeviceState(), installDefaultNodeDatabase(), installDefaultChannels(), + installDefaultConfig(bool preserveKey), installDefaultModuleConfig(); /// write to flash /// @return true if the save was successful @@ -197,6 +210,7 @@ class NodeDB bool saveChannelsToDisk(); bool saveDeviceStateToDisk(); + bool saveNodeDatabaseToDisk(); }; extern NodeDB *nodeDB; diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index 039b36d8d..f91c48560 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -23,8 +23,6 @@ #define MAX_NUM_NODES 100 #endif -#define MAX_NUM_NODES_FS 100 - /// Max number of channels allowed #define MAX_NUM_CHANNELS (member_size(meshtastic_ChannelFile, channels) / member_size(meshtastic_ChannelFile, channels[0]))