Updated changelog

Fixed Hop-number in docs

.gitignore

@@ -7,4 +7,8 @@ RNS/Utilities/RNS
 build
 dist
 docs/build
-rns*.egg-info
+rns*.egg-info
+profile.data
+tests/rnsconfig/storage
+*.data
+*.result

Changelog.md

@@ -0,0 +1,665 @@
+### 2023-01-14: RNS β 0.4.7
+
+This maintenance release adds support for using the `rnodeconf` utility to replicate RNode devices, and bootstrap device creation using only tools and software packages obtained from an RNode Bootstrap Console.
+
+**Changes**
+- Added the ability to use rnodeconf to bootstrap RNode creation without needing a connection to the Internet
+- Added ability for rnodeconf to extract firmwares from existing RNodes
+- Added ability for rnodeconf to use extracted firmwares for autoinstaller and updates
+- Updated documentation and manual
+
+**Release Hashes**
+```
+7ea22be8f4cc9504d8a612c5589132351cc0c6b474899204afd71367ab3fb226 rns-0.4.7-py3-none-any.whl
+3dc337b80df37c247abc9cee06c3ecba0f908449005d0eb365c2a9577d689e57 rnspure-0.4.7-py3-none-any.whl
+```
+
+### 2022-12-23: RNS β 0.4.6
+
+This maintenance release brings two bugfixes.
+
+**Changes**
+- Fixed missing path invalidation on failed link establishments made from a shared instance client
+- Fixed a memory leak in link handling
+
+**Release Hashes**
+```
+7f1b0b254dce5bb1bacc336b026dab2dda5859b43cb0f4ceed3f70ba825f8873 rns-0.4.6-py3-none-any.whl
+775c1b9b5bdf202524e50e58dc7c7bad9262ca3c16471cbfc6fb3a528e732460 rnspure-0.4.6-py3-none-any.whl
+```
+
+### 2022-12-22: RNS β 0.4.5
+
+This maintenance release significantly improves path rediscovery on roaming devices with multiple interfaces, and adds a few tweaks to interface handling, that are especially relevant on Android.
+
+**Changes**
+- Faster roaming path recovery for multiple interface non-transport instances
+- Fixed AutoInterface multicast echoes failing on interfaces with rolling MAC addresses on every re-connect
+- Added carrier change detection flag to AutoInterface
+- Adjusted loglevels for some items
+
+**Release Hashes**
+```
+6757d5d815d4d96c45c181daf321447914c0e90892d43e142f2bd3fffacda9d9 rns-0.4.5-py3-none-any.whl
+11669065091d67e3abaddb0096e5c92fc48080692b5644559226b2e2e6721060 rnspure-0.4.5-py3-none-any.whl
+```
+
+### 2022-12-22: RNS β 0.4.4
+
+This maintenance release improves path response handling and log output.
+
+**Release Hashes**
+```
+b0b59c25910151db0c2085d812bcc3d06cb930ddb8cd1e281b40cb592c1427eb rns-0.4.4-py3-none-any.whl
+fe29ce3eb9e55f6953312c8db8c350bd58a7777e8c8dffd5491b840254426332 rnspure-0.4.4-py3-none-any.whl
+```
+
+### 2022-12-22: RNS β 0.4.3
+
+This maintenance release brings faster path rediscovery and improves hardware support on Android, along with a few other minor tweaks and bugfixes.
+
+**Changes**
+- Added automatic path rediscovery on failed link establishments
+- Added signature validation for link request proof packets for every transport hop
+- Improved RNode hotplug support over Bluetooth on Android
+- Improved Resource transfer sequencing and retry handling
+- Fixed driver initialisation for Qinheng CH34x serial chips on Android
+- Updates to documentation
+
+**Release Hashes**
+```
+c035c2e21b8b207b00937ad57e947c7b4f17a02fe4f253d6e1fcc000479019b7 rns-0.4.3-py3-none-any.whl
+e367576893bada72329ad195ebaa1e295bbca8897241f258428e1957d2da9a55 rnspure-0.4.3-py3-none-any.whl
+```
+
+### 2022-11-24: RNS β 0.4.2
+
+This maintenance release brings a number of minor improvements, and fixes a few bugs related to hardware support on Android.
+
+**Changes**
+- Fixed AutoInterface roaming not working on Android devices that rotate Ethernet and WiFi MAC addresses on every physical connection change
+- Fixed RNode interface not working over Bluetooth on Android versions 10 and below
+- Greatly improved startup time for programs connecting to a shared Reticulum instance on slow or resource-limited systems
+- Improvements to internal utility-functions and logging
+- Added a public development roadmap
+- Updates and fixes to the documentation
+
+**Release Hashes**
+```
+ba541ead4194e7ae3e295bf2c84b609041e4dc82e1b5bfce0885396ee090e37f rns-0.4.2-py3-none-any.whl
+a352cb8d0862a1a23e66bda08357bf7e725b540bbdd3bb3b32914f3c0bb99a05 rnspure-0.4.2-py3-none-any.whl
+```
+
+### 2022-11-03: RNS β 0.4.1
+
+This maintenance release fixes few bugs, and improves I2P interface recovery on unresponsive I2P tunnels.
+
+**Changes**
+- Added better I2P tunnel state visibility to rnstatus util
+- Improved I2P recovery time on unresponsive tunnels
+- Improved I2P tunnel state detection
+- Fixed missing IFAC identity init on spawned TCP clients
+- Fixed missing IFAC identity init on spawned I2P interfaces
+- Fixed missing check for socket state on I2P interfaces
+
+**Release Hashes**
+```
+e28643a7396c3a41d859eb7d3a14f166e648003da36fc49094561fbf49c04b7e rns-0.4.1-py3-none-any.whl
+feaa326545c928f3d5dc7b6fdb31975517af15da0751927491c4ac23dac36edc rnspure-0.4.1-py3-none-any.whl
+```
+
+### 2022-11-03: RNS β 0.4.0
+
+This maintenance release fixes minor bug in the rnodeconf utility.
+
+**Changes**
+- Fixed incorrect storage location for local firmware cache in the rnodeconf utility
+
+**Release Hashes**
+```
+16dda7b087cff0c21b7b0460798cb433fc96f27d058eb7d50e38898a1a1e49c4 rns-0.4.0-py3-none-any.whl
+5f137cfd42ee9d9e7ae43b25d25849bd087145b7edf2c29ffdfd93d57ab34284 rnspure-0.4.0-py3-none-any.whl
+```
+
+### 2022-11-03: RNS β 0.3.19
+
+This release adds support for Bluetooth-connected RNode interfaces, and includes a few improvements to the rnodeconf utility.
+
+**Changes**
+- Added support for RNode interfaces connected over Bluetooth on Linux and Android
+- Improved rnodeconf install and update timing, which fixes installs sometimes failing on T-Beam devices
+
+**Release Hashes**
+```
+9d5bee8eb9b2160dab985017bfa3e3db9c35033cfae97653a9fa8faa6064f228 rns-0.3.19-py3-none-any.whl
+0f0996b5e401ca5d4e91080df3d6de326fc591164c9e6932a2eb79f1d2b8d375 rnspure-0.3.19-py3-none-any.whl
+```
+
+### 2022-11-03: RNS β 0.3.18
+
+This maintenance release includes the `rnodeconf` utility directly in the `rns` package, and brings a few improvements to interface handling and hardware interfacing.
+
+**Important!** The minimum supported RNode firmware version for this release is `1.51`, and the firmware will needs to be updated with `rnodeconf` version `2.0.0` or greater, since earlier versions won't be able to fetch the new release files.
+
+**Changes**
+- Added `rnodeconf` utility
+- Added more options for controlling log output
+- Added ability to write to the external framebuffer of RNode devices
+- Improved teardown handling on RNode interfaces
+
+**Release Hashes**
+```
+dc0c56950b85be763270695faf441029f7e6c31cdc44447c6c470e09c734aa45 rns-0.3.18-1-py3-none-any.whl
+760bfc52419a8c45a420df41c40a1bf96bd494dabd7efe461c7907b152bbf39c rnspure-0.3.18-1-py3-none-any.whl
+```
+
+### 2022-11-03: RNS β 0.3.17
+
+This maintenance release fixes a regression in the 0.3.16 release.
+
+**Changes**
+- Fixed an incorrect import that inadverdently caused Android-specific interfaces to be used on non Android operating systems.
+
+**Release Hashes**
+```
+SHA256 0e8327461e2d39f859059cc14e94fb33f21e1186c422bb766950f42ca1387656 rns-0.3.17-py3-none-any.whl
+SHA256 9e31160cc38e0d5531460d5eca7b3f6e6d8c3b2a7afb04338ee72cc488a2ba18 rnspure-0.3.17-py3-none-any.whl
+```
+
+### 2022-10-20: RNS β 0.3.16
+
+This maintenance release fixes a single bug that prevented running RNS in Termux (and similar) on Android.
+
+**Changes**
+- Fixed missing imports and module checks for API-limited environments on Android
+
+**Release Hashes**
+```
+SHA256 dc4202302b1f1503a0f1c8fef7123b31f7d5d7131ae5b9f988064ebe22e29ed8 rns-0.3.16-py3-none-any.whl
+SHA256 127624d2592745602d4a056c347fa6f5989f049275a5b8bfa97c296af9bc497f rnspure-0.3.16-py3-none-any.whl
+```
+
+### 2022-10-20: RNS β 0.3.15
+
+This maintenance release primarily adds support for external hardware interfaces on Android. A number of bugs have also been fixed, and improvements made to logging output consistency.
+
+**Changes**
+- Added support for RNode interfaces on Android
+- Added support for KISS interfaces on Android
+- Added support for Serial interfaces on Android
+- Added AutoInterface support for kernel network devices that rotate MAC addresses on roaming and/or reconnects
+- Updated various helper functions
+- Minor log output cleanup and fixes
+- Fixed missing lookup for locally running destinations in Identity.recall() when running as a shared transport instance
+- Fixed missing announce cap property on hot-plugged interfaces
+- Fixed incorrect behaviour in announce processing for instance-local destinations to roaming- or boundary-mode interfaces
+
+**Release Hashes**
+```
+SHA256 c56f32dbfd10fae1b5d2dddafe7d2a0f2127908827a71fce9e43fd051ea453bc rns-0.3.15-py3-none-any.whl
+SHA256 597d6df05b3586eaa1515c0215cec30d7a018a209e7900634345c39514efcd18 rnspure-0.3.15-py3-none-any.whl
+```
+
+### 2022-10-07: RNS β 0.3.14
+
+This maintenance release brings a few improvements, including optimised announce packet structure and updated documentation.
+
+**Please note!** While this is a small maintenance release, it includes changes to packe structure that breaks backwards compatibility with all previous RNS versions.
+
+**Changes**
+- Optimised announce packet structure
+- Reject mismatching public keys on hash collision.
+- Minor updates to documentation
+
+**Release Hashes**
+```
+SHA256 b761efc24d20c5719817bfefbbe8ce69f7c91d65bb8273cb02578f77d6f88bc5 rns-0.3.14-py3-none-any.whl
+SHA256 cc24a1f010431c8f193ec0ffc6dccade614a5be40c47ac12e3e9ae60b52f046e rnspure-0.3.14-py3-none-any.whl
+```
+
+### 2022-10-04: RNS β 0.3.13
+
+This maintenance release includes a single but important bugfix.
+
+**Changes**
+- Fixed missing hash construction step in announce emission and validation
+
+**Release Hashes**
+```
+SHA256 d6c8a7cb8ea7edc99800df92abff246e8159f2d9c9f1a2b57672385d49647c90 rns-0.3.13-py3-none-any.whl
+SHA256 c07c28942e374342c4e807a0b6e81d831737b87cf59651670b8c1c191030a326 rnspure-0.3.13-py3-none-any.whl
+```
+
+### 2022-09-30: RNS β 0.3.12
+
+This maintenance release includes a fix to the [serious security flaw discussed here](https://github.com/markqvist/Reticulum/discussions/103). **Please Note!** Updating to RNS 0.3.12 will intentionally break backwards compatibility with all previous verstions for link establishment. It is recommended to upgrade all your systems to 0.3.12 as soon as possible.
+
+Additionally, this release brings a range of small, but very useful improvements to reliability and user experience, along with a significant update to the documentation material.
+
+**Changes**
+- Fixed a [serious security flaw](https://github.com/markqvist/Reticulum/discussions/103) in link establishment key exchanges
+- Allow hot-plug of RNode devices
+- Better detachment handler for TCP clients on shutdown
+- Implemented better config directory path handling
+- Clarifications and improvements to various documentation chapters
+- Improved writing quality of documentation, courtesy of @huyndao
+- Improved overall presentation of documentation and manual
+- Improved reliability of data persistence in case of unexpected shutdowns or hardware crashes
+- Added rnsd warning on start as client
+- Fixed a rendering bug in the rnpath utility
+- Added initial connection timeout configuration option to TCP Client interfaces
+- Brought deprecated native python API calls up to date
+
+**Release Hashes**
+```
+SHA256 74a4881ebf8d805bffb43efef91769b1cbb87affe56ac630355946c7484cffbf rns-0.3.12-py3-none-any.whl
+SHA256 03429122b3b4133667632ba2404df7bbf57ea5df1b9c815d7608b1d59cd29a76 rnspure-0.3.12-py3-none-any.whl
+```
+
+### 2022-07-09: RNS β 0.3.11
+
+This maintenance release contains a single but important bug fix in resource transfers.
+
+**Changes**
+- Fixed a an incorrect size calculation for resource advertisements, that would lead to resources of specific sizes failing with an MTU error.
+
+**Release Hashes**
+```
+SHA256 7c03a003326bcd127226414b08cf48f87bcc6b88a7279c52e28415315668543c rns-0.3.11-py3-none-any.whl
+SHA256 1a6aaa3ba370ece28cc975ba94b0461c61497cf0797f92662472e0ec20576cb1 rnspure-0.3.11-py3-none-any.whl
+```
+
+### 2022-07-08: RNS β 0.3.10
+
+This maintenance release contains a single but important bug fix for systems running Reticulum Transport Instances.
+
+**Changes**
+- Fixed a potential race condition in link establishment flow, that could lead to links not being established over hops with very low latency.
+
+**Release Hashes**
+```
+SHA256 1c9fb56b967aed507694e6b5d5fca7a89b022cad9fa2058d248e359dc150fba7 rns-0.3.10-py3-none-any.whl
+SHA256 8eae07f9e6241ea1f3778430456225dee3ef73bb1c4df5e5362dd00226404628 rnspure-0.3.10-py3-none-any.whl
+```
+
+### 2022-07-05: RNS β 0.3.9
+
+This release expands the address space of Reticulum to 128 bits, and brings improvements to the documentation, along with a few bugfixes and updates.
+
+**Changes**
+- Expanded address space to 128 bits
+- Updated documentation
+- Improved rnx interactive mode
+- Improved readme file
+- Added reticulum.network website
+- Added periodic cache cleaning
+- Fixed a bug in the --no-auth option in rncp
+
+**Release Hashes**
+```
+SHA256 892005e95fc9eda4c4c5d9f94dd33cdc27d3ac6e228d1b0b2519e35069951b86 rns-0.3.9-1-py3-none-any.whl
+SHA256 cb7d873c51c746ecdb8963a6a7a0e8d010fb6c61ee785c5e97376d3779a7bae8 rnspure-0.3.9-1-py3-none-any.whl
+```
+
+### 2022-06-22: RNS β 0.3.8
+
+This release brings big improvements to compatibility with various system types, along with several convenient new features, and a lot of tuning, optimisation and stability improvements. In a continued effort, the documentation has also been updated, restructured, and had several new and informative sections added.
+
+**Changes**
+- Added ability to install and run RNS without any dependencies
+- Added backend abstraction for cryptographic primitives
+- Added pure-python implementations of all cryptographic primitives
+- Added accept option to Link API
+- Added several undocumented API calls to the documentation
+- Added option to filter interfaces to rnstatus utility
+- Added "Communications Hardware" chapter to the documentation
+- Improved multiple chapters and restructured documentation
+- Improved efficiency of Transport instances
+- Improved performance of Resource transfers
+- Improved Resource handling strategies over different physical link types
+- Improved link capacity and speed estimation calculations
+- Improved I2P interface error handling and stability
+- Tuned Resource and Link timeouts
+- Tuned TCP socket options for better reliability over intermittent links
+- Tuned I2P interface timeouts for better reliability over intermittent links
+- Fixed a missing check for zero-length packets on IFAC-enabled interfaces
+- Fixed a socket allocation leak in I2P interfaces
+- Added unit tests
+- Added performance profiling tools
+- Improved build system
+
+Release SHA-256 for `rns-0.3.8-py3-none-any.whl` is `fdb53aba14840edf3d71dde1a745f319e7f60d6993851b7651bf8ba3d5c53ba7`
+Release SHA-256 for `rnspure-0.3.8-py3-none-any.whl` is `b0eb004c3725bc20496b1c855e7d22729d8a39fd0cde957ab95aa8c7e13ee3a4`
+
+### 2022-05-29: RNS β 0.3.7
+
+This release comes with a big upgrade to reliability and resilience, with lots of small bug fixes and improvements, along with some significant new additions and features. The documentation and API reference has also seen several improvements for clarity.
+
+Users of I2P interfaces will see big improvements in reliability with better handling of errors from the I2P SAM API, and much better automatic recovery when I2P connectivity is intermittent.
+
+Reticulum is now able to perform network-wide discovery of unknown paths, using the new Gateway interface mode. The stability of established links has also been improved by using a better timeout calculation method.
+
+It is also worth mentioning the addition of the two new utilities, `rncp` and `rnx`, that allow you to transfer files to remote systems, and perform remote command execution.
+
+*Please Note!* For using 64-bit IFACs on RNode hardware, your RNodes must be running at least firmware version 1.28.
+
+**Changes**
+- Added gateway interface mode
+- Added `rncp` utility for transferring files to remote destinations
+- Added `rnx` utility for remotely executing commands and returning output
+- Implemented unknown path discovery
+- Implemented recursive path request loop avoidance
+- Implemented bandwidth cap for recursive path requests
+- Improved Link authentication callbacks
+- Improved Link stale time calculations and process
+- Improved error detection and handling in I2P interfaces
+- Improved automatic recovery and reliability on intermittent I2P interfaces
+- Added request size to receipts, and updated relevant API documentation
+- Added default identity storage folder
+- Fixed deprecated options in libi2p's asyncio calls
+- Fixed I2P controller startup when event loop is not immediately ready
+- Fixed bug in conditional resource acceptance callback
+- Fixed an invalid interface mode check
+- Fixed missing recursive progress callback allocation in segmented resource transfer
+- Fixed expired AP and Roaming interface mode paths not being removed at the correct time
+- Fixed announce rate targets not being set on I2PInterface peers
+- Fixed naming conflict in resource advertisements
+- Fixed link stale time calculation on newly created links without any actual traffic
+- Fixed a bug that caused large packets (over 492 bytes) with IFAC enabled to be dropped on RNode hardware
+- Improved output of `rnstatus` utility
+- Improved Destination and Link API documentation
+- Updated documentation and readme
+
+Release SHA-256 for Python Wheel is `2cd9a584d6b13bb478a43b49b7de3f2a8270c4b8979666b1ca40cd81daacbf42`
+
+### 2022-05-17: RNS β 0.3.6
+
+This release adds a number of improvements, a new interface type, and some very useful new interface modes.
+
+**Changes**
+- Added PipeInterface, create interfaces with any program over stdio
+- Added "roaming" and "boundary" interface modes
+- Added per-interface announce rate control
+- Added ability to drop announce queues to rnpath utility
+- Added announce rate information output to rnpath utility
+- Improved announce queue processing
+- Improved several documentation chapters
+- Improved logging output
+
+### 2022-04-28: RNS β 0.3.5
+
+This release brings major improvements and upgrades to Reticulum, along with better documentation and improved usability of the bundled utilities.
+
+**Changes**
+- Greatly improved convergence time. Even on huge networks, newly created destinations become globally reachable in less than a minute.
+- New announce propagation mechanism allows flexible scalability. Extremely slow network segments can now interconnect seamlessly with huge, high-bandwidth networks while still prioritising end-to-end connectivity for local nodes.
+- Reticulum can now scale to huge and complex networks with up to 128 hops, and billions of active endpoints.
+- Added virtual network segmentation for running multiple virtual networks over the same physical channel.
+- Added interface authentication for creating private access network interfaces and access points.
+- Updated documentation in accordance with current implementation of announce propagation mechanism.
+- Updated several outdated documentation chapters.
+- Added documentation for new interface features.
+- The output display of the rnstatus utility has been greatly improved.
+- Added ability to drop paths to the rnpath utility.
+- Added path table display to rnpath utility.
+- Added interface rate determination and estimation.
+- Added configurable bandwidth allocation for announce traffic.
+- Improved and cleaned logging output.
+- Various Transport optimisations.
+- Improved AutoInterface peering timing.
+- Updated manual in accordance with release.
+- Fixed a possible race condition in Transport startup when a local shared instance was restarted and apps reconnected.
+
+### 2022-03-28: RNS β 0.3.4
+
+This is a small maintenance release with a bugfix and some documentation and reliability improvements.
+
+**Changes**
+- Fixed https://github.com/markqvist/Reticulum/issues/18 that could potentially cause a routing loop if the API was used in an unintended way
+- Improved cryptography API compatibility
+- Improved documentation
+
+### 2022-02-26: RNS β 0.3.3
+
+This release adds major new functionality to Reticulum, including new connectivity options, improves stability, simplifies configuration and fixes a few bugs.
+
+**Improvements**
+ - Added the I2P Interface to Reticulum
+ - Added I2P tunneling support for TCP interfaces
+ - Improved recovery of AutoInterface on underlying medium carrier loss
+ - Improved AutoInterface timeouts and timing
+ - Enabled the "outbound" interface option as on by default
+ - Added the "Access Point" interface mode
+ - Simplified default configuration
+ - Added verbose configuration example to the "rnsd" program
+ - Improved documentation and manual
+ - Fixed a potential race condition in resource assembly
+ - Fixed a reference error in TCP interfaces
+ - Fixed a configuration keyword error
+
+### 2022-01-28: RNS β 0.3.2
+
+This maintenance release adds support for using a much wider range of devices as RNode LoRa interfaces with Reticulum, and also contains a few bugfixes and improvements.
+
+**Important!** From this release, RNodes used with Reticulum must have at least firmware version 1.26 installed, due to the new multiplatform RNode support.
+
+**Improvements**
+ - Added full support for RNodes based on ESP32 and ATmega2560 boards
+ - Fixed a bug in TCP interfaces on macOS
+ - Updated documentation and manual
+
+### 2022-01-26: RNS β 0.3.1
+
+This is a small maintenance and update release of Reticulum, including a few improvements. It also adds support for using ESP32-based T-Beam devices.
+
+Improvements:
+ - Added support for using T-Beam devices using the RNodeInterface
+ - Improved AutoInterface on Android
+ - Improved platform handling
+ - Improved malformed packet handling
+
+### 2021-12-11: RNS β 0.3.0
+
+This is a major release of Reticulum, including a range of stability and performance improvements, along with important new features, expanding the connectivity of Reticulum.
+
+An important improvement in this release is the addition of the AutoInterface, that will now be configured as the default interface on new installs. This interface automatically meshes with other Reticulum peers over any available system network devices, and doesn't require any existing IP infrastructure like a DHCP server or a router. For more information, consult the relevant section of the manual.
+
+**Improvements**
+ - Added new AutoInterface as default interface for new installs
+ - Serial port interfaces now automatically attempt to reconnect devices that are unplugged and replugged
+ - Added support for KISS over TCP in the TCPClientInterface
+ - Added support for running Reticulum as a systemd service
+ - Initial support for the Android operating system
+ - Added documentation for installing Reticulum on Android in Termux
+ - Improved documentation and manual
+ - Better path request handling for shared instances
+ - Better shutdown handling on external interrupts
+ - Many small stability and reliability improvements
+ - Fine-tuned various timing parameters for different link types
+
+### 2021-10-15: RNS β 0.2.9
+
+This beta release adds the fundamentals of RSSI and SNR functionality. It also implements timing improvements, allowing Reticulum to function on even lower bitrate physical links.
+
+**Improvements**
+ - Added RSSI and SNR reporting on supported interfaces
+ - Added RSSI and SNR to rnprobe utility
+ - Added RSSI and SNR to Echo example
+ - Support for physical layer throughput down to 500 bits per second.
+ - Improved callback handling
+
+### 2021-10-10: RNS β 0.2.8
+
+This beta release brings a single, but important improvement. Paths are now updated much more fluidly for peers moving around the network.
+
+Since updates were made to how tunnels and path table entries are represented in this release, it is recommended to delete the following files on Transport Nodes:
+
+~/.reticulum/storage/destination_table
+~/.reticulum/storage/packet_hashlist
+~/.reticulum/storage/tunnels
+~/.reticulum/storage/cache/*
+
+The files will be recreated when Reticulum is started.
+
+**Improvements**
+ - Improved path updates for peers moving around the network
+
+### 2021-10-08: RNS β 0.2.7
+
+This beta release brings a range of stability improvements and one bugfix.
+
+**Improvements**
+ - Improved output of the rnstatus utility
+ - Improved shared instance and local client handling
+ - Improved documentation
+ - Improved path restoration on tunnels
+ - Added log rotation
+
+**Fixed bugs**
+ - Fixed incorrect interface detachment on TCP client interfaces
+
+### 2021-09-25: RNS β 0.2.6
+
+This beta release brings a range of improvements and a few bugfixes.
+
+**Improvements**
+ - Added the "rnsd" utility for running Reticulum as a service
+ - Added the "rnstatus" utility for viewing interface status
+ - Added the "rnpath" utility for path lookups
+ - Added the "rnprobe" utility for testing connectivity
+ - Documentation has been improved and expanded
+ - Improved shutdown handling for shared instances
+ - Improved default configuration
+ - Improved recovery of TCP interfaces over unreliable links
+
+**Fixed bugs**
+ - Fixed a bug in reverse table culling
+ - Fixed a regression in TCP interface client spawner
+
+### 2021-09-18: RNS β 0.2.5
+
+This beta release brings a range of improvements and bugfixes.
+
+**Improvements**
+ - Added endpoint tunneling for path restoration over intermittent or roving link layer connections.
+ - Added ability for TCP client interfaces to automatically reconnect if TCP socket drops.
+ - Improved link teardown handling.
+ - Improved interface error handling on non-recoverable / hardware errors.
+
+**Fixed bugs**
+ - Fixed a bug that could cause path table entries to be culled two times in rare cases.
+ - Fixed a bug that could lead to the "outgoing" directive of interface configuration entries not being parsed correctly.
+
+### 2021-09-11: RNS β 0.2.4
+
+This beta release brings a range of improvements and bugfixes.
+
+**Improvements**
+ - Increased link MDU from 415 to 431 bytes by optimising transfer of Fernet tokens.
+ - All data lengths are now calculated dynamically from Reticulums base MTU, laying the groundwork for dynamic MTU interoperability.
+ - Disabled option to allow unencrypted links.
+ - Improved documentation.
+ - Improved request timeouts and handling.
+ - Improved link establishment.
+ - Improved resource transfer timing. 
+
+**Fixed bugs**
+ - Fixed a race condition in inbound proof handling.
+ - Fixed sequencing errors caused by duplicate HMU/request packets not being filtered.
+
+### 2021-08-29: RNS β 0.2.3
+
+This beta release brings a range of improvements and bugfixes.
+
+**Improvements**
+ - Improved resource handling.
+ - Improved timeout calculation for packets, links, resources and requests.
+ - Improved announce handling for shared instances.
+ - Improved default configuration template.
+ - Added example "Speedtest".
+
+**Fixed bugs**
+ - Fixed an issue that caused request timeout even though response had occurred.
+ - Fixed an issue that caused identity files to be written incorrectly.
+ - Fixed resource sequencing errors not being handled gracefully.
+
+### 2021-08-21: RNS β 0.2.2
+
+This beta release brings several new features to Reticulum along with two bugfixes.
+
+IMPORTANT! This version breaks wire-format compatibility with all previous versions of Reticulum. You must update *all* of your nodes at the same time.
+
+**New features**
+ - Link initiators can now identify to the remote peer over the link, once it has been set up. This can be used for authentication, amongst other things.
+ - Requests and responses of arbitrary sizes can now be carried out over links.
+ - UDP and TCP interfaces can now be bound to network device names (eth0, wlan0, etc.) instead of manually specifying listen IPs.
+
+**Fixed bugs**
+ - Fixed a race condition in outbound transport packet filtering.
+ - Fixed an issue where local UDP broadcast echoes could get processed as inbound packets.
+
+### 2021-05-20: RNS β 0.2.1
+
+This beta release sees significant improvements to bandwidth utilization and efficiency, while improving security by dropping RSA and moving completely to Curve25519.
+
+- All asymmetric cryptography migrated to X25519/Ed25519. This has greatly improved efficiency and reduced protocol overhead significantly.
+- Work has continued on the documentation, and the "Understanding Reticulum" chapters have been improved significantly in this release.
+- Class methods dealing with setting callbacks have been renamed to be more intuitive.
+
+As a few examples of the improved efficiency, a complete link establishment now only costs 240 bytes, down from 409 in the previous RSA version. An announce takes up 151 bytes vs 323.
+
+### 2021-05-18: RNS β 0.2.0
+
+This is the first beta release of RNS. This release also marks the publication of the Reticulum documentation, manual, and API documentation. All core features of Reticulum are now implemented, functional and ready to use in external programs. The wire-format and API will only change if there is a very good reason, though internals are still likely to be altered and optimised, and features are likely to be added.
+
+### 2021-05-13: RNS α 0.1.9
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-08-13: RNS α 0.1.8
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-08-13: RNS α 0.1.7
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-06-10: RNS α 0.1.6
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-06-09: RNS α 0.1.5
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-05-29: RNS α 0.1.4
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-05-21: RNS α 0.1.3
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-05-15: RNS α 0.1.2
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-05-14: RNS α 0.1.1
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-05-12: RNS α 0.1.0
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-04-28: RNS α 0.0.9
+
+This was a pre-release alpha version. No changelog available.
+
+### 2020-04-28: RNS α 0.0.8
+
+This was the first publicly available pre-release alpha of Reticulum.
+
+### 2016-05-29: Inintial Repository Commit
+
+The first commit to the Reticulum reference implementation was 9a9630cfd29e11ace3f12716ddb4dff0e5419b4b, which occurred on Sunday, the 22nd of May 2016.

Examples/Announce.py

@@ -130,10 +130,11 @@ class ExampleAnnounceHandler:
             RNS.prettyhexrep(destination_hash)
         )
 
-        RNS.log(
-            "The announce contained the following app data: "+
-            app_data.decode("utf-8")
-        )
+        if app_data:
+            RNS.log(
+                "The announce contained the following app data: "+
+                app_data.decode("utf-8")
+            )
 
 ##########################################################
 #### Program Startup #####################################

Examples/Echo.py

@@ -120,14 +120,16 @@ def client(destination_hexhash, configpath, timeout=None):
     # We need a binary representation of the destination
     # hash that was entered on the command line
     try:
-        if len(destination_hexhash) != 20:
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination_hexhash) != dest_len:
             raise ValueError(
-                "Destination length is invalid, must be 20 hexadecimal characters (10 bytes)"
+                "Destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2)
             )
 
         destination_hash = bytes.fromhex(destination_hexhash)
-    except:
-        RNS.log("Invalid destination entered. Check your input!\n")
+    except Exception as e:
+        RNS.log("Invalid destination entered. Check your input!")
+        RNS.log(str(e)+"\n")
         exit()
 
     # We must first initialise Reticulum

Examples/Filetransfer.py

@@ -215,8 +215,12 @@ def client(destination_hexhash, configpath):
     # We need a binary representation of the destination
     # hash that was entered on the command line
     try:
-        if len(destination_hexhash) != 20:
-            raise ValueError("Destination length is invalid, must be 20 hexadecimal characters (10 bytes)")
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination_hexhash) != dest_len:
+            raise ValueError(
+                "Destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2)
+            )
+            
         destination_hash = bytes.fromhex(destination_hexhash)
     except:
         RNS.log("Invalid destination entered. Check your input!\n")

Examples/Identify.py

@@ -84,7 +84,7 @@ def client_connected(link):
 def client_disconnected(link):
     RNS.log("Client disconnected")
 
-def remote_identified(identity):
+def remote_identified(link, identity):
     RNS.log("Remote identified as: "+str(identity))
 
 def server_packet_received(message, packet):
@@ -124,8 +124,12 @@ def client(destination_hexhash, configpath):
     # We need a binary representation of the destination
     # hash that was entered on the command line
     try:
-        if len(destination_hexhash) != 20:
-            raise ValueError("Destination length is invalid, must be 20 hexadecimal characters (10 bytes)")
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination_hexhash) != dest_len:
+            raise ValueError(
+                "Destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2)
+            )
+
         destination_hash = bytes.fromhex(destination_hexhash)
     except:
         RNS.log("Invalid destination entered. Check your input!\n")

Examples/Link.py

@@ -110,8 +110,12 @@ def client(destination_hexhash, configpath):
     # We need a binary representation of the destination
     # hash that was entered on the command line
     try:
-        if len(destination_hexhash) != 20:
-            raise ValueError("Destination length is invalid, must be 20 hexadecimal characters (10 bytes)")
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination_hexhash) != dest_len:
+            raise ValueError(
+                "Destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2)
+            )
+            
         destination_hash = bytes.fromhex(destination_hexhash)
     except:
         RNS.log("Invalid destination entered. Check your input!\n")

Examples/Request.py

@@ -110,8 +110,12 @@ def client(destination_hexhash, configpath):
     # We need a binary representation of the destination
     # hash that was entered on the command line
     try:
-        if len(destination_hexhash) != 20:
-            raise ValueError("Destination length is invalid, must be 20 hexadecimal characters (10 bytes)")
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination_hexhash) != dest_len:
+            raise ValueError(
+                "Destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2)
+            )
+            
         destination_hash = bytes.fromhex(destination_hexhash)
     except:
         RNS.log("Invalid destination entered. Check your input!\n")

Examples/Speedtest.py

@@ -166,8 +166,12 @@ def client(destination_hexhash, configpath):
     # We need a binary representation of the destination
     # hash that was entered on the command line
     try:
-        if len(destination_hexhash) != 20:
-            raise ValueError("Destination length is invalid, must be 20 hexadecimal characters (10 bytes)")
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination_hexhash) != dest_len:
+            raise ValueError(
+                "Destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2)
+            )
+            
         destination_hash = bytes.fromhex(destination_hexhash)
     except:
         RNS.log("Invalid destination entered. Check your input!\n")

Makefile

@@ -1,9 +1,28 @@
 all: release
 
+test:
+	@echo Running tests...
+	python -m tests.all
+
 clean:
 	@echo Cleaning...
-	-rm -r ./build
-	-rm -r ./dist
+	@-rm -rf ./build
+	@-rm -rf ./dist
+	@-rm -rf ./*.data
+	@-rm -rf ./__pycache__
+	@-rm -rf ./RNS/__pycache__
+	@-rm -rf ./RNS/Cryptography/__pycache__
+	@-rm -rf ./RNS/Cryptography/aes/__pycache__
+	@-rm -rf ./RNS/Cryptography/pure25519/__pycache__
+	@-rm -rf ./RNS/Interfaces/__pycache__
+	@-rm -rf ./RNS/Utilities/__pycache__
+	@-rm -rf ./RNS/vendor/__pycache__
+	@-rm -rf ./RNS/vendor/i2plib/__pycache__
+	@-rm -rf ./tests/__pycache__
+	@-rm -rf ./tests/rnsconfig/storage
+	@-rm -rf ./*.egg-info
+	@make -C docs clean
+	@echo Done
 
 remove_symlinks:
 	@echo Removing symlinks for build...
@@ -15,11 +34,28 @@ create_symlinks:
 	-ln -s ../RNS ./Examples/
 	-ln -s ../../RNS ./RNS/Utilities/
 
+build_sdist_only:
+	python3 setup.py sdist
+
 build_wheel:
 	python3 setup.py sdist bdist_wheel
 
-release: remove_symlinks build_wheel create_symlinks
+build_pure_wheel:
+	python3 setup.py sdist bdist_wheel --pure
+
+documentation:
+	make -C docs html
+
+manual:
+	make -C docs latexpdf
+
+release: test remove_symlinks build_wheel build_pure_wheel documentation manual create_symlinks
+
+debug: remove_symlinks build_wheel build_pure_wheel create_symlinks
 
 upload:
+	@echo Ready to publish release, hit enter to continue
+	@read VOID
 	@echo Uploading to PyPi...
 	twine upload dist/*
+	@echo Release published

README.md

@@ -1,31 +1,50 @@
-Reticulum Network Stack β
+Reticulum Network Stack β <img align="right" src="https://static.pepy.tech/personalized-badge/rns?period=total&units=international_system&left_color=grey&right_color=blue&left_text=Installs"/>
 ==========
 
-<p align="center"><img width="200" src="https://unsigned.io/wp-content/uploads/2022/03/reticulum_logo_512.png"></p>
+<p align="center"><img width="200" src="https://raw.githubusercontent.com/markqvist/Reticulum/master/docs/source/graphics/rns_logo_512.png"></p>
 
-Reticulum is the cryptography-based networking stack for wide-area networks built on readily available hardware. It can operate even with very high latency and extremely low bandwidth. Reticulum allows you to build wide-area networks with off-the-shelf tools, and offers end-to-end encryption and connectivity, initiator anonymity, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, unforgeable delivery acknowledgements and more.
+Reticulum is the cryptography-based networking stack for building local and wide-area
+networks with readily available hardware. It can operate even with very high latency
+and extremely low bandwidth. Reticulum allows you to build wide-area networks
+with off-the-shelf tools, and offers end-to-end encryption and connectivity,
+initiator anonymity, autoconfiguring cryptographically backed multi-hop
+transport, efficient addressing, unforgeable delivery acknowledgements and
+more.
 
-The vision of Reticulum is to allow anyone to be their own network operator, and to make it cheap and easy to cover vast areas with a myriad of independent, interconnectable and autonomous networks. Reticulum **is not** *one network*, it **is a tool** for building *thousands of networks*. Networks without kill-switches, surveillance, censorship and control. Networks that can freely interoperate, associate and disassociate with each other, and require no central oversight. Networks for human beings. *Networks for the people*.
+The vision of Reticulum is to allow anyone to be their own network operator,
+and to make it cheap and easy to cover vast areas with a myriad of independent,
+inter-connectable and autonomous networks. Reticulum **is not** *one* network.
+It is **a tool** for building *thousands of networks*. Networks without
+kill-switches, surveillance, censorship and control. Networks that can freely
+interoperate, associate and disassociate with each other, and require no
+central oversight. Networks for human beings. *Networks for the people*.
 
-Reticulum is a complete networking stack, and does not need IP or higher layers, although it is easy to use IP (with TCP or UDP) as the underlying carrier for Reticulum. It is therefore trivial to tunnel Reticulum over the Internet or private IP networks.
+Reticulum is a complete networking stack, and does not rely on IP or higher
+layers, but it is possible to use IP as the underlying carrier for Reticulum.
+It is therefore trivial to tunnel Reticulum over the Internet or private IP
+networks.
 
-Having no dependencies on traditional networking stacks free up overhead that has been utilised to implement a networking stack built directly on cryptographic principles, allowing resilience and stable functionality in open and trustless networks.
+Having no dependencies on traditional networking stacks frees up overhead that
+has been used to implement a networking stack built directly on cryptographic
+principles, allowing resilience and stable functionality, even in open and
+trustless networks.
 
-No kernel modules or drivers are required. Reticulum runs completely in userland, and can run on practically any system that runs Python 3.
+No kernel modules or drivers are required. Reticulum runs completely in
+userland, and can run on practically any system that runs Python 3.
 
 ## Read The Manual
 The full documentation for Reticulum is available at [markqvist.github.io/Reticulum/manual/](https://markqvist.github.io/Reticulum/manual/).
 
 You can also [download the Reticulum manual as a PDF](https://github.com/markqvist/Reticulum/raw/master/docs/Reticulum%20Manual.pdf)
 
-For more info, see [unsigned.io/projects/reticulum](https://unsigned.io/projects/reticulum/)
+For more info, see [reticulum.network](https://reticulum.network/)
 
 ## Notable Features
-- Coordination-less globally unique adressing and identification
+- Coordination-less globally unique addressing and identification
 - Fully self-configuring multi-hop routing
-- Complete initiator anonymity, communicate without revealing your identity
+- Initiator anonymity, communicate without revealing your identity
 - Asymmetric X25519 encryption and Ed25519 signatures as a basis for all communication
-- Forward Secrecy with ephemereal Elliptic Curve Diffie-Hellman keys on Curve25519
+- Forward Secrecy with ephemeral Elliptic Curve Diffie-Hellman keys on Curve25519
 - Reticulum uses the [Fernet](https://github.com/fernet/spec/blob/master/Spec.md) specification for on-the-wire / over-the-air encryption
   - Keys are ephemeral and derived from an ECDH key exchange on Curve25519
   - AES-128 in CBC mode with PKCS7 padding
@@ -34,57 +53,109 @@ For more info, see [unsigned.io/projects/reticulum](https://unsigned.io/projects
 - Unforgeable packet delivery confirmations
 - A variety of supported interface types
 - An intuitive and easy-to-use API
-- Reliable and efficient transfer of arbritrary amounts of data
+- Reliable and efficient transfer of arbitrary amounts of data
   - Reticulum can handle a few bytes of data or files of many gigabytes
-  - Sequencing, transfer coordination and checksumming is automatic
+  - Sequencing, transfer coordination and checksumming are automatic
   - The API is very easy to use, and provides transfer progress
 - Lightweight, flexible and expandable Request/Response mechanism
 - Efficient link establishment
-  - Total bandwidth cost of setting up an encrypted link is 3 packets totalling 237 bytes
-  - Low cost of keeping links open at only 0.62 bits per second
+  - Total bandwidth cost of setting up an encrypted link is 3 packets totaling 297 bytes
+  - Low cost of keeping links open at only 0.44 bits per second
+
+## Development Roadmap
+While Reticulum is already a fully featured and functional networking stack, many improvements and additions are actively being worked on, and planned for the future.
+
+To learn more about the direction and future of Reticulum, please see the [Development Roadmap](./Roadmap.md).
 
 ## Examples of Reticulum Applications
-If you want to quickly get an idea of what Reticulum can do, take a look at the following resources.
+If you want to quickly get an idea of what Reticulum can do, take a look at the
+following resources.
 
-- [LXMF](https://github.com/markqvist/lxmf) is a distributed, delay and disruption tolerant message transfer protocol built on Reticulum
 - For an off-grid, encrypted and resilient mesh communications platform, see [Nomad Network](https://github.com/markqvist/NomadNet)
-- The Android, Linux and macOS app [Sideband](https://unsigned.io/sideband) has a graphical interface and focuses on ease of use.
+- The Android, Linux and macOS app [Sideband](https://github.com/markqvist/Sideband) has a graphical interface and focuses on ease of use.
+- [LXMF](https://github.com/markqvist/lxmf) is a distributed, delay and disruption tolerant message transfer protocol built on Reticulum
 
 ## Where can Reticulum be used?
-Over practically any medium that can support at least a half-duplex channel with 500 bits per second throughput, and an MTU of 500 bytes. Data radios, modems, LoRa radios, serial lines, AX.25 TNCs, amateur radio digital modes, ad-hoc WiFi, free-space optical links and similar systems are all examples of the types of interfaces Reticulum was designed for.
+Over practically any medium that can support at least a half-duplex channel
+with 500 bits per second throughput, and an MTU of 500 bytes. Data radios,
+modems, LoRa radios, serial lines, AX.25 TNCs, amateur radio digital modes,
+WiFi and Ethernet devices, free-space optical links, and similar systems are
+all examples of the types of physical devices Reticulum can use.
 
-An open-source LoRa-based interface called [RNode](https://unsigned.io/projects/rnode/) has been designed specifically for use with Reticulum. It is possible to build yourself, or it can be purchased as a complete transceiver that just needs a USB connection to the host.
+An open-source LoRa-based interface called
+[RNode](https://markqvist.github.io/Reticulum/manual/hardware.html#rnode) has
+been designed specifically for use with Reticulum. It is possible to build
+yourself, or it can be purchased as a complete transceiver that just needs a
+USB connection to the host.
 
-Reticulum can also be encapsulated over existing IP networks, so there's nothing stopping you from using it over wired ethernet or your local WiFi network, where it'll work just as well. In fact, one of the strengths of Reticulum is how easily it allows you to connect different mediums into a self-configuring, resilient and encrypted mesh.
+Reticulum can also be encapsulated over existing IP networks, so there's
+nothing stopping you from using it over wired Ethernet, your local WiFi network
+or the Internet, where it'll work just as well. In fact, one of the strengths
+of Reticulum is how easily it allows you to connect different mediums into a
+self-configuring, resilient and encrypted mesh, using any available mixture of
+available infrastructure.
 
-As an example, it's possible to set up a Raspberry Pi connected to both a LoRa radio, a packet radio TNC and a WiFi network. Once the interfaces are configured, Reticulum will take care of the rest, and any device on the WiFi network can communicate with nodes on the LoRa and packet radio sides of the network, and vice versa.
+As an example, it's possible to set up a Raspberry Pi connected to both a LoRa
+radio, a packet radio TNC and a WiFi network. Once the interfaces are
+configured, Reticulum will take care of the rest, and any device on the WiFi
+network can communicate with nodes on the LoRa and packet radio sides of the
+network, and vice versa.
 
 ## How do I get started?
 The best way to get started with the Reticulum Network Stack depends on what
-you want to do. For full details and examples, have a look at the [Getting Started Fast](https://markqvist.github.io/Reticulum/manual/gettingstartedfast.html) section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
+you want to do. For full details and examples, have a look at the 
+[Getting Started Fast](https://markqvist.github.io/Reticulum/manual/gettingstartedfast.html) 
+section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
 
 To simply install Reticulum and related utilities on your system, the easiest way is via pip:
 
 ```bash
-pip3 install rns
+pip install rns
 ```
 
-You can then start any program that uses Reticulum, or start Reticulum as a system service with [the rnsd utility](https://markqvist.github.io/Reticulum/manual/using.html#the-rnsd-utility).
+You can then start any program that uses Reticulum, or start Reticulum as a
+system service with [the rnsd utility](https://markqvist.github.io/Reticulum/manual/using.html#the-rnsd-utility).
 
-When first started, Reticulum will create a default configuration file, providing basic connectivity to other Reticulum peers. The default config file contains examples for using Reticulum with LoRa transceivers (specifically [RNode](https://unsigned.io/projects/rnode/)), packet radio TNCs/modems, TCP and UDP.
+When first started, Reticulum will create a default configuration file,
+providing basic connectivity to other Reticulum peers that might be locally
+reachable. The default config file contains a few examples, and references for
+creating a more complex configuration.
 
-You can use the examples in the config file to expand communication over many mediums such as packet radio or LoRa (with [RNode](https://unsigned.io/projects/rnode/)), serial ports, or over fast IP links and the Internet using the UDP and TCP interfaces. For more detailed examples, take a look at the [Supported Interfaces](https://markqvist.github.io/Reticulum/manual/interfaces.html) section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
+If you have an old version of `pip` on your system, you may need to upgrade it first with `pip install pip --upgrade`. If you no not already have `pip` installed, you can install it using the package manager of your system with `sudo apt install python3-pip` or similar.
 
-## Current Status
-Reticulum should currently be considered beta software. All core protocol features are implemented and functioning, but additions will probably occur as real-world use is explored. There will be bugs. The API and wire-format can be considered relatively stable at the moment, but could change if warranted.
+For more detailed examples on how to expand communication over many mediums such 
+as packet radio or LoRa, serial ports, or over fast IP links and the Internet using 
+the UDP and TCP interfaces, take a look at the [Supported Interfaces](https://markqvist.github.io/Reticulum/manual/interfaces.html) 
+section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
+
+## Included Utilities
+Reticulum includes a range of useful utilities for managing your networks, 
+viewing status and information, and other tasks. You can read more about these 
+programs in the [Included Utility Programs](https://markqvist.github.io/Reticulum/manual/using.html#included-utility-programs) 
+section of the [Reticulum Manual](https://markqvist.github.io/Reticulum/manual/).
+
+- The system daemon `rnsd` for running Reticulum as an always-available service
+- An interface status utility called `rnstatus`, that displays information about interfaces
+- The path lookup and management tool `rnpath` letting you view and modify path tables
+- A diagnostics tool called `rnprobe` for checking connectivity to destinations
+- A simple file transfer program called `rncp` making it easy to copy files to remote systems
+- The remote command execution program `rnx` let's you run commands and
+  programs and retrieve output from remote systems
+
+All tools, including `rnx` and `rncp`, work reliably and well even over very
+low-bandwidth links like LoRa or Packet Radio.
 
 ## Supported interface types and devices
 
-Reticulum implements a range of generalised interface types that covers most of the communications hardware that Reticulum can run over. If your hardware is not supported, it's relatively simple to implement an interface class. I will gratefully accept pull requests for custom interfaces if they are generally useful.
+Reticulum implements a range of generalised interface types that covers most of
+the communications hardware that Reticulum can run over. If your hardware is
+not supported, it's relatively simple to implement an interface class. I will
+gratefully accept pull requests for custom interfaces if they are generally
+useful.
 
 Currently, the following interfaces are supported:
 
-- Any ethernet device
+- Any Ethernet device
 - LoRa using [RNode](https://unsigned.io/projects/rnode/)
 - Packet Radio TNCs (with or without AX.25)
 - KISS-compatible hardware and software modems
@@ -94,60 +165,93 @@ Currently, the following interfaces are supported:
 - External programs via stdio or pipes
 - Custom hardware via stdio or pipes
 
-## Development Roadmap
-- Version 0.3.6
-  - Improving [the manual](https://markqvist.github.io/Reticulum/manual/) with sections specifically for beginners
-- Version 0.3.7
-  - Support for radio and modem interfaces on Android
-  - GUI interface configuration tool
-  - Easy way to share interface configurations, see [#19](https://github.com/markqvist/Reticulum/discussions/19)
-  - More interface types for even broader compatibility
-    - Plain ESP32 devices (ESP-Now, WiFi, Bluetooth, etc.)
-    - More LoRa transceivers
-    - AT-compatible modems
-    - IR Transceivers
-    - AWDL / OWL
-    - HF Modems
-    - CAN-bus
-    - ZeroMQ
-    - MQTT
-    - SPI
-    - i²c
-- Planned, but not yet scheduled
-  - Globally routable multicast
-  - A portable Reticulum implementation in C, see [#21](https://github.com/markqvist/Reticulum/discussions/21)
+## Performance
+Reticulum targets a *very* wide usable performance envelope, but prioritises
+functionality and performance on low-bandwidth mediums. The goal is to
+provide a dynamic performance envelope from 250 bits per second, to 1 gigabit
+per second on normal hardware.
 
+Currently, the usable performance envelope is approximately 500 bits per second
+to 20 megabits per second, with physical mediums faster than that not being
+saturated. Performance beyond the current level is intended for future
+upgrades, but not highly prioritised at this point in time.
 
+## Current Status
+Reticulum should currently be considered beta software. All core protocol
+features are implemented and functioning, but additions will probably occur as
+real-world use is explored. There will be bugs. The API and wire-format can be
+considered relatively stable at the moment, but could change if warranted.
 
-## Dependencies:
-- Python 3.6
-- cryptography.io
-- netifaces
-- pyserial
+## Dependencies
+The installation of the default `rns` package requires the dependencies listed
+below. Almost all systems and distributions have readily available packages for
+these dependencies, and when the `rns` package is installed with `pip`, they
+will be downloaded and installed as well.
+
+- [PyCA/cryptography](https://github.com/pyca/cryptography)
+- [netifaces](https://github.com/al45tair/netifaces)
+- [pyserial](https://github.com/pyserial/pyserial)
+
+On more unusual systems, and in some rare cases, it might not be possible to
+install or even compile one or more of the above modules. In such situations,
+you can use the `rnspure` package instead, which require no external
+dependencies for installation. Please note that the contents of the `rns` and
+`rnspure` packages are *identical*. The only difference is that the `rnspure`
+package lists no dependencies required for installation.
+
+No matter how Reticulum is installed and started, it will load external
+dependencies only if they are *needed* and *available*. If for example you want
+to use Reticulum on a system that cannot support
+[pyserial](https://github.com/pyserial/pyserial), it is perfectly possible to
+do so using the `rnspure` package, but Reticulum will not be able to use
+serial-based interfaces. All other available modules will still be loaded when
+needed.
+
+**Please Note!** If you use the `rnspure` package to run Reticulum on systems
+that do not support [PyCA/cryptography](https://github.com/pyca/cryptography),
+it is important that you read and understand the [Cryptographic
+Primitives](#cryptographic-primitives) section of this document.
 
 ## Public Testnet
+If you just want to get started experimenting without building any physical
+networks, you are welcome to join the Unsigned.io RNS Testnet. The testnet is
+just that, an informal network for testing and experimenting. It will be up
+most of the time, and anyone can join, but it also means that there's no
+guarantees for service availability.
 
-If you just want to get started experimenting without building any physical networks, you are welcome to join the Unsigned.io RNS Testnet. The testnet is just that, an informal network for testing and experimenting. It will be up most of the time, and anyone can join, but it also means that there's no guarantees for service availability.
-
-The testnet runs the very latest version of Reticulum (often even a short while before it is publicly released). Sometimes experimental versions of Reticulum might be deployed to nodes on the testnet, which means strange behaviour might occur. If none of that scares you, you can join the testnet via eihter TCP or I2P. Just add one of the following interfaces to your Reticulum configuration file:
+The testnet runs the very latest version of Reticulum (often even a short while
+before it is publicly released). Sometimes experimental versions of Reticulum
+might be deployed to nodes on the testnet, which means strange behaviour might
+occur. If none of that scares you, you can join the testnet via either TCP or
+I2P. Just add one of the following interfaces to your Reticulum configuration
+file:
 
 ```
-# For connecting over TCP/IP:
-
-  [[RNS Testnet Frankfurt]]
+# TCP/IP interface to the Dublin Hub
+  [[RNS Testnet Dublin]]
     type = TCPClientInterface
-    interface_enabled = yes
-    outgoing = True
-    target_host = frankfurt.rns.unsigned.io
+    enabled = yes
+    target_host = dublin.connect.reticulum.network
     target_port = 4965
 
+# TCP/IP interface to the Frankfurt Hub
+  [[RNS Testnet Frankfurt]]
+    type = TCPClientInterface
+    enabled = yes
+    target_host = frankfurt.connect.reticulum.network
+    target_port = 5377
 
-# For connecting over I2P:
-
-  [[RNS Testnet I2P Node A]]
+# Interface to I2P Hub A
+  [[RNS Testnet I2P Hub A]]
     type = I2PInterface
-    interface_enabled = yes
-    peers = ykzlw5ujbaqc2xkec4cpvgyxj257wcrmmgkuxqmqcur7cq3w3lha.b32.i2p
+    enabled = yes
+    peers = pmlm3l5rpympihoy2o5ago43kluei2jjjzsalcuiuylbve3mwi2a.b32.i2p
+
+# Interface to I2P Hub B
+  [[RNS Testnet I2P Hub B]]
+    type = I2PInterface
+    enabled = yes
+    peers = iwoqtz22dsr73aemwpw7guocplsjjoamyl7sogj33qtcd6ds4mza.b32.i2p
 ```
 
 The testnet also contains a number of [Nomad Network](https://github.com/markqvist/nomadnet) nodes, and LXMF propagation nodes.
@@ -155,11 +259,89 @@ The testnet also contains a number of [Nomad Network](https://github.com/markqvi
 ## Support Reticulum
 You can help support the continued development of open, free and private communications systems by donating via one of the following channels:
 
-- Ethereum: 0x81F7B979fEa6134bA9FD5c701b3501A2e61E897a
-- Bitcoin: 3CPmacGm34qYvR6XWLVEJmi2aNe3PZqUuq
+- Monero:
+ ```
+  84FpY1QbxHcgdseePYNmhTHcrgMX4nFfBYtz2GKYToqHVVhJp8Eaw1Z1EedRnKD19b3B8NiLCGVxzKV17UMmmeEsCrPyA5w
+  ```
+- Ethereum
+  ```
+  0x81F7B979fEa6134bA9FD5c701b3501A2e61E897a
+  ```
+- Bitcoin
+  ```
+  3CPmacGm34qYvR6XWLVEJmi2aNe3PZqUuq
+  ```
 - Ko-Fi: https://ko-fi.com/markqvist
 
-Are certain features in the development roadmap are important to you or your organisation? Make them a reality quickly by sponsoring their implementation.
+Are certain features in the development roadmap are important to you or your
+organisation? Make them a reality quickly by sponsoring their implementation.
 
-## Caveat Emptor
-Reticulum is relatively young software, and should be considered as such. While it has been built with cryptography best-practices very foremost in mind, it _has not_ been externally security audited, and there could very well be privacy-breaking bugs. If you want to help out, or help sponsor an audit, please do get in touch.
+## Cryptographic Primitives
+Reticulum uses a simple suite of efficient, strong and modern cryptographic
+primitives, with widely available implementations that can be used both on
+general-purpose CPUs and on microcontrollers. The necessary primitives are:
+
+- Ed25519 for signatures
+- X22519 for ECDH key exchanges
+- HKDF for key derivation
+- Modified Fernet for encrypted tokens
+  - AES-128 in CBC mode
+  - HMAC for message authentication
+  - No Fernet version and timestamp fields
+- SHA-256
+- SHA-512
+
+In the default installation configuration, the `X25519`, `Ed25519` and
+`AES-128-CBC` primitives are provided by [OpenSSL](https://www.openssl.org/)
+(via the [PyCA/cryptography](https://github.com/pyca/cryptography) package).
+The hashing functions `SHA-256` and `SHA-512` are provided by the standard
+Python [hashlib](https://docs.python.org/3/library/hashlib.html). The `HKDF`,
+`HMAC`, `Fernet` primitives, and the `PKCS7` padding function are always
+provided by the following internal implementations:
+
+- [HKDF.py](RNS/Cryptography/HKDF.py)
+- [HMAC.py](RNS/Cryptography/HMAC.py)
+- [Fernet.py](RNS/Cryptography/Fernet.py)
+- [PKCS7.py](RNS/Cryptography/PKCS7.py)
+
+
+Reticulum also includes a complete implementation of all necessary primitives
+in pure Python. If OpenSSL & PyCA are not available on the system when
+Reticulum is started, Reticulum will instead use the internal pure-python
+primitives. A trivial consequence of this is performance, with the OpenSSL
+backend being *much* faster. The most important consequence however, is the
+potential loss of security by using primitives that has not seen the same
+amount of scrutiny, testing and review as those from OpenSSL.
+
+If you want to use the internal pure-python primitives, it is **highly
+advisable** that you have a good understanding of the risks that this pose, and
+make an informed decision on whether those risks are acceptable to you.
+
+Reticulum is relatively young software, and should be considered as such. While
+it has been built with cryptography best-practices very foremost in mind, it
+_has not_ been externally security audited, and there could very well be
+privacy or security breaking bugs. If you want to help out, or help sponsor an
+audit, please do get in touch.
+
+## Acknowledgements & Credits
+Reticulum can only exist because of the mountain of Open Source work it was
+built on top of, the contributions of everyone involved, and everyone that has
+supported the project through the years. To everyone who has helped, thank you
+so much.
+
+A number of other modules and projects are either part of, or used by
+Reticulum. Sincere thanks to the authors and contributors of the following
+projects:
+
+- [PyCA/cryptography](https://github.com/pyca/cryptography), *BSD License*
+- [Pure-25519](https://github.com/warner/python-pure25519) by [Brian Warner](https://github.com/warner), *MIT License*
+- [Pysha2](https://github.com/thomdixon/pysha2) by [Thom Dixon](https://github.com/thomdixon), *MIT License*
+- [Python-AES](https://github.com/orgurar/python-aes) by [Or Gur Arie](https://github.com/orgurar), *MIT License*
+- [Curve25519.py](https://gist.github.com/nickovs/cc3c22d15f239a2640c185035c06f8a3#file-curve25519-py) by [Nicko van Someren](https://gist.github.com/nickovs), *Public Domain*
+- [I2Plib](https://github.com/l-n-s/i2plib) by [Viktor Villainov](https://github.com/l-n-s)
+- [PySerial](https://github.com/pyserial/pyserial) by Chris Liechti, *BSD License*
+- [Netifaces](https://github.com/al45tair/netifaces) by [Alastair Houghton](https://github.com/al45tair), *MIT License*
+- [Configobj](https://github.com/DiffSK/configobj) by Michael Foord, Nicola Larosa, Rob Dennis & Eli Courtwright, *BSD License*
+- [Six](https://github.com/benjaminp/six) by [Benjamin Peterson](https://github.com/benjaminp), *MIT License*
+- [Umsgpack.py](https://github.com/vsergeev/u-msgpack-python) by [Ivan A. Sergeev](https://github.com/vsergeev)
+- [Python](https://www.python.org)

RNS/Cryptography/AES.py

@@ -0,0 +1,68 @@
+# MIT License
+#
+# Copyright (c) 2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import RNS.Cryptography.Provider as cp
+import RNS.vendor.platformutils as pu
+
+if cp.PROVIDER == cp.PROVIDER_INTERNAL:
+    from .aes import AES
+    
+elif cp.PROVIDER == cp.PROVIDER_PYCA:
+    from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
+
+    if pu.cryptography_old_api():
+        from cryptography.hazmat.backends import default_backend
+
+
+class AES_128_CBC:
+
+    @staticmethod
+    def encrypt(plaintext, key, iv):
+        if cp.PROVIDER == cp.PROVIDER_INTERNAL:
+            cipher = AES(key)
+            return cipher.encrypt(plaintext, iv)
+
+        elif cp.PROVIDER == cp.PROVIDER_PYCA:
+            if not pu.cryptography_old_api():
+                cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
+            else:
+                cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
+            
+            encryptor = cipher.encryptor()
+            ciphertext = encryptor.update(plaintext) + encryptor.finalize()
+            return ciphertext
+
+    @staticmethod
+    def decrypt(ciphertext, key, iv):
+        if cp.PROVIDER == cp.PROVIDER_INTERNAL:
+            cipher = AES(key)
+            return cipher.decrypt(ciphertext, iv)
+
+        elif cp.PROVIDER == cp.PROVIDER_PYCA:
+            if not pu.cryptography_old_api():
+                cipher = Cipher(algorithms.AES(key), modes.CBC(iv))
+            else:
+                cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend())
+
+            decryptor = cipher.decryptor()
+            plaintext = decryptor.update(ciphertext) + decryptor.finalize()
+            return plaintext

RNS/Cryptography/Ed25519.py

@@ -0,0 +1,41 @@
+import os
+from .pure25519 import ed25519_oop as ed25519
+
+class Ed25519PrivateKey:
+    def __init__(self, seed):
+        self.seed = seed
+        self.sk = ed25519.SigningKey(self.seed)
+        #self.vk = self.sk.get_verifying_key()
+
+    @classmethod
+    def generate(cls):
+        return cls.from_private_bytes(os.urandom(32))
+
+    @classmethod
+    def from_private_bytes(cls, data):
+        return cls(seed=data)
+
+    def private_bytes(self):
+        return self.seed
+
+    def public_key(self):
+        return Ed25519PublicKey.from_public_bytes(self.sk.vk_s)
+
+    def sign(self, message):
+        return self.sk.sign(message)
+
+
+class Ed25519PublicKey:
+    def __init__(self, seed):
+        self.seed = seed
+        self.vk = ed25519.VerifyingKey(self.seed)
+
+    @classmethod
+    def from_public_bytes(cls, data):
+        return cls(data)
+
+    def public_bytes(self):
+        return self.vk.to_bytes()
+
+    def verify(self, signature, message):
+        self.vk.verify(signature, message)

RNS/Cryptography/Fernet.py

@@ -0,0 +1,110 @@
+# MIT License
+#
+# Copyright (c) 2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+import time
+
+from RNS.Cryptography import HMAC
+from RNS.Cryptography import PKCS7
+from RNS.Cryptography.AES import AES_128_CBC
+
+class Fernet():
+    """
+    This class provides a slightly modified implementation of the Fernet spec
+    found at: https://github.com/fernet/spec/blob/master/Spec.md
+
+    According to the spec, a Fernet token includes a one byte VERSION and
+    eight byte TIMESTAMP field at the start of each token. These fields are
+    not relevant to Reticulum. They are therefore stripped from this
+    implementation, since they incur overhead and leak initiator metadata.
+    """
+    FERNET_OVERHEAD  = 48 # Bytes
+
+    @staticmethod
+    def generate_key():
+        return os.urandom(32)
+
+    def __init__(self, key = None):
+        if key == None:
+            raise ValueError("Fernet key cannot be None")
+
+        if len(key) != 32:
+            raise ValueError("Fernet key must be 32 bytes, not "+str(len(key)))
+            
+        self._signing_key = key[:16]
+        self._encryption_key = key[16:]
+
+
+    def verify_hmac(self, token):
+        if len(token) <= 32:
+            raise ValueError("Cannot verify HMAC on token of only "+str(len(token))+" bytes")
+        else:
+            received_hmac = token[-32:]
+            expected_hmac = HMAC.new(self._signing_key, token[:-32]).digest()
+
+            if received_hmac == expected_hmac:
+                return True
+            else:
+                return False
+
+
+    def encrypt(self, data = None):
+        iv = os.urandom(16)
+        current_time = int(time.time())
+
+        if not isinstance(data, bytes):
+            raise TypeError("Fernet token plaintext input must be bytes")
+
+        ciphertext = AES_128_CBC.encrypt(
+            plaintext = PKCS7.pad(data),
+            key = self._encryption_key,
+            iv = iv,
+        )
+
+        signed_parts = iv+ciphertext
+
+        return signed_parts + HMAC.new(self._signing_key, signed_parts).digest()
+
+
+    def decrypt(self, token = None):
+        if not isinstance(token, bytes):
+            raise TypeError("Fernet token must be bytes")
+
+        if not self.verify_hmac(token):
+            raise ValueError("Fernet token HMAC was invalid")
+
+        iv = token[:16]
+        ciphertext = token[16:-32]
+
+        try:
+            plaintext = PKCS7.unpad(
+                AES_128_CBC.decrypt(
+                    ciphertext,
+                    self._encryption_key,
+                    iv,
+                )
+            )
+
+            return plaintext
+
+        except Exception as e:
+            raise ValueError("Could not decrypt Fernet token")

RNS/Cryptography/HKDF.py

@@ -0,0 +1,57 @@
+# MIT License
+#
+# Copyright (c) 2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import hashlib
+from math import ceil
+from RNS.Cryptography import HMAC
+
+def hkdf(length=None, derive_from=None, salt=None, context=None):
+    hash_len = 32
+
+    def hmac_sha256(key, data):
+        return HMAC.new(key, data).digest()
+
+    if length == None or length < 1:
+        raise ValueError("Invalid output key length")
+
+    if derive_from == None or derive_from == "":
+        raise ValueError("Cannot derive key from empty input material")
+
+    if salt == None or len(salt) == 0:
+        salt = bytes([0] * hash_len)
+
+    if salt == None:
+        salt = b""
+
+    if context == None:
+        context = b""
+
+    pseudorandom_key = hmac_sha256(salt, derive_from)
+
+    block = b""
+    derived = b""
+
+    for i in range(ceil(length / hash_len)):
+        block = hmac_sha256(pseudorandom_key, block + context + bytes([i + 1]))
+        derived += block
+
+    return derived[:length]

RNS/Cryptography/HMAC.py

@@ -0,0 +1,183 @@
+# This HMAC implementation comes directly from the HMAC implementation
+# included in Python 3.10.4, and is almost completely identical. It has
+# been modified to be a pure Python implementation, that is not dependent
+# on the system having OpenSSL binaries installed.
+
+import warnings as _warnings
+import hashlib as _hashlib
+
+trans_5C = bytes((x ^ 0x5C) for x in range(256))
+trans_36 = bytes((x ^ 0x36) for x in range(256))
+
+# The size of the digests returned by HMAC depends on the underlying
+# hashing module used.  Use digest_size from the instance of HMAC instead.
+digest_size = None
+
+
+class HMAC:
+    """RFC 2104 HMAC class.  Also complies with RFC 4231.
+    This supports the API for Cryptographic Hash Functions (PEP 247).
+    """
+    blocksize = 64  # 512-bit HMAC; can be changed in subclasses.
+
+    __slots__ = (
+        "_hmac", "_inner", "_outer", "block_size", "digest_size"
+    )
+
+    def __init__(self, key, msg=None, digestmod=_hashlib.sha256):
+        """Create a new HMAC object.
+        key: bytes or buffer, key for the keyed hash object.
+        msg: bytes or buffer, Initial input for the hash or None.
+        digestmod: A hash name suitable for hashlib.new(). *OR*
+                   A hashlib constructor returning a new hash object. *OR*
+                   A module supporting PEP 247.
+                   Required as of 3.8, despite its position after the optional
+                   msg argument.  Passing it as a keyword argument is
+                   recommended, though not required for legacy API reasons.
+        """
+
+        if not isinstance(key, (bytes, bytearray)):
+            raise TypeError("key: expected bytes or bytearray, but got %r" % type(key).__name__)
+
+        if not digestmod:
+            raise TypeError("Missing required parameter 'digestmod'.")
+
+        self._hmac_init(key, msg, digestmod)
+
+    def _hmac_init(self, key, msg, digestmod):
+        if callable(digestmod):
+            digest_cons = digestmod
+        elif isinstance(digestmod, str):
+            digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
+        else:
+            digest_cons = lambda d=b'': digestmod.new(d)
+
+        self._hmac = None
+        self._outer = digest_cons()
+        self._inner = digest_cons()
+        self.digest_size = self._inner.digest_size
+
+        if hasattr(self._inner, 'block_size'):
+            blocksize = self._inner.block_size
+            if blocksize < 16:
+                _warnings.warn('block_size of %d seems too small; using our '
+                               'default of %d.' % (blocksize, self.blocksize),
+                               RuntimeWarning, 2)
+                blocksize = self.blocksize
+        else:
+            _warnings.warn('No block_size attribute on given digest object; '
+                           'Assuming %d.' % (self.blocksize),
+                           RuntimeWarning, 2)
+            blocksize = self.blocksize
+
+        if len(key) > blocksize:
+            key = digest_cons(key).digest()
+
+        # self.blocksize is the default blocksize. self.block_size is
+        # effective block size as well as the public API attribute.
+        self.block_size = blocksize
+
+        key = key.ljust(blocksize, b'\0')
+        self._outer.update(key.translate(trans_5C))
+        self._inner.update(key.translate(trans_36))
+        if msg is not None:
+            self.update(msg)
+
+    @property
+    def name(self):
+        if self._hmac:
+            return self._hmac.name
+        else:
+            return f"hmac-{self._inner.name}"
+
+    def update(self, msg):
+        """Feed data from msg into this hashing object."""
+        inst = self._hmac or self._inner
+        inst.update(msg)
+
+    def copy(self):
+        """Return a separate copy of this hashing object.
+        An update to this copy won't affect the original object.
+        """
+        # Call __new__ directly to avoid the expensive __init__.
+        other = self.__class__.__new__(self.__class__)
+        other.digest_size = self.digest_size
+        if self._hmac:
+            other._hmac = self._hmac.copy()
+            other._inner = other._outer = None
+        else:
+            other._hmac = None
+            other._inner = self._inner.copy()
+            other._outer = self._outer.copy()
+        return other
+
+    def _current(self):
+        """Return a hash object for the current state.
+        To be used only internally with digest() and hexdigest().
+        """
+        if self._hmac:
+            return self._hmac
+        else:
+            h = self._outer.copy()
+            h.update(self._inner.digest())
+            return h
+
+    def digest(self):
+        """Return the hash value of this hashing object.
+        This returns the hmac value as bytes.  The object is
+        not altered in any way by this function; you can continue
+        updating the object after calling this function.
+        """
+        h = self._current()
+        return h.digest()
+
+    def hexdigest(self):
+        """Like digest(), but returns a string of hexadecimal digits instead.
+        """
+        h = self._current()
+        return h.hexdigest()
+
+def new(key, msg=None, digestmod=_hashlib.sha256):
+    """Create a new hashing object and return it.
+    key: bytes or buffer, The starting key for the hash.
+    msg: bytes or buffer, Initial input for the hash, or None.
+    digestmod: A hash name suitable for hashlib.new(). *OR*
+               A hashlib constructor returning a new hash object. *OR*
+               A module supporting PEP 247.
+               Required as of 3.8, despite its position after the optional
+               msg argument.  Passing it as a keyword argument is
+               recommended, though not required for legacy API reasons.
+    You can now feed arbitrary bytes into the object using its update()
+    method, and can ask for the hash value at any time by calling its digest()
+    or hexdigest() methods.
+    """
+    return HMAC(key, msg, digestmod)
+
+
+def digest(key, msg, digest):
+    """Fast inline implementation of HMAC.
+    key: bytes or buffer, The key for the keyed hash object.
+    msg: bytes or buffer, Input message.
+    digest: A hash name suitable for hashlib.new() for best performance. *OR*
+            A hashlib constructor returning a new hash object. *OR*
+            A module supporting PEP 247.
+    """
+    if callable(digest):
+        digest_cons = digest
+    elif isinstance(digest, str):
+        digest_cons = lambda d=b'': _hashlib.new(digest, d)
+    else:
+        digest_cons = lambda d=b'': digest.new(d)
+
+    inner = digest_cons()
+    outer = digest_cons()
+    blocksize = getattr(inner, 'block_size', 64)
+    if len(key) > blocksize:
+        key = digest_cons(key).digest()
+
+    key = key + b'\x00' * (blocksize - len(key))
+    inner.update(key.translate(trans_36))
+    outer.update(key.translate(trans_5C))
+    inner.update(msg)
+    outer.update(inner.digest())
+    return outer.digest()

RNS/Cryptography/Hashes.py

@@ -0,0 +1,34 @@
+import importlib
+if importlib.util.find_spec('hashlib') != None:
+    import hashlib
+else:
+    hashlib = None
+
+if hasattr(hashlib, "sha512"):
+    from hashlib import sha512 as ext_sha512
+else:
+    from .SHA512 import sha512 as ext_sha512
+
+if hasattr(hashlib, "sha256"):
+    from hashlib import sha256 as ext_sha256
+else:
+    from .SHA256 import sha256 as ext_sha256
+
+"""
+The SHA primitives are abstracted here to allow platform-
+aware hardware acceleration in the future. Currently only
+uses Python's internal SHA-256 implementation. All SHA-256
+calls in RNS end up here.
+"""
+
+def sha256(data):
+    digest = ext_sha256()
+    digest.update(data)
+
+    return digest.digest()
+
+def sha512(data):
+    digest = ext_sha512()
+    digest.update(data)
+
+    return digest.digest()

RNS/Cryptography/PKCS7.py

@@ -0,0 +1,40 @@
+# MIT License
+#
+# Copyright (c) 2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+class PKCS7:
+    BLOCKSIZE = 16
+
+    @staticmethod
+    def pad(data, bs=BLOCKSIZE):
+        l = len(data)
+        n = bs-l%bs
+        v = bytes([n])
+        return data+v*n
+
+    @staticmethod
+    def unpad(data, bs=BLOCKSIZE):
+        l = len(data)
+        n = data[-1]
+        if n > bs:
+            raise ValueError("Cannot unpad, invalid padding length of "+str(n)+" bytes")
+        else:
+            return data[:l-n]

RNS/Cryptography/Provider.py

@@ -0,0 +1,38 @@
+import importlib
+
+PROVIDER_NONE     = 0x00
+PROVIDER_INTERNAL = 0x01
+PROVIDER_PYCA     = 0x02
+
+PROVIDER = PROVIDER_NONE
+
+pyca_v = None
+use_pyca = False
+
+try:
+    if importlib.util.find_spec('cryptography') != None:
+        import cryptography
+        pyca_v = cryptography.__version__
+        v = pyca_v.split(".")
+
+        if int(v[0]) == 2:
+            if int(v[1]) >= 8:
+                use_pyca = True
+        elif int(v[0]) >= 3:
+            use_pyca = True
+
+except Exception as e:
+    pass
+
+if use_pyca:
+    PROVIDER = PROVIDER_PYCA
+else:
+    PROVIDER = PROVIDER_INTERNAL
+
+def backend():
+    if PROVIDER == PROVIDER_NONE:
+        return "none"
+    elif PROVIDER == PROVIDER_INTERNAL:
+        return "internal"
+    elif PROVIDER == PROVIDER_PYCA:
+        return "openssl, PyCA "+str(pyca_v)

RNS/Cryptography/Proxies.py

@@ -0,0 +1,90 @@
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
+from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey
+
+# These proxy classes exist to create a uniform API accross
+# cryptography primitive providers.
+
+class X25519PrivateKeyProxy:
+    def __init__(self, real):
+        self.real = real
+
+    @classmethod
+    def generate(cls):
+        return cls(X25519PrivateKey.generate())
+
+    @classmethod
+    def from_private_bytes(cls, data):
+        return cls(X25519PrivateKey.from_private_bytes(data))
+
+    def private_bytes(self):
+        return self.real.private_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PrivateFormat.Raw,
+            encryption_algorithm=serialization.NoEncryption(),
+        )
+
+    def public_key(self):
+        return X25519PublicKeyProxy(self.real.public_key())
+
+    def exchange(self, peer_public_key):
+        return self.real.exchange(peer_public_key.real)
+
+
+class X25519PublicKeyProxy:
+    def __init__(self, real):
+        self.real = real
+
+    @classmethod
+    def from_public_bytes(cls, data):
+        return cls(X25519PublicKey.from_public_bytes(data))
+
+    def public_bytes(self):
+        return self.real.public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw
+        )
+
+
+class Ed25519PrivateKeyProxy:
+    def __init__(self, real):
+        self.real = real
+
+    @classmethod
+    def generate(cls):
+        return cls(Ed25519PrivateKey.generate())
+
+    @classmethod
+    def from_private_bytes(cls, data):
+        return cls(Ed25519PrivateKey.from_private_bytes(data))
+
+    def private_bytes(self):
+        return self.real.private_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PrivateFormat.Raw,
+            encryption_algorithm=serialization.NoEncryption()
+        )
+
+    def public_key(self):
+        return Ed25519PublicKeyProxy(self.real.public_key())
+
+    def sign(self, message):
+        return self.real.sign(message)
+
+
+class Ed25519PublicKeyProxy:
+    def __init__(self, real):
+        self.real = real
+
+    @classmethod
+    def from_public_bytes(cls, data):
+        return cls(Ed25519PublicKey.from_public_bytes(data))
+
+    def public_bytes(self):
+        return self.real.public_bytes(
+            encoding=serialization.Encoding.Raw,
+            format=serialization.PublicFormat.Raw
+        )
+
+    def verify(self, signature, message):
+        self.real.verify(signature, message)

RNS/Cryptography/SHA256.py

@@ -0,0 +1,129 @@
+# MIT License
+#
+# Copyright (c) 2017 Thomas Dixon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import copy
+import struct
+import sys
+
+
+def new(m=None):
+    return sha256(m)
+
+class sha256(object):
+    _k = (0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5,
+          0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
+          0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3,
+          0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
+          0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc,
+          0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
+          0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7,
+          0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
+          0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13,
+          0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
+          0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3,
+          0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
+          0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5,
+          0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
+          0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208,
+          0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2)
+    _h = (0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a,
+          0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19)
+    _output_size = 8
+    
+    blocksize = 1
+    block_size = 64
+    digest_size = 32
+    
+    def __init__(self, m=None):        
+        self._buffer = b""
+        self._counter = 0
+        
+        if m is not None:
+            if type(m) is not bytes:
+                raise TypeError('%s() argument 1 must be bytes, not %s' % (self.__class__.__name__, type(m).__name__))
+            self.update(m)
+        
+    def _rotr(self, x, y):
+        return ((x >> y) | (x << (32-y))) & 0xFFFFFFFF
+                    
+    def _sha256_process(self, c):
+        w = [0]*64
+        w[0:16] = struct.unpack('!16L', c)
+        
+        for i in range(16, 64):
+            s0 = self._rotr(w[i-15], 7) ^ self._rotr(w[i-15], 18) ^ (w[i-15] >> 3)
+            s1 = self._rotr(w[i-2], 17) ^ self._rotr(w[i-2], 19) ^ (w[i-2] >> 10)
+            w[i] = (w[i-16] + s0 + w[i-7] + s1) & 0xFFFFFFFF
+        
+        a,b,c,d,e,f,g,h = self._h
+        
+        for i in range(64):
+            s0 = self._rotr(a, 2) ^ self._rotr(a, 13) ^ self._rotr(a, 22)
+            maj = (a & b) ^ (a & c) ^ (b & c)
+            t2 = s0 + maj
+            s1 = self._rotr(e, 6) ^ self._rotr(e, 11) ^ self._rotr(e, 25)
+            ch = (e & f) ^ ((~e) & g)
+            t1 = h + s1 + ch + self._k[i] + w[i]
+            
+            h = g
+            g = f
+            f = e
+            e = (d + t1) & 0xFFFFFFFF
+            d = c
+            c = b
+            b = a
+            a = (t1 + t2) & 0xFFFFFFFF
+            
+        self._h = [(x+y) & 0xFFFFFFFF for x,y in zip(self._h, [a,b,c,d,e,f,g,h])]
+        
+    def update(self, m):
+        if not m:
+            return
+
+        if type(m) is not bytes:
+            raise TypeError('%s() argument 1 must be bytes, not %s' % (sys._getframe().f_code.co_name, type(m).__name__))
+        
+        self._buffer += m
+        self._counter += len(m)
+        
+        while len(self._buffer) >= 64:
+            self._sha256_process(self._buffer[:64])
+            self._buffer = self._buffer[64:]
+            
+    def digest(self):
+        mdi = self._counter & 0x3F
+        length = struct.pack('!Q', self._counter<<3)
+        
+        if mdi < 56:
+            padlen = 55-mdi
+        else:
+            padlen = 119-mdi
+        
+        r = self.copy()
+        r.update(b'\x80'+(b'\x00'*padlen)+length)
+        return b''.join([struct.pack('!L', i) for i in r._h[:self._output_size]])
+        
+    def hexdigest(self):
+        return self.digest().encode('hex')
+        
+    def copy(self):
+        return copy.deepcopy(self)

RNS/Cryptography/SHA512.py

@@ -0,0 +1,129 @@
+# MIT License
+#
+# Copyright (c) 2017 Thomas Dixon
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import copy, struct, sys
+
+def new(m=None):
+    return sha512(m)
+
+class sha512(object):
+    _k = (0x428a2f98d728ae22, 0x7137449123ef65cd, 0xb5c0fbcfec4d3b2f, 0xe9b5dba58189dbbc,
+          0x3956c25bf348b538, 0x59f111f1b605d019, 0x923f82a4af194f9b, 0xab1c5ed5da6d8118,
+          0xd807aa98a3030242, 0x12835b0145706fbe, 0x243185be4ee4b28c, 0x550c7dc3d5ffb4e2,
+          0x72be5d74f27b896f, 0x80deb1fe3b1696b1, 0x9bdc06a725c71235, 0xc19bf174cf692694,
+          0xe49b69c19ef14ad2, 0xefbe4786384f25e3, 0x0fc19dc68b8cd5b5, 0x240ca1cc77ac9c65,
+          0x2de92c6f592b0275, 0x4a7484aa6ea6e483, 0x5cb0a9dcbd41fbd4, 0x76f988da831153b5,
+          0x983e5152ee66dfab, 0xa831c66d2db43210, 0xb00327c898fb213f, 0xbf597fc7beef0ee4,
+          0xc6e00bf33da88fc2, 0xd5a79147930aa725, 0x06ca6351e003826f, 0x142929670a0e6e70,
+          0x27b70a8546d22ffc, 0x2e1b21385c26c926, 0x4d2c6dfc5ac42aed, 0x53380d139d95b3df,
+          0x650a73548baf63de, 0x766a0abb3c77b2a8, 0x81c2c92e47edaee6, 0x92722c851482353b,
+          0xa2bfe8a14cf10364, 0xa81a664bbc423001, 0xc24b8b70d0f89791, 0xc76c51a30654be30,
+          0xd192e819d6ef5218, 0xd69906245565a910, 0xf40e35855771202a, 0x106aa07032bbd1b8,
+          0x19a4c116b8d2d0c8, 0x1e376c085141ab53, 0x2748774cdf8eeb99, 0x34b0bcb5e19b48a8,
+          0x391c0cb3c5c95a63, 0x4ed8aa4ae3418acb, 0x5b9cca4f7763e373, 0x682e6ff3d6b2b8a3,
+          0x748f82ee5defb2fc, 0x78a5636f43172f60, 0x84c87814a1f0ab72, 0x8cc702081a6439ec,
+          0x90befffa23631e28, 0xa4506cebde82bde9, 0xbef9a3f7b2c67915, 0xc67178f2e372532b,
+          0xca273eceea26619c, 0xd186b8c721c0c207, 0xeada7dd6cde0eb1e, 0xf57d4f7fee6ed178,
+          0x06f067aa72176fba, 0x0a637dc5a2c898a6, 0x113f9804bef90dae, 0x1b710b35131c471b,
+          0x28db77f523047d84, 0x32caab7b40c72493, 0x3c9ebe0a15c9bebc, 0x431d67c49c100d4c,
+          0x4cc5d4becb3e42b6, 0x597f299cfc657e2a, 0x5fcb6fab3ad6faec, 0x6c44198c4a475817)
+    _h = (0x6a09e667f3bcc908, 0xbb67ae8584caa73b, 0x3c6ef372fe94f82b, 0xa54ff53a5f1d36f1,
+          0x510e527fade682d1, 0x9b05688c2b3e6c1f, 0x1f83d9abfb41bd6b, 0x5be0cd19137e2179)
+    _output_size = 8
+
+    blocksize = 1
+    block_size = 128
+    digest_size = 64
+
+    def __init__(self, m=None):
+        self._buffer = b''
+        self._counter = 0
+
+        if m is not None:
+            if type(m) is not bytes:
+                raise TypeError('%s() argument 1 must be bytes, not %s' % (self.__class__.__name__, type(m).__name__))
+            self.update(m)
+
+    def _rotr(self, x, y):
+        return ((x >> y) | (x << (64-y))) & 0xFFFFFFFFFFFFFFFF
+
+    def _sha512_process(self, chunk):
+        w = [0]*80
+        w[0:16] = struct.unpack('!16Q', chunk)
+
+        for i in range(16, 80):
+            s0 = self._rotr(w[i-15], 1) ^ self._rotr(w[i-15], 8) ^ (w[i-15] >> 7)
+            s1 = self._rotr(w[i-2], 19) ^ self._rotr(w[i-2], 61) ^ (w[i-2] >> 6)
+            w[i] = (w[i-16] + s0 + w[i-7] + s1) & 0xFFFFFFFFFFFFFFFF
+
+        a,b,c,d,e,f,g,h = self._h
+
+        for i in range(80):
+            s0 = self._rotr(a, 28) ^ self._rotr(a, 34) ^ self._rotr(a, 39)
+            maj = (a & b) ^ (a & c) ^ (b & c)
+            t2 = s0 + maj
+            s1 = self._rotr(e, 14) ^ self._rotr(e, 18) ^ self._rotr(e, 41)
+            ch = (e & f) ^ ((~e) & g)
+            t1 = h + s1 + ch + self._k[i] + w[i]
+
+            h = g
+            g = f
+            f = e
+            e = (d + t1) & 0xFFFFFFFFFFFFFFFF
+            d = c
+            c = b
+            b = a
+            a = (t1 + t2) & 0xFFFFFFFFFFFFFFFF
+
+        self._h = [(x+y) & 0xFFFFFFFFFFFFFFFF for x,y in zip(self._h, [a,b,c,d,e,f,g,h])]
+
+    def update(self, m):
+        if not m:
+            return
+        if type(m) is not bytes:
+            raise TypeError('%s() argument 1 must be bytes, not %s' % (sys._getframe().f_code.co_name, type(m).__name__))
+
+        self._buffer += m
+        self._counter += len(m)
+
+        while len(self._buffer) >= 128:
+            self._sha512_process(self._buffer[:128])
+            self._buffer = self._buffer[128:]
+
+    def digest(self):
+        mdi = self._counter & 0x7F
+        length = struct.pack('!Q', self._counter<<3)
+
+        if mdi < 112:
+            padlen = 111-mdi
+        else:
+            padlen = 239-mdi
+
+        r = self.copy()
+        r.update(b'\x80'+(b'\x00'*(padlen+8))+length)
+        return b''.join([struct.pack('!Q', i) for i in r._h[:self._output_size]])
+
+    def hexdigest(self):
+        return self.digest().encode('hex')
+
+    def copy(self):
+        return copy.deepcopy(self)

RNS/Cryptography/X25519.py

@@ -0,0 +1,171 @@
+# By Nicko van Someren, 2021. This code is released into the public domain.
+# Small modifications for use in Reticulum, and constant time key exchange
+# added by Mark Qvist in 2022.
+
+# WARNING! Only the X25519PrivateKey.exchange() method attempts to hide execution time.
+# In the context of Reticulum, this is sufficient, but it may not be in other systems. If
+# this code is to be used to provide cryptographic security in an environment where the
+# start and end times of the execution can be guessed, inferred or measured then it is
+# critical that steps are taken to hide the execution time, for instance by adding a
+# delay so that encrypted packets are not sent until a fixed time after the _start_ of
+# execution.
+
+
+import os
+import time
+
+P = 2 ** 255 - 19
+_A = 486662
+
+
+def _point_add(point_n, point_m, point_diff):
+    """Given the projection of two points and their difference, return their sum"""
+    (xn, zn) = point_n
+    (xm, zm) = point_m
+    (x_diff, z_diff) = point_diff
+    x = (z_diff << 2) * (xm * xn - zm * zn) ** 2
+    z = (x_diff << 2) * (xm * zn - zm * xn) ** 2
+    return x % P, z % P
+
+
+def _point_double(point_n):
+    """Double a point provided in projective coordinates"""
+    (xn, zn) = point_n
+    xn2 = xn ** 2
+    zn2 = zn ** 2
+    x = (xn2 - zn2) ** 2
+    xzn = xn * zn
+    z = 4 * xzn * (xn2 + _A * xzn + zn2)
+    return x % P, z % P
+
+
+def _const_time_swap(a, b, swap):
+    """Swap two values in constant time"""
+    index = int(swap) * 2
+    temp = (a, b, b, a)
+    return temp[index:index+2]
+
+
+def _raw_curve25519(base, n):
+    """Raise the point base to the power n"""
+    zero = (1, 0)
+    one = (base, 1)
+    mP, m1P = zero, one
+
+    for i in reversed(range(256)):
+        bit = bool(n & (1 << i))
+        mP, m1P = _const_time_swap(mP, m1P, bit)
+        mP, m1P = _point_double(mP), _point_add(mP, m1P, one)
+        mP, m1P = _const_time_swap(mP, m1P, bit)
+
+    x, z = mP
+    inv_z = pow(z, P - 2, P)
+    return (x * inv_z) % P
+
+
+def _unpack_number(s):
+    """Unpack 32 bytes to a 256 bit value"""
+    if len(s) != 32:
+        raise ValueError('Curve25519 values must be 32 bytes')
+    return int.from_bytes(s, "little")
+
+
+def _pack_number(n):
+    """Pack a value into 32 bytes"""
+    return n.to_bytes(32, "little")
+
+
+def _fix_secret(n):
+    """Mask a value to be an acceptable exponent"""
+    n &= ~7
+    n &= ~(128 << 8 * 31)
+    n |= 64 << 8 * 31
+    return n
+
+
+def curve25519(base_point_raw, secret_raw):
+    """Raise the base point to a given power"""
+    base_point = _unpack_number(base_point_raw)
+    secret = _fix_secret(_unpack_number(secret_raw))
+    return _pack_number(_raw_curve25519(base_point, secret))
+
+
+def curve25519_base(secret_raw):
+    """Raise the generator point to a given power"""
+    secret = _fix_secret(_unpack_number(secret_raw))
+    return _pack_number(_raw_curve25519(9, secret))
+
+
+class X25519PublicKey:
+    def __init__(self, x):
+        self.x = x
+
+    @classmethod
+    def from_public_bytes(cls, data):
+        return cls(_unpack_number(data))
+
+    def public_bytes(self):
+        return _pack_number(self.x)
+
+
+class X25519PrivateKey:
+    MIN_EXEC_TIME = 0.002
+    MAX_EXEC_TIME = 0.5
+    DELAY_WINDOW = 10
+
+    T_CLEAR = None
+    T_MAX = 0
+
+    def __init__(self, a):
+        self.a = a
+
+    @classmethod
+    def generate(cls):
+        return cls.from_private_bytes(os.urandom(32))
+
+    @classmethod
+    def from_private_bytes(cls, data):
+        return cls(_fix_secret(_unpack_number(data)))
+
+    def private_bytes(self):
+        return _pack_number(self.a)
+
+    def public_key(self):
+        return X25519PublicKey.from_public_bytes(_pack_number(_raw_curve25519(9, self.a)))
+
+    def exchange(self, peer_public_key):
+        if isinstance(peer_public_key, bytes):
+            peer_public_key = X25519PublicKey.from_public_bytes(peer_public_key)
+
+        start = time.time()
+        
+        shared = _pack_number(_raw_curve25519(peer_public_key.x, self.a))
+        
+        end = time.time()
+        duration = end-start
+
+        if X25519PrivateKey.T_CLEAR == None:
+            X25519PrivateKey.T_CLEAR = end + X25519PrivateKey.DELAY_WINDOW
+
+        if end > X25519PrivateKey.T_CLEAR:
+            X25519PrivateKey.T_CLEAR = end + X25519PrivateKey.DELAY_WINDOW
+            X25519PrivateKey.T_MAX = 0
+        
+        if duration < X25519PrivateKey.T_MAX or duration < X25519PrivateKey.MIN_EXEC_TIME:
+            target = start+X25519PrivateKey.T_MAX
+
+            if target > start+X25519PrivateKey.MAX_EXEC_TIME:
+                target = start+X25519PrivateKey.MAX_EXEC_TIME
+
+            if target < start+X25519PrivateKey.MIN_EXEC_TIME:
+                target = start+X25519PrivateKey.MIN_EXEC_TIME
+
+            try:
+                time.sleep(target-time.time())
+            except Exception as e:
+                pass
+
+        elif duration > X25519PrivateKey.T_MAX:
+            X25519PrivateKey.T_MAX = duration
+
+        return shared

RNS/Cryptography/__init__.py

@@ -0,0 +1,24 @@
+import os
+import glob
+
+from .Hashes import sha256
+from .Hashes import sha512
+from .HKDF import hkdf
+from .PKCS7 import PKCS7
+from .Fernet import Fernet
+from .Provider import backend
+
+import RNS.Cryptography.Provider as cp
+
+if cp.PROVIDER == cp.PROVIDER_INTERNAL:
+    from RNS.Cryptography.X25519 import X25519PrivateKey, X25519PublicKey
+    from RNS.Cryptography.Ed25519 import Ed25519PrivateKey, Ed25519PublicKey
+
+elif cp.PROVIDER == cp.PROVIDER_PYCA:
+    from RNS.Cryptography.Proxies import X25519PrivateKeyProxy as X25519PrivateKey
+    from RNS.Cryptography.Proxies import X25519PublicKeyProxy as X25519PublicKey
+    from RNS.Cryptography.Proxies import Ed25519PrivateKeyProxy as Ed25519PrivateKey
+    from RNS.Cryptography.Proxies import Ed25519PublicKeyProxy as Ed25519PublicKey
+
+modules = glob.glob(os.path.dirname(__file__)+"/*.py")
+__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]

RNS/Cryptography/aes/__init__.py

@@ -0,0 +1 @@
+from .aes import AES

RNS/Cryptography/aes/aes.py

@@ -0,0 +1,271 @@
+# MIT License
+
+# Copyright (c) 2021 Or Gur Arie
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from .utils import *
+
+
+class AES:
+    # AES-128 block size
+    block_size = 16
+    # AES-128 encrypts messages with 10 rounds
+    _rounds = 10
+
+
+    # initiate the AES objecy
+    def __init__(self, key):
+        """
+        Initializes the object with a given key.
+        """
+        # make sure key length is right
+        assert len(key) == AES.block_size
+
+        # ExpandKey
+        self._round_keys = self._expand_key(key)
+
+
+    # will perform the AES ExpandKey phase
+    def _expand_key(self, master_key):
+        """
+        Expands and returns a list of key matrices for the given master_key.
+        """
+
+        # Initialize round keys with raw key material.
+        key_columns = bytes2matrix(master_key)
+        iteration_size = len(master_key) // 4
+
+        # Each iteration has exactly as many columns as the key material.
+        i = 1
+        while len(key_columns) < (self._rounds + 1) * 4:
+            # Copy previous word.
+            word = list(key_columns[-1])
+
+            # Perform schedule_core once every "row".
+            if len(key_columns) % iteration_size == 0:
+                # Circular shift.
+                word.append(word.pop(0))
+                # Map to S-BOX.
+                word = [s_box[b] for b in word]
+                # XOR with first byte of R-CON, since the others bytes of R-CON are 0.
+                word[0] ^= r_con[i]
+                i += 1
+            elif len(master_key) == 32 and len(key_columns) % iteration_size == 4:
+                # Run word through S-box in the fourth iteration when using a
+                # 256-bit key.
+                word = [s_box[b] for b in word]
+
+            # XOR with equivalent word from previous iteration.
+            word = bytes(i^j for i, j in zip(word, key_columns[-iteration_size]))
+            key_columns.append(word)
+
+        # Group key words in 4x4 byte matrices.
+        return [key_columns[4*i : 4*(i+1)] for i in range(len(key_columns) // 4)]
+
+
+    # encrypt a single block of data with AES
+    def _encrypt_block(self, plaintext):
+        """
+        Encrypts a single block of 16 byte long plaintext.
+        """
+        # length of a single block
+        assert len(plaintext) == AES.block_size
+
+        # perform on a matrix
+        state = bytes2matrix(plaintext)
+
+        # AddRoundKey
+        add_round_key(state, self._round_keys[0])
+
+        # 9 main rounds
+        for i in range(1, self._rounds):
+            # SubBytes
+            sub_bytes(state)
+            # ShiftRows
+            shift_rows(state)
+            # MixCols
+            mix_columns(state)
+            # AddRoundKey
+            add_round_key(state, self._round_keys[i])
+
+        # last round, w/t AddRoundKey step
+        sub_bytes(state)
+        shift_rows(state)
+        add_round_key(state, self._round_keys[-1])
+
+        # return the encrypted matrix as bytes
+        return matrix2bytes(state)
+
+
+    # decrypt a single block of data with AES
+    def _decrypt_block(self, ciphertext):
+        """
+        Decrypts a single block of 16 byte long ciphertext.
+        """
+        # length of a single block
+        assert len(ciphertext) == AES.block_size
+
+        # perform on a matrix
+        state = bytes2matrix(ciphertext)
+
+        # in reverse order, last round is first
+        add_round_key(state, self._round_keys[-1])
+        inv_shift_rows(state)
+        inv_sub_bytes(state)
+
+        for i in range(self._rounds - 1, 0, -1):
+            # nain rounds
+            add_round_key(state, self._round_keys[i])
+            inv_mix_columns(state)
+            inv_shift_rows(state)
+            inv_sub_bytes(state)
+
+        # initial AddRoundKey phase
+        add_round_key(state, self._round_keys[0])
+
+        # return bytes
+        return matrix2bytes(state)
+
+
+    # will encrypt the entire data 
+    def encrypt(self, plaintext, iv):
+        """
+        Encrypts `plaintext` using CBC mode and PKCS#7 padding, with the given
+        initialization vector (iv).
+        """
+        # iv length must be same as block size
+        assert len(iv) == AES.block_size
+
+        assert len(plaintext) % AES.block_size == 0
+
+        ciphertext_blocks = []
+
+        previous = iv
+        for plaintext_block in split_blocks(plaintext):
+            # in CBC mode every block is XOR'd with the previous block
+            xorred = xor_bytes(plaintext_block, previous)
+
+            # encrypt current block
+            block = self._encrypt_block(xorred)
+            previous = block
+
+            # append to ciphertext
+            ciphertext_blocks.append(block)
+
+        # return as bytes
+        return b''.join(ciphertext_blocks)
+
+
+    # will decrypt the entire data 
+    def decrypt(self, ciphertext, iv):
+        """
+        Decrypts `ciphertext` using CBC mode and PKCS#7 padding, with the given
+        initialization vector (iv).
+        """
+        # iv length must be same as block size
+        assert len(iv) == AES.block_size
+
+        plaintext_blocks = []
+
+        previous = iv
+        for ciphertext_block in split_blocks(ciphertext):
+            # in CBC mode every block is XOR'd with the previous block
+            xorred = xor_bytes(previous, self._decrypt_block(ciphertext_block))
+            
+            # append plaintext
+            plaintext_blocks.append(xorred)
+            previous = ciphertext_block
+
+        return b''.join(plaintext_blocks)
+
+
+def test():
+    # modules and classes requiered for test only
+    import os
+    class bcolors:
+        OK = '\033[92m' #GREEN
+        WARNING = '\033[93m' #YELLOW
+        FAIL = '\033[91m' #RED
+        RESET = '\033[0m' #RESET COLOR
+
+    # will test AES class by performing an encryption / decryption
+    print("AES Tests")
+    print("=========")
+
+    # generate a secret key and print details
+    key = os.urandom(AES.block_size)
+    _aes = AES(key)
+    print(f"Algorithm: AES-CBC-{AES.block_size*8}")
+    print(f"Secret Key: {key.hex()}")
+    print()
+
+    # test single block encryption / decryption
+    iv = os.urandom(AES.block_size)
+
+    single_block_text = b"SingleBlock Text"
+    print("Single Block Tests")
+    print("------------------")
+    print(f"iv: {iv.hex()}")
+    
+    print(f"plain text: '{single_block_text.decode()}'")
+    ciphertext_block = _aes._encrypt_block(single_block_text)
+    plaintext_block = _aes._decrypt_block(ciphertext_block)
+    print(f"Ciphertext Hex: {ciphertext_block.hex()}")
+    print(f"Plaintext: {plaintext_block.decode()}")
+    assert plaintext_block == single_block_text
+    print(bcolors.OK + "Single Block Test Passed Successfully" + bcolors.RESET)
+    print()
+
+    # test a less than a block length phrase
+    iv = os.urandom(AES.block_size)
+
+    short_text = b"Just Text"
+    print("Short Text Tests")
+    print("----------------")
+    print(f"iv: {iv.hex()}")
+    print(f"plain text: '{short_text.decode()}'")
+    ciphertext_short = _aes.encrypt(short_text, iv)
+    plaintext_short = _aes.decrypt(ciphertext_short, iv)
+    print(f"Ciphertext Hex: {ciphertext_short.hex()}")
+    print(f"Plaintext: {plaintext_short.decode()}")
+    assert short_text == plaintext_short
+    print(bcolors.OK + "Short Text Test Passed Successfully" + bcolors.RESET)
+    print()
+
+    # test an arbitrary length phrase
+    iv = os.urandom(AES.block_size)
+
+    text = b"This Text is longer than one block"
+    print("Arbitrary Length Tests")
+    print("----------------------")
+    print(f"iv: {iv.hex()}")
+    print(f"plain text: '{text.decode()}'")
+    ciphertext = _aes.encrypt(text, iv)
+    plaintext = _aes.decrypt(ciphertext, iv)
+    print(f"Ciphertext Hex: {ciphertext.hex()}")
+    print(f"Plaintext: {plaintext.decode()}")
+    assert text == plaintext
+    print(bcolors.OK + "Arbitrary Length Text Test Passed Successfully" + bcolors.RESET)
+    print()
+
+
+if __name__ == "__main__":
+    # test AES class
+    test()    

RNS/Cryptography/aes/utils.py

@@ -0,0 +1,159 @@
+# MIT License
+
+# Copyright (c) 2021 Or Gur Arie
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+'''
+Utils class for AES encryption / decryption
+'''
+
+## AES lookup tables
+# resource: https://en.wikipedia.org/wiki/Rijndael_S-box
+s_box = (
+    0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0xFE, 0xD7, 0xAB, 0x76,
+    0xCA, 0x82, 0xC9, 0x7D, 0xFA, 0x59, 0x47, 0xF0, 0xAD, 0xD4, 0xA2, 0xAF, 0x9C, 0xA4, 0x72, 0xC0,
+    0xB7, 0xFD, 0x93, 0x26, 0x36, 0x3F, 0xF7, 0xCC, 0x34, 0xA5, 0xE5, 0xF1, 0x71, 0xD8, 0x31, 0x15,
+    0x04, 0xC7, 0x23, 0xC3, 0x18, 0x96, 0x05, 0x9A, 0x07, 0x12, 0x80, 0xE2, 0xEB, 0x27, 0xB2, 0x75,
+    0x09, 0x83, 0x2C, 0x1A, 0x1B, 0x6E, 0x5A, 0xA0, 0x52, 0x3B, 0xD6, 0xB3, 0x29, 0xE3, 0x2F, 0x84,
+    0x53, 0xD1, 0x00, 0xED, 0x20, 0xFC, 0xB1, 0x5B, 0x6A, 0xCB, 0xBE, 0x39, 0x4A, 0x4C, 0x58, 0xCF,
+    0xD0, 0xEF, 0xAA, 0xFB, 0x43, 0x4D, 0x33, 0x85, 0x45, 0xF9, 0x02, 0x7F, 0x50, 0x3C, 0x9F, 0xA8,
+    0x51, 0xA3, 0x40, 0x8F, 0x92, 0x9D, 0x38, 0xF5, 0xBC, 0xB6, 0xDA, 0x21, 0x10, 0xFF, 0xF3, 0xD2,
+    0xCD, 0x0C, 0x13, 0xEC, 0x5F, 0x97, 0x44, 0x17, 0xC4, 0xA7, 0x7E, 0x3D, 0x64, 0x5D, 0x19, 0x73,
+    0x60, 0x81, 0x4F, 0xDC, 0x22, 0x2A, 0x90, 0x88, 0x46, 0xEE, 0xB8, 0x14, 0xDE, 0x5E, 0x0B, 0xDB,
+    0xE0, 0x32, 0x3A, 0x0A, 0x49, 0x06, 0x24, 0x5C, 0xC2, 0xD3, 0xAC, 0x62, 0x91, 0x95, 0xE4, 0x79,
+    0xE7, 0xC8, 0x37, 0x6D, 0x8D, 0xD5, 0x4E, 0xA9, 0x6C, 0x56, 0xF4, 0xEA, 0x65, 0x7A, 0xAE, 0x08,
+    0xBA, 0x78, 0x25, 0x2E, 0x1C, 0xA6, 0xB4, 0xC6, 0xE8, 0xDD, 0x74, 0x1F, 0x4B, 0xBD, 0x8B, 0x8A,
+    0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E,
+    0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF,
+    0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16,
+)
+
+inv_s_box = (
+    0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38, 0xBF, 0x40, 0xA3, 0x9E, 0x81, 0xF3, 0xD7, 0xFB,
+    0x7C, 0xE3, 0x39, 0x82, 0x9B, 0x2F, 0xFF, 0x87, 0x34, 0x8E, 0x43, 0x44, 0xC4, 0xDE, 0xE9, 0xCB,
+    0x54, 0x7B, 0x94, 0x32, 0xA6, 0xC2, 0x23, 0x3D, 0xEE, 0x4C, 0x95, 0x0B, 0x42, 0xFA, 0xC3, 0x4E,
+    0x08, 0x2E, 0xA1, 0x66, 0x28, 0xD9, 0x24, 0xB2, 0x76, 0x5B, 0xA2, 0x49, 0x6D, 0x8B, 0xD1, 0x25,
+    0x72, 0xF8, 0xF6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xD4, 0xA4, 0x5C, 0xCC, 0x5D, 0x65, 0xB6, 0x92,
+    0x6C, 0x70, 0x48, 0x50, 0xFD, 0xED, 0xB9, 0xDA, 0x5E, 0x15, 0x46, 0x57, 0xA7, 0x8D, 0x9D, 0x84,
+    0x90, 0xD8, 0xAB, 0x00, 0x8C, 0xBC, 0xD3, 0x0A, 0xF7, 0xE4, 0x58, 0x05, 0xB8, 0xB3, 0x45, 0x06,
+    0xD0, 0x2C, 0x1E, 0x8F, 0xCA, 0x3F, 0x0F, 0x02, 0xC1, 0xAF, 0xBD, 0x03, 0x01, 0x13, 0x8A, 0x6B,
+    0x3A, 0x91, 0x11, 0x41, 0x4F, 0x67, 0xDC, 0xEA, 0x97, 0xF2, 0xCF, 0xCE, 0xF0, 0xB4, 0xE6, 0x73,
+    0x96, 0xAC, 0x74, 0x22, 0xE7, 0xAD, 0x35, 0x85, 0xE2, 0xF9, 0x37, 0xE8, 0x1C, 0x75, 0xDF, 0x6E,
+    0x47, 0xF1, 0x1A, 0x71, 0x1D, 0x29, 0xC5, 0x89, 0x6F, 0xB7, 0x62, 0x0E, 0xAA, 0x18, 0xBE, 0x1B,
+    0xFC, 0x56, 0x3E, 0x4B, 0xC6, 0xD2, 0x79, 0x20, 0x9A, 0xDB, 0xC0, 0xFE, 0x78, 0xCD, 0x5A, 0xF4,
+    0x1F, 0xDD, 0xA8, 0x33, 0x88, 0x07, 0xC7, 0x31, 0xB1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xEC, 0x5F,
+    0x60, 0x51, 0x7F, 0xA9, 0x19, 0xB5, 0x4A, 0x0D, 0x2D, 0xE5, 0x7A, 0x9F, 0x93, 0xC9, 0x9C, 0xEF,
+    0xA0, 0xE0, 0x3B, 0x4D, 0xAE, 0x2A, 0xF5, 0xB0, 0xC8, 0xEB, 0xBB, 0x3C, 0x83, 0x53, 0x99, 0x61,
+    0x17, 0x2B, 0x04, 0x7E, 0xBA, 0x77, 0xD6, 0x26, 0xE1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0C, 0x7D,
+)
+
+
+## AES AddRoundKey
+# Round constants https://en.wikipedia.org/wiki/AES_key_schedule#Round_constants
+r_con = (
+    0x00, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40,
+    0x80, 0x1B, 0x36, 0x6C, 0xD8, 0xAB, 0x4D, 0x9A,
+    0x2F, 0x5E, 0xBC, 0x63, 0xC6, 0x97, 0x35, 0x6A,
+    0xD4, 0xB3, 0x7D, 0xFA, 0xEF, 0xC5, 0x91, 0x39,
+)
+
+def add_round_key(s, k):
+    for i in range(4):
+        for j in range(4):
+            s[i][j] ^= k[i][j]
+
+
+## AES SubBytes
+def sub_bytes(s):
+    for i in range(4):
+        for j in range(4):
+            s[i][j] = s_box[s[i][j]]
+
+
+def inv_sub_bytes(s):
+    for i in range(4):
+        for j in range(4):
+            s[i][j] = inv_s_box[s[i][j]]
+
+
+## AES ShiftRows
+def shift_rows(s):
+    s[0][1], s[1][1], s[2][1], s[3][1] = s[1][1], s[2][1], s[3][1], s[0][1]
+    s[0][2], s[1][2], s[2][2], s[3][2] = s[2][2], s[3][2], s[0][2], s[1][2]
+    s[0][3], s[1][3], s[2][3], s[3][3] = s[3][3], s[0][3], s[1][3], s[2][3]
+
+
+def inv_shift_rows(s):
+    s[0][1], s[1][1], s[2][1], s[3][1] = s[3][1], s[0][1], s[1][1], s[2][1]
+    s[0][2], s[1][2], s[2][2], s[3][2] = s[2][2], s[3][2], s[0][2], s[1][2]
+    s[0][3], s[1][3], s[2][3], s[3][3] = s[1][3], s[2][3], s[3][3], s[0][3]
+
+
+## AES MixColumns
+# learned from http://cs.ucsb.edu/~koc/cs178/projects/JT/aes.c
+xtime = lambda a: (((a << 1) ^ 0x1B) & 0xFF) if (a & 0x80) else (a << 1)
+
+
+def mix_single_column(a):
+    # see Sec 4.1.2 in The Design of Rijndael
+    t = a[0] ^ a[1] ^ a[2] ^ a[3]
+    u = a[0]
+    a[0] ^= t ^ xtime(a[0] ^ a[1])
+    a[1] ^= t ^ xtime(a[1] ^ a[2])
+    a[2] ^= t ^ xtime(a[2] ^ a[3])
+    a[3] ^= t ^ xtime(a[3] ^ u)
+
+
+def mix_columns(s):
+    for i in range(4):
+        mix_single_column(s[i])
+
+
+def inv_mix_columns(s):
+    # see Sec 4.1.3 in The Design of Rijndael
+    for i in range(4):
+        u = xtime(xtime(s[i][0] ^ s[i][2]))
+        v = xtime(xtime(s[i][1] ^ s[i][3]))
+        s[i][0] ^= u
+        s[i][1] ^= v
+        s[i][2] ^= u
+        s[i][3] ^= v
+
+    mix_columns(s)
+
+
+## AES Bytes
+def bytes2matrix(text):
+    """ Converts a 16-byte array into a 4x4 matrix.  """
+    return [list(text[i:i+4]) for i in range(0, len(text), 4)]
+
+def matrix2bytes(matrix):
+    """ Converts a 4x4 matrix into a 16-byte array.  """
+    return bytes(sum(matrix, []))
+
+
+def xor_bytes(a, b):
+    """ Returns a new byte array with the elements xor'ed. """
+    return bytes(i^j for i, j in zip(a, b))
+
+
+def split_blocks(message, block_size=16, require_padding=True):
+        assert len(message) % block_size == 0 or not require_padding
+        return [message[i:i+16] for i in range(0, len(message), block_size)]

RNS/Cryptography/pure25519/_ed25519.py

@@ -0,0 +1,58 @@
+# MIT License
+#
+# Copyright (c) 2015 Brian Warner and other contributors
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from . import eddsa
+
+class BadSignatureError(Exception):
+    pass
+
+SECRETKEYBYTES = 64
+PUBLICKEYBYTES = 32
+SIGNATUREKEYBYTES = 64
+
+def publickey(seed32):
+    assert len(seed32) == 32
+    vk32 = eddsa.publickey(seed32)
+    return vk32, seed32+vk32
+
+def sign(msg, skvk):
+    assert len(skvk) == 64
+    sk = skvk[:32]
+    vk = skvk[32:]
+    sig = eddsa.signature(msg, sk, vk)
+    return sig+msg
+
+def open(sigmsg, vk):
+    assert len(vk) == 32
+    sig = sigmsg[:64]
+    msg = sigmsg[64:]
+    try:
+        valid = eddsa.checkvalid(sig, msg, vk)
+    except ValueError as e:
+        raise BadSignatureError(e)
+    except Exception as e:
+        if str(e) == "decoding point that is not on curve":
+            raise BadSignatureError(e)
+        raise
+    if not valid:
+        raise BadSignatureError()
+    return msg

RNS/Cryptography/pure25519/basic.py

@@ -0,0 +1,368 @@
+# MIT License
+#
+# Copyright (c) 2015 Brian Warner and other contributors
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import binascii, hashlib, itertools
+
+Q = 2**255 - 19
+L = 2**252 + 27742317777372353535851937790883648493
+
+def inv(x):
+    return pow(x, Q-2, Q)
+
+d = -121665 * inv(121666)
+I = pow(2,(Q-1)//4,Q)
+
+def xrecover(y):
+    xx = (y*y-1) * inv(d*y*y+1)
+    x = pow(xx,(Q+3)//8,Q)
+    if (x*x - xx) % Q != 0: x = (x*I) % Q
+    if x % 2 != 0: x = Q-x
+    return x
+
+By = 4 * inv(5)
+Bx = xrecover(By)
+B = [Bx % Q,By % Q]
+
+# Extended Coordinates: x=X/Z, y=Y/Z, x*y=T/Z
+# http://www.hyperelliptic.org/EFD/g1p/auto-twisted-extended-1.html
+
+def xform_affine_to_extended(pt):
+    (x, y) = pt
+    return (x%Q, y%Q, 1, (x*y)%Q) # (X,Y,Z,T)
+
+def xform_extended_to_affine(pt):
+    (x, y, z, _) = pt
+    return ((x*inv(z))%Q, (y*inv(z))%Q)
+
+def double_element(pt): # extended->extended
+    # dbl-2008-hwcd
+    (X1, Y1, Z1, _) = pt
+    A = (X1*X1)
+    B = (Y1*Y1)
+    C = (2*Z1*Z1)
+    D = (-A) % Q
+    J = (X1+Y1) % Q
+    E = (J*J-A-B) % Q
+    G = (D+B) % Q
+    F = (G-C) % Q
+    H = (D-B) % Q
+    X3 = (E*F) % Q
+    Y3 = (G*H) % Q
+    Z3 = (F*G) % Q
+    T3 = (E*H) % Q
+    return (X3, Y3, Z3, T3)
+
+def add_elements(pt1, pt2): # extended->extended
+    # add-2008-hwcd-3 . Slightly slower than add-2008-hwcd-4, but -3 is
+    # unified, so it's safe for general-purpose addition
+    (X1, Y1, Z1, T1) = pt1
+    (X2, Y2, Z2, T2) = pt2
+    A = ((Y1-X1)*(Y2-X2)) % Q
+    B = ((Y1+X1)*(Y2+X2)) % Q
+    C = T1*(2*d)*T2 % Q
+    D = Z1*2*Z2 % Q
+    E = (B-A) % Q
+    F = (D-C) % Q
+    G = (D+C) % Q
+    H = (B+A) % Q
+    X3 = (E*F) % Q
+    Y3 = (G*H) % Q
+    T3 = (E*H) % Q
+    Z3 = (F*G) % Q
+    return (X3, Y3, Z3, T3)
+
+def scalarmult_element_safe_slow(pt, n):
+    # this form is slightly slower, but tolerates arbitrary points, including
+    # those which are not in the main 1*L subgroup. This includes points of
+    # order 1 (the neutral element Zero), 2, 4, and 8.
+    assert n >= 0
+    if n==0:
+        return xform_affine_to_extended((0,1))
+    _ = double_element(scalarmult_element_safe_slow(pt, n>>1))
+    return add_elements(_, pt) if n&1 else _
+
+def _add_elements_nonunfied(pt1, pt2): # extended->extended
+    # add-2008-hwcd-4 : NOT unified, only for pt1!=pt2. About 10% faster than
+    # the (unified) add-2008-hwcd-3, and safe to use inside scalarmult if you
+    # aren't using points of order 1/2/4/8
+    (X1, Y1, Z1, T1) = pt1
+    (X2, Y2, Z2, T2) = pt2
+    A = ((Y1-X1)*(Y2+X2)) % Q
+    B = ((Y1+X1)*(Y2-X2)) % Q
+    C = (Z1*2*T2) % Q
+    D = (T1*2*Z2) % Q
+    E = (D+C) % Q
+    F = (B-A) % Q
+    G = (B+A) % Q
+    H = (D-C) % Q
+    X3 = (E*F) % Q
+    Y3 = (G*H) % Q
+    Z3 = (F*G) % Q
+    T3 = (E*H) % Q
+    return (X3, Y3, Z3, T3)
+
+def scalarmult_element(pt, n): # extended->extended
+    # This form only works properly when given points that are a member of
+    # the main 1*L subgroup. It will give incorrect answers when called with
+    # the points of order 1/2/4/8, including point Zero. (it will also work
+    # properly when given points of order 2*L/4*L/8*L)
+    assert n >= 0
+    if n==0:
+        return xform_affine_to_extended((0,1))
+    _ = double_element(scalarmult_element(pt, n>>1))
+    return _add_elements_nonunfied(_, pt) if n&1 else _
+
+# points are encoded as 32-bytes little-endian, b255 is sign, b2b1b0 are 0
+
+def encodepoint(P):
+    x = P[0]
+    y = P[1]
+    # MSB of output equals x.b0 (=x&1)
+    # rest of output is little-endian y
+    assert 0 <= y < (1<<255) # always < 0x7fff..ff
+    if x & 1:
+        y += 1<<255
+    return binascii.unhexlify("%064x" % y)[::-1]
+
+def isoncurve(P):
+    x = P[0]
+    y = P[1]
+    return (-x*x + y*y - 1 - d*x*x*y*y) % Q == 0
+
+class NotOnCurve(Exception):
+    pass
+
+def decodepoint(s):
+    unclamped = int(binascii.hexlify(s[:32][::-1]), 16)
+    clamp = (1 << 255) - 1
+    y = unclamped & clamp # clear MSB
+    x = xrecover(y)
+    if bool(x & 1) != bool(unclamped & (1<<255)): x = Q-x
+    P = [x,y]
+    if not isoncurve(P): raise NotOnCurve("decoding point that is not on curve")
+    return P
+
+# scalars are encoded as 32-bytes little-endian
+
+def bytes_to_scalar(s):
+    assert len(s) == 32, len(s)
+    return int(binascii.hexlify(s[::-1]), 16)
+
+def bytes_to_clamped_scalar(s):
+    # Ed25519 private keys clamp the scalar to ensure two things:
+    #   1: integer value is in L/2 .. L, to avoid small-logarithm
+    #      non-wraparaound
+    #   2: low-order 3 bits are zero, so a small-subgroup attack won't learn
+    #      any information
+    # set the top two bits to 01, and the bottom three to 000
+    a_unclamped = bytes_to_scalar(s)
+    AND_CLAMP = (1<<254) - 1 - 7
+    OR_CLAMP = (1<<254)
+    a_clamped = (a_unclamped & AND_CLAMP) | OR_CLAMP
+    return a_clamped
+
+def random_scalar(entropy_f): # 0..L-1 inclusive
+    # reduce the bias to a safe level by generating 256 extra bits
+    oversized = int(binascii.hexlify(entropy_f(32+32)), 16)
+    return oversized % L
+
+def password_to_scalar(pw):
+    oversized = hashlib.sha512(pw).digest()
+    return int(binascii.hexlify(oversized), 16) % L
+
+def scalar_to_bytes(y):
+    y = y % L
+    assert 0 <= y < 2**256
+    return binascii.unhexlify("%064x" % y)[::-1]
+
+# Elements, of various orders
+
+def is_extended_zero(XYTZ):
+    # catch Zero
+    (X, Y, Z, T) = XYTZ
+    Y = Y % Q
+    Z = Z % Q
+    if X==0 and Y==Z and Y!=0:
+        return True
+    return False
+
+class ElementOfUnknownGroup:
+    # This is used for points of order 2,4,8,2*L,4*L,8*L
+    def __init__(self, XYTZ):
+        assert isinstance(XYTZ, tuple)
+        assert len(XYTZ) == 4
+        self.XYTZ = XYTZ
+
+    def add(self, other):
+        if not isinstance(other, ElementOfUnknownGroup):
+            raise TypeError("elements can only be added to other elements")
+        sum_XYTZ = add_elements(self.XYTZ, other.XYTZ)
+        if is_extended_zero(sum_XYTZ):
+            return Zero
+        return ElementOfUnknownGroup(sum_XYTZ)
+
+    def scalarmult(self, s):
+        if isinstance(s, ElementOfUnknownGroup):
+            raise TypeError("elements cannot be multiplied together")
+        assert s >= 0
+        product = scalarmult_element_safe_slow(self.XYTZ, s)
+        return ElementOfUnknownGroup(product)
+
+    def to_bytes(self):
+        return encodepoint(xform_extended_to_affine(self.XYTZ))
+    def __eq__(self, other):
+        return self.to_bytes() == other.to_bytes()
+    def __ne__(self, other):
+        return not self == other
+
+class Element(ElementOfUnknownGroup):
+    # this only holds elements in the main 1*L subgroup. It never holds Zero,
+    # or elements of order 1/2/4/8, or 2*L/4*L/8*L.
+
+    def add(self, other):
+        if not isinstance(other, ElementOfUnknownGroup):
+            raise TypeError("elements can only be added to other elements")
+        sum_element = ElementOfUnknownGroup.add(self, other)
+        if sum_element is Zero:
+            return sum_element
+        if isinstance(other, Element):
+            # adding two subgroup elements results in another subgroup
+            # element, or Zero, and we've already excluded Zero
+            return Element(sum_element.XYTZ)
+        # not necessarily a subgroup member, so assume not
+        return sum_element
+
+    def scalarmult(self, s):
+        if isinstance(s, ElementOfUnknownGroup):
+            raise TypeError("elements cannot be multiplied together")
+        # scalarmult of subgroup members can be done modulo the subgroup
+        # order, and using the faster non-unified function.
+        s = s % L
+        # scalarmult(s=0) gets you Zero
+        if s == 0:
+            return Zero
+        # scalarmult(s=1) gets you self, which is a subgroup member
+        # scalarmult(s<grouporder) gets you a different subgroup member
+        return Element(scalarmult_element(self.XYTZ, s))
+
+    # negation and subtraction only make sense for the main subgroup
+    def negate(self):
+        # slow. Prefer e.scalarmult(-pw) to e.scalarmult(pw).negate()
+        return Element(scalarmult_element(self.XYTZ, L-2))
+    def subtract(self, other):
+        return self.add(other.negate())
+
+class _ZeroElement(ElementOfUnknownGroup):
+    def add(self, other):
+        return other # zero+anything = anything
+    def scalarmult(self, s):
+        return self # zero*anything = zero
+    def negate(self):
+        return self # -zero = zero
+    def subtract(self, other):
+        return self.add(other.negate())
+
+
+Base = Element(xform_affine_to_extended(B))
+Zero = _ZeroElement(xform_affine_to_extended((0,1))) # the neutral (identity) element
+
+_zero_bytes = Zero.to_bytes()
+
+
+def arbitrary_element(seed): # unknown DL
+    # TODO: if we don't need uniformity, maybe use just sha256 here?
+    hseed = hashlib.sha512(seed).digest()
+    y = int(binascii.hexlify(hseed), 16) % Q
+
+    # we try successive Y values until we find a valid point
+    for plus in itertools.count(0):
+        y_plus = (y + plus) % Q
+        x = xrecover(y_plus)
+        Pa = [x,y_plus] # no attempt to use both "positive" and "negative" X
+
+        # only about 50% of Y coordinates map to valid curve points (I think
+        # the other half give you points on the "twist").
+        if not isoncurve(Pa):
+            continue
+
+        P = ElementOfUnknownGroup(xform_affine_to_extended(Pa))
+        # even if the point is on our curve, it may not be in our particular
+        # (order=L) subgroup. The curve has order 8*L, so an arbitrary point
+        # could have order 1,2,4,8,1*L,2*L,4*L,8*L (everything which divides
+        # the group order).
+
+        # [I MAY BE COMPLETELY WRONG ABOUT THIS, but my brief statistical
+        # tests suggest it's not too far off] There are phi(x) points with
+        # order x, so:
+        #  1 element of order 1: [(x=0,y=1)=Zero]
+        #  1 element of order 2 [(x=0,y=-1)]
+        #  2 elements of order 4
+        #  4 elements of order 8
+        #  L-1 elements of order L (including Base)
+        #  L-1 elements of order 2*L
+        #  2*(L-1) elements of order 4*L
+        #  4*(L-1) elements of order 8*L
+
+        # So 50% of random points will have order 8*L, 25% will have order
+        # 4*L, 13% order 2*L, and 13% will have our desired order 1*L (and a
+        # vanishingly small fraction will have 1/2/4/8). If we multiply any
+        # of the 8*L points by 2, we're sure to get an 4*L point (and
+        # multiplying a 4*L point by 2 gives us a 2*L point, and so on).
+        # Multiplying a 1*L point by 2 gives us a different 1*L point. So
+        # multiplying by 8 gets us from almost any point into a uniform point
+        # on the correct 1*L subgroup.
+
+        P8 = P.scalarmult(8)
+
+        # if we got really unlucky and picked one of the 8 low-order points,
+        # multiplying by 8 will get us to the identity (Zero), which we check
+        # for explicitly.
+        if is_extended_zero(P8.XYTZ):
+            continue
+
+        # Test that we're finally in the right group. We want to scalarmult
+        # by L, and we want to *not* use the trick in Group.scalarmult()
+        # which does x%L, because that would bypass the check we care about.
+        # P is still an _ElementOfUnknownGroup, which doesn't use x%L because
+        # that's not correct for points outside the main group.
+        assert is_extended_zero(P8.scalarmult(L).XYTZ)
+
+        return Element(P8.XYTZ)
+    # never reached
+
+def bytes_to_unknown_group_element(bytes):
+    # this accepts all elements, including Zero and wrong-subgroup ones
+    if bytes == _zero_bytes:
+        return Zero
+    XYTZ = xform_affine_to_extended(decodepoint(bytes))
+    return ElementOfUnknownGroup(XYTZ)
+
+def bytes_to_element(bytes):
+    # this strictly only accepts elements in the right subgroup
+    P = bytes_to_unknown_group_element(bytes)
+    if P is Zero:
+        raise ValueError("element was Zero")
+    if not is_extended_zero(P.scalarmult(L).XYTZ):
+        raise ValueError("element is not in the right group")
+    # the point is in the expected 1*L subgroup, not in the 2/4/8 groups,
+    # or in the 2*L/4*L/8*L groups. Promote it to a correct-group Element.
+    return Element(P.XYTZ)

RNS/Cryptography/pure25519/ed25519_oop.py

@@ -0,0 +1,213 @@
+# MIT License
+#
+# Copyright (c) 2015 Brian Warner and other contributors
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+import base64
+from . import _ed25519
+BadSignatureError = _ed25519.BadSignatureError
+
+def create_keypair(entropy=os.urandom):
+    SEEDLEN = int(_ed25519.SECRETKEYBYTES/2)
+    assert SEEDLEN == 32
+    seed = entropy(SEEDLEN)
+    sk = SigningKey(seed)
+    vk = sk.get_verifying_key()
+    return sk, vk
+
+class BadPrefixError(Exception):
+    pass
+
+def remove_prefix(s_bytes, prefix):
+    assert(type(s_bytes) == type(prefix))
+    if s_bytes[:len(prefix)] != prefix:
+        raise BadPrefixError("did not see expected '%s' prefix" % (prefix,))
+    return s_bytes[len(prefix):]
+
+def to_ascii(s_bytes, prefix="", encoding="base64"):
+    """Return a version-prefixed ASCII representation of the given binary
+    string. 'encoding' indicates how to do the encoding, and can be one of:
+     * base64
+     * base32
+     * base16 (or hex)
+
+    This function handles bytes, not bits, so it does not append any trailing
+    '=' (unlike standard base64.b64encode). It also lowercases the base32
+    output.
+
+    'prefix' will be prepended to the encoded form, and is useful for
+    distinguishing the purpose and version of the binary string. E.g. you
+    could prepend 'pub0-' to a VerifyingKey string to allow the receiving
+    code to raise a useful error if someone pasted in a signature string by
+    mistake.
+    """
+    assert isinstance(s_bytes, bytes)
+    if not isinstance(prefix, bytes):
+        prefix = prefix.encode('ascii')
+    if encoding == "base64":
+        s_ascii = base64.b64encode(s_bytes).decode('ascii').rstrip("=")
+    elif encoding == "base32":
+        s_ascii = base64.b32encode(s_bytes).decode('ascii').rstrip("=").lower()
+    elif encoding in ("base16", "hex"):
+        s_ascii = base64.b16encode(s_bytes).decode('ascii').lower()
+    else:
+        raise NotImplementedError
+    return prefix+s_ascii.encode('ascii')
+
+def from_ascii(s_ascii, prefix="", encoding="base64"):
+    """This is the opposite of to_ascii. It will throw BadPrefixError if
+    the prefix is not found.
+    """
+    if isinstance(s_ascii, bytes):
+        s_ascii = s_ascii.decode('ascii')
+    if isinstance(prefix, bytes):
+        prefix = prefix.decode('ascii')
+    s_ascii = remove_prefix(s_ascii.strip(), prefix)
+    if encoding == "base64":
+        s_ascii += "="*((4 - len(s_ascii)%4)%4)
+        s_bytes = base64.b64decode(s_ascii)
+    elif encoding == "base32":
+        s_ascii += "="*((8 - len(s_ascii)%8)%8)
+        s_bytes = base64.b32decode(s_ascii.upper())
+    elif encoding in ("base16", "hex"):
+        s_bytes = base64.b16decode(s_ascii.upper())
+    else:
+        raise NotImplementedError
+    return s_bytes
+
+class SigningKey(object):
+    # this can only be used to reconstruct a key created by create_keypair().
+    def __init__(self, sk_s, prefix="", encoding=None):
+        assert isinstance(sk_s, bytes)
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        sk_s = remove_prefix(sk_s, prefix)
+        if encoding is not None:
+            sk_s = from_ascii(sk_s, encoding=encoding)
+        if len(sk_s) == 32:
+            # create from seed
+            vk_s, sk_s = _ed25519.publickey(sk_s)
+        else:
+            if len(sk_s) != 32+32:
+                raise ValueError("SigningKey takes 32-byte seed or 64-byte string")
+        self.sk_s = sk_s # seed+pubkey
+        self.vk_s = sk_s[32:] # just pubkey
+
+    def to_bytes(self, prefix=""):
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        return prefix+self.sk_s
+
+    def to_ascii(self, prefix="", encoding=None):
+        assert encoding
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        return to_ascii(self.to_seed(), prefix, encoding)
+
+    def to_seed(self, prefix=""):
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        return prefix+self.sk_s[:32]
+
+    def __eq__(self, them):
+        if not isinstance(them, object): return False
+        return (them.__class__ == self.__class__
+                and them.sk_s == self.sk_s)
+
+    def get_verifying_key(self):
+        return VerifyingKey(self.vk_s)
+
+    def sign(self, msg, prefix="", encoding=None):
+        assert isinstance(msg, bytes)
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        sig_and_msg = _ed25519.sign(msg, self.sk_s)
+        # the response is R+S+msg
+        sig_R = sig_and_msg[0:32]
+        sig_S = sig_and_msg[32:64]
+        msg_out = sig_and_msg[64:]
+        sig_out = sig_R + sig_S
+        assert msg_out == msg
+        if encoding:
+            return to_ascii(sig_out, prefix, encoding)
+        return prefix+sig_out
+
+class VerifyingKey(object):
+    def __init__(self, vk_s, prefix="", encoding=None):
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        if not isinstance(vk_s, bytes):
+            vk_s = vk_s.encode('ascii')
+        assert isinstance(vk_s, bytes)
+        vk_s = remove_prefix(vk_s, prefix)
+        if encoding is not None:
+            vk_s = from_ascii(vk_s, encoding=encoding)
+
+        assert len(vk_s) == 32
+        self.vk_s = vk_s
+
+    def to_bytes(self, prefix=""):
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        return prefix+self.vk_s
+
+    def to_ascii(self, prefix="", encoding=None):
+        assert encoding
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        return to_ascii(self.vk_s, prefix, encoding)
+
+    def __eq__(self, them):
+        if not isinstance(them, object): return False
+        return (them.__class__ == self.__class__
+                and them.vk_s == self.vk_s)
+
+    def verify(self, sig, msg, prefix="", encoding=None):
+        if not isinstance(sig, bytes):
+            sig = sig.encode('ascii')
+        if not isinstance(prefix, bytes):
+            prefix = prefix.encode('ascii')
+        assert isinstance(sig, bytes)
+        assert isinstance(msg, bytes)
+        if encoding:
+            sig = from_ascii(sig, prefix, encoding)
+        else:
+            sig = remove_prefix(sig, prefix)
+        assert len(sig) == 64
+        sig_R = sig[:32]
+        sig_S = sig[32:]
+        sig_and_msg = sig_R + sig_S + msg
+        # this might raise BadSignatureError
+        msg2 = _ed25519.open(sig_and_msg, self.vk_s)
+        assert msg2 == msg
+
+def selftest():
+    message = b"crypto libraries should always test themselves at powerup"
+    sk = SigningKey(b"priv0-VIsfn5OFGa09Un2MR6Hm7BQ5++xhcQskU2OGXG8jSJl4cWLZrRrVcSN2gVYMGtZT+3354J5jfmqAcuRSD9KIyg",
+                    prefix="priv0-", encoding="base64")
+    vk = VerifyingKey(b"pub0-eHFi2a0a1XEjdoFWDBrWU/t9+eCeY35qgHLkUg/SiMo",
+                      prefix="pub0-", encoding="base64")
+    assert sk.get_verifying_key() == vk
+    sig = sk.sign(message, prefix="sig0-", encoding="base64")
+    assert sig == b"sig0-E/QrwtSF52x8+q0l4ahA7eJbRKc777ClKNg217Q0z4fiYMCdmAOI+rTLVkiFhX6k3D+wQQfKdJYMxaTUFfv1DQ", sig
+    vk.verify(sig, message, prefix="sig0-", encoding="base64")
+
+selftest()

RNS/Cryptography/pure25519/eddsa.py

@@ -0,0 +1,94 @@
+# MIT License
+#
+# Copyright (c) 2015 Brian Warner and other contributors
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from RNS.Cryptography.Hashes import sha512
+from .basic import (bytes_to_clamped_scalar,
+                    bytes_to_scalar, scalar_to_bytes,
+                    bytes_to_element, Base)
+import hashlib, binascii
+
+def H(m):
+    return sha512(m)
+
+def publickey(seed):
+    # turn first half of SHA512(seed) into scalar, then into point
+    assert len(seed) == 32
+    a = bytes_to_clamped_scalar(H(seed)[:32])
+    A = Base.scalarmult(a)
+    return A.to_bytes()
+
+def Hint(m):
+    h = H(m)
+    return int(binascii.hexlify(h[::-1]), 16)
+
+def signature(m,sk,pk):
+    assert len(sk) == 32 # seed
+    assert len(pk) == 32
+    h = H(sk[:32])
+    a_bytes, inter = h[:32], h[32:]
+    a = bytes_to_clamped_scalar(a_bytes)
+    r = Hint(inter + m)
+    R = Base.scalarmult(r)
+    R_bytes = R.to_bytes()
+    S = r + Hint(R_bytes + pk + m) * a
+    return R_bytes + scalar_to_bytes(S)
+
+def checkvalid(s, m, pk):
+    if len(s) != 64: raise Exception("signature length is wrong")
+    if len(pk) != 32: raise Exception("public-key length is wrong")
+    R = bytes_to_element(s[:32])
+    A = bytes_to_element(pk)
+    S = bytes_to_scalar(s[32:])
+    h = Hint(s[:32] + pk + m)
+    v1 = Base.scalarmult(S)
+    v2 = R.add(A.scalarmult(h))
+    return v1==v2
+
+# wrappers
+
+import os
+
+def create_signing_key():
+    seed = os.urandom(32)
+    return seed
+
+def create_verifying_key(signing_key):
+    return publickey(signing_key)
+
+def sign(skbytes, msg):
+    """Return just the signature, given the message and just the secret
+    key."""
+    if len(skbytes) != 32:
+        raise ValueError("Bad signing key length %d" % len(skbytes))
+    vkbytes = create_verifying_key(skbytes)
+    sig = signature(msg, skbytes, vkbytes)
+    return sig
+
+def verify(vkbytes, sig, msg):
+    if len(vkbytes) != 32:
+        raise ValueError("Bad verifying key length %d" % len(vkbytes))
+    if len(sig) != 64:
+        raise ValueError("Bad signature length %d" % len(sig))
+    rc = checkvalid(sig, msg, vkbytes)
+    if not rc:
+        raise ValueError("rc != 0", rc)
+    return True

RNS/Destination.py

@@ -20,14 +20,11 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import base64
 import math
 import time
 import RNS
 
-from cryptography.fernet import Fernet
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.backends import default_backend
+from RNS.Cryptography import Fernet
 
 class Callbacks:
     def __init__(self):
@@ -72,8 +69,10 @@ class Destination:
     OUT        = 0x12;
     directions = [IN, OUT]
 
+    PR_TAG_WINDOW = 30
+
     @staticmethod
-    def full_name(app_name, *aspects):
+    def expand_name(identity, app_name, *aspects):
         """
         :returns: A string containing the full human-readable name of the destination, for an app_name and a number of aspects.
         """
@@ -84,23 +83,25 @@ class Destination:
         name = app_name
         for aspect in aspects:
             if "." in aspect: raise ValueError("Dots can't be used in aspects")
-            name = name + "." + aspect
+            name += "." + aspect
+
+        if identity != None:
+            name += "." + identity.hexhash
 
         return name
 
 
     @staticmethod
-    def hash(app_name, *aspects):
+    def hash(identity, app_name, *aspects):
         """
         :returns: A destination name in adressable hash form, for an app_name and a number of aspects.
         """
-        name = Destination.full_name(app_name, *aspects)
+        name_hash = RNS.Identity.full_hash(Destination.expand_name(None, app_name, *aspects).encode("utf-8"))[:(RNS.Identity.NAME_HASH_LENGTH//8)]
+        addr_hash_material = name_hash
+        if identity != None:
+            addr_hash_material += identity.hash
 
-        # Create a digest for the destination
-        digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
-        digest.update(name.encode("UTF-8"))
-
-        return digest.finalize()[:10]
+        return RNS.Identity.full_hash(addr_hash_material)[:RNS.Reticulum.TRUNCATED_HASHLENGTH//8]
 
     @staticmethod
     def app_and_aspects_from_name(full_name):
@@ -116,14 +117,16 @@ class Destination:
         :returns: A destination name in adressable hash form, for a full name string and Identity instance.
         """
         app_name, aspects = Destination.app_and_aspects_from_name(full_name)
-        aspects.append(identity.hexhash)
-        return Destination.hash(app_name, *aspects)
+
+        return Destination.hash(identity, app_name, *aspects)
 
     def __init__(self, identity, direction, type, app_name, *aspects):
         # Check input values and build name string
         if "." in app_name: raise ValueError("Dots can't be used in app names") 
         if not type in Destination.types: raise ValueError("Unknown destination type")
         if not direction in Destination.directions: raise ValueError("Unknown destination direction")
+
+        self.accept_link_requests = True
         self.callbacks = Callbacks()
         self.request_handlers = {}
         self.type = type
@@ -131,11 +134,9 @@ class Destination:
         self.proof_strategy = Destination.PROVE_NONE
         self.mtu = 0
 
+        self.path_responses = {}
         self.links = []
 
-        if identity != None and type == Destination.SINGLE:
-            aspects = aspects+(identity.hexhash,)
-
         if identity == None and direction == Destination.IN and self.type != Destination.PLAIN:
             identity = RNS.Identity()
             aspects = aspects+(identity.hexhash,)
@@ -144,12 +145,14 @@ class Destination:
             raise TypeError("Selected destination type PLAIN cannot hold an identity")
 
         self.identity = identity
+        self.name = Destination.expand_name(identity, app_name, *aspects)
 
-        self.name = Destination.full_name(app_name, *aspects)      
-        self.hash = Destination.hash(app_name, *aspects)
+        # Generate the destination address hash
+        self.hash = Destination.hash(self.identity, app_name, *aspects)
+        self.name_hash = RNS.Identity.full_hash(self.expand_name(None, app_name, *aspects).encode("utf-8"))[:(RNS.Identity.NAME_HASH_LENGTH//8)]
         self.hexhash = self.hash.hex()
-        self.default_app_data = None
 
+        self.default_app_data = None
         self.callback = None
         self.proofcallback = None
 
@@ -163,7 +166,7 @@ class Destination:
         return "<"+self.name+"/"+self.hexhash+">"
 
 
-    def announce(self, app_data=None, path_response=False):
+    def announce(self, app_data=None, path_response=False, attached_interface=None, tag=None, send=True):
         """
         Creates an announce packet for this destination and broadcasts it on all
         relevant interfaces. Application specific data can be added to the announce.
@@ -173,43 +176,87 @@ class Destination:
         """
         if self.type != Destination.SINGLE:
             raise TypeError("Only SINGLE destination types can be announced")
-            
-        destination_hash = self.hash
-        random_hash = RNS.Identity.get_random_hash()[0:5]+int(time.time()).to_bytes(5, "big")
-
-        if app_data == None and self.default_app_data != None:
-            if isinstance(self.default_app_data, bytes):
-                app_data = self.default_app_data
-            elif callable(self.default_app_data):
-                returned_app_data = self.default_app_data()
-                if isinstance(returned_app_data, bytes):
-                    app_data = returned_app_data
         
-        signed_data = self.hash+self.identity.get_public_key()+random_hash
-        if app_data != None:
-            signed_data += app_data
+        now = time.time()
+        stale_responses = []
+        for entry_tag in self.path_responses:
+            entry = self.path_responses[entry_tag]
+            if now > entry[0]+Destination.PR_TAG_WINDOW:
+                stale_responses.append(entry_tag)
 
-        signature = self.identity.sign(signed_data)
+        for entry_tag in stale_responses:
+            self.path_responses.pop(entry_tag)
 
-        announce_data = self.identity.get_public_key()+random_hash+signature
+        if (path_response == True and tag != None) and tag in self.path_responses:
+            # This code is currently not used, since Transport will block duplicate
+            # path requests based on tags. When multi-path support is implemented in
+            # Transport, this will allow Transport to detect redundant paths to the
+            # same destination, and select the best one based on chosen criteria,
+            # since it will be able to detect that a single emitted announce was
+            # received via multiple paths. The difference in reception time will
+            # potentially also be useful in determining characteristics of the
+            # multiple available paths, and to choose the best one.
+            RNS.log("Using cached announce data for answering path request with tag "+RNS.prettyhexrep(tag), RNS.LOG_EXTREME)
+            announce_data = self.path_responses[tag][1]
+        
+        else:
+            destination_hash = self.hash
+            random_hash = RNS.Identity.get_random_hash()[0:5]+int(time.time()).to_bytes(5, "big")
 
-        if app_data != None:
-            announce_data += app_data
+            if app_data == None and self.default_app_data != None:
+                if isinstance(self.default_app_data, bytes):
+                    app_data = self.default_app_data
+                elif callable(self.default_app_data):
+                    returned_app_data = self.default_app_data()
+                    if isinstance(returned_app_data, bytes):
+                        app_data = returned_app_data
+            
+            signed_data = self.hash+self.identity.get_public_key()+self.name_hash+random_hash
+            if app_data != None:
+                signed_data += app_data
+
+            signature = self.identity.sign(signed_data)
+
+            announce_data = self.identity.get_public_key()+self.name_hash+random_hash+signature
+
+            if app_data != None:
+                announce_data += app_data
+
+            self.path_responses[tag] = [time.time(), announce_data]
 
         if path_response:
             announce_context = RNS.Packet.PATH_RESPONSE
         else:
             announce_context = RNS.Packet.NONE
 
-        RNS.Packet(self, announce_data, RNS.Packet.ANNOUNCE, context = announce_context).send()
+        announce_packet = RNS.Packet(self, announce_data, RNS.Packet.ANNOUNCE, context = announce_context, attached_interface = attached_interface)
 
+        if send:
+            announce_packet.send()
+        else:
+            return announce_packet
+
+    def accepts_links(self, accepts = None):
+        """
+        Set or query whether the destination accepts incoming link requests.
+
+        :param accepts: If ``True`` or ``False``, this method sets whether the destination accepts incoming link requests. If not provided or ``None``, the method returns whether the destination currently accepts link requests.
+        :returns: ``True`` or ``False`` depending on whether the destination accepts incoming link requests, if the *accepts* parameter is not provided or ``None``.
+        """
+        if accepts == None:
+            return self.accept_link_requests
+
+        if accepts:
+            self.accept_link_requests = True
+        else:
+            self.accept_link_requests = False
 
     def set_link_established_callback(self, callback):
         """
         Registers a function to be called when a link has been established to
         this destination.
 
-        :param callback: A function or method to be called.
+        :param callback: A function or method with the signature *callback(link)* to be called when a new link is established with this destination.
         """
         self.callbacks.link_established = callback
 
@@ -218,7 +265,7 @@ class Destination:
         Registers a function to be called when a packet has been received by
         this destination.
 
-        :param callback: A function or method to be called.
+        :param callback: A function or method with the signature *callback(data, packet)* to be called when this destination receives a packet.
         """
         self.callbacks.packet = callback
 
@@ -228,7 +275,7 @@ class Destination:
         a packet sent to this destination. Allows control over when and if
         proofs should be returned for received packets.
 
-        :param callback: A function or method to be called. The callback must return one of True or False. If the callback returns True, a proof will be sent. If it returns False, a proof will not be sent.
+        :param callback: A function or method to with the signature *callback(packet)* be called when a packet that requests a proof is received. The callback must return one of True or False. If the callback returns True, a proof will be sent. If it returns False, a proof will not be sent.
         """
         self.callbacks.proof_requested = callback
 
@@ -298,9 +345,10 @@ class Destination:
 
 
     def incoming_link_request(self, data, packet):
-        link = RNS.Link.validate_request(self, data, packet)
-        if link != None:
-            self.links.append(link)
+        if self.accept_link_requests:
+            link = RNS.Link.validate_request(self, data, packet)
+            if link != None:
+                self.links.append(link)
 
     def create_keys(self):
         """
@@ -315,8 +363,8 @@ class Destination:
             raise TypeError("A single destination holds keys through an Identity instance")
 
         if self.type == Destination.GROUP:
-            self.prv_bytes = base64.urlsafe_b64decode(Fernet.generate_key())
-            self.prv = Fernet(base64.urlsafe_b64encode(self.prv_bytes))
+            self.prv_bytes = Fernet.generate_key()
+            self.prv = Fernet(self.prv_bytes)
 
 
     def get_private_key(self):
@@ -348,7 +396,7 @@ class Destination:
 
         if self.type == Destination.GROUP:
             self.prv_bytes = key
-            self.prv = Fernet(base64.urlsafe_b64encode(self.prv_bytes))
+            self.prv = Fernet(self.prv_bytes)
 
     def load_public_key(self, key):
         if self.type != Destination.SINGLE:
@@ -373,7 +421,7 @@ class Destination:
         if self.type == Destination.GROUP:
             if hasattr(self, "prv") and self.prv != None:
                 try:
-                    return base64.urlsafe_b64decode(self.prv.encrypt(plaintext))
+                    return self.prv.encrypt(plaintext)
                 except Exception as e:
                     RNS.log("The GROUP destination could not encrypt data", RNS.LOG_ERROR)
                     RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
@@ -398,7 +446,7 @@ class Destination:
         if self.type == Destination.GROUP:
             if hasattr(self, "prv") and self.prv != None:
                 try:
-                    return self.prv.decrypt(base64.urlsafe_b64encode(ciphertext))
+                    return self.prv.decrypt(ciphertext)
                 except Exception as e:
                     RNS.log("The GROUP destination could not decrypt data", RNS.LOG_ERROR)
                     RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)

RNS/Identity.py

@@ -20,23 +20,18 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-import base64
 import math
 import os
 import RNS
 import time
 import atexit
-import base64
-from .vendor import umsgpack as umsgpack
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
-from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey
-from cryptography.hazmat.primitives.kdf.hkdf import HKDF
-from cryptography.fernet import Fernet
+import hashlib
+
+from .vendor import umsgpack as umsgpack
+
+from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey
+from RNS.Cryptography import Fernet
 
-cio_default_backend = default_backend()
 
 class Identity:
     """
@@ -58,13 +53,12 @@ class Identity:
     """   
 
     # Non-configurable constants
-    FERNET_VERSION            = 0x80
-    FERNET_OVERHEAD           = 57          # In bytes
-    OPTIMISED_FERNET_OVERHEAD = 54          # In bytes
+    FERNET_OVERHEAD           = RNS.Cryptography.Fernet.FERNET_OVERHEAD
     AES128_BLOCKSIZE          = 16          # In bytes
     HASHLENGTH                = 256         # In bits
     SIGLENGTH                 = KEYSIZE     # In bits
 
+    NAME_HASH_LENGTH     = 80
     TRUNCATED_HASHLENGTH = RNS.Reticulum.TRUNCATED_HASHLENGTH
     """
     Constant specifying the truncated hash length (in bits) used by Reticulum
@@ -97,6 +91,13 @@ class Identity:
             identity.app_data = identity_data[3]
             return identity
         else:
+            for registered_destination in RNS.Transport.destinations:
+                if destination_hash == registered_destination.hash:
+                    identity = Identity(create_keys=False)
+                    identity.load_public_key(registered_destination.identity.get_public_key())
+                    identity.app_data = None
+                    return identity
+
             return None
 
     @staticmethod
@@ -115,7 +116,26 @@ class Identity:
 
     @staticmethod
     def save_known_destinations():
+        # TODO: Improve the storage method so we don't have to
+        # deserialize and serialize the entire table on every
+        # save, but the only changes. It might be possible to
+        # simply overwrite on exit now that every local client
+        # disconnect triggers a data persist.
+        
         try:
+            if hasattr(Identity, "saving_known_destinations"):
+                wait_interval = 0.2
+                wait_timeout = 5
+                wait_start = time.time()
+                while Identity.saving_known_destinations:
+                    time.sleep(wait_interval)
+                    if time.time() > wait_start+wait_timeout:
+                        RNS.log("Could not save known destinations to storage, waiting for previous save operation timed out.", RNS.LOG_ERROR)
+                        return False
+
+            Identity.saving_known_destinations = True
+            save_start = time.time()
+
             storage_known_destinations = {}
             if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"):
                 try:
@@ -129,21 +149,37 @@ class Identity:
                 if not destination_hash in Identity.known_destinations:
                     Identity.known_destinations[destination_hash] = storage_known_destinations[destination_hash]
 
-            RNS.log("Saving known destinations to storage...", RNS.LOG_VERBOSE)
+            RNS.log("Saving "+str(len(Identity.known_destinations))+" known destinations to storage...", RNS.LOG_DEBUG)
             file = open(RNS.Reticulum.storagepath+"/known_destinations","wb")
             umsgpack.dump(Identity.known_destinations, file)
             file.close()
-            RNS.log("Done saving known destinations to storage", RNS.LOG_VERBOSE)
+
+            save_time = time.time() - save_start
+            if save_time < 1:
+                time_str = str(round(save_time*1000,2))+"ms"
+            else:
+                time_str = str(round(save_time,2))+"s"
+
+            RNS.log("Saved known destinations to storage in "+time_str, RNS.LOG_DEBUG)
+
         except Exception as e:
             RNS.log("Error while saving known destinations to disk, the contained exception was: "+str(e), RNS.LOG_ERROR)
 
+        Identity.saving_known_destinations = False
+
     @staticmethod
     def load_known_destinations():
         if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"):
             try:
                 file = open(RNS.Reticulum.storagepath+"/known_destinations","rb")
-                Identity.known_destinations = umsgpack.load(file)
+                loaded_known_destinations = umsgpack.load(file)
                 file.close()
+
+                Identity.known_destinations = {}
+                for known_destination in loaded_known_destinations:
+                    if len(known_destination) == RNS.Reticulum.TRUNCATED_HASHLENGTH//8:
+                        Identity.known_destinations[known_destination] = loaded_known_destinations[known_destination]
+
                 RNS.log("Loaded "+str(len(Identity.known_destinations))+" known destination from storage", RNS.LOG_VERBOSE)
             except:
                 RNS.log("Error loading known destinations from disk, file will be recreated on exit", RNS.LOG_ERROR)
@@ -158,10 +194,7 @@ class Identity:
         :param data: Data to be hashed as *bytes*.
         :returns: SHA-256 hash as *bytes*
         """
-        digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
-        digest.update(data)
-
-        return digest.finalize()
+        return RNS.Cryptography.sha256(data)
 
     @staticmethod
     def truncated_hash(data):
@@ -189,33 +222,64 @@ class Identity:
             if packet.packet_type == RNS.Packet.ANNOUNCE:
                 destination_hash = packet.destination_hash
                 public_key = packet.data[:Identity.KEYSIZE//8]
-                random_hash = packet.data[Identity.KEYSIZE//8:Identity.KEYSIZE//8+10]
-                signature = packet.data[Identity.KEYSIZE//8+10:Identity.KEYSIZE//8+10+Identity.KEYSIZE//8]
+                name_hash = packet.data[Identity.KEYSIZE//8:Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8]
+                random_hash = packet.data[Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8:Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10]
+                signature = packet.data[Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10:Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10+Identity.SIGLENGTH//8]
                 app_data = b""
-                if len(packet.data) > Identity.KEYSIZE//8+10+Identity.KEYSIZE//8:
-                    app_data = packet.data[Identity.KEYSIZE//8+10+Identity.KEYSIZE//8:]
+                if len(packet.data) > Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10+Identity.SIGLENGTH//8:
+                    app_data = packet.data[Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10+Identity.SIGLENGTH//8:]
 
-                signed_data = destination_hash+public_key+random_hash+app_data
+                signed_data = destination_hash+public_key+name_hash+random_hash+app_data
 
-                if not len(packet.data) > Identity.KEYSIZE//8+10+Identity.KEYSIZE//8:
+                if not len(packet.data) > Identity.KEYSIZE//8+Identity.NAME_HASH_LENGTH//8+10+Identity.SIGLENGTH//8:
                     app_data = None
 
                 announced_identity = Identity(create_keys=False)
                 announced_identity.load_public_key(public_key)
 
                 if announced_identity.pub != None and announced_identity.validate(signature, signed_data):
-                    RNS.Identity.remember(packet.get_hash(), destination_hash, public_key, app_data)
-                    del announced_identity
+                    hash_material = name_hash+announced_identity.hash
+                    expected_hash = RNS.Identity.full_hash(hash_material)[:RNS.Reticulum.TRUNCATED_HASHLENGTH//8]
+
+                    if destination_hash == expected_hash:
+                        # Check if we already have a public key for this destination
+                        # and make sure the public key is not different.
+                        if destination_hash in Identity.known_destinations:
+                            if public_key != Identity.known_destinations[destination_hash][2]:
+                                # In reality, this should never occur, but in the odd case
+                                # that someone manages a hash collision, we reject the announce.
+                                RNS.log("Received announce with valid signature and destination hash, but announced public key does not match already known public key.", RNS.LOG_CRITICAL)
+                                RNS.log("This may indicate an attempt to modify network paths, or a random hash collision. The announce was rejected.", RNS.LOG_CRITICAL)
+                                return False
+
+                        RNS.Identity.remember(packet.get_hash(), destination_hash, public_key, app_data)
+                        del announced_identity
+
+                        if packet.rssi != None or packet.snr != None:
+                            signal_str = " ["
+                            if packet.rssi != None:
+                                signal_str += "RSSI "+str(packet.rssi)+"dBm"
+                                if packet.snr != None:
+                                    signal_str += ", "
+                            if packet.snr != None:
+                                signal_str += "SNR "+str(packet.snr)+"dB"
+                            signal_str += "]"
+                        else:
+                            signal_str = ""
+
+                        if hasattr(packet, "transport_id") and packet.transport_id != None:
+                            RNS.log("Valid announce for "+RNS.prettyhexrep(destination_hash)+" "+str(packet.hops)+" hops away, received via "+RNS.prettyhexrep(packet.transport_id)+" on "+str(packet.receiving_interface)+signal_str, RNS.LOG_EXTREME)
+                        else:
+                            RNS.log("Valid announce for "+RNS.prettyhexrep(destination_hash)+" "+str(packet.hops)+" hops away, received on "+str(packet.receiving_interface)+signal_str, RNS.LOG_EXTREME)
+
+                        return True
 
-                    if hasattr(packet, "transport_id") and packet.transport_id != None:
-                        RNS.log("Valid announce for "+RNS.prettyhexrep(destination_hash)+" "+str(packet.hops)+" hops away, received via "+RNS.prettyhexrep(packet.transport_id)+" on "+str(packet.receiving_interface), RNS.LOG_EXTREME)
                     else:
-                        RNS.log("Valid announce for "+RNS.prettyhexrep(destination_hash)+" "+str(packet.hops)+" hops away, received on "+str(packet.receiving_interface), RNS.LOG_EXTREME)
-
-                    return True
+                        RNS.log("Received invalid announce for "+RNS.prettyhexrep(destination_hash)+": Destination mismatch.", RNS.LOG_DEBUG)
+                        return False
 
                 else:
-                    RNS.log("Received invalid announce for "+RNS.prettyhexrep(destination_hash), RNS.LOG_DEBUG)
+                    RNS.log("Received invalid announce for "+RNS.prettyhexrep(destination_hash)+": Invalid signature.", RNS.LOG_DEBUG)
                     del announced_identity
                     return False
         
@@ -223,9 +287,14 @@ class Identity:
             RNS.log("Error occurred while validating announce. The contained exception was: "+str(e), RNS.LOG_ERROR)
             return False
 
+    @staticmethod
+    def persist_data():
+        if not RNS.Transport.owner.is_connected_to_shared_instance:
+            Identity.save_known_destinations()
+
     @staticmethod
     def exit_handler():
-        Identity.save_known_destinations()
+        Identity.persist_data()
 
 
     @staticmethod
@@ -297,30 +366,16 @@ class Identity:
 
     def create_keys(self):
         self.prv           = X25519PrivateKey.generate()
-        self.prv_bytes     = self.prv.private_bytes(
-            encoding=serialization.Encoding.Raw,
-            format=serialization.PrivateFormat.Raw,
-            encryption_algorithm=serialization.NoEncryption()
-        )
+        self.prv_bytes     = self.prv.private_bytes()
 
         self.sig_prv       = Ed25519PrivateKey.generate()
-        self.sig_prv_bytes = self.sig_prv.private_bytes(
-            encoding=serialization.Encoding.Raw,
-            format=serialization.PrivateFormat.Raw,
-            encryption_algorithm=serialization.NoEncryption()
-        )
+        self.sig_prv_bytes = self.sig_prv.private_bytes()
 
         self.pub           = self.prv.public_key()
-        self.pub_bytes     = self.pub.public_bytes(
-            encoding=serialization.Encoding.Raw,
-            format=serialization.PublicFormat.Raw
-        )
+        self.pub_bytes     = self.pub.public_bytes()
 
         self.sig_pub       = self.sig_prv.public_key()
-        self.sig_pub_bytes = self.sig_pub.public_bytes(
-            encoding=serialization.Encoding.Raw,
-            format=serialization.PublicFormat.Raw
-        )
+        self.sig_pub_bytes = self.sig_pub.public_bytes()
 
         self.update_hashes()
 
@@ -352,16 +407,10 @@ class Identity:
             self.sig_prv       = Ed25519PrivateKey.from_private_bytes(self.sig_prv_bytes)
             
             self.pub           = self.prv.public_key()
-            self.pub_bytes     = self.pub.public_bytes(
-                encoding=serialization.Encoding.Raw,
-                format=serialization.PublicFormat.Raw
-            )
+            self.pub_bytes     = self.pub.public_bytes()
 
             self.sig_pub       = self.sig_prv.public_key()
-            self.sig_pub_bytes = self.sig_pub.public_bytes(
-                encoding=serialization.Encoding.Raw,
-                format=serialization.PublicFormat.Raw
-            )
+            self.sig_pub_bytes = self.sig_pub.public_bytes()
 
             self.update_hashes()
 
@@ -421,24 +470,19 @@ class Identity:
         """
         if self.pub != None:
             ephemeral_key = X25519PrivateKey.generate()
-            ephemeral_pub_bytes = ephemeral_key.public_key().public_bytes(
-                encoding=serialization.Encoding.Raw,
-                format=serialization.PublicFormat.Raw
-            )
+            ephemeral_pub_bytes = ephemeral_key.public_key().public_bytes()
 
             shared_key = ephemeral_key.exchange(self.pub)
             
-            # TODO: Improve this re-allocation of HKDF
-            derived_key = HKDF(
-                algorithm=hashes.SHA256(),
+            derived_key = RNS.Cryptography.hkdf(
                 length=32,
+                derive_from=shared_key,
                 salt=self.get_salt(),
-                info=self.get_context(),
-                backend=cio_default_backend,
-            ).derive(shared_key)
+                context=self.get_context(),
+            )
 
-            fernet = Fernet(base64.urlsafe_b64encode(derived_key))
-            ciphertext = base64.urlsafe_b64decode(fernet.encrypt(plaintext))
+            fernet = Fernet(derived_key)
+            ciphertext = fernet.encrypt(plaintext)
             token = ephemeral_pub_bytes+ciphertext
 
             return token
@@ -463,18 +507,16 @@ class Identity:
 
                     shared_key = self.prv.exchange(peer_pub)
 
-                    # TODO: Improve this re-allocation of HKDF
-                    derived_key = HKDF(
-                        algorithm=hashes.SHA256(),
+                    derived_key = RNS.Cryptography.hkdf(
                         length=32,
+                        derive_from=shared_key,
                         salt=self.get_salt(),
-                        info=self.get_context(),
-                        backend=cio_default_backend,
-                    ).derive(shared_key)
+                        context=self.get_context(),
+                    )
 
-                    fernet = Fernet(base64.urlsafe_b64encode(derived_key))
+                    fernet = Fernet(derived_key)
                     ciphertext = ciphertext_token[Identity.KEYSIZE//8//2:]
-                    plaintext = fernet.decrypt(base64.urlsafe_b64encode(ciphertext))
+                    plaintext = fernet.decrypt(ciphertext)
 
                 except Exception as e:
                     RNS.log("Decryption by "+RNS.prettyhexrep(self.hash)+" failed: "+str(e), RNS.LOG_DEBUG)

RNS/Interfaces/AX25KISSInterface.py

@@ -79,6 +79,8 @@ class AX25KISSInterface(Interface):
 
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 564
         
         self.pyserial = serial
         self.serial   = None
@@ -151,7 +153,7 @@ class AX25KISSInterface(Interface):
         # Allow time for interface to initialise before config
         sleep(2.0)
         thread = threading.Thread(target=self.readLoop)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
         self.online = True
         RNS.log("Serial port "+self.port+" is now open")
@@ -304,7 +306,7 @@ class AX25KISSInterface(Interface):
                         in_frame = True
                         command = KISS.CMD_UNKNOWN
                         data_buffer = b""
-                    elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU+AX25.HEADER_SIZE):
+                    elif (in_frame and len(data_buffer) < self.HW_MTU+AX25.HEADER_SIZE):
                         if (len(data_buffer) == 0 and command == KISS.CMD_UNKNOWN):
                             # We only support one HDLC port for now, so
                             # strip off the port nibble

RNS/Interfaces/Android/KISSInterface.py

@@ -0,0 +1,407 @@
+# MIT License
+#
+# Copyright (c) 2016-2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from RNS.Interfaces.Interface import Interface
+from time import sleep
+import sys
+import threading
+import time
+import RNS
+
+class KISS():
+    FEND              = 0xC0
+    FESC              = 0xDB
+    TFEND             = 0xDC
+    TFESC             = 0xDD
+    CMD_UNKNOWN       = 0xFE
+    CMD_DATA          = 0x00
+    CMD_TXDELAY       = 0x01
+    CMD_P             = 0x02
+    CMD_SLOTTIME      = 0x03
+    CMD_TXTAIL        = 0x04
+    CMD_FULLDUPLEX    = 0x05
+    CMD_SETHARDWARE   = 0x06
+    CMD_READY         = 0x0F
+    CMD_RETURN        = 0xFF
+
+    @staticmethod
+    def escape(data):
+        data = data.replace(bytes([0xdb]), bytes([0xdb, 0xdd]))
+        data = data.replace(bytes([0xc0]), bytes([0xdb, 0xdc]))
+        return data
+
+class KISSInterface(Interface):
+    MAX_CHUNK = 32768
+    BITRATE_GUESS = 1200
+
+    owner    = None
+    port     = None
+    speed    = None
+    databits = None
+    parity   = None
+    stopbits = None
+    serial   = None
+
+    def __init__(self, owner, name, port, speed, databits, parity, stopbits, preamble, txtail, persistence, slottime, flow_control, beacon_interval, beacon_data):
+        import importlib
+        if RNS.vendor.platformutils.is_android():
+            self.on_android  = True
+            if importlib.util.find_spec('usbserial4a') != None:
+                if importlib.util.find_spec('jnius') == None:
+                    RNS.log("Could not load jnius API wrapper for Android, KISS interface cannot be created.", RNS.LOG_CRITICAL)
+                    RNS.log("This probably means you are trying to use an USB-based interface from within Termux or similar.", RNS.LOG_CRITICAL)
+                    RNS.log("This is currently not possible, due to this environment limiting access to the native Android APIs.", RNS.LOG_CRITICAL)
+                    RNS.panic()
+
+                from usbserial4a import serial4a as serial
+                self.parity = "N"
+            
+            else:
+                RNS.log("Could not load USB serial module for Android, KISS interface cannot be created.", RNS.LOG_CRITICAL)
+                RNS.log("You can install this module by issuing: pip install usbserial4a", RNS.LOG_CRITICAL)
+                RNS.panic()
+        else:
+            raise SystemError("Android-specific interface was used on non-Android OS")
+
+        self.rxb = 0
+        self.txb = 0
+        
+        self.HW_MTU = 564
+        
+        if beacon_data == None:
+            beacon_data = ""
+
+        self.pyserial = serial
+        self.serial   = None
+        self.owner    = owner
+        self.name     = name
+        self.port     = port
+        self.speed    = speed
+        self.databits = databits
+        self.parity   = "N"
+        self.stopbits = stopbits
+        self.timeout  = 100
+        self.online   = False
+        self.beacon_i = beacon_interval
+        self.beacon_d = beacon_data.encode("utf-8")
+        self.first_tx = None
+        self.bitrate  = KISSInterface.BITRATE_GUESS
+
+        self.packet_queue    = []
+        self.flow_control    = flow_control
+        self.interface_ready = False
+        self.flow_control_timeout = 5
+        self.flow_control_locked  = time.time()
+
+        self.preamble    = preamble if preamble != None else 350;
+        self.txtail      = txtail if txtail != None else 20;
+        self.persistence = persistence if persistence != None else 64;
+        self.slottime    = slottime if slottime != None else 20;
+
+        if parity.lower() == "e" or parity.lower() == "even":
+            self.parity = "E"
+
+        if parity.lower() == "o" or parity.lower() == "odd":
+            self.parity = "O"
+
+        try:
+            self.open_port()
+        except Exception as e:
+            RNS.log("Could not open serial port "+self.port, RNS.LOG_ERROR)
+            raise e
+
+        if self.serial.is_open:
+            self.configure_device()
+        else:
+            raise IOError("Could not open serial port")
+
+
+    def open_port(self):
+        RNS.log("Opening serial port "+self.port+"...")
+        # Get device parameters
+        from usb4a import usb
+        device = usb.get_usb_device(self.port)
+        if device:
+            vid = device.getVendorId()
+            pid = device.getProductId()
+
+            # Driver overrides for speficic chips
+            proxy = self.pyserial.get_serial_port
+            if vid == 0x1A86 and pid == 0x55D4:
+                # Force CDC driver for Qinheng CH34x
+                RNS.log(str(self)+" using CDC driver for "+RNS.hexrep(vid)+":"+RNS.hexrep(pid), RNS.LOG_DEBUG)
+                from usbserial4a.cdcacmserial4a import CdcAcmSerial
+                proxy = CdcAcmSerial
+
+            self.serial = proxy(
+                self.port,
+                baudrate = self.speed,
+                bytesize = self.databits,
+                parity = self.parity,
+                stopbits = self.stopbits,
+                xonxoff = False,
+                rtscts = False,
+                timeout = None,
+                inter_byte_timeout = None,
+                # write_timeout = wtimeout,
+                dsrdtr = False,
+            )
+
+            if vid == 0x0403:
+                # Hardware parameters for FTDI devices @ 115200 baud
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 16 * 1024
+                self.serial.USB_READ_TIMEOUT_MILLIS = 100
+                self.serial.timeout = 0.1
+            elif vid == 0x10C4:
+                # Hardware parameters for SiLabs CP210x @ 115200 baud
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 64 
+                self.serial.USB_READ_TIMEOUT_MILLIS = 12
+                self.serial.timeout = 0.012
+            elif vid == 0x1A86 and pid == 0x55D4:
+                # Hardware parameters for Qinheng CH34x @ 115200 baud
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 64
+                self.serial.USB_READ_TIMEOUT_MILLIS = 12
+                self.serial.timeout = 0.1
+            else:
+                # Default values
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 1 * 1024
+                self.serial.USB_READ_TIMEOUT_MILLIS = 100
+                self.serial.timeout = 0.1
+
+            RNS.log(str(self)+" USB read buffer size set to "+RNS.prettysize(self.serial.DEFAULT_READ_BUFFER_SIZE), RNS.LOG_DEBUG)
+            RNS.log(str(self)+" USB read timeout set to "+str(self.serial.USB_READ_TIMEOUT_MILLIS)+"ms", RNS.LOG_DEBUG)
+            RNS.log(str(self)+" USB write timeout set to "+str(self.serial.USB_WRITE_TIMEOUT_MILLIS)+"ms", RNS.LOG_DEBUG)
+
+    def configure_device(self):
+        # Allow time for interface to initialise before config
+        sleep(2.0)
+        thread = threading.Thread(target=self.readLoop)
+        thread.daemon = True
+        thread.start()
+        self.online = True
+        RNS.log("Serial port "+self.port+" is now open")
+        RNS.log("Configuring KISS interface parameters...")
+        self.setPreamble(self.preamble)
+        self.setTxTail(self.txtail)
+        self.setPersistence(self.persistence)
+        self.setSlotTime(self.slottime)
+        self.setFlowControl(self.flow_control)
+        self.interface_ready = True
+        RNS.log("KISS interface configured")
+
+    def setPreamble(self, preamble):
+        preamble_ms = preamble
+        preamble = int(preamble_ms / 10)
+        if preamble < 0:
+            preamble = 0
+        if preamble > 255:
+            preamble = 255
+
+        kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_TXDELAY])+bytes([preamble])+bytes([KISS.FEND])
+        written = self.serial.write(kiss_command)
+        if written != len(kiss_command):
+            raise IOError("Could not configure KISS interface preamble to "+str(preamble_ms)+" (command value "+str(preamble)+")")
+
+    def setTxTail(self, txtail):
+        txtail_ms = txtail
+        txtail = int(txtail_ms / 10)
+        if txtail < 0:
+            txtail = 0
+        if txtail > 255:
+            txtail = 255
+
+        kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_TXTAIL])+bytes([txtail])+bytes([KISS.FEND])
+        written = self.serial.write(kiss_command)
+        if written != len(kiss_command):
+            raise IOError("Could not configure KISS interface TX tail to "+str(txtail_ms)+" (command value "+str(txtail)+")")
+
+    def setPersistence(self, persistence):
+        if persistence < 0:
+            persistence = 0
+        if persistence > 255:
+            persistence = 255
+
+        kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_P])+bytes([persistence])+bytes([KISS.FEND])
+        written = self.serial.write(kiss_command)
+        if written != len(kiss_command):
+            raise IOError("Could not configure KISS interface persistence to "+str(persistence))
+
+    def setSlotTime(self, slottime):
+        slottime_ms = slottime
+        slottime = int(slottime_ms / 10)
+        if slottime < 0:
+            slottime = 0
+        if slottime > 255:
+            slottime = 255
+
+        kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_SLOTTIME])+bytes([slottime])+bytes([KISS.FEND])
+        written = self.serial.write(kiss_command)
+        if written != len(kiss_command):
+            raise IOError("Could not configure KISS interface slot time to "+str(slottime_ms)+" (command value "+str(slottime)+")")
+
+    def setFlowControl(self, flow_control):
+        kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_READY])+bytes([0x01])+bytes([KISS.FEND])
+        written = self.serial.write(kiss_command)
+        if written != len(kiss_command):
+            if (flow_control):
+                raise IOError("Could not enable KISS interface flow control")
+            else:
+                raise IOError("Could not enable KISS interface flow control")
+
+
+    def processIncoming(self, data):
+        self.rxb += len(data)
+        def af():
+            self.owner.inbound(data, self)
+        threading.Thread(target=af, daemon=True).start()
+
+    def processOutgoing(self,data):
+        datalen = len(data)
+        if self.online:
+            if self.interface_ready:
+                if self.flow_control:
+                    self.interface_ready = False
+                    self.flow_control_locked = time.time()
+
+                data = data.replace(bytes([0xdb]), bytes([0xdb])+bytes([0xdd]))
+                data = data.replace(bytes([0xc0]), bytes([0xdb])+bytes([0xdc]))
+                frame = bytes([KISS.FEND])+bytes([0x00])+data+bytes([KISS.FEND])
+
+                written = self.serial.write(frame)
+                self.txb += datalen
+
+                if data == self.beacon_d:
+                    self.first_tx = None
+                else:
+                    if self.first_tx == None:
+                        self.first_tx = time.time()
+
+                if written != len(frame):
+                    raise IOError("Serial interface only wrote "+str(written)+" bytes of "+str(len(data)))
+
+            else:
+                self.queue(data)
+
+    def queue(self, data):
+        self.packet_queue.append(data)
+
+    def process_queue(self):
+        if len(self.packet_queue) > 0:
+            data = self.packet_queue.pop(0)
+            self.interface_ready = True
+            self.processOutgoing(data)
+        elif len(self.packet_queue) == 0:
+            self.interface_ready = True
+
+    def readLoop(self):
+        try:
+            in_frame = False
+            escape = False
+            command = KISS.CMD_UNKNOWN
+            data_buffer = b""
+            last_read_ms = int(time.time()*1000)
+
+            while self.serial.is_open:
+                serial_bytes = self.serial.read()
+                got = len(serial_bytes)
+
+                for byte in serial_bytes:
+                    last_read_ms = int(time.time()*1000)
+
+                    if (in_frame and byte == KISS.FEND and command == KISS.CMD_DATA):
+                        in_frame = False
+                        self.processIncoming(data_buffer)
+                    elif (byte == KISS.FEND):
+                        in_frame = True
+                        command = KISS.CMD_UNKNOWN
+                        data_buffer = b""
+                    elif (in_frame and len(data_buffer) < self.HW_MTU):
+                        if (len(data_buffer) == 0 and command == KISS.CMD_UNKNOWN):
+                            # We only support one HDLC port for now, so
+                            # strip off the port nibble
+                            byte = byte & 0x0F
+                            command = byte
+                        elif (command == KISS.CMD_DATA):
+                            if (byte == KISS.FESC):
+                                escape = True
+                            else:
+                                if (escape):
+                                    if (byte == KISS.TFEND):
+                                        byte = KISS.FEND
+                                    if (byte == KISS.TFESC):
+                                        byte = KISS.FESC
+                                    escape = False
+                                data_buffer = data_buffer+bytes([byte])
+                        elif (command == KISS.CMD_READY):
+                            self.process_queue()
+                
+                if got == 0:
+                    time_since_last = int(time.time()*1000) - last_read_ms
+                    if len(data_buffer) > 0 and time_since_last > self.timeout:
+                        data_buffer = b""
+                        in_frame = False
+                        command = KISS.CMD_UNKNOWN
+                        escape = False
+                    sleep(0.05)
+
+                    if self.flow_control:
+                        if not self.interface_ready:
+                            if time.time() > self.flow_control_locked + self.flow_control_timeout:
+                                RNS.log("Interface "+str(self)+" is unlocking flow control due to time-out. This should not happen. Your hardware might have missed a flow-control READY command, or maybe it does not support flow-control.", RNS.LOG_WARNING)
+                                self.process_queue()
+
+                    if self.beacon_i != None and self.beacon_d != None:
+                        if self.first_tx != None:
+                            if time.time() > self.first_tx + self.beacon_i:
+                                RNS.log("Interface "+str(self)+" is transmitting beacon data: "+str(self.beacon_d.decode("utf-8")), RNS.LOG_DEBUG)
+                                self.first_tx = None
+                                self.processOutgoing(self.beacon_d)
+
+        except Exception as e:
+            self.online = False
+            RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
+            RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR)
+            
+            if RNS.Reticulum.panic_on_interface_error:
+                RNS.panic()
+
+            RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
+
+        self.online = False
+        self.serial.close()
+        self.reconnect_port()
+
+    def reconnect_port(self):
+        while not self.online:
+            try:
+                time.sleep(5)
+                RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
+                self.open_port()
+                if self.serial.is_open:
+                    self.configure_device()
+            except Exception as e:
+                RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
+
+        RNS.log("Reconnected serial port for "+str(self))
+
+    def __str__(self):
+        return "KISSInterface["+self.name+"]"

RNS/Interfaces/Android/SerialInterface.py

@@ -0,0 +1,258 @@
+# MIT License
+#
+# Copyright (c) 2016-2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from RNS.Interfaces.Interface import Interface
+from time import sleep
+import sys
+import threading
+import time
+import RNS
+
+class HDLC():
+    # The Serial Interface packetizes data using
+    # simplified HDLC framing, similar to PPP
+    FLAG              = 0x7E
+    ESC               = 0x7D
+    ESC_MASK          = 0x20
+
+    @staticmethod
+    def escape(data):
+        data = data.replace(bytes([HDLC.ESC]), bytes([HDLC.ESC, HDLC.ESC^HDLC.ESC_MASK]))
+        data = data.replace(bytes([HDLC.FLAG]), bytes([HDLC.ESC, HDLC.FLAG^HDLC.ESC_MASK]))
+        return data
+
+class SerialInterface(Interface):
+    MAX_CHUNK = 32768
+
+    owner    = None
+    port     = None
+    speed    = None
+    databits = None
+    parity   = None
+    stopbits = None
+    serial   = None
+
+    def __init__(self, owner, name, port, speed, databits, parity, stopbits):
+        import importlib
+        if RNS.vendor.platformutils.is_android():
+            self.on_android  = True
+            if importlib.util.find_spec('usbserial4a') != None:
+                if importlib.util.find_spec('jnius') == None:
+                    RNS.log("Could not load jnius API wrapper for Android, Serial interface cannot be created.", RNS.LOG_CRITICAL)
+                    RNS.log("This probably means you are trying to use an USB-based interface from within Termux or similar.", RNS.LOG_CRITICAL)
+                    RNS.log("This is currently not possible, due to this environment limiting access to the native Android APIs.", RNS.LOG_CRITICAL)
+                    RNS.panic()
+
+                from usbserial4a import serial4a as serial
+                self.parity = "N"
+            
+            else:
+                RNS.log("Could not load USB serial module for Android, Serial interface cannot be created.", RNS.LOG_CRITICAL)
+                RNS.log("You can install this module by issuing: pip install usbserial4a", RNS.LOG_CRITICAL)
+                RNS.panic()
+        else:
+            raise SystemError("Android-specific interface was used on non-Android OS")
+
+        self.rxb = 0
+        self.txb = 0
+
+        self.HW_MTU = 564
+        
+        self.pyserial = serial
+        self.serial   = None
+        self.owner    = owner
+        self.name     = name
+        self.port     = port
+        self.speed    = speed
+        self.databits = databits
+        self.parity   = "N"
+        self.stopbits = stopbits
+        self.timeout  = 100
+        self.online   = False
+        self.bitrate  = self.speed
+
+        if parity.lower() == "e" or parity.lower() == "even":
+            self.parity = "E"
+
+        if parity.lower() == "o" or parity.lower() == "odd":
+            self.parity = "O"
+
+        try:
+            self.open_port()
+        except Exception as e:
+            RNS.log("Could not open serial port for interface "+str(self), RNS.LOG_ERROR)
+            raise e
+
+        if self.serial.is_open:
+            self.configure_device()
+        else:
+            raise IOError("Could not open serial port")
+
+
+    def open_port(self):
+        RNS.log("Opening serial port "+self.port+"...")
+        # Get device parameters
+        from usb4a import usb
+        device = usb.get_usb_device(self.port)
+        if device:
+            vid = device.getVendorId()
+            pid = device.getProductId()
+
+            # Driver overrides for speficic chips
+            proxy = self.pyserial.get_serial_port
+            if vid == 0x1A86 and pid == 0x55D4:
+                # Force CDC driver for Qinheng CH34x
+                RNS.log(str(self)+" using CDC driver for "+RNS.hexrep(vid)+":"+RNS.hexrep(pid), RNS.LOG_DEBUG)
+                from usbserial4a.cdcacmserial4a import CdcAcmSerial
+                proxy = CdcAcmSerial
+
+            self.serial = proxy(
+                self.port,
+                baudrate = self.speed,
+                bytesize = self.databits,
+                parity = self.parity,
+                stopbits = self.stopbits,
+                xonxoff = False,
+                rtscts = False,
+                timeout = None,
+                inter_byte_timeout = None,
+                # write_timeout = wtimeout,
+                dsrdtr = False,
+            )
+
+            if vid == 0x0403:
+                # Hardware parameters for FTDI devices @ 115200 baud
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 16 * 1024
+                self.serial.USB_READ_TIMEOUT_MILLIS = 100
+                self.serial.timeout = 0.1
+            elif vid == 0x10C4:
+                # Hardware parameters for SiLabs CP210x @ 115200 baud
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 64 
+                self.serial.USB_READ_TIMEOUT_MILLIS = 12
+                self.serial.timeout = 0.012
+            elif vid == 0x1A86 and pid == 0x55D4:
+                # Hardware parameters for Qinheng CH34x @ 115200 baud
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 64
+                self.serial.USB_READ_TIMEOUT_MILLIS = 12
+                self.serial.timeout = 0.1
+            else:
+                # Default values
+                self.serial.DEFAULT_READ_BUFFER_SIZE = 1 * 1024
+                self.serial.USB_READ_TIMEOUT_MILLIS = 100
+                self.serial.timeout = 0.1
+
+            RNS.log(str(self)+" USB read buffer size set to "+RNS.prettysize(self.serial.DEFAULT_READ_BUFFER_SIZE), RNS.LOG_DEBUG)
+            RNS.log(str(self)+" USB read timeout set to "+str(self.serial.USB_READ_TIMEOUT_MILLIS)+"ms", RNS.LOG_DEBUG)
+            RNS.log(str(self)+" USB write timeout set to "+str(self.serial.USB_WRITE_TIMEOUT_MILLIS)+"ms", RNS.LOG_DEBUG)
+
+    def configure_device(self):
+        sleep(0.5)
+        thread = threading.Thread(target=self.readLoop)
+        thread.daemon = True
+        thread.start()
+        self.online = True
+        RNS.log("Serial port "+self.port+" is now open", RNS.LOG_VERBOSE)
+
+
+    def processIncoming(self, data):
+        self.rxb += len(data)
+        def af():
+            self.owner.inbound(data, self)
+        threading.Thread(target=af, daemon=True).start()
+
+    def processOutgoing(self,data):
+        if self.online:
+            data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
+            written = self.serial.write(data)
+            self.txb += len(data)            
+            if written != len(data):
+                raise IOError("Serial interface only wrote "+str(written)+" bytes of "+str(len(data)))
+
+    def readLoop(self):
+        try:
+            in_frame = False
+            escape = False
+            data_buffer = b""
+            last_read_ms = int(time.time()*1000)
+
+            while self.serial.is_open:
+                serial_bytes = self.serial.read()
+                got = len(serial_bytes)
+
+                for byte in serial_bytes:
+                    last_read_ms = int(time.time()*1000)
+
+                    if (in_frame and byte == HDLC.FLAG):
+                        in_frame = False
+                        self.processIncoming(data_buffer)
+                    elif (byte == HDLC.FLAG):
+                        in_frame = True
+                        data_buffer = b""
+                    elif (in_frame and len(data_buffer) < self.HW_MTU):
+                        if (byte == HDLC.ESC):
+                            escape = True
+                        else:
+                            if (escape):
+                                if (byte == HDLC.FLAG ^ HDLC.ESC_MASK):
+                                    byte = HDLC.FLAG
+                                if (byte == HDLC.ESC  ^ HDLC.ESC_MASK):
+                                    byte = HDLC.ESC
+                                escape = False
+                            data_buffer = data_buffer+bytes([byte])
+                        
+                if got == 0:
+                    time_since_last = int(time.time()*1000) - last_read_ms
+                    if len(data_buffer) > 0 and time_since_last > self.timeout:
+                        data_buffer = b""
+                        in_frame = False
+                        escape = False
+                    # sleep(0.08)
+                    
+        except Exception as e:
+            self.online = False
+            RNS.log("A serial port error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
+            RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is now offline.", RNS.LOG_ERROR)
+            
+            if RNS.Reticulum.panic_on_interface_error:
+                RNS.panic()
+
+            RNS.log("Reticulum will attempt to reconnect the interface periodically.", RNS.LOG_ERROR)
+
+        self.online = False
+        self.serial.close()
+        self.reconnect_port()
+
+    def reconnect_port(self):
+        while not self.online:
+            try:
+                time.sleep(5)
+                RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
+                self.open_port()
+                if self.serial.is_open:
+                    self.configure_device()
+            except Exception as e:
+                RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
+
+        RNS.log("Reconnected serial port for "+str(self))
+
+    def __str__(self):
+        return "SerialInterface["+self.name+"]"

RNS/Interfaces/Android/__init__.py

@@ -0,0 +1,27 @@
+# MIT License
+#
+# Copyright (c) 2016-2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import os
+import glob
+
+modules = glob.glob(os.path.dirname(__file__)+"/*.py")
+__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]

RNS/Interfaces/AutoInterface.py

@@ -48,6 +48,11 @@ class AutoInterface(Interface):
 
     BITRATE_GUESS      = 10*1000*1000
 
+    def handler_factory(self, callback):
+        def create_handler(*args, **keys):
+            return AutoInterfaceHandler(callback, *args, **keys)
+        return create_handler
+
     def __init__(self, owner, name, group_id=None, discovery_scope=None, discovery_port=None, data_port=None, allowed_interfaces=None, ignored_interfaces=None, configured_bitrate=None):
         import importlib
         if importlib.util.find_spec('netifaces') != None:
@@ -60,6 +65,9 @@ class AutoInterface(Interface):
         self.netifaces = netifaces
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 1064
+
         self.IN  = True
         self.OUT = False
         self.name = name
@@ -67,11 +75,14 @@ class AutoInterface(Interface):
         self.peers = {}
         self.link_local_addresses = []
         self.adopted_interfaces = {}
+        self.interface_servers = {}
         self.multicast_echoes = {}
         self.timed_out_interfaces = {}
+        self.carrier_changed = False
 
         self.outbound_udp_socket = None
 
+        self.announce_rate_target = None
         self.announce_interval = AutoInterface.PEERING_TIMEOUT/6.0
         self.peer_job_interval = AutoInterface.PEERING_TIMEOUT*1.1
         self.peering_timeout   = AutoInterface.PEERING_TIMEOUT
@@ -165,7 +176,8 @@ class AutoInterface(Interface):
                             # Set up multicast socket
                             discovery_socket = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM)
                             discovery_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-                            discovery_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
+                            if hasattr(socket, "SO_REUSEPORT"):
+                                discovery_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
                             discovery_socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, if_struct)
 
                             # Join multicast group
@@ -181,7 +193,7 @@ class AutoInterface(Interface):
                                 self.discovery_handler(discovery_socket, ifname)
 
                             thread = threading.Thread(target=discovery_loop)
-                            thread.setDaemon(True)
+                            thread.daemon = True
                             thread.start()
 
                             suitable_interfaces += 1
@@ -194,11 +206,6 @@ class AutoInterface(Interface):
             peering_wait = self.announce_interval*1.2
             RNS.log(str(self)+" discovering peers for "+str(round(peering_wait, 2))+" seconds...", RNS.LOG_VERBOSE)
 
-            def handlerFactory(callback):
-                def createHandler(*args, **keys):
-                    return AutoInterfaceHandler(callback, *args, **keys)
-                return createHandler
-
             self.owner = owner
             socketserver.UDPServer.address_family = socket.AF_INET6
 
@@ -207,14 +214,15 @@ class AutoInterface(Interface):
                 addr_info = socket.getaddrinfo(local_addr, self.data_port, socket.AF_INET6, socket.SOCK_DGRAM)
                 address = addr_info[0][4]
 
-                self.server = socketserver.UDPServer(address, handlerFactory(self.processIncoming))
+                udp_server = socketserver.UDPServer(address, self.handler_factory(self.processIncoming))
+                self.interface_servers[ifname] = udp_server
                 
-                thread = threading.Thread(target=self.server.serve_forever)
-                thread.setDaemon(True)
+                thread = threading.Thread(target=udp_server.serve_forever)
+                thread.daemon = True
                 thread.start()
 
             job_thread = threading.Thread(target=self.peer_jobs)
-            job_thread.setDaemon(True)
+            job_thread.daemon = True
             job_thread.start()
 
             time.sleep(peering_wait)
@@ -232,7 +240,7 @@ class AutoInterface(Interface):
             self.announce_handler(ifname)
             
         thread = threading.Thread(target=announce_loop)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
         
         while True:
@@ -262,16 +270,60 @@ class AutoInterface(Interface):
                 RNS.log(str(self)+" removed peer "+str(peer_addr)+" on "+str(removed_peer[0]), RNS.LOG_DEBUG)
 
             for ifname in self.adopted_interfaces:
+                # Check that the link-local address has not changed
+                try:
+                    addresses = self.netifaces.ifaddresses(ifname)
+                    if self.netifaces.AF_INET6 in addresses:
+                        link_local_addr = None
+                        for address in addresses[self.netifaces.AF_INET6]:
+                            if "addr" in address:
+                                if address["addr"].startswith("fe80:"):
+                                    link_local_addr = address["addr"].split("%")[0]
+                                    if link_local_addr != self.adopted_interfaces[ifname]:
+                                        old_link_local_address = self.adopted_interfaces[ifname]
+                                        RNS.log("Replacing link-local address "+str(old_link_local_address)+" for "+str(ifname)+" with "+str(link_local_addr), RNS.LOG_DEBUG)
+                                        self.adopted_interfaces[ifname] = link_local_addr
+                                        self.link_local_addresses.append(link_local_addr)
+
+                                        if old_link_local_address in self.link_local_addresses:
+                                            self.link_local_addresses.remove(old_link_local_address)
+
+                                        local_addr = link_local_addr+"%"+ifname
+                                        addr_info = socket.getaddrinfo(local_addr, self.data_port, socket.AF_INET6, socket.SOCK_DGRAM)
+                                        listen_address = addr_info[0][4]
+
+                                        if ifname in self.interface_servers:
+                                            RNS.log("Shutting down previous UDP listener for "+str(self)+" "+str(ifname), RNS.LOG_DEBUG)
+                                            previous_server = self.interface_servers[ifname]
+                                            def shutdown_server():
+                                                previous_server.shutdown()
+                                            threading.Thread(target=shutdown_server, daemon=True).start()
+
+                                        RNS.log("Starting new UDP listener for "+str(self)+" "+str(ifname), RNS.LOG_DEBUG)
+
+                                        udp_server = socketserver.UDPServer(listen_address, self.handler_factory(self.processIncoming))
+                                        self.interface_servers[ifname] = udp_server
+
+                                        thread = threading.Thread(target=udp_server.serve_forever)
+                                        thread.daemon = True
+                                        thread.start()
+
+                except Exception as e:
+                    RNS.log("Could not get device information while updating link-local addresses for "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
+
+                # Check multicast echo timeouts
                 last_multicast_echo = 0
                 if ifname in self.multicast_echoes:
                     last_multicast_echo = self.multicast_echoes[ifname]
 
                 if now - last_multicast_echo > self.multicast_echo_timeout:
                     if ifname in self.timed_out_interfaces and self.timed_out_interfaces[ifname] == False:
+                        self.carrier_changed = True
                         RNS.log("Multicast echo timeout for "+str(ifname)+". Carrier lost.", RNS.LOG_WARNING)
                     self.timed_out_interfaces[ifname] = True
                 else:
                     if ifname in self.timed_out_interfaces and self.timed_out_interfaces[ifname] == True:
+                        self.carrier_changed = True
                         RNS.log(str(self)+" Carrier recovered on "+str(ifname), RNS.LOG_WARNING)
                     self.timed_out_interfaces[ifname] = False
                 
@@ -291,6 +343,8 @@ class AutoInterface(Interface):
             ifis = struct.pack("I", socket.if_nametoindex(ifname))
             announce_socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_IF, ifis)
             announce_socket.sendto(discovery_token, addr_info[0][4])
+            announce_socket.close()
+            
         except Exception as e:
             if (ifname in self.timed_out_interfaces and self.timed_out_interfaces[ifname] == False) or not ifname in self.timed_out_interfaces:
                 RNS.log(str(self)+" Detected possible carrier loss on "+str(ifname)+": "+str(e), RNS.LOG_WARNING)
@@ -332,6 +386,7 @@ class AutoInterface(Interface):
                     peer_addr = str(peer)+"%"+str(self.peers[peer][0])
                     addr_info = socket.getaddrinfo(peer_addr, self.data_port, socket.AF_INET6, socket.SOCK_DGRAM)
                     self.outbound_udp_socket.sendto(data, addr_info[0][4])
+
                 except Exception as e:
                     RNS.log("Could not transmit on "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
 

RNS/Interfaces/I2PInterface.py

@@ -72,10 +72,12 @@ class I2PController:
 
         self.client_tunnels = {}
         self.server_tunnels = {}
+        self.i2plib_tunnels = {}
         self.loop = None
         self.i2plib = i2plib
         self.utils = i2plib.utils
         self.sam_address = i2plib.get_sam_address()
+        self.ready = False
 
         self.storagepath = rns_storagepath+"/i2p"
         if not os.path.isdir(self.storagepath):
@@ -85,17 +87,35 @@ class I2PController:
     def start(self):
         asyncio.set_event_loop(asyncio.new_event_loop())
         self.loop = asyncio.get_event_loop()
+
+        time.sleep(0.10)
+        if self.loop == None:
+            RNS.log("Could not get event loop for "+str(self)+", waiting for event loop to appear", RNS.LOG_VERBOSE)
+
+        while self.loop == None:
+            self.loop = asyncio.get_event_loop()
+            sleep(0.25)
+
         try:
+            self.ready = True
             self.loop.run_forever()
         except Exception as e:
+            self.ready = False
             RNS.log("Exception on event loop for "+str(self)+": "+str(e), RNS.LOG_ERROR)
         finally:
             self.loop.close()
 
 
     def stop(self):
-        for task in asyncio.Task.all_tasks(loop=self.loop):
-            task.cancel()
+        for i2ptunnel in self.i2plib_tunnels:
+            if hasattr(i2ptunnel, "stop") and callable(i2ptunnel.stop):
+                i2ptunnel.stop()
+
+        if hasattr(asyncio.Task, "all_tasks") and callable(asyncio.Task.all_tasks):
+            for task in asyncio.Task.all_tasks(loop=self.loop):
+                task.cancel()
+
+        time.sleep(0.2)
 
         self.loop.stop()
 
@@ -104,8 +124,13 @@ class I2PController:
         return self.i2plib.utils.get_free_port()
 
 
+    def stop_tunnel(self, i2ptunnel):
+        if hasattr(i2ptunnel, "stop") and callable(i2ptunnel.stop):
+            i2ptunnel.stop()
+
     def client_tunnel(self, owner, i2p_destination):
         self.client_tunnels[i2p_destination] = False
+        self.i2plib_tunnels[i2p_destination] = None
 
         while True:
             if not self.client_tunnels[i2p_destination]:
@@ -113,29 +138,138 @@ class I2PController:
                     async def tunnel_up():
                         RNS.log("Bringing up I2P tunnel to "+str(owner)+", this may take a while...", RNS.LOG_INFO)
                         tunnel = self.i2plib.ClientTunnel(i2p_destination, owner.local_addr, sam_address=self.sam_address, loop=self.loop)
+                        self.i2plib_tunnels[i2p_destination] = tunnel
                         await tunnel.run()
-                        owner.awaiting_i2p_tunnel = False
-                        RNS.log(str(owner)+ " tunnel setup complete", RNS.LOG_VERBOSE)
 
-                    try:
-                        self.loop.ext_owner = self
-                        future = asyncio.run_coroutine_threadsafe(tunnel_up(), self.loop).result()
-                        self.client_tunnels[i2p_destination] = True
+                    self.loop.ext_owner = self
+                    result = asyncio.run_coroutine_threadsafe(tunnel_up(), self.loop).result()
+                    
+                    if not i2p_destination in self.i2plib_tunnels:
+                        raise IOError("No tunnel control instance was created")
 
-                    except Exception as e:
-                        RNS.log("Error while setting up I2P tunnel: "+str(e))
-                        raise e
+                    else: 
+                        tn = self.i2plib_tunnels[i2p_destination]
+                        if tn != None and hasattr(tn, "status"):
 
+                            RNS.log("Waiting for status from I2P control process", RNS.LOG_EXTREME)
+                            while not tn.status["setup_ran"]:
+                                time.sleep(0.1)
+                            RNS.log("Got status from I2P control process", RNS.LOG_EXTREME)
+
+                            if tn.status["setup_failed"]:
+                                self.stop_tunnel(tn)
+                                raise tn.status["exception"]
+
+                            else:
+                                if owner.socket != None:
+                                    if hasattr(owner.socket, "close"):
+                                        if callable(owner.socket.close):
+                                            try:
+                                                owner.socket.shutdown(socket.SHUT_RDWR)
+                                            except Exception as e:
+                                                RNS.log("Error while shutting down socket for "+str(owner)+": "+str(e))
+
+                                            try:
+                                                owner.socket.close()
+                                            except Exception as e:
+                                                RNS.log("Error while closing socket for "+str(owner)+": "+str(e))
+                                self.client_tunnels[i2p_destination] = True
+                                owner.awaiting_i2p_tunnel = False
+
+                                RNS.log(str(owner)+" tunnel setup complete", RNS.LOG_VERBOSE)
+
+                        else:
+                            raise IOError("Got no status response from SAM API")
+
+                except ConnectionRefusedError as e:
+                    raise e
+                    
+                except ConnectionAbortedError as e:
+                    raise e
 
                 except Exception as e:
-                    raise IOError("Could not connect to I2P SAM API while configuring to "+str(owner)+". Check that I2P is running and SAM is enabled.")
+                    RNS.log("Unexpected error type from I2P SAM: "+str(e), RNS.LOG_ERROR)
+                    raise e
 
+            else:
+                i2ptunnel = self.i2plib_tunnels[i2p_destination]
+                if hasattr(i2ptunnel, "status"):
+                    i2p_exception = i2ptunnel.status["exception"]
+
+                    if i2ptunnel.status["setup_ran"] == False:
+                        RNS.log(str(self)+" I2P tunnel setup did not complete", RNS.LOG_ERROR)
+
+                        self.stop_tunnel(i2ptunnel)
+                        return False
+
+                    elif i2p_exception != None:
+                        RNS.log("An error ocurred while setting up I2P tunnel to "+str(i2p_destination), RNS.LOG_ERROR)
+
+                        if isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.CantReachPeer):
+                            RNS.log("The I2P daemon can't reach peer "+str(i2p_destination), RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedDest):
+                            RNS.log("The I2P daemon reported that the destination is already in use", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedId):
+                            RNS.log("The I2P daemon reported that the ID is arleady in use", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidId):
+                            RNS.log("The I2P daemon reported that the stream session ID doesn't exist", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidKey):
+                            RNS.log("The I2P daemon reported that the key for "+str(i2p_destination)+" is invalid", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.KeyNotFound):
+                            RNS.log("The I2P daemon could not find the key for "+str(i2p_destination), RNS.LOG_ERROR)
+                        
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.PeerNotFound):
+                            RNS.log("The I2P daemon mould not find the peer "+str(i2p_destination), RNS.LOG_ERROR)
+                        
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.I2PError):
+                            RNS.log("The I2P daemon experienced an unspecified error", RNS.LOG_ERROR)
+                        
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.Timeout):
+                            RNS.log("I2P daemon timed out while setting up client tunnel to "+str(i2p_destination), RNS.LOG_ERROR)
+
+                        RNS.log("Resetting I2P tunnel and retrying later", RNS.LOG_ERROR)
+
+                        self.stop_tunnel(i2ptunnel)
+                        return False
+
+                    elif i2ptunnel.status["setup_failed"] == True:
+                        RNS.log(str(self)+" Unspecified I2P tunnel setup error, resetting I2P tunnel", RNS.LOG_ERROR)
+
+                        self.stop_tunnel(i2ptunnel)
+                        return False
+
+                else:
+                    RNS.log(str(self)+" Got no status from SAM API, resetting I2P tunnel", RNS.LOG_ERROR)
+
+                    self.stop_tunnel(i2ptunnel)
+                    return False
+
+            # Wait for status from I2P control process
             time.sleep(5)
 
 
     def server_tunnel(self, owner):
-        i2p_dest_hash = RNS.Identity.full_hash(RNS.Identity.full_hash(owner.name.encode("utf-8")))
-        i2p_keyfile   = self.storagepath+"/"+RNS.hexrep(i2p_dest_hash, delimit=False)+".i2p"
+        while RNS.Transport.identity == None:
+            time.sleep(1)
+
+        # Old format
+        i2p_dest_hash_of = RNS.Identity.full_hash(RNS.Identity.full_hash(owner.name.encode("utf-8")))
+        i2p_keyfile_of   = self.storagepath+"/"+RNS.hexrep(i2p_dest_hash_of, delimit=False)+".i2p"
+
+        # New format
+        i2p_dest_hash_nf = RNS.Identity.full_hash(RNS.Identity.full_hash(owner.name.encode("utf-8"))+RNS.Identity.full_hash(RNS.Transport.identity.hash))
+        i2p_keyfile_nf   = self.storagepath+"/"+RNS.hexrep(i2p_dest_hash_nf, delimit=False)+".i2p"
+
+        # Use old format if a key is already present
+        if os.path.isfile(i2p_keyfile_of):
+            i2p_keyfile = i2p_keyfile_of
+        else:
+            i2p_keyfile = i2p_keyfile_nf
 
         i2p_dest = None
         if not os.path.isfile(i2p_keyfile):
@@ -154,20 +288,82 @@ class I2PController:
         owner.b32 = i2p_b32
 
         self.server_tunnels[i2p_b32] = False
+        self.i2plib_tunnels[i2p_b32] = None
 
-        while self.server_tunnels[i2p_b32] == False:
-            try:
-                async def tunnel_up():
-                    RNS.log(str(owner)+" Bringing up I2P endpoint, this may take a while...", RNS.LOG_INFO)
-                    tunnel = self.i2plib.ServerTunnel((owner.bind_ip, owner.bind_port), loop=self.loop, destination=i2p_dest, sam_address=self.sam_address)
-                    await tunnel.run()
-                    RNS.log(str(owner)+ " endpoint setup complete. Now reachable at: "+str(i2p_dest.base32)+".b32.i2p", RNS.LOG_VERBOSE)
+        while True:
+            if self.server_tunnels[i2p_b32] == False:
+                try:
+                    async def tunnel_up():
+                        RNS.log(str(owner)+" Bringing up I2P endpoint, this may take a while...", RNS.LOG_INFO)
+                        tunnel = self.i2plib.ServerTunnel((owner.bind_ip, owner.bind_port), loop=self.loop, destination=i2p_dest, sam_address=self.sam_address)
+                        self.i2plib_tunnels[i2p_b32] = tunnel
+                        await tunnel.run()
+                        owner.online = True
+                        RNS.log(str(owner)+ " endpoint setup complete. Now reachable at: "+str(i2p_dest.base32)+".b32.i2p", RNS.LOG_VERBOSE)
 
-                asyncio.run_coroutine_threadsafe(tunnel_up(), self.loop).result()
-                self.server_tunnels[i2p_b32] = True
+                    asyncio.run_coroutine_threadsafe(tunnel_up(), self.loop).result()
+                    self.server_tunnels[i2p_b32] = True
 
-            except Exception as e:
-                raise IOError("Could not connect to I2P SAM API while configuring "+str(self)+". Check that I2P is running and SAM is enabled.")
+                except Exception as e:
+                    raise e
+
+            else:
+                i2ptunnel = self.i2plib_tunnels[i2p_b32]
+                if hasattr(i2ptunnel, "status"):
+                    i2p_exception = i2ptunnel.status["exception"]
+
+                    if i2ptunnel.status["setup_ran"] == False:
+                        RNS.log(str(self)+" I2P tunnel setup did not complete", RNS.LOG_ERROR)
+
+                        self.stop_tunnel(i2ptunnel)
+                        return False
+
+                    elif i2p_exception != None:
+                        RNS.log("An error ocurred while setting up I2P tunnel", RNS.LOG_ERROR)
+                        
+                        if isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.CantReachPeer):
+                            RNS.log("The I2P daemon can't reach peer "+str(i2p_destination), RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedDest):
+                            RNS.log("The I2P daemon reported that the destination is already in use", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.DuplicatedId):
+                            RNS.log("The I2P daemon reported that the ID is arleady in use", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidId):
+                            RNS.log("The I2P daemon reported that the stream session ID doesn't exist", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.InvalidKey):
+                            RNS.log("The I2P daemon reported that the key for "+str(i2p_destination)+" is invalid", RNS.LOG_ERROR)
+
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.KeyNotFound):
+                            RNS.log("The I2P daemon could not find the key for "+str(i2p_destination), RNS.LOG_ERROR)
+                        
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.PeerNotFound):
+                            RNS.log("The I2P daemon mould not find the peer "+str(i2p_destination), RNS.LOG_ERROR)
+                        
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.I2PError):
+                            RNS.log("The I2P daemon experienced an unspecified error", RNS.LOG_ERROR)
+                        
+                        elif isinstance(i2p_exception, RNS.vendor.i2plib.exceptions.Timeout):
+                            RNS.log("I2P daemon timed out while setting up client tunnel to "+str(i2p_destination), RNS.LOG_ERROR)
+
+                        RNS.log("Resetting I2P tunnel and retrying later", RNS.LOG_ERROR)
+
+                        self.stop_tunnel(i2ptunnel)
+                        return False
+
+                    elif i2ptunnel.status["setup_failed"] == True:
+                        RNS.log(str(self)+" Unspecified I2P tunnel setup error, resetting I2P tunnel", RNS.LOG_ERROR)
+
+                        self.stop_tunnel(i2ptunnel)
+                        return False
+
+                else:
+                    RNS.log(str(self)+" Got no status from SAM API, resetting I2P tunnel", RNS.LOG_ERROR)
+
+                    self.stop_tunnel(i2ptunnel)
+                    return False
 
             time.sleep(5)
 
@@ -183,14 +379,21 @@ class I2PInterfacePeer(Interface):
     RECONNECT_MAX_TRIES = None
 
     # TCP socket options
-    I2P_USER_TIMEOUT = 40
+    I2P_USER_TIMEOUT = 45
     I2P_PROBE_AFTER = 10
-    I2P_PROBE_INTERVAL = 5
-    I2P_PROBES = 6
+    I2P_PROBE_INTERVAL = 9
+    I2P_PROBES = 5
+    I2P_READ_TIMEOUT = (I2P_PROBE_INTERVAL * I2P_PROBES + I2P_PROBE_AFTER)*2
+
+    TUNNEL_STATE_INIT    = 0x00
+    TUNNEL_STATE_ACTIVE  = 0x01
+    TUNNEL_STATE_STALE   = 0x02
 
     def __init__(self, parent_interface, owner, name, target_i2p_dest=None, connected_socket=None, max_reconnect_tries=None):
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 1064
         
         self.IN               = True
         self.OUT              = False
@@ -211,6 +414,34 @@ class I2PInterfacePeer(Interface):
         self.i2p_tunnel_ready = False
         self.mode             = RNS.Interfaces.Interface.Interface.MODE_FULL
         self.bitrate          = I2PInterface.BITRATE_GUESS
+        self.last_read        = 0
+        self.last_write       = 0
+        self.wd_reset         = False
+        self.i2p_tunnel_state = I2PInterfacePeer.TUNNEL_STATE_INIT
+
+        self.ifac_size = self.parent_interface.ifac_size
+        self.ifac_netname = self.parent_interface.ifac_netname
+        self.ifac_netkey = self.parent_interface.ifac_netkey
+        if self.ifac_netname != None or self.ifac_netkey != None:
+            ifac_origin = b""
+            if self.ifac_netname != None:
+                ifac_origin += RNS.Identity.full_hash(self.ifac_netname.encode("utf-8"))
+            if self.ifac_netkey != None:
+                ifac_origin += RNS.Identity.full_hash(self.ifac_netkey.encode("utf-8"))
+
+            ifac_origin_hash = RNS.Identity.full_hash(ifac_origin)
+            self.ifac_key = RNS.Cryptography.hkdf(
+                length=64,
+                derive_from=ifac_origin_hash,
+                salt=RNS.Reticulum.IFAC_SALT,
+                context=None
+            )
+            self.ifac_identity = RNS.Identity.from_bytes(self.ifac_key)
+            self.ifac_signature = self.ifac_identity.sign(RNS.Identity.full_hash(self.ifac_key))
+
+        self.announce_rate_target  = None
+        self.announce_rate_grace   = None
+        self.announce_rate_penalty = None
 
         if max_reconnect_tries == None:
             self.max_reconnect_tries = I2PInterfacePeer.RECONNECT_MAX_TRIES
@@ -233,54 +464,59 @@ class I2PInterfacePeer(Interface):
             self.initiator   = True
 
             self.bind_ip     = "127.0.0.1"
-            self.bind_port   = self.parent_interface.i2p.get_free_port()
-            self.local_addr  = (self.bind_ip, self.bind_port)
-            self.target_ip = self.bind_ip
-            self.target_port = self.bind_port
 
             self.awaiting_i2p_tunnel = True
 
             def tunnel_job():
-                self.parent_interface.i2p.client_tunnel(self, target_i2p_dest)
+                while self.awaiting_i2p_tunnel:
+                    try:
+                        self.bind_port   = self.parent_interface.i2p.get_free_port()
+                        self.local_addr  = (self.bind_ip, self.bind_port)
+                        self.target_ip = self.bind_ip
+                        self.target_port = self.bind_port
+
+                        if not self.parent_interface.i2p.client_tunnel(self, target_i2p_dest):
+                            RNS.log(str(self)+" I2P control process experienced an error, requesting new tunnel...", RNS.LOG_ERROR)
+                            self.awaiting_i2p_tunnel = True
+
+                    except Exception as e:
+                        RNS.log("Error while while configuring "+str(self)+": "+str(e), RNS.LOG_ERROR)
+                        RNS.log("Check that I2P is installed and running, and that SAM is enabled. Retrying tunnel setup later.", RNS.LOG_ERROR)
+
+                    time.sleep(8)
 
             thread = threading.Thread(target=tunnel_job)
-            thread.setDaemon(True)
+            thread.daemon = True
             thread.start()
 
             def wait_job():
                 while self.awaiting_i2p_tunnel:
                     time.sleep(0.25)
+                time.sleep(2)
                 
                 if not self.kiss_framing:
                     self.wants_tunnel = True
 
                 if not self.connect(initial=True):
                     thread = threading.Thread(target=self.reconnect)
-                    thread.setDaemon(True)
+                    thread.daemon = True
                     thread.start()
                 else:
                     thread = threading.Thread(target=self.read_loop)
-                    thread.setDaemon(True)
+                    thread.daemon = True
                     thread.start()
 
             thread = threading.Thread(target=wait_job)
-            thread.setDaemon(True)
+            thread.daemon = True
             thread.start()
 
 
     def set_timeouts_linux(self):
-        if not self.i2p_tunneled:
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(I2PInterfacePeer.TCP_USER_TIMEOUT * 1000))
-            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, int(I2PInterfacePeer.TCP_PROBE_AFTER))
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, int(I2PInterfacePeer.TCP_PROBE_INTERVAL))
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, int(I2PInterfacePeer.TCP_PROBES))
-        else:
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(I2PInterfacePeer.I2P_USER_TIMEOUT * 1000))
-            self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, int(I2PInterfacePeer.I2P_PROBE_AFTER))
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, int(I2PInterfacePeer.I2P_PROBE_INTERVAL))
-            self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, int(I2PInterfacePeer.I2P_PROBES))
+        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(I2PInterfacePeer.I2P_USER_TIMEOUT * 1000))
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, int(I2PInterfacePeer.I2P_PROBE_AFTER))
+        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, int(I2PInterfacePeer.I2P_PROBE_INTERVAL))
+        self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, int(I2PInterfacePeer.I2P_PROBES))
 
     def set_timeouts_osx(self):
         if hasattr(socket, "TCP_KEEPALIVE"):
@@ -289,17 +525,27 @@ class I2PInterfacePeer(Interface):
             TCP_KEEPIDLE = 0x10
 
         self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
-        
-        if not self.i2p_tunneled:
-            self.socket.setsockopt(socket.IPPROTO_TCP, TCP_KEEPIDLE, int(I2PInterfacePeer.TCP_PROBE_AFTER))
-        else:
-            self.socket.setsockopt(socket.IPPROTO_TCP, TCP_KEEPIDLE, int(I2PInterfacePeer.I2P_PROBE_AFTER))
-        
+        self.socket.setsockopt(socket.IPPROTO_TCP, TCP_KEEPIDLE, int(I2PInterfacePeer.I2P_PROBE_AFTER))
+    
+    def shutdown_socket(self, target_socket):
+        if callable(target_socket.close):
+            try:
+                if socket != None:
+                    target_socket.shutdown(socket.SHUT_RDWR)
+            except Exception as e:
+                RNS.log("Error while shutting down socket for "+str(self)+": "+str(e))
+
+            try:
+                if socket != None:
+                    target_socket.close()
+            except Exception as e:
+                RNS.log("Error while closing socket for "+str(self)+": "+str(e))    
+    
     def detach(self):
+        RNS.log("Detaching "+str(self), RNS.LOG_DEBUG)
         if self.socket != None:
             if hasattr(self.socket, "close"):
                 if callable(self.socket.close):
-                    RNS.log("Detaching "+str(self), RNS.LOG_DEBUG)
                     self.detached = True
                     
                     try:
@@ -345,7 +591,6 @@ class I2PInterfacePeer(Interface):
 
         return True
 
-
     def reconnect(self):
         if self.initiator:
             if not self.reconnecting:
@@ -374,7 +619,7 @@ class I2PInterfacePeer(Interface):
 
                 self.reconnecting = False
                 thread = threading.Thread(target=self.read_loop)
-                thread.setDaemon(True)
+                thread.daemon = True
                 thread.start()
                 if not self.kiss_framing:
                     RNS.Transport.synthesize_tunnel(self)
@@ -393,7 +638,7 @@ class I2PInterfacePeer(Interface):
     def processOutgoing(self, data):
         if self.online:
             while self.writing:
-                time.sleep(0.01)
+                time.sleep(0.001)
 
             try:
                 self.writing = True
@@ -406,6 +651,8 @@ class I2PInterfacePeer(Interface):
                 self.socket.sendall(data)
                 self.writing = False
                 self.txb += len(data)
+                self.last_write = time.time()
+                
                 if hasattr(self, "parent_interface") and self.parent_interface != None and self.parent_count:
                     self.parent_interface.txb += len(data)
 
@@ -415,8 +662,59 @@ class I2PInterfacePeer(Interface):
                 self.teardown()
 
 
+    def read_watchdog(self):
+        while self.wd_reset:
+            time.sleep(0.25)
+
+        should_run = True
+        try:
+            while should_run and not self.wd_reset:
+                time.sleep(1)
+
+                if (time.time()-self.last_read > I2PInterfacePeer.I2P_PROBE_AFTER*2):
+                    if self.i2p_tunnel_state != I2PInterfacePeer.TUNNEL_STATE_STALE:
+                        RNS.log("I2P tunnel became unresponsive", RNS.LOG_DEBUG)
+
+                    self.i2p_tunnel_state = I2PInterfacePeer.TUNNEL_STATE_STALE
+                else:
+                    self.i2p_tunnel_state = I2PInterfacePeer.TUNNEL_STATE_ACTIVE
+
+                if (time.time()-self.last_write > I2PInterfacePeer.I2P_PROBE_AFTER*1):
+                    try:
+                        if self.socket != None:
+                            self.socket.sendall(bytes([HDLC.FLAG, HDLC.FLAG]))
+                    except Exception as e:
+                        RNS.log("An error ocurred while sending I2P keepalive. The contained exception was: "+str(e), RNS.LOG_ERROR)
+                        self.shutdown_socket(self.socket)
+                        should_run = False
+                
+                if (time.time()-self.last_read > I2PInterfacePeer.I2P_READ_TIMEOUT):
+                    RNS.log("I2P socket is unresponsive, restarting...", RNS.LOG_WARNING)
+                    if self.socket != None:
+                        try:
+                            self.socket.shutdown(socket.SHUT_RDWR)
+                        except Exception as e:
+                            RNS.log("Error while shutting down socket for "+str(self)+": "+str(e))
+
+                        try:
+                            self.socket.close()
+                        except Exception as e:
+                            RNS.log("Error while closing socket for "+str(self)+": "+str(e))
+
+                    should_run = False
+
+                self.wd_reset = False
+
+        finally:
+            self.wd_reset = False
+
     def read_loop(self):
         try:
+            self.last_read  = time.time()
+            self.last_write = time.time()
+
+            wd_thread = threading.Thread(target=self.read_watchdog, daemon=True).start()
+
             in_frame = False
             escape = False
             data_buffer = b""
@@ -426,6 +724,7 @@ class I2PInterfacePeer(Interface):
                 data_in = self.socket.recv(4096)
                 if len(data_in) > 0:
                     pointer = 0
+                    self.last_read = time.time()
                     while pointer < len(data_in):
                         byte = data_in[pointer]
                         pointer += 1
@@ -439,7 +738,7 @@ class I2PInterfacePeer(Interface):
                                 in_frame = True
                                 command = KISS.CMD_UNKNOWN
                                 data_buffer = b""
-                            elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                            elif (in_frame and len(data_buffer) < self.HW_MTU):
                                 if (len(data_buffer) == 0 and command == KISS.CMD_UNKNOWN):
                                     # We only support one HDLC port for now, so
                                     # strip off the port nibble
@@ -465,7 +764,7 @@ class I2PInterfacePeer(Interface):
                             elif (byte == HDLC.FLAG):
                                 in_frame = True
                                 data_buffer = b""
-                            elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                            elif (in_frame and len(data_buffer) < self.HW_MTU):
                                 if (byte == HDLC.ESC):
                                     escape = True
                                 else:
@@ -478,6 +777,11 @@ class I2PInterfacePeer(Interface):
                                     data_buffer = data_buffer+bytes([byte])
                 else:
                     self.online = False
+
+                    self.wd_reset = True
+                    time.sleep(2)
+                    self.wd_reset = False
+
                     if self.initiator and not self.detached:
                         RNS.log("Socket for "+str(self)+" was closed, attempting to reconnect...", RNS.LOG_WARNING)
                         self.reconnect()
@@ -512,7 +816,8 @@ class I2PInterfacePeer(Interface):
         self.IN = False
 
         if hasattr(self, "parent_interface") and self.parent_interface != None:
-            self.parent_interface.clients -= 1
+            if self.parent_interface.clients > 0:
+                self.parent_interface.clients -= 1
 
         if self in RNS.Transport.interfaces:
             if not self.initiator:
@@ -526,9 +831,12 @@ class I2PInterfacePeer(Interface):
 class I2PInterface(Interface):
     BITRATE_GUESS      = 256*1000
 
-    def __init__(self, owner, name, rns_storagepath, peers, connectable = True):
+    def __init__(self, owner, name, rns_storagepath, peers, connectable = False, ifac_size = 16, ifac_netname = None, ifac_netkey = None):
         self.rxb = 0
         self.txb = 0
+        
+        self.HW_MTU = 1064
+
         self.online = False
         self.clients = 0
         self.owner = owner
@@ -549,11 +857,29 @@ class I2PInterface(Interface):
         self.bind_port   = self.i2p.get_free_port()
         self.address = (self.bind_ip, self.bind_port)
         self.bitrate = I2PInterface.BITRATE_GUESS
+        self.ifac_size = ifac_size
+        self.ifac_netname = ifac_netname
+        self.ifac_netkey = ifac_netkey
+
+        self.online = False
 
         i2p_thread = threading.Thread(target=self.i2p.start)
-        i2p_thread.setDaemon(True)
+        i2p_thread.daemon = True
         i2p_thread.start()
 
+        i2p_notready_warning = False
+        time.sleep(0.25)
+
+        if not self.i2p.ready:
+            RNS.log("I2P controller did not become available in time, waiting for controller", RNS.LOG_VERBOSE)
+            i2p_notready_warning = True
+
+        while not self.i2p.ready:
+            time.sleep(0.25)
+
+        if i2p_notready_warning == True:
+            RNS.log("I2P controller ready, continuing setup", RNS.LOG_VERBOSE)
+
         def handlerFactory(callback):
             def createHandler(*args, **keys):
                 return I2PInterfaceHandler(callback, *args, **keys)
@@ -563,20 +889,31 @@ class I2PInterface(Interface):
         self.server = ThreadingI2PServer(self.address, handlerFactory(self.incoming_connection))
 
         thread = threading.Thread(target=self.server.serve_forever)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
 
         if self.connectable:
             def tunnel_job():
-                self.i2p.server_tunnel(self)
+                while True:
+                    try:
+                        if not self.i2p.server_tunnel(self):
+                            RNS.log(str(self)+" I2P control process experienced an error, requesting new tunnel...", RNS.LOG_ERROR)
+                            self.online = False
+
+                    except Exception as e:
+                        RNS.log("Error while while configuring "+str(self)+": "+str(e), RNS.LOG_ERROR)
+                        RNS.log("Check that I2P is installed and running, and that SAM is enabled. Retrying tunnel setup later.", RNS.LOG_ERROR)
+
+                    time.sleep(15)
+
 
             thread = threading.Thread(target=tunnel_job)
-            thread.setDaemon(True)
+            thread.daemon = True
             thread.start()
 
         if peers != None:
             for peer_addr in peers:
-                interface_name = peer_addr
+                interface_name = self.name+" to "+peer_addr
                 peer_interface = I2PInterfacePeer(self, self.owner, interface_name, peer_addr)
                 peer_interface.OUT = True
                 peer_interface.IN  = True
@@ -584,9 +921,6 @@ class I2PInterface(Interface):
                 peer_interface.parent_count = False
                 RNS.Transport.interfaces.append(peer_interface)
 
-        self.online = True
-
-
     def incoming_connection(self, handler):
         RNS.log("Accepting incoming I2P connection", RNS.LOG_VERBOSE)
         interface_name = "Connected peer on "+self.name
@@ -596,12 +930,32 @@ class I2PInterface(Interface):
         spawned_interface.parent_interface = self
         spawned_interface.online = True
         spawned_interface.bitrate = self.bitrate
+
         spawned_interface.ifac_size = self.ifac_size
         spawned_interface.ifac_netname = self.ifac_netname
         spawned_interface.ifac_netkey = self.ifac_netkey
+        if spawned_interface.ifac_netname != None or spawned_interface.ifac_netkey != None:
+            ifac_origin = b""
+            if spawned_interface.ifac_netname != None:
+                ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netname.encode("utf-8"))
+            if spawned_interface.ifac_netkey != None:
+                ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netkey.encode("utf-8"))
+
+            ifac_origin_hash = RNS.Identity.full_hash(ifac_origin)
+            spawned_interface.ifac_key = RNS.Cryptography.hkdf(
+                length=64,
+                derive_from=ifac_origin_hash,
+                salt=RNS.Reticulum.IFAC_SALT,
+                context=None
+            )
+            spawned_interface.ifac_identity = RNS.Identity.from_bytes(spawned_interface.ifac_key)
+            spawned_interface.ifac_signature = spawned_interface.ifac_identity.sign(RNS.Identity.full_hash(spawned_interface.ifac_key))
+
         spawned_interface.announce_rate_target = self.announce_rate_target
         spawned_interface.announce_rate_grace = self.announce_rate_grace
         spawned_interface.announce_rate_penalty = self.announce_rate_penalty
+        spawned_interface.mode = self.mode
+        spawned_interface.HW_MTU = self.HW_MTU
         RNS.log("Spawned new I2PInterface Peer: "+str(spawned_interface), RNS.LOG_VERBOSE)
         RNS.Transport.interfaces.append(spawned_interface)
         self.clients += 1
@@ -611,6 +965,7 @@ class I2PInterface(Interface):
         pass
 
     def detach(self):
+        RNS.log("Detaching "+str(self), RNS.LOG_DEBUG)
         self.i2p.stop()
 
     def __str__(self):

RNS/Interfaces/Interface.py

@@ -31,11 +31,17 @@ class Interface:
     RPT = False
     name = None
 
+    # Interface mode definitions
     MODE_FULL           = 0x01
     MODE_POINT_TO_POINT = 0x02
     MODE_ACCESS_POINT   = 0x03
     MODE_ROAMING        = 0x04
     MODE_BOUNDARY       = 0x05
+    MODE_GATEWAY        = 0x06
+
+    # Which interface modes a Transport Node
+    # should actively discover paths for.
+    DISCOVER_PATHS_FOR  = [MODE_ACCESS_POINT, MODE_GATEWAY]
 
     def __init__(self):
         self.rxb = 0

RNS/Interfaces/KISSInterface.py

@@ -73,6 +73,8 @@ class KISSInterface(Interface):
         self.rxb = 0
         self.txb = 0
         
+        self.HW_MTU = 564
+        
         if beacon_data == None:
             beacon_data = ""
 
@@ -142,7 +144,7 @@ class KISSInterface(Interface):
         # Allow time for interface to initialise before config
         sleep(2.0)
         thread = threading.Thread(target=self.readLoop)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
         self.online = True
         RNS.log("Serial port "+self.port+" is now open")
@@ -279,7 +281,7 @@ class KISSInterface(Interface):
                         in_frame = True
                         command = KISS.CMD_UNKNOWN
                         data_buffer = b""
-                    elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                    elif (in_frame and len(data_buffer) < self.HW_MTU):
                         if (len(data_buffer) == 0 and command == KISS.CMD_UNKNOWN):
                             # We only support one HDLC port for now, so
                             # strip off the port nibble

RNS/Interfaces/LocalInterface.py

@@ -49,6 +49,12 @@ class LocalClientInterface(Interface):
     def __init__(self, owner, name, target_port = None, connected_socket=None):
         self.rxb = 0
         self.txb = 0
+
+        # TODO: Remove at some point
+        # self.rxptime = 0
+        
+        self.HW_MTU = 1064
+
         self.online  = False
         
         self.IN               = True
@@ -86,7 +92,7 @@ class LocalClientInterface(Interface):
 
         if connected_socket == None:
             thread = threading.Thread(target=self.read_loop)
-            thread.setDaemon(True)
+            thread.daemon = True
             thread.start()
 
     def connect(self):
@@ -117,11 +123,11 @@ class LocalClientInterface(Interface):
                         RNS.log("Connection attempt for "+str(self)+" failed: "+str(e), RNS.LOG_DEBUG)
 
                 if not self.never_connected:
-                    RNS.log("Reconnected TCP socket for "+str(self)+".", RNS.LOG_INFO)
+                    RNS.log("Reconnected socket for "+str(self)+".", RNS.LOG_INFO)
 
                 self.reconnecting = False
                 thread = threading.Thread(target=self.read_loop)
-                thread.setDaemon(True)
+                thread.daemon = True
                 thread.start()
                 RNS.Transport.shared_connection_reappeared()
         
@@ -134,15 +140,18 @@ class LocalClientInterface(Interface):
         self.rxb += len(data)
         if hasattr(self, "parent_interface") and self.parent_interface != None:
             self.parent_interface.rxb += len(data)
-            
+        
+        # TODO: Remove at some point
+        # processing_start = time.time()
+        
         self.owner.inbound(data, self)
 
+        # TODO: Remove at some point
+        # duration = time.time() - processing_start
+        # self.rxptime += duration
 
     def processOutgoing(self, data):
         if self.online:
-            while self.writing:
-                time.sleep(0.01)
-
             try:
                 self.writing = True
                 data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
@@ -177,7 +186,7 @@ class LocalClientInterface(Interface):
                         elif (byte == HDLC.FLAG):
                             in_frame = True
                             data_buffer = b""
-                        elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                        elif (in_frame and len(data_buffer) < self.HW_MTU):
                             if (byte == HDLC.ESC):
                                 escape = True
                             else:
@@ -237,6 +246,7 @@ class LocalClientInterface(Interface):
             RNS.Transport.local_client_interfaces.remove(self)
             if hasattr(self, "parent_interface") and self.parent_interface != None:
                 self.parent_interface.clients -= 1
+                RNS.Transport.owner._should_persist_data()
 
         if nowarning == False:
             RNS.log("The interface "+str(self)+" experienced an unrecoverable error and is being torn down. Restart Reticulum to attempt to open this interface again.", RNS.LOG_ERROR)
@@ -286,7 +296,7 @@ class LocalServerInterface(Interface):
             self.server = ThreadingTCPServer(address, handlerFactory(self.incoming_connection))
 
             thread = threading.Thread(target=self.server.serve_forever)
-            thread.setDaemon(True)
+            thread.daemon = True
             thread.start()
 
             self.announce_rate_target  = None

RNS/Interfaces/PipeInterface.py

@@ -56,6 +56,8 @@ class PipeInterface(Interface):
 
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 1064
         
         self.owner    = owner
         self.name     = name
@@ -94,7 +96,7 @@ class PipeInterface(Interface):
     def configure_pipe(self):
         sleep(0.01)
         thread = threading.Thread(target=self.readLoop)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
         self.online = True
         RNS.log("Subprocess pipe for "+str(self)+" is now connected", RNS.LOG_VERBOSE)
@@ -137,7 +139,7 @@ class PipeInterface(Interface):
                     elif (byte == HDLC.FLAG):
                         in_frame = True
                         data_buffer = b""
-                    elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                    elif (in_frame and len(data_buffer) < self.HW_MTU):
                         if (byte == HDLC.ESC):
                             escape = True
                         else:

RNS/Interfaces/RNodeInterface.py

@@ -44,6 +44,7 @@ class KISS():
     CMD_RADIO_STATE = 0x06
     CMD_RADIO_LOCK  = 0x07
     CMD_DETECT      = 0x08
+    CMD_LEAVE       = 0x0A
     CMD_READY       = 0x0F
     CMD_STAT_RX     = 0x21
     CMD_STAT_TX     = 0x22
@@ -51,6 +52,9 @@ class KISS():
     CMD_STAT_SNR    = 0x24
     CMD_BLINK       = 0x30
     CMD_RANDOM      = 0x40
+    CMD_FB_EXT      = 0x41
+    CMD_FB_READ     = 0x42
+    CMD_FB_WRITE    = 0x43
     CMD_PLATFORM    = 0x48
     CMD_MCU         = 0x49
     CMD_FW_VERSION  = 0x50
@@ -82,14 +86,6 @@ class KISS():
 class RNodeInterface(Interface):
     MAX_CHUNK = 32768
 
-    owner    = None
-    port     = None
-    speed    = None
-    databits = None
-    parity   = None
-    stopbits = None
-    serial   = None
-
     FREQ_MIN = 137000000
     FREQ_MAX = 1020000000
 
@@ -98,9 +94,14 @@ class RNodeInterface(Interface):
     CALLSIGN_MAX_LEN    = 32
 
     REQUIRED_FW_VER_MAJ = 1
-    REQUIRED_FW_VER_MIN = 26
+    REQUIRED_FW_VER_MIN = 52
+
+    RECONNECT_WAIT = 5
 
     def __init__(self, owner, name, port, frequency = None, bandwidth = None, txpower = None, sf = None, cr = None, flow_control = False, id_interval = None, id_callsign = None):
+        if RNS.vendor.platformutils.is_android():
+            raise SystemError("Invlaid interface type. The Android-specific RNode interface must be used on Android")
+
         import importlib
         if importlib.util.find_spec('serial') != None:
             import serial
@@ -111,6 +112,8 @@ class RNodeInterface(Interface):
 
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 508
         
         self.pyserial    = serial
         self.serial      = None
@@ -119,7 +122,6 @@ class RNodeInterface(Interface):
         self.port        = port
         self.speed       = 115200
         self.databits    = 8
-        self.parity      = serial.PARITY_NONE
         self.stopbits    = 1
         self.timeout     = 100
         self.online      = False
@@ -132,6 +134,7 @@ class RNodeInterface(Interface):
         self.state       = KISS.RADIO_STATE_OFF
         self.bitrate     = 0
         self.platform    = None
+        self.display     = None
         self.mcu         = None
         self.detected    = False
         self.firmware_ok = False
@@ -140,6 +143,7 @@ class RNodeInterface(Interface):
 
         self.last_id     = 0
         self.first_tx    = None
+        self.reconnect_w = RNodeInterface.RECONNECT_WAIT
 
         self.r_frequency = None
         self.r_bandwidth = None
@@ -156,6 +160,7 @@ class RNodeInterface(Interface):
         self.packet_queue    = []
         self.flow_control    = flow_control
         self.interface_ready = False
+        self.announce_rate_target = None
 
         self.validcfg  = True
         if (self.frequency < RNodeInterface.FREQ_MIN or self.frequency > RNodeInterface.FREQ_MAX):
@@ -195,14 +200,20 @@ class RNodeInterface(Interface):
 
         try:
             self.open_port()
+
+            if self.serial.is_open:
+                self.configure_device()
+            else:
+                raise IOError("Could not open serial port")
+
         except Exception as e:
             RNS.log("Could not open serial port for interface "+str(self), RNS.LOG_ERROR)
-            raise e
+            RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+            RNS.log("Reticulum will attempt to bring up this interface periodically", RNS.LOG_ERROR)
+            thread = threading.Thread(target=self.reconnect_port)
+            thread.daemon = True
+            thread.start()
 
-        if self.serial.is_open:
-            self.configure_device()
-        else:
-            raise IOError("Could not open serial port")
 
     def open_port(self):
         RNS.log("Opening serial port "+self.port+"...")
@@ -210,7 +221,7 @@ class RNodeInterface(Interface):
             port = self.port,
             baudrate = self.speed,
             bytesize = self.databits,
-            parity = self.parity,
+            parity = self.pyserial.PARITY_NONE,
             stopbits = self.stopbits,
             xonxoff = False,
             rtscts = False,
@@ -224,27 +235,26 @@ class RNodeInterface(Interface):
     def configure_device(self):
         sleep(2.0)
         thread = threading.Thread(target=self.readLoop)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
 
         self.detect()
-        sleep(0.1)
+        sleep(0.2)
         
         if not self.detected:
             raise IOError("Could not detect device")
         else:
             if self.platform == KISS.PLATFORM_ESP32:
-                RNS.log("Resetting ESP32-based device before configuration...", RNS.LOG_VERBOSE)
-                self.hard_reset()
+                self.display = True
 
-        self.online = True
         RNS.log("Serial port "+self.port+" is now open")
         RNS.log("Configuring RNode interface...", RNS.LOG_VERBOSE)
         self.initRadio()
         if (self.validateRadioState()):
             self.interface_ready = True
             RNS.log(str(self)+" is configured and powered up")
-            sleep(1.0)
+            sleep(0.3)
+            self.online = True
         else:
             RNS.log("After configuring "+str(self)+", the reported radio parameters did not match your configuration.", RNS.LOG_ERROR)
             RNS.log("Make sure that your hardware actually supports the parameters specified in the configuration", RNS.LOG_ERROR)
@@ -265,7 +275,51 @@ class RNodeInterface(Interface):
         kiss_command = bytes([KISS.FEND, KISS.CMD_DETECT, KISS.DETECT_REQ, KISS.FEND, KISS.CMD_FW_VERSION, 0x00, KISS.FEND, KISS.CMD_PLATFORM, 0x00, KISS.FEND, KISS.CMD_MCU, 0x00, KISS.FEND])
         written = self.serial.write(kiss_command)
         if written != len(kiss_command):
-            raise IOError("An IO error occurred while detecting hardware for "+self(str))
+            raise IOError("An IO error occurred while detecting hardware for "+str(self))
+    
+    def leave(self):
+        kiss_command = bytes([KISS.FEND, KISS.CMD_LEAVE, 0xFF, KISS.FEND])
+        written = self.serial.write(kiss_command)
+        if written != len(kiss_command):
+            raise IOError("An IO error occurred while sending host left command to device")
+    
+    def enable_external_framebuffer(self):
+        if self.display != None:
+            kiss_command = bytes([KISS.FEND, KISS.CMD_FB_EXT, 0x01, KISS.FEND])
+            written = self.serial.write(kiss_command)
+            if written != len(kiss_command):
+                raise IOError("An IO error occurred while enabling external framebuffer on device")
+
+    def disable_external_framebuffer(self):
+        if self.display != None:
+            kiss_command = bytes([KISS.FEND, KISS.CMD_FB_EXT, 0x00, KISS.FEND])
+            written = self.serial.write(kiss_command)
+            if written != len(kiss_command):
+                raise IOError("An IO error occurred while disabling external framebuffer on device")
+
+    FB_PIXEL_WIDTH     = 64
+    FB_BITS_PER_PIXEL  = 1
+    FB_PIXELS_PER_BYTE = 8//FB_BITS_PER_PIXEL
+    FB_BYTES_PER_LINE  = FB_PIXEL_WIDTH//FB_PIXELS_PER_BYTE
+    def display_image(self, imagedata):
+        if self.display != None:
+            lines = len(imagedata)//8
+            for line in range(lines):
+                line_start = line*RNodeInterface.FB_BYTES_PER_LINE
+                line_end   = line_start+RNodeInterface.FB_BYTES_PER_LINE
+                line_data = bytes(imagedata[line_start:line_end])
+                self.write_framebuffer(line, line_data)
+
+    def write_framebuffer(self, line, line_data):
+        if self.display != None:
+            line_byte = line.to_bytes(1, byteorder="big", signed=False)
+            data = line_byte+line_data
+            escaped_data = KISS.escape(data)
+            kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_FB_WRITE])+escaped_data+bytes([KISS.FEND])
+            
+            written = self.serial.write(kiss_command)
+            if written != len(kiss_command):
+                raise IOError("An IO error occurred while writing framebuffer data device")
 
     def hard_reset(self):
         kiss_command = bytes([KISS.FEND, KISS.CMD_RESET, 0xf8, KISS.FEND])
@@ -284,7 +338,7 @@ class RNodeInterface(Interface):
         kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_FREQUENCY])+data+bytes([KISS.FEND])
         written = self.serial.write(kiss_command)
         if written != len(kiss_command):
-            raise IOError("An IO error occurred while configuring frequency for "+self(str))
+            raise IOError("An IO error occurred while configuring frequency for "+str(self))
 
     def setBandwidth(self):
         c1 = self.bandwidth >> 24
@@ -296,35 +350,35 @@ class RNodeInterface(Interface):
         kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_BANDWIDTH])+data+bytes([KISS.FEND])
         written = self.serial.write(kiss_command)
         if written != len(kiss_command):
-            raise IOError("An IO error occurred while configuring bandwidth for "+self(str))
+            raise IOError("An IO error occurred while configuring bandwidth for "+str(self))
 
     def setTXPower(self):
         txp = bytes([self.txpower])
         kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_TXPOWER])+txp+bytes([KISS.FEND])
         written = self.serial.write(kiss_command)
         if written != len(kiss_command):
-            raise IOError("An IO error occurred while configuring TX power for "+self(str))
+            raise IOError("An IO error occurred while configuring TX power for "+str(self))
 
     def setSpreadingFactor(self):
         sf = bytes([self.sf])
         kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_SF])+sf+bytes([KISS.FEND])
         written = self.serial.write(kiss_command)
         if written != len(kiss_command):
-            raise IOError("An IO error occurred while configuring spreading factor for "+self(str))
+            raise IOError("An IO error occurred while configuring spreading factor for "+str(self))
 
     def setCodingRate(self):
         cr = bytes([self.cr])
         kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_CR])+cr+bytes([KISS.FEND])
         written = self.serial.write(kiss_command)
         if written != len(kiss_command):
-            raise IOError("An IO error occurred while configuring coding rate for "+self(str))
+            raise IOError("An IO error occurred while configuring coding rate for "+str(self))
 
     def setRadioState(self, state):
         self.state = state
         kiss_command = bytes([KISS.FEND])+bytes([KISS.CMD_RADIO_STATE])+bytes([state])+bytes([KISS.FEND])
         written = self.serial.write(kiss_command)
         if written != len(kiss_command):
-            raise IOError("An IO error occurred while configuring radio state for "+self(str))
+            raise IOError("An IO error occurred while configuring radio state for "+str(self))
 
     def validate_firmware(self):
         if (self.maj_version >= RNodeInterface.REQUIRED_FW_VER_MAJ):
@@ -336,14 +390,16 @@ class RNodeInterface(Interface):
 
         RNS.log("The firmware version of the connected RNode is "+str(self.maj_version)+"."+str(self.min_version), RNS.LOG_ERROR)
         RNS.log("This version of Reticulum requires at least version "+str(RNodeInterface.REQUIRED_FW_VER_MAJ)+"."+str(RNodeInterface.REQUIRED_FW_VER_MIN), RNS.LOG_ERROR)
-        RNS.log("Please update your RNode firmware with rnodeconf (https://github.com/markqvist/rnodeconfigutil/)")
+        RNS.log("Please update your RNode firmware with rnodeconf from https://github.com/markqvist/rnodeconfigutil/")
         RNS.panic()
 
 
     def validateRadioState(self):
         RNS.log("Wating for radio configuration validation for "+str(self)+"...", RNS.LOG_VERBOSE)
         sleep(0.25);
-        if (self.frequency != self.r_frequency):
+
+        self.validcfg = True
+        if (self.r_frequency != None and abs(self.frequency - int(self.r_frequency)) > 100):
             RNS.log("Frequency mismatch", RNS.LOG_ERROR)
             self.validcfg = False
         if (self.bandwidth != self.r_bandwidth):
@@ -374,7 +430,7 @@ class RNodeInterface(Interface):
             self.bitrate = 0
 
     def processIncoming(self, data):
-        self.rxb += len(data)            
+        self.rxb += len(data)
         self.owner.inbound(data, self)
         self.r_stat_rssi = None
         self.r_stat_snr = None
@@ -439,7 +495,7 @@ class RNodeInterface(Interface):
                         command = KISS.CMD_UNKNOWN
                         data_buffer = b""
                         command_buffer = b""
-                    elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                    elif (in_frame and len(data_buffer) < self.HW_MTU):
                         if (len(data_buffer) == 0 and command == KISS.CMD_UNKNOWN):
                             command = byte
                         elif (command == KISS.CMD_DATA):
@@ -618,7 +674,7 @@ class RNodeInterface(Interface):
     def reconnect_port(self):
         while not self.online:
             try:
-                time.sleep(3.5)
+                time.sleep(5)
                 RNS.log("Attempting to reconnect serial port "+str(self.port)+" for "+str(self)+"...", RNS.LOG_VERBOSE)
                 self.open_port()
                 if self.serial.is_open:
@@ -626,8 +682,13 @@ class RNodeInterface(Interface):
             except Exception as e:
                 RNS.log("Error while reconnecting port, the contained exception was: "+str(e), RNS.LOG_ERROR)
 
-        RNS.log("Reconnected serial port for "+str(self))
+        if self.online:
+            RNS.log("Reconnected serial port for "+str(self))
+
+    def detach(self):
+        self.disable_external_framebuffer()
+        self.setRadioState(KISS.RADIO_STATE_OFF)
+        self.leave()
 
     def __str__(self):
         return "RNodeInterface["+str(self.name)+"]"
-

RNS/Interfaces/SerialInterface.py

@@ -62,6 +62,8 @@ class SerialInterface(Interface):
 
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 564
         
         self.pyserial = serial
         self.serial   = None
@@ -114,7 +116,7 @@ class SerialInterface(Interface):
     def configure_device(self):
         sleep(0.5)
         thread = threading.Thread(target=self.readLoop)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
         self.online = True
         RNS.log("Serial port "+self.port+" is now open", RNS.LOG_VERBOSE)
@@ -152,7 +154,7 @@ class SerialInterface(Interface):
                     elif (byte == HDLC.FLAG):
                         in_frame = True
                         data_buffer = b""
-                    elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                    elif (in_frame and len(data_buffer) < self.HW_MTU):
                         if (byte == HDLC.ESC):
                             escape = True
                         else:

RNS/Interfaces/TCPInterface.py

@@ -65,20 +65,25 @@ class TCPClientInterface(Interface):
     RECONNECT_MAX_TRIES = None
 
     # TCP socket options
-    TCP_USER_TIMEOUT = 20
+    TCP_USER_TIMEOUT = 24
     TCP_PROBE_AFTER = 5
-    TCP_PROBE_INTERVAL = 3
-    TCP_PROBES = 5
+    TCP_PROBE_INTERVAL = 2
+    TCP_PROBES = 12
 
-    I2P_USER_TIMEOUT = 40
+    INITIAL_CONNECT_TIMEOUT = 5
+    SYNCHRONOUS_START = True
+
+    I2P_USER_TIMEOUT = 45
     I2P_PROBE_AFTER = 10
-    I2P_PROBE_INTERVAL = 5
-    I2P_PROBES = 6
+    I2P_PROBE_INTERVAL = 9
+    I2P_PROBES = 5
 
-    def __init__(self, owner, name, target_ip=None, target_port=None, connected_socket=None, max_reconnect_tries=None, kiss_framing=False, i2p_tunneled = False):
+    def __init__(self, owner, name, target_ip=None, target_port=None, connected_socket=None, max_reconnect_tries=None, kiss_framing=False, i2p_tunneled = False, connect_timeout = None):
         self.rxb = 0
         self.txb = 0
         
+        self.HW_MTU = 1064
+        
         self.IN               = True
         self.OUT              = False
         self.socket           = None
@@ -117,18 +122,30 @@ class TCPClientInterface(Interface):
             self.target_ip   = target_ip
             self.target_port = target_port
             self.initiator   = True
-            
-            if not self.connect(initial=True):
-                thread = threading.Thread(target=self.reconnect)
-                thread.setDaemon(True)
-                thread.start()
-            else:
-                thread = threading.Thread(target=self.read_loop)
-                thread.setDaemon(True)
-                thread.start()
-                if not self.kiss_framing:
-                    self.wants_tunnel = True
 
+            if connect_timeout != None:
+                self.connect_timeout = connect_timeout
+            else:
+                self.connect_timeout = TCPClientInterface.INITIAL_CONNECT_TIMEOUT
+            
+            if TCPClientInterface.SYNCHRONOUS_START:
+                self.initial_connect()
+            else:
+                thread = threading.Thread(target=self.initial_connect)
+                thread.daemon = True
+                thread.start()
+            
+    def initial_connect(self):
+        if not self.connect(initial=True):
+            thread = threading.Thread(target=self.reconnect)
+            thread.daemon = True
+            thread.start()
+        else:
+            thread = threading.Thread(target=self.read_loop)
+            thread.daemon = True
+            thread.start()
+            if not self.kiss_framing:
+                self.wants_tunnel = True
 
     def set_timeouts_linux(self):
         if not self.i2p_tunneled:
@@ -137,6 +154,7 @@ class TCPClientInterface(Interface):
             self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, int(TCPClientInterface.TCP_PROBE_AFTER))
             self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, int(TCPClientInterface.TCP_PROBE_INTERVAL))
             self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, int(TCPClientInterface.TCP_PROBES))
+
         else:
             self.socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_USER_TIMEOUT, int(TCPClientInterface.I2P_USER_TIMEOUT * 1000))
             self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
@@ -178,9 +196,17 @@ class TCPClientInterface(Interface):
 
     def connect(self, initial=False):
         try:
+            if initial:
+                RNS.log("Establishing TCP connection for "+str(self)+"...", RNS.LOG_DEBUG)
+
             self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.socket.settimeout(TCPClientInterface.INITIAL_CONNECT_TIMEOUT)
             self.socket.connect((self.target_ip, self.target_port))
+            self.socket.settimeout(None)
             self.online  = True
+
+            if initial:
+                RNS.log("TCP connection for "+str(self)+" established", RNS.LOG_DEBUG)
         
         except Exception as e:
             if initial:
@@ -224,11 +250,11 @@ class TCPClientInterface(Interface):
                         RNS.log("Connection attempt for "+str(self)+" failed: "+str(e), RNS.LOG_DEBUG)
 
                 if not self.never_connected:
-                    RNS.log("Reconnected TCP socket for "+str(self)+".", RNS.LOG_INFO)
+                    RNS.log("Reconnected socket for "+str(self)+".", RNS.LOG_INFO)
 
                 self.reconnecting = False
                 thread = threading.Thread(target=self.read_loop)
-                thread.setDaemon(True)
+                thread.daemon = True
                 thread.start()
                 if not self.kiss_framing:
                     RNS.Transport.synthesize_tunnel(self)
@@ -246,8 +272,8 @@ class TCPClientInterface(Interface):
 
     def processOutgoing(self, data):
         if self.online:
-            while self.writing:
-                time.sleep(0.01)
+            # while self.writing:
+            #     time.sleep(0.01)
 
             try:
                 self.writing = True
@@ -293,7 +319,7 @@ class TCPClientInterface(Interface):
                                 in_frame = True
                                 command = KISS.CMD_UNKNOWN
                                 data_buffer = b""
-                            elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                            elif (in_frame and len(data_buffer) < self.HW_MTU):
                                 if (len(data_buffer) == 0 and command == KISS.CMD_UNKNOWN):
                                     # We only support one HDLC port for now, so
                                     # strip off the port nibble
@@ -319,7 +345,7 @@ class TCPClientInterface(Interface):
                             elif (byte == HDLC.FLAG):
                                 in_frame = True
                                 data_buffer = b""
-                            elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                            elif (in_frame and len(data_buffer) < self.HW_MTU):
                                 if (byte == HDLC.ESC):
                                     escape = True
                                 else:
@@ -333,10 +359,10 @@ class TCPClientInterface(Interface):
                 else:
                     self.online = False
                     if self.initiator and not self.detached:
-                        RNS.log("TCP socket for "+str(self)+" was closed, attempting to reconnect...", RNS.LOG_WARNING)
+                        RNS.log("The socket for "+str(self)+" was closed, attempting to reconnect...", RNS.LOG_WARNING)
                         self.reconnect()
                     else:
-                        RNS.log("TCP socket for remote client "+str(self)+" was closed.", RNS.LOG_VERBOSE)
+                        RNS.log("The socket for remote client "+str(self)+" was closed.", RNS.LOG_VERBOSE)
                         self.teardown()
 
                     break
@@ -405,12 +431,16 @@ class TCPServerInterface(Interface):
     def __init__(self, owner, name, device=None, bindip=None, bindport=None, i2p_tunneled=False):
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 1064
+
         self.online = False
         self.clients = 0
         
         self.IN  = True
         self.OUT = False
         self.name = name
+        self.detached = False
 
         self.i2p_tunneled = i2p_tunneled
         self.mode         = RNS.Interfaces.Interface.Interface.MODE_FULL
@@ -437,7 +467,7 @@ class TCPServerInterface(Interface):
             self.bitrate = TCPServerInterface.BITRATE_GUESS
 
             thread = threading.Thread(target=self.server.serve_forever)
-            thread.setDaemon(True)
+            thread.daemon = True
             thread.start()
 
             self.online = True
@@ -453,12 +483,32 @@ class TCPServerInterface(Interface):
         spawned_interface.target_port = str(handler.client_address[1])
         spawned_interface.parent_interface = self
         spawned_interface.bitrate = self.bitrate
+        
         spawned_interface.ifac_size = self.ifac_size
         spawned_interface.ifac_netname = self.ifac_netname
         spawned_interface.ifac_netkey = self.ifac_netkey
+        if spawned_interface.ifac_netname != None or spawned_interface.ifac_netkey != None:
+            ifac_origin = b""
+            if spawned_interface.ifac_netname != None:
+                ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netname.encode("utf-8"))
+            if spawned_interface.ifac_netkey != None:
+                ifac_origin += RNS.Identity.full_hash(spawned_interface.ifac_netkey.encode("utf-8"))
+
+            ifac_origin_hash = RNS.Identity.full_hash(ifac_origin)
+            spawned_interface.ifac_key = RNS.Cryptography.hkdf(
+                length=64,
+                derive_from=ifac_origin_hash,
+                salt=RNS.Reticulum.IFAC_SALT,
+                context=None
+            )
+            spawned_interface.ifac_identity = RNS.Identity.from_bytes(spawned_interface.ifac_key)
+            spawned_interface.ifac_signature = spawned_interface.ifac_identity.sign(RNS.Identity.full_hash(spawned_interface.ifac_key))
+
         spawned_interface.announce_rate_target = self.announce_rate_target
         spawned_interface.announce_rate_grace = self.announce_rate_grace
         spawned_interface.announce_rate_penalty = self.announce_rate_penalty
+        spawned_interface.mode = self.mode
+        spawned_interface.HW_MTU = self.HW_MTU
         spawned_interface.online = True
         RNS.log("Spawned new TCPClient Interface: "+str(spawned_interface), RNS.LOG_VERBOSE)
         RNS.Transport.interfaces.append(spawned_interface)
@@ -468,9 +518,25 @@ class TCPServerInterface(Interface):
     def processOutgoing(self, data):
         pass
 
+
+    def detach(self):
+        if self.server != None:
+            if hasattr(self.server, "shutdown"):
+                if callable(self.server.shutdown):
+                    try:
+                        RNS.log("Detaching "+str(self), RNS.LOG_DEBUG)
+                        self.server.shutdown()
+                        self.detached = True
+                        self.server = None
+
+                    except Exception as e:
+                        RNS.log("Error while shutting down server for "+str(self)+": "+str(e))
+
+
     def __str__(self):
         return "TCPServerInterface["+self.name+"/"+self.bind_ip+":"+str(self.bind_port)+"]"
 
+
 class TCPInterfaceHandler(socketserver.BaseRequestHandler):
     def __init__(self, callback, *args, **keys):
         self.callback = callback

RNS/Interfaces/UDPInterface.py

@@ -57,6 +57,9 @@ class UDPInterface(Interface):
     def __init__(self, owner, name, device=None, bindip=None, bindport=None, forwardip=None, forwardport=None):
         self.rxb = 0
         self.txb = 0
+
+        self.HW_MTU = 1064
+
         self.IN  = True
         self.OUT = False
         self.name = name
@@ -86,7 +89,7 @@ class UDPInterface(Interface):
             self.server = socketserver.UDPServer(address, handlerFactory(self.processIncoming))
 
             thread = threading.Thread(target=self.server.serve_forever)
-            thread.setDaemon(True)
+            thread.daemon = True
             thread.start()
 
             self.online = True

RNS/Interfaces/__init__.py

@@ -22,6 +22,7 @@
 
 import os
 import glob
+import RNS.Interfaces.Android
 
 modules = glob.glob(os.path.dirname(__file__)+"/*.py")
-__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]
+__all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]

RNS/Link.py

@@ -20,24 +20,16 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
-from cryptography.hazmat.primitives import serialization
-from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey, Ed25519PublicKey
-from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey
-from cryptography.hazmat.primitives.kdf.hkdf import HKDF
-from cryptography.fernet import Fernet
+from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey
+from RNS.Cryptography import Fernet
+
 from time import sleep
 from .vendor import umsgpack as umsgpack
 import threading
-import base64
 import math
 import time
 import RNS
 
-import traceback
-
-cio_default_backend = default_backend()
 
 class LinkCallbacks:
     def __init__(self):
@@ -53,7 +45,7 @@ class Link:
     """
     This class is used to establish and manage links to other peers. When a
     link instance is created, Reticulum will attempt to establish verified
-    connectivity with the specified destination.
+    and encrypted connectivity with the specified destination.
 
     :param destination: A :ref:`RNS.Destination<api-destination>` instance which to establish a link to.
     :param established_callback: An optional function or method with the signature *callback(link)* to be called when the link has been established.
@@ -67,20 +59,34 @@ class Link:
     ECPUBSIZE         = 32+32
     KEYSIZE           = 32
 
-    MDU = math.floor((RNS.Reticulum.MTU-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.OPTIMISED_FERNET_OVERHEAD)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1
+    MDU = math.floor((RNS.Reticulum.MTU-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.FERNET_OVERHEAD)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1
 
     ESTABLISHMENT_TIMEOUT_PER_HOP = RNS.Reticulum.DEFAULT_PER_HOP_TIMEOUT
     """
-    Default timeout for link establishment in seconds per hop to destination.
+    Timeout for link establishment in seconds per hop to destination.
     """
 
     TRAFFIC_TIMEOUT_FACTOR = 6
     KEEPALIVE_TIMEOUT_FACTOR = 4
+    """
+    RTT timeout factor used in link timeout calculation.
+    """
     STALE_GRACE = 2
+    """
+    Grace period in seconds used in link timeout calculation.
+    """
     KEEPALIVE = 360
     """
     Interval for sending keep-alive packets on established links in seconds.
     """
+    STALE_TIME = 2*KEEPALIVE
+    """
+    If no traffic or keep-alive packets are received within this period, the
+    link will be marked as stale, and a final keep-alive packet will be sent.
+    If after this no traffic or keep-alive packets are received within ``RTT`` *
+    ``KEEPALIVE_TIMEOUT_FACTOR`` + ``STALE_GRACE``, the link is considered timed out,
+    and will be torn down.
+    """
 
     PENDING   = 0x00
     HANDSHAKE = 0x01
@@ -105,6 +111,7 @@ class Link:
                 link.set_link_id(packet)
                 link.destination = packet.destination
                 link.establishment_timeout = Link.ESTABLISHMENT_TIMEOUT_PER_HOP * max(1, packet.hops)
+                link.establishment_cost += len(packet.raw)
                 RNS.log("Validating link request "+RNS.prettyhexrep(link.link_id), RNS.LOG_VERBOSE)
                 link.handshake()
                 link.attached_interface = packet.receiving_interface
@@ -131,6 +138,7 @@ class Link:
         if destination != None and destination.type != RNS.Destination.SINGLE:
             raise TypeError("Links can only be established to the \"single\" destination type")
         self.rtt = None
+        self.establishment_cost = 0
         self.callbacks = LinkCallbacks()
         self.resource_strategy = Link.ACCEPT_NONE
         self.outgoing_resources = []
@@ -145,6 +153,7 @@ class Link:
         self.traffic_timeout_factor = Link.TRAFFIC_TIMEOUT_FACTOR
         self.keepalive_timeout_factor = Link.KEEPALIVE_TIMEOUT_FACTOR
         self.keepalive = Link.KEEPALIVE
+        self.stale_time = Link.STALE_TIME
         self.watchdog_lock = False
         self.status = Link.PENDING
         self.activated_at = None
@@ -155,7 +164,7 @@ class Link:
         self.__remote_identity = None
         if self.destination == None:
             self.initiator = False
-            self.prv     = self.owner.identity.prv
+            self.prv     = X25519PrivateKey.generate()
             self.sig_prv = self.owner.identity.sig_prv
         else:
             self.initiator = True
@@ -166,16 +175,10 @@ class Link:
         self.fernet  = None
         
         self.pub = self.prv.public_key()
-        self.pub_bytes = self.pub.public_bytes(
-            encoding=serialization.Encoding.Raw,
-            format=serialization.PublicFormat.Raw
-        )
+        self.pub_bytes = self.pub.public_bytes()
 
         self.sig_pub = self.sig_prv.public_key()
-        self.sig_pub_bytes = self.sig_pub.public_bytes(
-            encoding=serialization.Encoding.Raw,
-            format=serialization.PublicFormat.Raw
-        )
+        self.sig_pub_bytes = self.sig_pub.public_bytes()
 
         if peer_pub_bytes == None:
             self.peer_pub = None
@@ -190,14 +193,11 @@ class Link:
             self.set_link_closed_callback(closed_callback)
 
         if (self.initiator):
-            peer_pub_bytes = self.destination.identity.get_public_key()[:Link.ECPUBSIZE//2]
-            peer_sig_pub_bytes = self.destination.identity.get_public_key()[Link.ECPUBSIZE//2:Link.ECPUBSIZE]
             self.request_data = self.pub_bytes+self.sig_pub_bytes
             self.packet = RNS.Packet(destination, self.request_data, packet_type=RNS.Packet.LINKREQUEST)
             self.packet.pack()
+            self.establishment_cost += len(self.packet.raw)
             self.set_link_id(self.packet)
-            self.load_peer(peer_pub_bytes, peer_sig_pub_bytes)
-            self.handshake()
             RNS.Transport.register_link(self)
             self.request_time = time.time()
             self.start_watchdog()
@@ -224,22 +224,22 @@ class Link:
         self.status = Link.HANDSHAKE
         self.shared_key = self.prv.exchange(self.peer_pub)
 
-        # TODO: Improve this re-allocation of HKDF
-        self.derived_key = HKDF(
-            algorithm=hashes.SHA256(),
+        self.derived_key = RNS.Cryptography.hkdf(
             length=32,
+            derive_from=self.shared_key,
             salt=self.get_salt(),
-            info=self.get_context(),
-            backend=cio_default_backend,
-        ).derive(self.shared_key)
+            context=self.get_context(),
+        )
+
 
     def prove(self):
         signed_data = self.link_id+self.pub_bytes+self.sig_pub_bytes
         signature = self.owner.identity.sign(signed_data)
 
-        proof_data = signature
+        proof_data = signature+self.pub_bytes
         proof = RNS.Packet(self, proof_data, packet_type=RNS.Packet.PROOF, context=RNS.Packet.LRPROOF)
         proof.send()
+        self.establishment_cost += len(proof.raw)
         self.had_outbound()
 
 
@@ -257,8 +257,14 @@ class Link:
         self.had_outbound()
 
     def validate_proof(self, packet):
-        if self.status == Link.HANDSHAKE:
-            if self.initiator and len(packet.data) == RNS.Identity.SIGLENGTH//8:
+        if self.status == Link.PENDING:
+            if self.initiator and len(packet.data) == RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2:
+                peer_pub_bytes = packet.data[RNS.Identity.SIGLENGTH//8:RNS.Identity.SIGLENGTH//8+Link.ECPUBSIZE//2]
+                peer_sig_pub_bytes = self.destination.identity.get_public_key()[Link.ECPUBSIZE//2:Link.ECPUBSIZE]
+                self.load_peer(peer_pub_bytes, peer_sig_pub_bytes)
+                self.handshake()
+
+                self.establishment_cost += len(packet.raw)
                 signed_data = self.link_id+self.peer_pub_bytes+self.peer_sig_pub_bytes
                 signature = packet.data[:RNS.Identity.SIGLENGTH//8]
                 
@@ -266,6 +272,8 @@ class Link:
                     self.rtt = time.time() - self.request_time
                     self.attached_interface = packet.receiving_interface
                     self.__remote_identity = self.destination.identity
+                    self.status = Link.ACTIVE
+                    self.activated_at = time.time()
                     RNS.Transport.activate_link(self)
                     RNS.log("Link "+str(self)+" established with "+str(self.destination)+", RTT is "+str(round(self.rtt, 3))+"s", RNS.LOG_VERBOSE)
                     rtt_data = umsgpack.packb(self.rtt)
@@ -273,11 +281,9 @@ class Link:
                     rtt_packet.send()
                     self.had_outbound()
 
-                    self.status = Link.ACTIVE
-                    self.activated_at = time.time()
                     if self.callbacks.link_established != None:
                         thread = threading.Thread(target=self.callbacks.link_established, args=(self,))
-                        thread.setDaemon(True)
+                        thread.daemon = True
                         thread.start()
                 else:
                     RNS.log("Invalid link proof signature received by "+str(self)+". Ignoring.", RNS.LOG_DEBUG)
@@ -334,7 +340,8 @@ class Link:
                     response_callback = response_callback,
                     failed_callback = failed_callback,
                     progress_callback = progress_callback,
-                    timeout = timeout
+                    timeout = timeout,
+                    request_size = len(packed_request),
                 )
             
         else:
@@ -348,7 +355,8 @@ class Link:
                 response_callback = response_callback,
                 failed_callback = failed_callback,
                 progress_callback = progress_callback,
-                timeout = timeout
+                timeout = timeout,
+                request_size = len(packed_request),
             )
 
 
@@ -383,7 +391,9 @@ class Link:
         """
         :returns: The time in seconds since last inbound packet on the link.
         """
-        return time.time() - self.last_inbound
+        activated_at = self.activated_at if self.activated_at != None else 0
+        last_inbound = max(self.last_inbound, activated_at)
+        return time.time() - last_inbound
 
     def no_outbound_for(self):
         """
@@ -399,7 +409,7 @@ class Link:
 
     def get_remote_identity(self):
         """
-        :returns: The identity of the remote peer, if it is known
+        :returns: The identity of the remote peer, if it is known. Calling this method will not query the remote initiator to reveal its identity. Returns ``None`` if the link initiator has not already independently called the ``identify(identity)`` method.
         """
         return self.__remote_identity
 
@@ -461,7 +471,7 @@ class Link:
 
     def start_watchdog(self):
         thread = threading.Thread(target=self.__watchdog_job)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
 
     def __watchdog_job(self):
@@ -497,13 +507,21 @@ class Link:
                         sleep_time = 0.001
 
                 elif self.status == Link.ACTIVE:
-                    if time.time() >= self.last_inbound + self.keepalive:
-                        sleep_time = self.rtt * self.keepalive_timeout_factor + Link.STALE_GRACE
-                        self.status = Link.STALE
+                    activated_at = self.activated_at if self.activated_at != None else 0
+                    last_inbound = max(self.last_inbound, activated_at)
+
+                    if time.time() >= last_inbound + self.keepalive:
                         if self.initiator:
                             self.send_keepalive()
+
+                        if time.time() >= last_inbound + self.stale_time:
+                            sleep_time = self.rtt * self.keepalive_timeout_factor + Link.STALE_GRACE
+                            self.status = Link.STALE
+                        else:
+                            sleep_time = self.keepalive
+                    
                     else:
-                        sleep_time = (self.last_inbound + self.keepalive) - time.time()
+                        sleep_time = (last_inbound + self.keepalive) - time.time()
 
                 elif self.status == Link.STALE:
                     sleep_time = 0.001
@@ -543,7 +561,7 @@ class Link:
                 allowed = False
                 if not allow == RNS.Destination.ALLOW_NONE:
                     if allow == RNS.Destination.ALLOW_LIST:
-                        if self.__remote_identity.hash in allowed_list:
+                        if self.__remote_identity != None and self.__remote_identity.hash in allowed_list:
                             allowed = True
                     elif allow == RNS.Destination.ALLOW_ALL:
                         allowed = True
@@ -623,7 +641,7 @@ class Link:
                         plaintext = self.decrypt(packet.data)
                         if self.callbacks.packet != None:
                             thread = threading.Thread(target=self.callbacks.packet, args=(plaintext, packet))
-                            thread.setDaemon(True)
+                            thread.daemon = True
                             thread.start()
                         
                         if self.destination.proof_strategy == RNS.Destination.PROVE_ALL:
@@ -650,7 +668,7 @@ class Link:
                                 self.__remote_identity = identity
                                 if self.callbacks.remote_identified != None:
                                     try:
-                                        self.callbacks.remote_identified(self.__remote_identity)
+                                        self.callbacks.remote_identified(self, self.__remote_identity)
                                     except Exception as e:
                                         RNS.log("Error while executing remote identified callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
 
@@ -687,19 +705,21 @@ class Link:
                         if RNS.ResourceAdvertisement.is_request(packet):
                             RNS.Resource.accept(packet, callback=self.request_resource_concluded)
                         elif RNS.ResourceAdvertisement.is_response(packet):
-                            request_id = RNS.ResourceAdvertisement.get_request_id(packet)
+                            request_id = RNS.ResourceAdvertisement.read_request_id(packet)
                             for pending_request in self.pending_requests:
                                 if pending_request.request_id == request_id:
                                     RNS.Resource.accept(packet, callback=self.response_resource_concluded, progress_callback=pending_request.response_resource_progress, request_id = request_id)
-                                    pending_request.response_size = RNS.ResourceAdvertisement.get_size(packet)
-                                    pending_request.response_transfer_size = RNS.ResourceAdvertisement.get_transfer_size(packet)
+                                    pending_request.response_size = RNS.ResourceAdvertisement.read_size(packet)
+                                    pending_request.response_transfer_size = RNS.ResourceAdvertisement.read_transfer_size(packet)
                                     pending_request.started_at = time.time()
                         elif self.resource_strategy == Link.ACCEPT_NONE:
                             pass
                         elif self.resource_strategy == Link.ACCEPT_APP:
                             if self.callbacks.resource != None:
                                 try:
-                                    if self.callbacks.resource(resource):
+                                    resource_advertisement = RNS.ResourceAdvertisement.unpack(packet.plaintext)
+                                    resource_advertisement.link = self
+                                    if self.callbacks.resource(resource_advertisement):
                                         RNS.Resource.accept(packet, self.callbacks.resource_concluded)
                                 except Exception as e:
                                     RNS.log("Error while executing resource accept callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
@@ -712,6 +732,7 @@ class Link:
                             resource_hash = plaintext[1+RNS.Resource.MAPHASH_LEN:RNS.Identity.HASHLENGTH//8+1+RNS.Resource.MAPHASH_LEN]
                         else:
                             resource_hash = plaintext[1:RNS.Identity.HASHLENGTH//8+1]
+
                         for resource in self.outgoing_resources:
                             if resource.hash == resource_hash:
                                 # We need to check that this request has not been
@@ -763,21 +784,12 @@ class Link:
         try:
             if not self.fernet:
                 try:
-                    self.fernet = Fernet(base64.urlsafe_b64encode(self.derived_key))
+                    self.fernet = Fernet(self.derived_key)
                 except Exception as e:
                     RNS.log("Could not "+str(self)+" instantiate Fernet while performin encryption on link. The contained exception was: "+str(e), RNS.LOG_ERROR)
                     raise e
 
-            # The fernet token VERSION field is stripped here and
-            # reinserted on the receiving end, since it is always
-            # set to 0x80.
-            #
-            # Since we're also quite content with supporting time-
-            # stamps until the year 8921556 AD, we'll also strip 2
-            # bytes from the timestamp field and reinsert those as
-            # 0x00 when received.
-            ciphertext = base64.urlsafe_b64decode(self.fernet.encrypt(plaintext))[3:]
-            return ciphertext
+            return self.fernet.encrypt(plaintext)
 
         except Exception as e:
             RNS.log("Encryption on link "+str(self)+" failed. The contained exception was: "+str(e), RNS.LOG_ERROR)
@@ -787,15 +799,12 @@ class Link:
     def decrypt(self, ciphertext):
         try:
             if not self.fernet:
-                self.fernet = Fernet(base64.urlsafe_b64encode(self.derived_key))
+                self.fernet = Fernet(self.derived_key)
                 
-            plaintext = self.fernet.decrypt(base64.urlsafe_b64encode(bytes([RNS.Identity.FERNET_VERSION, 0x00, 0x00])+ciphertext))
-            return plaintext
+            return self.fernet.decrypt(ciphertext)
+
         except Exception as e:
             RNS.log("Decryption failed on link "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
-            # RNS.log(traceback.format_exc(), RNS.LOG_ERROR)
-            # TODO: Think long about implications here
-            # self.teardown()
 
 
     def sign(self, message):
@@ -812,6 +821,12 @@ class Link:
         self.callbacks.link_established = callback
 
     def set_link_closed_callback(self, callback):
+        """
+        Registers a function to be called when a link has been
+        torn down.
+
+        :param callback: A function or method with the signature *callback(link)* to be called.
+        """
         self.callbacks.link_closed = callback
 
     def set_packet_callback(self, callback):
@@ -830,7 +845,7 @@ class Link:
         the resource will be accepted. If it returns *False* it will
         be ignored.
 
-        :param callback: A function or method with the signature *callback(resource)* to be called.
+        :param callback: A function or method with the signature *callback(resource)* to be called. Please note that only the basic information of the resource is available at this time, such as *get_transfer_size()*, *get_data_size()*, *get_parts()* and *is_compressed()*.
         """
         self.callbacks.resource = callback
 
@@ -857,7 +872,7 @@ class Link:
         Registers a function to be called when an initiating peer has
         identified over this link.
 
-        :param callback: A function or method with the signature *callback(identity)* to be called.
+        :param callback: A function or method with the signature *callback(link, identity)* to be called.
         """
         self.callbacks.remote_identified = callback
 
@@ -885,6 +900,13 @@ class Link:
     def register_incoming_resource(self, resource):
         self.incoming_resources.append(resource)
 
+    def has_incoming_resource(self, resource):
+        for incoming_resource in self.incoming_resources:
+            if incoming_resource.hash == resource.hash:
+                return True
+
+        return False
+
     def cancel_outgoing_resource(self, resource):
         if resource in self.outgoing_resources:
             self.outgoing_resources.remove(resource)
@@ -920,7 +942,7 @@ class RequestReceipt():
     RECEIVING = 0x03
     READY     = 0x04
 
-    def __init__(self, link, packet_receipt = None, resource = None, response_callback = None, failed_callback = None, progress_callback = None, timeout = None):
+    def __init__(self, link, packet_receipt = None, resource = None, response_callback = None, failed_callback = None, progress_callback = None, timeout = None, request_size = None):
         self.packet_receipt = packet_receipt
         self.resource = resource
         self.started_at = None
@@ -936,6 +958,7 @@ class RequestReceipt():
         
         self.link                   = link
         self.request_id             = self.hash
+        self.request_size           = request_size
 
         self.response               = None
         self.response_transfer_size = None
@@ -966,7 +989,7 @@ class RequestReceipt():
             self.status = RequestReceipt.DELIVERED
             self.__resource_response_timeout = time.time()+self.timeout
             response_timeout_thread = threading.Thread(target=self.__response_timeout_job)
-            response_timeout_thread.setDaemon(True)
+            response_timeout_thread.daemon = True
             response_timeout_thread.start()
         else:
             RNS.log("Sending request "+RNS.prettyhexrep(self.request_id)+" as resource failed with status: "+RNS.hexrep([resource.status]), RNS.LOG_DEBUG)

RNS/Packet.py

@@ -29,15 +29,19 @@ import RNS
 class Packet:
     """
     The Packet class is used to create packet instances that can be sent
-    over a Reticulum network. Packets to will automatically be encrypted if
+    over a Reticulum network. Packets will automatically be encrypted if
     they are adressed to a ``RNS.Destination.SINGLE`` destination,
     ``RNS.Destination.GROUP`` destination or a :ref:`RNS.Link<api-link>`.
 
     For ``RNS.Destination.GROUP`` destinations, Reticulum will use the
-    pre-shared key configured for the destination.
+    pre-shared key configured for the destination. All packets to group
+    destinations are encrypted with the same AES-128 key.
 
-    For ``RNS.Destination.SINGLE`` destinations and :ref:`RNS.Link<api-link>`
-    destinations, reticulum will use ephemeral keys, and offers **Forward Secrecy**.
+    For ``RNS.Destination.SINGLE`` destinations, Reticulum will use a newly
+    derived ephemeral AES-128 key for every packet.
+
+    For :ref:`RNS.Link<api-link>` destinations, Reticulum will use per-link
+    ephemeral keys, and offers **Forward Secrecy**.
 
     :param destination: A :ref:`RNS.Destination<api-destination>` instance to which the packet will be sent.
     :param data: The data payload to be included in the packet as *bytes*.
@@ -216,16 +220,18 @@ class Packet:
             self.destination_type = (self.flags & 0b00001100) >> 2
             self.packet_type      = (self.flags & 0b00000011)
 
+            DST_LEN = RNS.Reticulum.TRUNCATED_HASHLENGTH//8
+
             if self.header_type == Packet.HEADER_2:
-                self.transport_id = self.raw[2:12]
-                self.destination_hash = self.raw[12:22]
-                self.context = ord(self.raw[22:23])
-                self.data = self.raw[23:]
+                self.transport_id = self.raw[2:DST_LEN+2]
+                self.destination_hash = self.raw[DST_LEN+2:2*DST_LEN+2]
+                self.context = ord(self.raw[2*DST_LEN+2:2*DST_LEN+3])
+                self.data = self.raw[2*DST_LEN+3:]
             else:
                 self.transport_id = None
-                self.destination_hash = self.raw[2:12]
-                self.context = ord(self.raw[12:13])
-                self.data = self.raw[13:]
+                self.destination_hash = self.raw[2:DST_LEN+2]
+                self.context = ord(self.raw[DST_LEN+2:DST_LEN+3])
+                self.data = self.raw[DST_LEN+3:]
 
             self.packed = False
             self.update_hash()
@@ -252,7 +258,7 @@ class Packet:
 
             if not self.packed:
                 self.pack()
-    
+
             if RNS.Transport.outbound(self):
                 return self.receipt
             else:
@@ -313,7 +319,7 @@ class Packet:
     def get_hashable_part(self):
         hashable_part = bytes([self.raw[0] & 0b00001111])
         if self.header_type == Packet.HEADER_2:
-            hashable_part += self.raw[12:]
+            hashable_part += self.raw[(RNS.Identity.TRUNCATED_HASHLENGTH//8)+2:]
         else:
             hashable_part += self.raw[2:]
 
@@ -321,7 +327,7 @@ class Packet:
 
 class ProofDestination:
     def __init__(self, packet):
-        self.hash = packet.get_hash()[:10];
+        self.hash = packet.get_hash()[:RNS.Reticulum.TRUNCATED_HASHLENGTH//8];
         self.type = RNS.Destination.SINGLE
 
     def encrypt(self, plaintext):
@@ -488,7 +494,7 @@ class PacketReceipt:
 
             if self.callbacks.timeout:
                 thread = threading.Thread(target=self.callbacks.timeout, args=(self,))
-                thread.setDaemon(True)
+                thread.daemon = True
                 thread.start()
 
 

RNS/Resource.py

@@ -42,10 +42,39 @@ class Resource:
     :param callback: An optional *callable* with the signature *callback(resource)*. Will be called when the resource transfer concludes.
     :param progress_callback: An optional *callable* with the signature *callback(resource)*. Will be called whenever the resource transfer progress is updated.
     """
-    WINDOW_FLEXIBILITY   = 4
-    WINDOW_MIN           = 1
-    WINDOW_MAX           = 10
+
+    # The initial window size at beginning of transfer
     WINDOW               = 4
+
+    # Absolute minimum window size during transfer
+    WINDOW_MIN           = 1
+
+    # The maximum window size for transfers on slow links
+    WINDOW_MAX_SLOW      = 10
+
+    # The maximum window size for transfers on fast links
+    WINDOW_MAX_FAST      = 75
+    
+    # For calculating maps and guard segments, this
+    # must be set to the global maximum window.
+    WINDOW_MAX           = WINDOW_MAX_FAST
+    
+    # If the fast rate is sustained for this many request
+    # rounds, the fast link window size will be allowed.
+    FAST_RATE_THRESHOLD  = WINDOW_MAX_SLOW - WINDOW - 2
+
+    # If the RTT rate is higher than this value,
+    # the max window size for fast links will be used.
+    # The default is 50 Kbps (the value is stored in
+    # bytes per second, hence the "/ 8").
+    RATE_FAST            = (50*1000) / 8
+
+    # The minimum allowed flexibility of the window size.
+    # The difference between window_max and window_min
+    # will never be smaller than this value.
+    WINDOW_FLEXIBILITY   = 4
+
+    # Number of bytes in a map hash
     MAPHASH_LEN          = 4
     SDU                  = RNS.Packet.MDU
     RANDOM_HASH_SIZE     = 4
@@ -74,9 +103,11 @@ class Resource:
 
     PART_TIMEOUT_FACTOR           = 4
     PART_TIMEOUT_FACTOR_AFTER_RTT = 2
-    MAX_RETRIES                   = 5
+    MAX_RETRIES                   = 8
+    MAX_ADV_RETRIES               = 4
     SENDER_GRACE_TIME             = 10
     RETRY_GRACE_TIME              = 0.25
+    PER_RETRY_DELAY               = 0.5
 
     WATCHDOG_MAX_SLEEP            = 1
 
@@ -120,7 +151,7 @@ class Resource:
             resource.outstanding_parts   = 0
             resource.parts                 = [None] * resource.total_parts
             resource.window              = Resource.WINDOW
-            resource.window_max          = Resource.WINDOW_MAX
+            resource.window_max          = Resource.WINDOW_MAX_SLOW
             resource.window_min          = Resource.WINDOW_MIN
             resource.window_flexibility  = Resource.WINDOW_FLEXIBILITY
             resource.last_activity       = time.time()
@@ -141,20 +172,26 @@ class Resource:
 
             resource.consecutive_completed_height = 0
             
-            resource.link.register_incoming_resource(resource)
+            if not resource.link.has_incoming_resource(resource):
+                resource.link.register_incoming_resource(resource)
 
-            RNS.log("Accepting resource advertisement for "+RNS.prettyhexrep(resource.hash), RNS.LOG_DEBUG)
-            if resource.link.callbacks.resource_started != None:
-                try:
-                    resource.link.callbacks.resource_started(resource)
-                except Exception as e:
-                    RNS.log("Error while executing resource started callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
+                RNS.log("Accepting resource advertisement for "+RNS.prettyhexrep(resource.hash), RNS.LOG_DEBUG)
+                if resource.link.callbacks.resource_started != None:
+                    try:
+                        resource.link.callbacks.resource_started(resource)
+                    except Exception as e:
+                        RNS.log("Error while executing resource started callback from "+str(resource)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
 
-            resource.hashmap_update(0, resource.hashmap_raw)
+                resource.hashmap_update(0, resource.hashmap_raw)
 
-            resource.watchdog_job()
+                resource.watchdog_job()
+
+                return resource
+
+            else:
+                RNS.log("Ignoring resource advertisement for "+RNS.prettyhexrep(resource.hash)+", resource already transferring", RNS.LOG_DEBUG)
+                return None
 
-            return resource
         except Exception as e:
             RNS.log("Could not decode resource advertisement, dropping resource", RNS.LOG_DEBUG)
             return None
@@ -210,6 +247,7 @@ class Resource:
         self.status = Resource.NONE
         self.link = link
         self.max_retries = Resource.MAX_RETRIES
+        self.max_adv_retries = Resource.MAX_ADV_RETRIES
         self.retries_left = self.max_retries
         self.timeout_factor = self.link.traffic_timeout_factor
         self.part_timeout_factor = Resource.PART_TIMEOUT_FACTOR
@@ -219,6 +257,11 @@ class Resource:
         self.__watchdog_job_id = 0
         self.__progress_callback = progress_callback
         self.rtt = None
+        self.rtt_rxd_bytes = 0
+        self.req_sent = 0
+        self.req_resp_rtt_rate = 0
+        self.rtt_rxd_bytes_at_part_req = 0
+        self.fast_rate_rounds = 0
         self.request_id = request_id
         self.is_response = is_response
 
@@ -356,12 +399,11 @@ class Resource:
         the resource advertisement it will begin transferring.
         """
         thread = threading.Thread(target=self.__advertise_job)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
 
     def __advertise_job(self):
-        data = ResourceAdvertisement(self).pack()
-        self.advertisement_packet = RNS.Packet(self.link, data, context=RNS.Packet.RESOURCE_ADV)
+        self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self).pack(), context=RNS.Packet.RESOURCE_ADV)
         while not self.link.ready_for_new_resource():
             self.status = Resource.QUEUED
             sleep(0.25)
@@ -372,6 +414,7 @@ class Resource:
             self.adv_sent = self.last_activity
             self.rtt = None
             self.status = Resource.ADVERTISED
+            self.retries_left = self.max_adv_retries
             self.link.register_outgoing_resource(self)
             RNS.log("Sent resource advertisement for "+RNS.prettyhexrep(self.hash), RNS.LOG_DEBUG)
         except Exception as e:
@@ -383,7 +426,7 @@ class Resource:
 
     def watchdog_job(self):
         thread = threading.Thread(target=self.__watchdog_job)
-        thread.setDaemon(True)
+        thread.daemon = True
         thread.start()
 
     def __watchdog_job(self):
@@ -407,7 +450,8 @@ class Resource:
                         try:
                             RNS.log("No part requests received, retrying resource advertisement...", RNS.LOG_DEBUG)
                             self.retries_left -= 1
-                            self.advertisement_packet.resend()
+                            self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self).pack(), context=RNS.Packet.RESOURCE_ADV)
+                            self.advertisement_packet.send()
                             self.last_activity = time.time()
                             self.adv_sent = self.last_activity
                             sleep_time = 0.001
@@ -426,7 +470,9 @@ class Resource:
 
                     window_remaining = self.outstanding_parts
 
-                    sleep_time = self.last_activity + (rtt*(self.part_timeout_factor+window_remaining)) + Resource.RETRY_GRACE_TIME - time.time()
+                    retries_used = self.max_retries - self.retries_left
+                    extra_wait = retries_used * Resource.PER_RETRY_DELAY
+                    sleep_time = self.last_activity + (rtt*(self.part_timeout_factor+window_remaining)) + Resource.RETRY_GRACE_TIME + extra_wait - time.time()
                     
                     if sleep_time < 0:
                         if self.retries_left > 0:
@@ -446,7 +492,8 @@ class Resource:
                             self.cancel()
                             sleep_time = 0.001
                 else:
-                    max_wait = self.rtt * self.timeout_factor * self.max_retries + self.sender_grace_time
+                    max_extra_wait = sum([(r+1) * Resource.PER_RETRY_DELAY for r in range(self.MAX_RETRIES)])
+                    max_wait = self.rtt * self.timeout_factor * self.max_retries + self.sender_grace_time + max_extra_wait
                     sleep_time = self.last_activity + max_wait - time.time()
                     if sleep_time < 0:
                         RNS.log("Resource timed out waiting for part requests", RNS.LOG_DEBUG)
@@ -526,8 +573,11 @@ class Resource:
                         RNS.log("Error while executing resource assembled callback from "+str(self)+". The contained exception was: "+str(e), RNS.LOG_ERROR)
 
                 try:
-                    self.data.close()
+                    if hasattr(self.data, "close") and callable(self.data.close):
+                        self.data.close()
+
                     os.unlink(self.storagepath)
+
                 except Exception as e:
                     RNS.log("Error while cleaning up resource files, the contained exception was:", RNS.LOG_ERROR)
                     RNS.log(str(e))
@@ -564,7 +614,7 @@ class Resource:
                     else:
                         # Otherwise we'll recursively create the
                         # next segment of the resource
-                        Resource(self.input_file, self.link, callback = self.callback, segment_index = self.segment_index+1, original_hash=self.original_hash)
+                        Resource(self.input_file, self.link, callback = self.callback, segment_index = self.segment_index+1, original_hash=self.original_hash, progress_callback = self.__progress_callback)
                 else:
                     pass
             else:
@@ -582,7 +632,7 @@ class Resource:
         if self.req_resp == None:
             self.req_resp = self.last_activity
             rtt = self.req_resp-self.req_sent
-
+            
             self.part_timeout_factor = Resource.PART_TIMEOUT_FACTOR_AFTER_RTT
             if self.rtt == None:
                 self.rtt = self.link.rtt
@@ -592,6 +642,16 @@ class Resource:
             elif rtt > self.rtt:
                 self.rtt = min(self.rtt + self.rtt*0.05, rtt)
 
+            if rtt > 0:
+                req_resp_cost = len(packet.raw)+self.req_sent_bytes
+                self.req_resp_rtt_rate = req_resp_cost / rtt
+
+                if self.req_resp_rtt_rate > Resource.RATE_FAST and self.fast_rate_rounds < Resource.FAST_RATE_THRESHOLD:
+                    self.fast_rate_rounds += 1
+
+                    if self.fast_rate_rounds == Resource.FAST_RATE_THRESHOLD:
+                        self.window_max = Resource.WINDOW_MAX_FAST
+
         if not self.status == Resource.FAILED:
             self.status = Resource.TRANSFERRING
             part_data = packet.data
@@ -603,6 +663,7 @@ class Resource:
                     if self.parts[i] == None:
                         # Insert data into parts list
                         self.parts[i] = part_data
+                        self.rtt_rxd_bytes += len(part_data)
                         self.received_count += 1
                         self.outstanding_parts -= 1
 
@@ -636,6 +697,20 @@ class Resource:
                     if (self.window - self.window_min) > (self.window_flexibility-1):
                         self.window_min += 1
 
+                if self.req_sent != 0:
+                    rtt = time.time()-self.req_sent
+                    req_transferred = self.rtt_rxd_bytes - self.rtt_rxd_bytes_at_part_req
+
+                    if rtt != 0:
+                        self.req_data_rtt_rate = req_transferred/rtt
+                        self.rtt_rxd_bytes_at_part_req = self.rtt_rxd_bytes
+
+                        if self.req_data_rtt_rate > Resource.RATE_FAST and self.fast_rate_rounds < Resource.FAST_RATE_THRESHOLD:
+                            self.fast_rate_rounds += 1
+
+                            if self.fast_rate_rounds == Resource.FAST_RATE_THRESHOLD:
+                                self.window_max = Resource.WINDOW_MAX_FAST
+
                 self.request_next()
         else:
             self.receiving_part = False
@@ -683,6 +758,7 @@ class Resource:
                     request_packet.send()
                     self.last_activity = time.time()
                     self.req_sent = self.last_activity
+                    self.req_sent_bytes = len(request_packet.raw)
                     self.req_resp = None
                 except Exception as e:
                     RNS.log("Could not send resource request packet, cancelling resource", RNS.LOG_DEBUG)
@@ -707,27 +783,34 @@ class Resource:
 
             requested_hashes = request_data[pad+RNS.Identity.HASHLENGTH//8:]
 
-            for i in range(0,len(requested_hashes)//Resource.MAPHASH_LEN):
-                requested_hash = requested_hashes[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN]
-                
-                search_start = self.receiver_min_consecutive_height
-                search_end   = self.receiver_min_consecutive_height+ResourceAdvertisement.COLLISION_GUARD_SIZE
-                for part in self.parts[search_start:search_end]:
-                    if part.map_hash == requested_hash:
-                        try:
-                            if not part.sent:
-                                part.send()
-                                self.sent_parts += 1
-                            else:
-                                part.resend()
-                            self.last_activity = time.time()
-                            self.last_part_sent = self.last_activity
-                            break
-                        except Exception as e:
-                            RNS.log("Resource could not send parts, cancelling transfer!", RNS.LOG_DEBUG)
-                            RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG)
-                            self.cancel()
+            # Define the search scope
+            search_start = self.receiver_min_consecutive_height
+            search_end   = self.receiver_min_consecutive_height+ResourceAdvertisement.COLLISION_GUARD_SIZE
 
+            map_hashes = []
+            for i in range(0,len(requested_hashes)//Resource.MAPHASH_LEN):
+                map_hash = requested_hashes[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN]
+                map_hashes.append(map_hash)
+
+            search_scope = self.parts[search_start:search_end]
+            requested_parts = list(filter(lambda part: part.map_hash in map_hashes, search_scope))
+
+            for part in requested_parts:
+                try:
+                    if not part.sent:
+                        part.send()
+                        self.sent_parts += 1
+                    else:
+                        part.resend()
+
+                    self.last_activity = time.time()
+                    self.last_part_sent = self.last_activity
+
+                except Exception as e:
+                    RNS.log("Resource could not send parts, cancelling transfer!", RNS.LOG_DEBUG)
+                    RNS.log("The contained exception was: "+str(e), RNS.LOG_DEBUG)
+                    self.cancel()
+            
             if wants_more_hashmap:
                 last_map_hash = request_data[1:Resource.MAPHASH_LEN+1]
                 
@@ -826,12 +909,48 @@ class Resource:
         progress = self.processed_parts / self.progress_total_parts
         return progress
 
+    def get_transfer_size(self):
+        """
+        :returns: The number of bytes needed to transfer the resource.
+        """
+        return self.size
+
+    def get_data_size(self):
+        """
+        :returns: The total data size of the resource.
+        """
+        return self.total_size
+
+    def get_parts(self):
+        """
+        :returns: The number of parts the resource will be transferred in.
+        """
+        return self.total_parts
+
+    def get_segments(self):
+        """
+        :returns: The number of segments the resource is divided into.
+        """
+        return self.total_segments
+
+    def get_hash(self):
+        """
+        :returns: The hash of the resource.
+        """
+        return self.hash
+
+    def is_compressed(self):
+        """
+        :returns: Whether the resource is compressed.
+        """
+        return self.compressed
+
     def __str__(self):
-        return "<"+RNS.hexrep(self.hash)+"/"+RNS.hexrep(self.link.link_id)+">"
+        return "<"+RNS.hexrep(self.hash,delimit=False)+"/"+RNS.hexrep(self.link.link_id,delimit=False)+">"
 
 
 class ResourceAdvertisement:
-    OVERHEAD             = 128
+    OVERHEAD             = 134
     HASHMAP_MAX_LEN      = math.floor((RNS.Link.MDU-OVERHEAD)/Resource.MAPHASH_LEN)
     COLLISION_GUARD_SIZE = 2*Resource.WINDOW_MAX+HASHMAP_MAX_LEN
 
@@ -857,19 +976,19 @@ class ResourceAdvertisement:
 
 
     @staticmethod
-    def get_request_id(advertisement_packet):
+    def read_request_id(advertisement_packet):
         adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
         return adv.q
 
 
     @staticmethod
-    def get_transfer_size(advertisement_packet):
+    def read_transfer_size(advertisement_packet):
         adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
         return adv.t
 
 
     @staticmethod
-    def get_size(advertisement_packet):
+    def read_size(advertisement_packet):
         adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext)
         return adv.d
 
@@ -903,6 +1022,23 @@ class ResourceAdvertisement:
             # Flags
             self.f = 0x00 | self.p << 4 | self.u << 3 | self.s << 2 | self.c << 1 | self.e
 
+    def get_transfer_size(self):
+        return self.t
+
+    def get_data_size(self):
+        return self.d
+
+    def get_parts(self):
+        return self.n
+
+    def get_segments(self):
+        return self.l
+
+    def get_hash(self):
+        return self.h
+
+    def is_compressed(self):
+        return self.c
 
     def pack(self, segment=0):
         hashmap_start = segment*ResourceAdvertisement.HASHMAP_MAX_LEN

RNS/Utilities/rncp.py

@@ -0,0 +1,389 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2016-2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import RNS
+import argparse
+import time
+import sys
+import os
+
+from RNS._version import __version__
+
+APP_NAME = "rncp"
+allow_all = False
+allowed_identity_hashes = []
+
+def receive(configdir, verbosity = 0, quietness = 0, allowed = [], display_identity = False, limit = None, disable_auth = None, disable_announce = False):
+    global allow_all, allowed_identity_hashes
+    identity = None
+
+    targetloglevel = 3+verbosity-quietness
+    reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
+
+    identity_path = RNS.Reticulum.identitypath+"/"+APP_NAME
+    if os.path.isfile(identity_path):
+        identity = RNS.Identity.from_file(identity_path)                
+
+    if identity == None:
+        RNS.log("No valid saved identity found, creating new...", RNS.LOG_INFO)
+        identity = RNS.Identity()
+        identity.to_file(identity_path)
+
+    destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "receive")
+
+    if display_identity:
+        print("Identity     : "+str(identity))
+        print("Receiving on : "+RNS.prettyhexrep(destination.hash))
+        exit(0)
+
+    if disable_auth:
+        allow_all = True
+    else:
+        if allowed != None:
+            for a in allowed:
+                try:
+                    dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+                    if len(a) != dest_len:
+                        raise ValueError("Allowed destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2))
+                    try:
+                        destination_hash = bytes.fromhex(a)
+                        allowed_identity_hashes.append(destination_hash)
+                    except Exception as e:
+                        raise ValueError("Invalid destination entered. Check your input.")
+                except Exception as e:
+                    print(str(e))
+                    exit(1)
+
+    if len(allowed_identity_hashes) < 1 and not disable_auth:
+        print("Warning: No allowed identities configured, rncp will not accept any files!")
+
+    destination.set_link_established_callback(receive_link_established)
+    print("rncp ready to receive on "+RNS.prettyhexrep(destination.hash))
+
+    if not disable_announce:
+        destination.announce()
+    
+    while True:
+        time.sleep(1)
+
+def receive_link_established(link):
+    RNS.log("Incoming link established", RNS.LOG_VERBOSE)
+    link.set_remote_identified_callback(receive_sender_identified)
+    link.set_resource_strategy(RNS.Link.ACCEPT_APP)
+    link.set_resource_callback(receive_resource_callback)
+    link.set_resource_started_callback(receive_resource_started)
+    link.set_resource_concluded_callback(receive_resource_concluded)
+
+def receive_sender_identified(link, identity):
+    global allow_all
+
+    if identity.hash in allowed_identity_hashes:
+        RNS.log("Authenticated sender", RNS.LOG_VERBOSE)
+    else:
+        if not allow_all:
+            RNS.log("Sender not allowed, tearing down link", RNS.LOG_VERBOSE)
+            link.teardown()
+        else:
+            pass
+
+def receive_resource_callback(resource):
+    global allow_all
+    
+    sender_identity = resource.link.get_remote_identity()
+
+    if sender_identity != None:
+        if sender_identity.hash in allowed_identity_hashes:
+            return True
+
+    if allow_all:
+        return True
+
+    return False
+
+def receive_resource_started(resource):
+    if resource.link.get_remote_identity():
+        id_str = " from "+RNS.prettyhexrep(resource.link.get_remote_identity().hash)
+    else:
+        id_str = ""
+
+    print("Starting resource transfer "+RNS.prettyhexrep(resource.hash)+id_str)
+
+def receive_resource_concluded(resource):
+    if resource.status == RNS.Resource.COMPLETE:
+        print(str(resource)+" completed")
+
+        if resource.total_size > 4:
+            filename_len = int.from_bytes(resource.data.read(2), "big")
+            filename = resource.data.read(filename_len).decode("utf-8")
+
+            counter = 0
+            saved_filename = filename
+            while os.path.isfile(saved_filename):
+                counter += 1
+                saved_filename = filename+"."+str(counter)
+            
+            file = open(saved_filename, "wb")
+            file.write(resource.data.read())
+            file.close()
+
+        else:
+            print("Invalid data received, ignoring resource")
+
+    else:
+        print("Resource failed")
+
+resource_done = False
+current_resource = None
+stats = []
+speed = 0.0
+def sender_progress(resource):
+    stats_max = 32
+    global current_resource, stats, speed, resource_done
+    current_resource = resource
+    now = time.time()
+    got = current_resource.get_progress()*current_resource.total_size
+    entry = [now, got]
+    stats.append(entry)
+    while len(stats) > stats_max:
+        stats.pop(0)
+
+    span = now - stats[0][0]
+    if span == 0:
+        speed = 0
+    else:
+        diff = got - stats[0][1]
+        speed = diff/span
+
+    if resource.status < RNS.Resource.COMPLETE:
+        resource_done = False
+    else:
+        resource_done = True
+
+link = None
+def send(configdir, verbosity = 0, quietness = 0, destination = None, file = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT):
+    global current_resource, resource_done, link, speed
+    from tempfile import TemporaryFile
+    targetloglevel = 3+verbosity-quietness
+
+    try:
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination) != dest_len:
+            raise ValueError("Allowed destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2))
+        try:
+            destination_hash = bytes.fromhex(destination)
+        except Exception as e:
+            raise ValueError("Invalid destination entered. Check your input.")
+    except Exception as e:
+        print(str(e))
+        exit(1)
+
+    
+    file_path = os.path.expanduser(file)
+    if not os.path.isfile(file_path):
+        print("File not found")
+        exit(1)
+
+    temp_file = TemporaryFile()
+    real_file = open(file_path, "rb")
+    filename_bytes = os.path.basename(file_path).encode("utf-8")
+    filename_len = len(filename_bytes)
+
+    if filename_len > 0xFFFF:
+        print("Filename exceeds max size, cannot send")
+        exit(1)
+    else:
+        print("Preparing file...", end=" ")
+
+    temp_file.write(filename_len.to_bytes(2, "big"))
+    temp_file.write(filename_bytes)
+    temp_file.write(real_file.read())
+    temp_file.seek(0)
+
+    print("\r                                                            \r", end="")
+
+    reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
+
+    identity_path = RNS.Reticulum.identitypath+"/"+APP_NAME
+    if os.path.isfile(identity_path):
+        identity = RNS.Identity.from_file(identity_path)                
+
+    if identity == None:
+        RNS.log("No valid saved identity found, creating new...", RNS.LOG_INFO)
+        identity = RNS.Identity()
+        identity.to_file(identity_path)
+
+    if not RNS.Transport.has_path(destination_hash):
+        RNS.Transport.request_path(destination_hash)
+        print("Path to "+RNS.prettyhexrep(destination_hash)+" requested  ", end=" ")
+        sys.stdout.flush()
+
+    i = 0
+    syms = "⢄⢂⢁⡁⡈⡐⡠"
+    estab_timeout = time.time()+timeout
+    while not RNS.Transport.has_path(destination_hash) and time.time() < estab_timeout:
+        time.sleep(0.1)
+        print(("\b\b"+syms[i]+" "), end="")
+        sys.stdout.flush()
+        i = (i+1)%len(syms)
+
+    if not RNS.Transport.has_path(destination_hash):
+        print("\r                                                            \rPath not found")
+        exit(1)
+    else:
+        print("\r                                                            \rEstablishing link with "+RNS.prettyhexrep(destination_hash)+" ", end=" ")
+
+    receiver_identity = RNS.Identity.recall(destination_hash)
+    receiver_destination = RNS.Destination(
+        receiver_identity,
+        RNS.Destination.OUT,
+        RNS.Destination.SINGLE,
+        APP_NAME,
+        "receive"
+    )
+
+    link = RNS.Link(receiver_destination)
+    while link.status != RNS.Link.ACTIVE and time.time() < estab_timeout:
+        time.sleep(0.1)
+        print(("\b\b"+syms[i]+" "), end="")
+        sys.stdout.flush()
+        i = (i+1)%len(syms)
+
+    if not RNS.Transport.has_path(destination_hash):
+        print("\r                                                            \rCould not establish link with "+RNS.prettyhexrep(destination_hash))
+        exit(1)
+    else:
+        print("\r                                                            \rAdvertising file resource  ", end=" ")
+
+    link.identify(identity)
+    resource = RNS.Resource(temp_file, link, callback = sender_progress, progress_callback = sender_progress)
+    current_resource = resource
+
+    while resource.status < RNS.Resource.TRANSFERRING:
+        time.sleep(0.1)
+        print(("\b\b"+syms[i]+" "), end="")
+        sys.stdout.flush()
+        i = (i+1)%len(syms)
+
+    
+    if resource.status > RNS.Resource.COMPLETE:
+        print("\r                                                            \rFile was not accepted by "+RNS.prettyhexrep(destination_hash))
+        exit(1)
+    else:
+        print("\r                                                            \rTransferring file  ", end=" ")
+
+    while not resource_done:
+        time.sleep(0.1)
+        prg = current_resource.get_progress()
+        percent = round(prg * 100.0, 1)
+        stat_str = str(percent)+"% - " + size_str(int(prg*current_resource.total_size)) + " of " + size_str(current_resource.total_size) + " - " +size_str(speed, "b")+"ps"
+        print("\r                                                                                  \rTransferring file "+syms[i]+" "+stat_str, end=" ")
+        sys.stdout.flush()
+        i = (i+1)%len(syms)
+
+    if current_resource.status != RNS.Resource.COMPLETE:
+        print("\r                                                            \rThe transfer failed")
+        exit(1)
+    else:
+        print("\r                                                                                  \r"+str(file_path)+" copied to "+RNS.prettyhexrep(destination_hash))
+        link.teardown()
+        time.sleep(0.25)
+        real_file.close()
+        temp_file.close()
+        exit(0)
+
+def main():
+    try:
+        parser = argparse.ArgumentParser(description="Reticulum File Transfer Utility")
+        parser.add_argument("file", nargs="?", default=None, help="file to be transferred", type=str)
+        parser.add_argument("destination", nargs="?", default=None, help="hexadecimal hash of the receiver", type=str)
+        parser.add_argument("--config", metavar="path", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
+        parser.add_argument('-v', '--verbose', action='count', default=0, help="increase verbosity")
+        parser.add_argument('-q', '--quiet', action='count', default=0, help="decrease verbosity")
+        parser.add_argument('-p', '--print-identity', action='store_true', default=False, help="print identity and destination info and exit")
+        parser.add_argument("-r", '--receive', action='store_true', default=False, help="wait for incoming files")
+        parser.add_argument("-b", '--no-announce', action='store_true', default=False, help="don't announce at program start")
+        parser.add_argument('-a', metavar="allowed_hash", dest="allowed", action='append', help="accept from this identity", type=str)
+        parser.add_argument('-n', '--no-auth', action='store_true', default=False, help="accept files from anyone")
+        parser.add_argument("-w", action="store", metavar="seconds", type=float, help="sender timeout before giving up", default=RNS.Transport.PATH_REQUEST_TIMEOUT)
+        # parser.add_argument("--limit", action="store", metavar="files", type=float, help="maximum number of files to accept", default=None)
+        parser.add_argument("--version", action="version", version="rncp {version}".format(version=__version__))
+        
+        args = parser.parse_args()
+
+        if args.receive or args.print_identity:
+            receive(
+                configdir = args.config,
+                verbosity=args.verbose,
+                quietness=args.quiet,
+                allowed = args.allowed,
+                display_identity=args.print_identity,
+                # limit=args.limit,
+                disable_auth=args.no_auth,
+                disable_announce=args.no_announce,
+            )
+
+        elif args.destination != None and args.file != None:
+            send(
+                configdir = args.config,
+                verbosity = args.verbose,
+                quietness = args.quiet,
+                destination = args.destination,
+                file = args.file,
+                timeout = args.w,
+            )
+
+        else:
+            print("")
+            parser.print_help()
+            print("")
+
+    except KeyboardInterrupt:
+        print("")
+        if resource != None:
+            resource.cancel()
+        if link != None:
+            link.teardown()
+        exit()
+
+def size_str(num, suffix='B'):
+    units = ['','K','M','G','T','P','E','Z']
+    last_unit = 'Y'
+
+    if suffix == 'b':
+        num *= 8
+        units = ['','K','M','G','T','P','E','Z']
+        last_unit = 'Y'
+
+    for unit in units:
+        if abs(num) < 1000.0:
+            if unit == "":
+                return "%.0f %s%s" % (num, unit, suffix)
+            else:
+                return "%.2f %s%s" % (num, unit, suffix)
+        num /= 1000.0
+
+    return "%.2f%s%s" % (num, last_unit, suffix)
+
+if __name__ == "__main__":
+    main()

RNS/Utilities/rnpath.py

@@ -197,7 +197,7 @@ def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity,
 
             print("\rPath found, destination "+RNS.prettyhexrep(destination_hash)+" is "+str(hops)+" hop"+ms+" away via "+next_hop+" on "+next_hop_interface)
         else:
-            print("\r                                                \rPath not found")
+            print("\r                                                       \rPath not found")
             sys.exit(1)
 
     
@@ -257,7 +257,7 @@ def main():
             metavar="seconds",
             type=float,
             help="timeout before giving up",
-            default=15
+            default=RNS.Transport.PATH_REQUEST_TIMEOUT
         )
 
         parser.add_argument(
@@ -321,7 +321,7 @@ def pretty_date(time=False):
         if second_diff < 3600:
             return str(int(second_diff / 60)) + " minutes"
         if second_diff < 7200:
-            return "an hour ago"
+            return "an hour"
         if second_diff < 86400:
             return str(int(second_diff / 3600)) + " hours"
     if day_diff == 1:

RNS/Utilities/rnsd.py

@@ -33,12 +33,17 @@ def program_setup(configdir, verbosity = 0, quietness = 0, service = False):
     targetloglevel = 3+verbosity-quietness
 
     if service:
-        RNS.logdest = RNS.LOG_FILE
-        RNS.logfile = RNS.Reticulum.configdir+"/logfile"
+        targetlogdest  = RNS.LOG_FILE
         targetloglevel = None
+    else:
+        targetlogdest  = RNS.LOG_STDOUT
+
+    reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel, logdest=targetlogdest)
+    if reticulum.is_connected_to_shared_instance:
+        RNS.log("Started rnsd version {version} connected to another shared local instance, this is probably NOT what you want!".format(version=__version__), RNS.LOG_WARNING)
+    else:
+        RNS.log("Started rnsd version {version}".format(version=__version__), RNS.LOG_NOTICE)
 
-    reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
-    RNS.log("Started rnsd version {version}".format(version=__version__), RNS.LOG_NOTICE)
     while True:
         time.sleep(1)
 
@@ -147,7 +152,7 @@ loglevel = 4
 
   [[Default Interface]]
     type = AutoInterface
-    interface_enabled = True
+    enabled = yes
 
 
   # The following example enables communication with other
@@ -155,7 +160,7 @@ loglevel = 4
   
   [[UDP Interface]]
     type = UDPInterface
-    interface_enabled = False
+    enabled = no
     listen_ip = 0.0.0.0
     listen_port = 4242
     forward_ip = 255.255.255.255
@@ -198,7 +203,7 @@ loglevel = 4
   
   [[TCP Server Interface]]
     type = TCPServerInterface
-    interface_enabled = False
+    enabled = no
 
     # This configuration will listen on all IP
     # interfaces on port 4242
@@ -224,7 +229,7 @@ loglevel = 4
 
   [[TCP Client Interface]]
     type = TCPClientInterface
-    interface_enabled = False
+    enabled = no
     target_host = 127.0.0.1
     target_port = 4242
 
@@ -237,9 +242,9 @@ loglevel = 4
 
   [[I2P]]
     type = I2PInterface
-    interface_enabled = yes
+    enabled = no
     connectable = yes
-    peers = 5urvjicpzi7q3ybztsef4i5ow2aq4soktfj7zedz53s47r54jnqq.b32.i2p
+    peers = ykzlw5ujbaqc2xkec4cpvgyxj257wcrmmgkuxqmqcur7cq3w3lha.b32.i2p
 
 
   # Here's an example of how to add a LoRa interface
@@ -249,7 +254,7 @@ loglevel = 4
     type = RNodeInterface
 
     # Enable interface if you want use it!
-    interface_enabled = False
+    enabled = no
 
     # Serial port for the device
     port = /dev/ttyUSB0
@@ -301,7 +306,7 @@ loglevel = 4
     type = KISSInterface
 
     # Enable interface if you want use it!
-    interface_enabled = False
+    enabled = no
 
     # Serial port for the device
     port = /dev/ttyUSB1
@@ -372,7 +377,7 @@ loglevel = 4
     ssid = 0
 
     # Enable interface if you want use it!
-    interface_enabled = False
+    enabled = no
 
     # Serial port for the device
     port = /dev/ttyUSB2

RNS/Utilities/rnstatus.py

@@ -46,7 +46,7 @@ def size_str(num, suffix='B'):
 
     return "%.2f%s%s" % (num, last_unit, suffix)
 
-def program_setup(configdir, dispall=False, verbosity = 0):
+def program_setup(configdir, dispall=False, verbosity=0, name_filter=None):
     reticulum = RNS.Reticulum(configdir = configdir, loglevel = 3+verbosity)
 
     stats = None
@@ -59,79 +59,103 @@ def program_setup(configdir, dispall=False, verbosity = 0):
         for ifstat in stats["interfaces"]:
             name = ifstat["name"]
 
-            if dispall or not (name.startswith("LocalInterface[") or name.startswith("TCPInterface[Client")):
-                print("")
+            if dispall or not (
+                name.startswith("LocalInterface[") or
+                name.startswith("TCPInterface[Client") or
+                name.startswith("I2PInterfacePeer[Connected peer") or
+                (name.startswith("I2PInterface[") and ("i2p_connectable" in ifstat and ifstat["i2p_connectable"] == False))
+                ):
 
-                if ifstat["status"]:
-                    ss = "Up"
-                else:
-                    ss = "Down"
+                if not (name.startswith("I2PInterface[") and ("i2p_connectable" in ifstat and ifstat["i2p_connectable"] == False)):
+                    if name_filter == None or name_filter.lower() in name.lower():
+                        print("")
 
-                if ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT:
-                    modestr = "Access Point"
-                elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_POINT_TO_POINT:
-                    modestr = "Point-to-Point"
-                elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ROAMING:
-                    modestr = "Roaming"
-                elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_BOUNDARY:
-                    modestr = "Boundary"
-                else:
-                    modestr = "Full"
-
-
-                if ifstat["clients"] != None:
-                    clients = ifstat["clients"]
-                    if name.startswith("Shared Instance["):
-                        cnum = max(clients-1,0)
-                        if cnum == 1:
-                            spec_str = " program"
+                        if ifstat["status"]:
+                            ss = "Up"
                         else:
-                            spec_str = " programs"
+                            ss = "Down"
 
-                        clients_string = "Serving : "+str(cnum)+spec_str
-                    else:
-                        clients_string = "Clients : "+str(clients)
+                        if ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT:
+                            modestr = "Access Point"
+                        elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_POINT_TO_POINT:
+                            modestr = "Point-to-Point"
+                        elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_ROAMING:
+                            modestr = "Roaming"
+                        elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_BOUNDARY:
+                            modestr = "Boundary"
+                        elif ifstat["mode"] == RNS.Interfaces.Interface.Interface.MODE_GATEWAY:
+                            modestr = "Gateway"
+                        else:
+                            modestr = "Full"
 
-                else:
-                    clients = None
 
-                print(" {n}".format(n=ifstat["name"]))
+                        if ifstat["clients"] != None:
+                            clients = ifstat["clients"]
+                            if name.startswith("Shared Instance["):
+                                cnum = max(clients-1,0)
+                                if cnum == 1:
+                                    spec_str = " program"
+                                else:
+                                    spec_str = " programs"
 
-                if "ifac_netname" in ifstat and ifstat["ifac_netname"] != None:
-                    print("    Network : {nn}".format(nn=ifstat["ifac_netname"]))
+                                clients_string = "Serving : "+str(cnum)+spec_str
+                            elif name.startswith("I2PInterface["):
+                                if "i2p_connectable" in ifstat and ifstat["i2p_connectable"] == True:
+                                    cnum = clients
+                                    if cnum == 1:
+                                        spec_str = " connected I2P endpoint"
+                                    else:
+                                        spec_str = " connected I2P endpoints"
 
-                print("    Status  : {ss}".format(ss=ss))
+                                    clients_string = "Peers   : "+str(cnum)+spec_str
+                                else:
+                                    clients_string = ""
+                            else:
+                                clients_string = "Clients : "+str(clients)
 
-                if clients != None:
-                    print("    "+clients_string)
+                        else:
+                            clients = None
 
-                if not (name.startswith("Shared Instance[") or name.startswith("TCPInterface[Client") or name.startswith("LocalInterface[")):
-                    print("    Mode    : {mode}".format(mode=modestr))
+                        print(" {n}".format(n=ifstat["name"]))
 
-                if "bitrate" in ifstat and ifstat["bitrate"] != None:
-                    print("    Rate    : {ss}".format(ss=speed_str(ifstat["bitrate"])))
-                
-                if "peers" in ifstat and ifstat["peers"] != None:
-                    print("    Peers   : {np} reachable".format(np=ifstat["peers"]))
+                        if "ifac_netname" in ifstat and ifstat["ifac_netname"] != None:
+                            print("    Network : {nn}".format(nn=ifstat["ifac_netname"]))
 
-                if "ifac_signature" in ifstat and ifstat["ifac_signature"] != None:
-                    sigstr = "<…"+RNS.hexrep(ifstat["ifac_signature"][-5:], delimit=False)+">"
-                    print("    Access  : {nb}-bit IFAC by {sig}".format(nb=ifstat["ifac_size"]*8, sig=sigstr))
-                
-                if "i2p_b32" in ifstat and ifstat["i2p_b32"] != None:
-                    print("    I2P B32 : {ep}".format(ep=str(ifstat["i2p_b32"])))
+                        print("    Status  : {ss}".format(ss=ss))
 
-                if "announce_queue" in ifstat and ifstat["announce_queue"] != None and ifstat["announce_queue"] > 0:
-                    aqn = ifstat["announce_queue"]
-                    if aqn == 1:
-                        print("    Queued  : {np} announce".format(np=aqn))
-                    else:
-                        print("    Queued  : {np} announces".format(np=aqn))
-                
-                print("    Traffic : {txb}↑\n              {rxb}↓".format(rxb=size_str(ifstat["rxb"]), txb=size_str(ifstat["txb"])))
+                        if clients != None and clients_string != "":
+                            print("    "+clients_string)
+
+                        if not (name.startswith("Shared Instance[") or name.startswith("TCPInterface[Client") or name.startswith("LocalInterface[")):
+                            print("    Mode    : {mode}".format(mode=modestr))
+
+                        if "bitrate" in ifstat and ifstat["bitrate"] != None:
+                            print("    Rate    : {ss}".format(ss=speed_str(ifstat["bitrate"])))
+                        
+                        if "peers" in ifstat and ifstat["peers"] != None:
+                            print("    Peers   : {np} reachable".format(np=ifstat["peers"]))
+
+                        if "tunnelstate" in ifstat and ifstat["tunnelstate"] != None:
+                            print("    I2P     : {ts}".format(ts=ifstat["tunnelstate"]))
+
+                        if "ifac_signature" in ifstat and ifstat["ifac_signature"] != None:
+                            sigstr = "<…"+RNS.hexrep(ifstat["ifac_signature"][-5:], delimit=False)+">"
+                            print("    Access  : {nb}-bit IFAC by {sig}".format(nb=ifstat["ifac_size"]*8, sig=sigstr))
+                        
+                        if "i2p_b32" in ifstat and ifstat["i2p_b32"] != None:
+                            print("    I2P B32 : {ep}".format(ep=str(ifstat["i2p_b32"])))
+
+                        if "announce_queue" in ifstat and ifstat["announce_queue"] != None and ifstat["announce_queue"] > 0:
+                            aqn = ifstat["announce_queue"]
+                            if aqn == 1:
+                                print("    Queued  : {np} announce".format(np=aqn))
+                            else:
+                                print("    Queued  : {np} announces".format(np=aqn))
+                        
+                        print("    Traffic : {txb}↑\n              {rxb}↓".format(rxb=size_str(ifstat["rxb"]), txb=size_str(ifstat["txb"])))
 
         if "transport_id" in stats and stats["transport_id"] != None:
-            print("\n Reticulum Transport Instance "+RNS.prettyhexrep(stats["transport_id"])+" running")
+            print("\n Reticulum Transport Instance "+RNS.prettyhexrep(stats["transport_id"])+" is running")
 
         print("")
                 
@@ -153,6 +177,8 @@ def main():
         )
 
         parser.add_argument('-v', '--verbose', action='count', default=0)
+
+        parser.add_argument("filter", nargs="?", default=None, help="only display interfaces with names including filter", type=str)
         
         args = parser.parse_args()
 
@@ -161,7 +187,7 @@ def main():
         else:
             configarg = None
 
-        program_setup(configdir = configarg, dispall = args.all, verbosity=args.verbose)
+        program_setup(configdir = configarg, dispall = args.all, verbosity=args.verbose, name_filter=args.filter)
 
     except KeyboardInterrupt:
         print("")

RNS/Utilities/rnx.py

@@ -0,0 +1,714 @@
+#!/usr/bin/env python3
+
+# MIT License
+#
+# Copyright (c) 2016-2022 Mark Qvist / unsigned.io
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import RNS
+import subprocess
+import argparse
+import shlex
+import time
+import sys
+import tty
+import os
+
+from RNS._version import __version__
+
+APP_NAME = "rnx"
+identity = None
+reticulum = None
+allow_all = False
+allowed_identity_hashes = []
+
+def prepare_identity(identity_path):
+    global identity
+    if identity_path == None:
+        identity_path = RNS.Reticulum.identitypath+"/"+APP_NAME
+
+    if os.path.isfile(identity_path):
+        identity = RNS.Identity.from_file(identity_path)                
+
+    if identity == None:
+        RNS.log("No valid saved identity found, creating new...", RNS.LOG_INFO)
+        identity = RNS.Identity()
+        identity.to_file(identity_path)
+
+def listen(configdir, identitypath = None, verbosity = 0, quietness = 0, allowed = [], print_identity = False, disable_auth = None, disable_announce=False):
+    global identity, allow_all, allowed_identity_hashes, reticulum
+
+    targetloglevel = 3+verbosity-quietness
+    reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
+    
+    prepare_identity(identitypath)
+    destination = RNS.Destination(identity, RNS.Destination.IN, RNS.Destination.SINGLE, APP_NAME, "execute")
+
+    if print_identity:
+        print("Identity     : "+str(identity))
+        print("Listening on : "+RNS.prettyhexrep(destination.hash))
+        exit(0)
+
+    if disable_auth:
+        allow_all = True
+    else:
+        if allowed != None:
+            for a in allowed:
+                try:
+                    dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+                    if len(a) != dest_len:
+                        raise ValueError("Allowed destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2))
+                    try:
+                        destination_hash = bytes.fromhex(a)
+                        allowed_identity_hashes.append(destination_hash)
+                    except Exception as e:
+                        raise ValueError("Invalid destination entered. Check your input.")
+                except Exception as e:
+                    print(str(e))
+                    exit(1)
+
+    if len(allowed_identity_hashes) < 1 and not disable_auth:
+        print("Warning: No allowed identities configured, rncx will not accept any commands!")
+
+    destination.set_link_established_callback(command_link_established)
+
+    if not allow_all:
+        destination.register_request_handler(
+            path = "command",
+            response_generator = execute_received_command,
+            allow = RNS.Destination.ALLOW_LIST,
+            allowed_list = allowed_identity_hashes
+        )
+    else:
+        destination.register_request_handler(
+            path = "command",
+            response_generator = execute_received_command,
+            allow = RNS.Destination.ALLOW_ALL,
+        )
+
+    RNS.log("rnx listening for commands on "+RNS.prettyhexrep(destination.hash))
+
+    if not disable_announce:
+        destination.announce()
+    
+    while True:
+        time.sleep(1)
+
+def command_link_established(link):
+    link.set_remote_identified_callback(initiator_identified)
+    link.set_link_closed_callback(command_link_closed)
+    RNS.log("Command link "+str(link)+" established")
+
+def command_link_closed(link):
+    RNS.log("Command link "+str(link)+" closed")
+
+def initiator_identified(link, identity):
+    global allow_all
+    RNS.log("Initiator of link "+str(link)+" identified as "+RNS.prettyhexrep(identity.hash))
+    if not allow_all and not identity.hash in allowed_identity_hashes:
+        RNS.log("Identity "+RNS.prettyhexrep(identity.hash)+" not allowed, tearing down link")
+        link.teardown()
+
+def execute_received_command(path, data, request_id, remote_identity, requested_at):
+    command = data[0].decode("utf-8")  # Command to execute
+    timeout = data[1]                  # Timeout in seconds
+    o_limit = data[2]                  # Size limit for stdout
+    e_limit = data[3]                  # Size limit for stderr
+    stdin   = data[4]                  # Data passed to stdin
+
+    if remote_identity != None:
+        RNS.log("Executing command ["+command+"] for "+RNS.prettyhexrep(remote_identity.hash))
+    else:
+        RNS.log("Executing command ["+command+"] for unknown requestor")
+
+    result    = [
+        False,                         # 0: Command was executed
+        None,                          # 1: Return value
+        None,                          # 2: Stdout
+        None,                          # 3: Stderr
+        None,                          # 4: Total stdout length
+        None,                          # 5: Total stderr length
+        time.time(),                   # 6: Started
+        None,                          # 7: Concluded
+    ]
+
+    try:
+        process = subprocess.Popen(shlex.split(command), stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        result[0] = True
+
+    except Exception as e:
+        result[0] = False
+        return result
+
+    stdout = b""
+    stderr = b""
+    timed_out = False
+
+    if stdin != None:
+        process.stdin.write(stdin)
+
+    while True:
+        try:
+            stdout, stderr = process.communicate(timeout=1)
+            if process.poll() != None:
+                break
+
+            if len(stdout) > 0:
+                print(str(stdout))
+                sys.stdout.flush()
+
+        except subprocess.TimeoutExpired:
+            pass
+
+        if timeout != None and time.time() > result[6]+timeout:
+            RNS.log("Command ["+command+"] timed out and is being killed...")
+            process.terminate()
+            process.wait()
+            if process.poll() != None:
+                stdout, stderr = process.communicate()
+            else:
+                stdout = None
+                stderr = None
+
+            break
+
+    if timeout != None and time.time() < result[6]+timeout:
+        result[7] = time.time()
+
+    # Deliver result
+    result[1] = process.returncode
+
+    if o_limit != None and len(stdout) > o_limit:
+        if o_limit == 0:
+            result[2] = b""
+        else:
+            result[2] = stdout[0:o_limit]
+    else:
+        result[2] = stdout
+
+    if e_limit != None and len(stderr) > e_limit:
+        if e_limit == 0:
+            result[3] = b""
+        else:
+            result[3] = stderr[0:e_limit]
+    else:
+        result[3] = stderr
+
+    result[4] = len(stdout)
+    result[5] = len(stderr)
+
+    if timed_out:
+        RNS.log("Command timed out")
+        return result
+
+    if remote_identity != None:
+        RNS.log("Delivering result of command ["+str(command)+"] to "+RNS.prettyhexrep(remote_identity.hash))
+    else:
+        RNS.log("Delivering result of command ["+str(command)+"] to unknown requestor")
+
+    return result
+
+def spin(until=None, msg=None, timeout=None):
+    i = 0
+    syms = "⢄⢂⢁⡁⡈⡐⡠"
+    if timeout != None:
+        timeout = time.time()+timeout
+
+    print(msg+"  ", end=" ")
+    while (timeout == None or time.time()<timeout) and not until():
+        time.sleep(0.1)
+        print(("\b\b"+syms[i]+" "), end="")
+        sys.stdout.flush()
+        i = (i+1)%len(syms)
+
+    print("\r"+" "*len(msg)+"  \r", end="")
+
+    if timeout != None and time.time() > timeout:
+        return False
+    else:
+        return True
+
+current_progress = 0.0
+stats = []
+speed = 0.0
+def spin_stat(until=None, timeout=None):
+    global current_progress, response_transfer_size, speed
+    i = 0
+    syms = "⢄⢂⢁⡁⡈⡐⡠"
+    if timeout != None:
+        timeout = time.time()+timeout
+
+    while (timeout == None or time.time()<timeout) and not until():
+        time.sleep(0.1)
+        prg = current_progress
+        percent = round(prg * 100.0, 1)
+        stat_str = str(percent)+"% - " + size_str(int(prg*response_transfer_size)) + " of " + size_str(response_transfer_size) + " - " +size_str(speed, "b")+"ps"
+        print("\r                                                                                  \rReceiving result "+syms[i]+" "+stat_str, end=" ")
+
+        sys.stdout.flush()
+        i = (i+1)%len(syms)
+
+    print("\r                                                                                  \r", end="")
+
+    if timeout != None and time.time() > timeout:
+        return False
+    else:
+        return True
+
+def remote_execution_done(request_receipt):
+    pass
+
+def remote_execution_progress(request_receipt):
+    stats_max = 32
+    global current_progress, response_transfer_size, speed
+    current_progress = request_receipt.progress
+    response_transfer_size = request_receipt.response_transfer_size
+    now = time.time()
+    got = current_progress*response_transfer_size
+    entry = [now, got]
+    stats.append(entry)
+    while len(stats) > stats_max:
+        stats.pop(0)
+
+    span = now - stats[0][0]
+    if span == 0:
+        speed = 0
+    else:
+        diff = got - stats[0][1]
+        speed = diff/span
+
+link = None
+listener_destination = None
+remote_exec_grace = 2.0
+def execute(configdir, identitypath = None, verbosity = 0, quietness = 0, detailed = False, mirror = False, noid = False, destination = None, command = None, stdin = None, stdoutl = None, stderrl = None, timeout = RNS.Transport.PATH_REQUEST_TIMEOUT, result_timeout = None, interactive = False):
+    global identity, reticulum, link, listener_destination, remote_exec_grace
+
+    try:
+        dest_len = (RNS.Reticulum.TRUNCATED_HASHLENGTH//8)*2
+        if len(destination) != dest_len:
+            raise ValueError("Allowed destination length is invalid, must be {hex} hexadecimal characters ({byte} bytes).".format(hex=dest_len, byte=dest_len//2))
+        try:
+            destination_hash = bytes.fromhex(destination)
+        except Exception as e:
+            raise ValueError("Invalid destination entered. Check your input.")
+    except Exception as e:
+        print(str(e))
+        exit(241)
+
+    if reticulum == None:
+        targetloglevel = 3+verbosity-quietness
+        reticulum = RNS.Reticulum(configdir=configdir, loglevel=targetloglevel)
+
+    if identity == None:
+        prepare_identity(identitypath)
+
+    if not RNS.Transport.has_path(destination_hash):
+        RNS.Transport.request_path(destination_hash)
+        if not spin(until=lambda: RNS.Transport.has_path(destination_hash), msg="Path to "+RNS.prettyhexrep(destination_hash)+" requested", timeout=timeout):
+            print("Path not found")
+            exit(242)
+
+    if listener_destination == None:
+        listener_identity = RNS.Identity.recall(destination_hash)
+        listener_destination = RNS.Destination(
+            listener_identity,
+            RNS.Destination.OUT,
+            RNS.Destination.SINGLE,
+            APP_NAME,
+            "execute"
+        )
+
+    if link == None or link.status == RNS.Link.CLOSED or link.status == RNS.Link.PENDING:
+        link = RNS.Link(listener_destination)
+        link.did_identify = False
+    
+    if not spin(until=lambda: link.status == RNS.Link.ACTIVE, msg="Establishing link with "+RNS.prettyhexrep(destination_hash), timeout=timeout):
+        print("Could not establish link with "+RNS.prettyhexrep(destination_hash))
+        exit(243)
+
+    if not noid and not link.did_identify:
+        link.identify(identity)
+        link.did_identify = True
+
+    if stdin != None:
+        stdin = stdin.encode("utf-8")
+
+    request_data = [
+        command.encode("utf-8"),  # Command to execute
+        timeout,                  # Timeout in seconds
+        stdoutl,                  # Size limit for stdout
+        stderrl,                  # Size limit for stderr
+        stdin,                    # Data passed to stdin
+    ]
+
+    # TODO: Tune
+    rexec_timeout = timeout+link.rtt*4+remote_exec_grace
+
+    request_receipt = link.request(
+        path="command",
+        data=request_data,
+        response_callback=remote_execution_done,
+        failed_callback=remote_execution_done,
+        progress_callback=remote_execution_progress,
+        timeout=rexec_timeout
+    )
+
+    spin(
+        until=lambda:link.status == RNS.Link.CLOSED or (request_receipt.status != RNS.RequestReceipt.FAILED and request_receipt.status != RNS.RequestReceipt.SENT),
+        msg="Sending execution request",
+        timeout=rexec_timeout+0.5
+    )
+
+    if link.status == RNS.Link.CLOSED:
+        print("Could not request remote execution, link was closed")
+        exit(244)
+
+    if request_receipt.status == RNS.RequestReceipt.FAILED:
+        print("Could not request remote execution")
+        if interactive:
+            return
+        else:
+            exit(244)
+
+    spin(
+        until=lambda:request_receipt.status != RNS.RequestReceipt.DELIVERED,
+        msg="Command delivered, awaiting result",
+        timeout=timeout
+    )
+
+    if request_receipt.status == RNS.RequestReceipt.FAILED:
+        print("No result was received")
+        if interactive:
+            return
+        else:
+            exit(245)
+
+    spin_stat(
+        until=lambda:request_receipt.status != RNS.RequestReceipt.RECEIVING,
+        timeout=result_timeout
+    )
+
+    if request_receipt.status == RNS.RequestReceipt.FAILED:
+        print("Receiving result failed")
+        if interactive:
+            return
+        else:
+            exit(246)
+
+    if request_receipt.response != None:
+        try:
+            executed = request_receipt.response[0]
+            retval = request_receipt.response[1]
+            stdout = request_receipt.response[2]
+            stderr = request_receipt.response[3]
+            outlen = request_receipt.response[4]
+            errlen = request_receipt.response[5]
+            started = request_receipt.response[6]
+            concluded = request_receipt.response[7]
+
+        except Exception as e:
+            print("Received invalid result")
+            if interactive:
+                return
+            else:
+                exit(247)
+
+        if executed:
+            if detailed:
+                if stdout != None and len(stdout) > 0:
+                    print(stdout.decode("utf-8"), end="")
+                if stderr != None and len(stderr) > 0:
+                    print(stderr.decode("utf-8"), file=sys.stderr, end="")
+
+                sys.stdout.flush()
+                sys.stderr.flush()
+
+                print("\n--- End of remote output, rnx done ---")
+                if started != None and concluded != None:
+                    cmd_duration = round(concluded - started, 3)
+                    print("Remote command execution took "+str(cmd_duration)+" seconds")
+
+                    total_size = request_receipt.response_size
+                    if request_receipt.request_size != None:
+                        total_size += request_receipt.request_size
+
+                    transfer_duration = round(request_receipt.response_concluded_at - request_receipt.sent_at - cmd_duration, 3)
+                    if transfer_duration == 1:
+                        tdstr = " in 1 second"
+                    elif transfer_duration < 10:
+                        tdstr = " in "+str(transfer_duration)+" seconds"
+                    else:
+                        tdstr = " in "+pretty_time(transfer_duration)
+
+                    spdstr = ", effective rate "+size_str(total_size/transfer_duration, "b")+"ps"
+
+                    print("Transferred "+size_str(total_size)+tdstr+spdstr)
+
+                if outlen != None and stdout != None:
+                    if len(stdout) < outlen:
+                        tstr = ", "+str(len(stdout))+" bytes displayed"
+                    else:
+                        tstr = ""
+                    print("Remote wrote "+str(outlen)+" bytes to stdout"+tstr)
+                
+                if errlen != None and stderr != None:
+                    if len(stderr) < errlen:
+                        tstr = ", "+str(len(stderr))+" bytes displayed"
+                    else:
+                        tstr = ""
+                    print("Remote wrote "+str(errlen)+" bytes to stderr"+tstr)
+
+            else:
+                if stdout != None and len(stdout) > 0:
+                    print(stdout.decode("utf-8"), end="")
+                if stderr != None and len(stderr) > 0:
+                    print(stderr.decode("utf-8"), file=sys.stderr, end="")
+
+
+                if (stdoutl != 0 and len(stdout) < outlen) or (stderrl != 0 and len(stderr) < errlen):
+                    sys.stdout.flush()
+                    sys.stderr.flush()
+                    print("\nOutput truncated before being returned:")
+                    if len(stdout) != 0 and len(stdout) < outlen:
+                        print("  stdout truncated to "+str(len(stdout))+" bytes")
+                    if len(stderr) != 0 and len(stderr) < errlen:
+                        print("  stderr truncated to "+str(len(stderr))+" bytes")
+        else:
+            print("Remote could not execute command")
+            if interactive:
+                return
+            else:
+                exit(248)
+    else:
+        print("No response")
+        if interactive:
+            return
+        else:
+            exit(249)
+
+    try:
+        if not interactive:
+            link.teardown()
+
+    except Exception as e:
+        pass
+
+    if not interactive and mirror:
+        if request_receipt.response[1] != None:
+            exit(request_receipt.response[1])
+        else:
+            exit(240)
+    else:
+        if interactive:
+            if mirror:
+                return request_receipt.response[1]
+            else:
+                return None
+        else:
+            exit(0)
+
+def main():
+    try:
+        parser = argparse.ArgumentParser(description="Reticulum Remote Execution Utility")
+        parser.add_argument("destination", nargs="?", default=None, help="hexadecimal hash of the listener", type=str)
+        parser.add_argument("command", nargs="?", default=None, help="command to be execute", type=str)
+        parser.add_argument("--config", metavar="path", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
+        parser.add_argument('-v', '--verbose', action='count', default=0, help="increase verbosity")
+        parser.add_argument('-q', '--quiet', action='count', default=0, help="decrease verbosity")
+        parser.add_argument('-p', '--print-identity', action='store_true', default=False, help="print identity and destination info and exit")
+        parser.add_argument("-l", '--listen', action='store_true', default=False, help="listen for incoming commands")
+        parser.add_argument('-i', metavar="identity", action='store', dest="identity", default=None, help="path to identity to use", type=str)
+        parser.add_argument("-x", '--interactive', action='store_true', default=False, help="enter interactive mode")
+        parser.add_argument("-b", '--no-announce', action='store_true', default=False, help="don't announce at program start")
+        parser.add_argument('-a', metavar="allowed_hash", dest="allowed", action='append', help="accept from this identity", type=str)
+        parser.add_argument('-n', '--noauth', action='store_true', default=False, help="accept commands from anyone")
+        parser.add_argument('-N', '--noid', action='store_true', default=False, help="don't identify to listener")
+        parser.add_argument("-d", '--detailed', action='store_true', default=False, help="show detailed result output")
+        parser.add_argument("-m", action='store_true', dest="mirror", default=False, help="mirror exit code of remote command")
+        parser.add_argument("-w", action="store", metavar="seconds", type=float, help="connect and request timeout before giving up", default=RNS.Transport.PATH_REQUEST_TIMEOUT)
+        parser.add_argument("-W", action="store", metavar="seconds", type=float, help="max result download time", default=None)
+        parser.add_argument("--stdin", action='store', default=None, help="pass input to stdin", type=str)
+        parser.add_argument("--stdout", action='store', default=None, help="max size in bytes of returned stdout", type=int)
+        parser.add_argument("--stderr", action='store', default=None, help="max size in bytes of returned stderr", type=int)
+        parser.add_argument("--version", action="version", version="rnx {version}".format(version=__version__))
+        
+        args = parser.parse_args()
+
+        if args.listen or args.print_identity:
+            listen(
+                configdir = args.config,
+                identitypath = args.identity,
+                verbosity=args.verbose,
+                quietness=args.quiet,
+                allowed = args.allowed,
+                print_identity=args.print_identity,
+                disable_auth=args.noauth,
+                disable_announce=args.no_announce,
+            )
+
+        elif args.destination != None and args.command != None:
+            execute(
+                configdir = args.config,
+                identitypath = args.identity,
+                verbosity = args.verbose,
+                quietness = args.quiet,
+                detailed = args.detailed,
+                mirror = args.mirror,
+                noid = args.noid,
+                destination = args.destination,
+                command = args.command,
+                stdin = args.stdin,
+                stdoutl = args.stdout,
+                stderrl = args.stderr,
+                timeout = args.w,
+                result_timeout = args.W,
+                interactive = args.interactive,
+            )
+
+        if args.destination != None and args.interactive:
+            # command_history_max = 5000
+            # command_history = []
+            # command_current = ""
+            # history_idx = 0
+            # tty.setcbreak(sys.stdin.fileno())
+
+            code = None
+            while True:
+                try:
+                    cstr = str(code) if code and code != 0 else ""
+                    prompt = cstr+"> "
+                    print(prompt,end="")
+
+                    # cmdbuf = b""
+                    # while True:
+                    #     ch = sys.stdin.read(1)
+                    #     cmdbuf += ch.encode("utf-8")
+                    #     print("\r"+prompt+cmdbuf.decode("utf-8"), end="")    
+                    
+                    command = input()
+                    if command.lower() == "exit" or command.lower() == "quit":
+                        exit(0)
+
+                except KeyboardInterrupt:
+                    exit(0)
+                except EOFError:
+                    exit(0)
+
+                if command.lower() == "clear":
+                    print('\033c', end='')
+
+                # command_history.append(command)
+                # while len(command_history) > command_history_max:
+                #     command_history.pop(0)
+
+                else:
+                    code = execute(
+                        configdir = args.config,
+                        identitypath = args.identity,
+                        verbosity = args.verbose,
+                        quietness = args.quiet,
+                        detailed = args.detailed,
+                        mirror = args.mirror,
+                        noid = args.noid,
+                        destination = args.destination,
+                        command = command,
+                        stdin = None,
+                        stdoutl = args.stdout,
+                        stderrl = args.stderr,
+                        timeout = args.w,
+                        result_timeout = args.W,
+                        interactive = True,
+                    )
+
+        else:
+            print("")
+            parser.print_help()
+            print("")
+
+    except KeyboardInterrupt:
+        # tty.setnocbreak(sys.stdin.fileno())
+        print("")
+        if link != None:
+            link.teardown()
+        exit()
+
+def size_str(num, suffix='B'):
+    units = ['','K','M','G','T','P','E','Z']
+    last_unit = 'Y'
+
+    if suffix == 'b':
+        num *= 8
+        units = ['','K','M','G','T','P','E','Z']
+        last_unit = 'Y'
+
+    for unit in units:
+        if abs(num) < 1000.0:
+            if unit == "":
+                return "%.0f %s%s" % (num, unit, suffix)
+            else:
+                return "%.2f %s%s" % (num, unit, suffix)
+        num /= 1000.0
+
+    return "%.2f%s%s" % (num, last_unit, suffix)
+
+def pretty_time(time, verbose=False):
+    days = int(time // (24 * 3600))
+    time = time % (24 * 3600)
+    hours = int(time // 3600)
+    time %= 3600
+    minutes = int(time // 60)
+    time %= 60
+    seconds = round(time, 2)
+    
+    ss = "" if seconds == 1 else "s"
+    sm = "" if minutes == 1 else "s"
+    sh = "" if hours == 1 else "s"
+    sd = "" if days == 1 else "s"
+
+    components = []
+    if days > 0:
+        components.append(str(days)+" day"+sd if verbose else str(days)+"d")
+
+    if hours > 0:
+        components.append(str(hours)+" hour"+sh if verbose else str(hours)+"h")
+
+    if minutes > 0:
+        components.append(str(minutes)+" minute"+sm if verbose else str(minutes)+"m")
+
+    if seconds > 0:
+        components.append(str(seconds)+" second"+ss if verbose else str(seconds)+"s")
+
+    i = 0
+    tstr = ""
+    for c in components:
+        i += 1
+        if i == 1:
+            pass
+        elif i < len(components):
+            tstr += ", "
+        elif i == len(components):
+            tstr += " and "
+
+        tstr += c
+
+    return tstr
+
+if __name__ == "__main__":
+    main()

RNS/__init__.py

@@ -37,6 +37,8 @@ from .Destination import Destination
 from .Packet import Packet
 from .Packet import PacketReceipt
 from .Resource import Resource, ResourceAdvertisement
+from .Cryptography import HKDF
+from .Cryptography import Hashes
 
 modules = glob.glob(os.path.dirname(__file__)+"/*.py")
 __all__ = [ os.path.basename(f)[:-3] for f in modules if not f.endswith('__init__.py')]
@@ -55,10 +57,11 @@ LOG_FILE     = 0x92
 
 LOG_MAXSIZE  = 5*1024*1024
 
-loglevel     = LOG_NOTICE
-logfile      = None
-logdest      = LOG_STDOUT
-logtimefmt   = "%Y-%m-%d %H:%M:%S"
+loglevel        = LOG_NOTICE
+logfile         = None
+logdest         = LOG_STDOUT
+logtimefmt      = "%Y-%m-%d %H:%M:%S"
+compact_log_fmt = False
 
 instance_random = random.Random()
 instance_random.seed(os.urandom(10))
@@ -99,10 +102,14 @@ def timestamp_str(time_s):
     return time.strftime(logtimefmt, timestamp)
 
 def log(msg, level=3, _override_destination = False):
-    global _always_override_destination
+    global _always_override_destination, compact_log_fmt
     
     if loglevel >= level:
-        logstring = "["+timestamp_str(time.time())+"] ["+loglevelname(level)+"] "+msg
+        if not compact_log_fmt:
+            logstring = "["+timestamp_str(time.time())+"] ["+loglevelname(level)+"] "+msg
+        else:
+            logstring = "["+timestamp_str(time.time())+"] "+msg
+
         logging_lock.acquire()
 
         if (logdest == LOG_STDOUT or _always_override_destination or _override_destination):
@@ -151,6 +158,79 @@ def prettyhexrep(data):
     hexrep = "<"+delimiter.join("{:02x}".format(c) for c in data)+">"
     return hexrep
 
+def prettysize(num, suffix='B'):
+    units = ['','K','M','G','T','P','E','Z']
+    last_unit = 'Y'
+
+    if suffix == 'b':
+        num *= 8
+        units = ['','K','M','G','T','P','E','Z']
+        last_unit = 'Y'
+
+    for unit in units:
+        if abs(num) < 1000.0:
+            if unit == "":
+                return "%.0f %s%s" % (num, unit, suffix)
+            else:
+                return "%.2f %s%s" % (num, unit, suffix)
+        num /= 1000.0
+
+    return "%.2f%s%s" % (num, last_unit, suffix)
+
+def prettytime(time, verbose=False):
+    days = int(time // (24 * 3600))
+    time = time % (24 * 3600)
+    hours = int(time // 3600)
+    time %= 3600
+    minutes = int(time // 60)
+    time %= 60
+    seconds = round(time, 2)
+    
+    ss = "" if seconds == 1 else "s"
+    sm = "" if minutes == 1 else "s"
+    sh = "" if hours == 1 else "s"
+    sd = "" if days == 1 else "s"
+
+    components = []
+    if days > 0:
+        components.append(str(days)+" day"+sd if verbose else str(days)+"d")
+
+    if hours > 0:
+        components.append(str(hours)+" hour"+sh if verbose else str(hours)+"h")
+
+    if minutes > 0:
+        components.append(str(minutes)+" minute"+sm if verbose else str(minutes)+"m")
+
+    if seconds > 0:
+        components.append(str(seconds)+" second"+ss if verbose else str(seconds)+"s")
+
+    i = 0
+    tstr = ""
+    for c in components:
+        i += 1
+        if i == 1:
+            pass
+        elif i < len(components):
+            tstr += ", "
+        elif i == len(components):
+            tstr += " and "
+
+        tstr += c
+
+    if tstr == "":
+        return "0s"
+    else:
+        return tstr
+
+def phyparams():
+    print("Required Physical Layer MTU : "+str(Reticulum.MTU)+" bytes")
+    print("Plaintext Packet MDU        : "+str(Packet.PLAIN_MDU)+" bytes")
+    print("Encrypted Packet MDU        : "+str(Packet.ENCRYPTED_MDU)+" bytes")
+    print("Link Curve                  : "+str(Link.CURVE))
+    print("Link Packet MDU             : "+str(Packet.ENCRYPTED_MDU)+" bytes")
+    print("Link Public Key Size        : "+str(Link.ECPUBSIZE*8)+" bits")
+    print("Link Private Key Size       : "+str(Link.KEYSIZE*8)+" bits")
+
 def panic():
     os._exit(255)
 

RNS/_version.py

@@ -1 +1 @@
-__version__ = "0.3.6"
+__version__ = "0.4.7"

RNS/vendor/i2plib/aiosam.py

@@ -25,7 +25,7 @@ async def get_sam_socket(sam_address=sam.DEFAULT_ADDRESS, loop=None):
     :param loop: (optional) event loop instance
     :return: A (reader, writer) pair
     """
-    reader, writer = await asyncio.open_connection(*sam_address, loop=loop)
+    reader, writer = await asyncio.open_connection(*sam_address)
     writer.write(sam.hello("3.1", "3.1"))
     reply = parse_reply(await reader.readline())
     if reply.ok:

RNS/vendor/i2plib/tunnel.py

@@ -85,17 +85,35 @@ class ClientTunnel(I2PTunnel):
         """A coroutine used to run the tunnel"""
         await self._pre_run()
 
+        self.status = { "setup_ran": False, "setup_failed": False, "exception": None, "connect_tasks": [] }
         async def handle_client(client_reader, client_writer):
             """Handle local client connection"""
-            remote_reader, remote_writer = await aiosam.stream_connect(
-                    self.session_name, self.remote_destination, 
-                    sam_address=self.sam_address, loop=self.loop)
-            asyncio.ensure_future(proxy_data(remote_reader, client_writer), 
-                                  loop=self.loop)
-            asyncio.ensure_future(proxy_data(client_reader, remote_writer),
-                                  loop=self.loop)
+            try:
+                sc_task = aiosam.stream_connect(
+                        self.session_name, self.remote_destination, 
+                        sam_address=self.sam_address, loop=self.loop)
+                self.status["connect_tasks"].append(sc_task)
+                
+                remote_reader, remote_writer = await sc_task
+                asyncio.ensure_future(proxy_data(remote_reader, client_writer), 
+                                      loop=self.loop)
+                asyncio.ensure_future(proxy_data(client_reader, remote_writer),
+                                      loop=self.loop)
 
-        self.server = await asyncio.start_server(handle_client, *self.local_address, loop=self.loop)
+            except Exception as e:
+                self.status["setup_ran"] = True
+                self.status["setup_failed"] = True
+                self.status["exception"] = e
+
+
+        try:
+            self.server = await asyncio.start_server(handle_client, *self.local_address)
+            self.status["setup_ran"] = True
+
+        except Exception as e:
+            self.status["setup_ran"] = True
+            self.status["setup_failed"] = True
+            self.status["exception"] = e
 
     def stop(self):
         super().stop()
@@ -117,26 +135,39 @@ class ServerTunnel(I2PTunnel):
         """A coroutine used to run the tunnel"""
         await self._pre_run()
 
+        self.status = { "setup_ran": False, "setup_failed": False, "exception": None, "connect_tasks": [] }
         async def handle_client(incoming, client_reader, client_writer):
-            # data and dest may come in one chunk
-            dest, data = incoming.split(b"\n", 1) 
-            remote_destination = sam.Destination(dest.decode())
-            logger.debug("{} client connected: {}.b32.i2p".format(
-                self.session_name, remote_destination.base32))
+            try:
+                # data and dest may come in one chunk
+                dest, data = incoming.split(b"\n", 1) 
+                remote_destination = sam.Destination(dest.decode())
+                logger.debug("{} client connected: {}.b32.i2p".format(
+                    self.session_name, remote_destination.base32))
+            
+            except Exception as e:
+                self.status["exception"] = e
+                self.status["setup_failed"] = True
+                data = None
 
             try:
-                remote_reader, remote_writer = await asyncio.wait_for(
+                sc_task = asyncio.wait_for(
                         asyncio.open_connection(
                            host=self.local_address[0], 
-                           port=self.local_address[1], loop=self.loop),
-                        timeout=5, loop=self.loop)
+                           port=self.local_address[1]),
+                        timeout=5)
+                self.status["connect_tasks"].append(sc_task)
+
+                remote_reader, remote_writer = await sc_task
                 if data: remote_writer.write(data)
                 asyncio.ensure_future(proxy_data(remote_reader, client_writer),
                                       loop=self.loop)
                 asyncio.ensure_future(proxy_data(client_reader, remote_writer),
                                       loop=self.loop)
+
             except ConnectionRefusedError:
                 client_writer.close()
+                self.status["exception"] = e
+                self.status["setup_failed"] = True
 
         async def server_loop():
             try:
@@ -151,6 +182,7 @@ class ServerTunnel(I2PTunnel):
                 pass
 
         self.server_loop = asyncio.ensure_future(server_loop(), loop=self.loop)
+        self.status["setup_ran"] = True
 
     def stop(self):
         super().stop()

RNS/vendor/platformutils.py

@@ -8,6 +8,12 @@ def get_platform():
         import sys
         return sys.platform
 
+def is_linux():
+    if get_platform() == "linux":
+        return True
+    else:
+        return False
+
 def is_darwin():
     if get_platform() == "darwin":
         return True
@@ -42,4 +48,4 @@ def cryptography_old_api():
     if cryptography.__version__ == "2.8":
         return True
     else:
-        return False
+        return False

Roadmap.md

@@ -0,0 +1,134 @@
+# Reticulum Development Roadmap
+This document outlines the currently established development roadmap for Reticulum.
+
+1. [Currently Active Work Areas](#currently-active-work-areas)
+2. [Primary Efforts](#primary-efforts)
+    - [Comprehensibility](#comprehensibility)
+    - [Universality](#universality)
+    - [Functionality](#functionality)
+    - [Usability & Utility](#usability--utility)
+    - [Interfaceability](#interfaceability)
+3. [Auxillary Efforts](#auxillary-efforts)
+4. [Release History](#release-history)
+
+## Currently Active Work Areas
+For each release cycle of Reticulum, improvements and additions from the five [Primary Efforts](#primary-efforts) are selected as active work areas, and can be expected to be included in the upcoming releases within that cycle. While not entirely set in stone for each release cycle, they serve as a pointer of what to expect in the near future.
+
+- The current `0.4.x` release cycle aims at completing:
+  - [x] Improve storage persist call on local client connect/disconnect
+  - [x] Better path invalidation on roaming interfaces
+  - [x] Improved roaming support on Android
+  - [ ] Updating the documentation to reflect recent changes and improvements
+  - [ ] Add bluetooth pairing code output to rnodeconf
+  - [ ] Transit traffic display in rnstatus
+  - [ ] JSON output mode for rnstatus
+  - [ ] Add `rnid` utility
+  - [ ] Add `rnsign` utility
+  - [ ] Add `rncrypt` utility
+  - [ ] Create a standalone RNS Daemon app for Android
+- Targets for related applications
+  - [x] Add offline & paper message transport to LXMF
+  - [x] Implement paper messaging in Nomad Network
+  - [x] Implement paper messaging in Sideband
+  - [x] Add spatial and multi-interface roaming support in Sideband
+  - [x] Expand device support in Sideband to support older Android devices
+
+## Primary Efforts
+The development path for Reticulum is currently laid out in five distinct areas: *Comprehensibility*, *Universality*, *Functionality*, *Usability & Utility* and *Interfaceability*. Conceptualising the development of Reticulum into these areas serves to advance the implementation and work towards the Foundational Goals & Values of Reticulum.
+
+### Comprehensibility
+These efforts are aimed at improving the ease of which Reticulum is understood, and lowering the barrier to entry for people who wish to start building systems on Reticulum.
+
+- Improving [the manual](https://markqvist.github.io/Reticulum/manual/) with tutorials specifically for beginners
+- Updating the documentation to reflect recent changes and improvements
+    - Update descriptions of protocol mechanics
+        - Update announce description
+        - Add in-depth explanation of the IFAC system
+    - Software
+        - Update Sideband screenshots
+        - Update Sideband description
+        - Update NomadNet screenshots
+        - Update Sideband screenshots
+    - Installation
+        - Install docs for fedora, needs `python3-netifaces`
+        - Add a *Reticulum On Raspberry Pi* section
+        - Update *Reticulum On Android* section if necessary
+        - Update Android install documentation.
+    - Communications hardware section
+        - Add information about RNode external displays.
+        - Packet radio modems.
+        - Possibly add other relevant types here as well.
+    - Setup *Best Practices For...* / *Installation Examples* section.
+        - Home or office (example)
+        - Vehicles (example)
+        - No-grid/solar/remote sites (example)
+
+### Universality
+These efforts seek to broaden the universality of the Reticulum software and hardware ecosystem by continously diversifying platform support, and by improving the overall availability and ease of deployment of the Reticulum stack.
+
+- Improved roaming support on Android
+- OpenWRT support
+- Create a standalone RNS Daemon app for Android
+- A lightweight and portable C implementation for microcontrollers, µRNS
+- A portable, high-performance Reticulum implementation in C/C++, see [#21](https://github.com/markqvist/Reticulum/discussions/21)
+- Performance and memory optimisations of the Python implementation
+- Bindings for other programming languages
+
+### Functionality
+These efforts aim to expand and improve the core functionality and reliability of Reticulum.
+
+- Improve storage persist call on local client connect/disconnect
+- Faster path invalidation on physical topography changes
+- Better path invalidation on roaming interfaces
+- Add a `Buffer` class to the API, for handling stream-like buffers over Reticulum
+- Network-wide path balancing
+- Distributed Destination Naming System
+- Globally routable multicast
+- Destination proxying
+- [Metric-based path selection and multiple paths](https://github.com/markqvist/Reticulum/discussions/86)
+
+### Usability & Utility
+These effors seek to make Reticulum easier to use and operate, and to expand the utility of the stack on deployed systems.
+
+- Add bluetooth pairing code output to rnodeconf
+- Easy way to share interface configurations, see [#19](https://github.com/markqvist/Reticulum/discussions/19)
+- Transit traffic display in rnstatus
+- JSON output mode for rnstatus
+- rnid utility
+- rnsign utility
+- rncrypt utility
+- rnsconfig utility
+- Expand rnx utility to true interactive remote shell
+
+### Interfaceability
+These efforts aim to expand the types of physical and virtual interfaces that Reticulum can natively use to transport data.
+
+- Filesystem interface
+- Plain ESP32 devices (ESP-Now, WiFi, Bluetooth, etc.)
+- More LoRa transceivers
+- AT-compatible modems
+- Direct SDR Support
+- Optical mediums
+- IR Transceivers
+- AWDL / OWL
+- HF Modems
+- GNU Radio
+- CAN-bus
+- Raw SPI
+- Raw i²c
+- MQTT
+- XBee
+- Tor
+
+## Auxillary Efforts
+The Reticulum ecosystem is enriched by several other software and hardware projects, and the support and improvement of these, in symbiosis with the core Reticulum project helps expand the reach and utility of Reticulum itself.
+
+This section lists, in no particular order, various important efforts that would be beneficial to the goals of Reticulum.
+
+- The [RNode](https://unsigned.io/rnode/) project
+    - [ ] Evolve RNode into a self-replicating system, so that anyone can use an existing RNode to create more RNodes, and bootstrap functional networks based on Reticulum, even in absence of the Internet.
+    - [ ] Create a WebUSB-based bootstrapping utility, and integrate this directly into the [RNode Bootstrap Console](#), both on-device, and on an Internet-reachable copy. This will make it much easier to create new RNodes for average users.
+
+## Release History
+
+Please see the [Changelog](./Changelog.md) for a complete release history and changelog of Reticulum.

docs/manual/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: cb185315f69a86407f1dca0f5d7d25ef
+config: afc56b8a3660cd0f3eeb6ca077eaba56
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/manual/_sources/gettingstartedfast.rst.txt

@@ -7,16 +7,35 @@ you want to do. This guide will outline sensible starting paths for different
 scenarios.
 
 
+Standalone Reticulum Installation
+=============================================
+If you simply want to install Reticulum and related utilities on a system,
+the easiest way is via ``pip``:
+
+.. code::
+
+   pip install rns
+
+If you no not already have pip installed, you can install it using the package manager
+of your system with a command like ``sudo apt install python3-pip``,
+``sudo pamac install python-pip`` or similar. You can also dowload the Reticulum release
+wheels from GitHub, or other release channels, and install them offline using ``pip``:
+
+.. code::
+
+   pip install ./rns-0.4.6-py3-none-any.whl
+
+
 Try Using a Reticulum-based Program
 =============================================
 
 If you simply want to try using a program built with Reticulum, a few different
-programs exist that allow basic communication and a range of other useful functions
-over even extremely low-bandwidth Reticulum networks.
+programs exist that allow basic communication and a range of other useful functions,
+even over extremely low-bandwidth Reticulum networks.
 
 These programs will let you get a feel for how Reticulum works. They have been designed
-to run well over networks based on LoRa or packet radio, but can also be used completely
-over local WiFi, wired ethernet, the Internet, or any combination.
+to run well over networks based on LoRa or packet radio, but can also be used over fast
+links, such as local WiFi, wired Ethernet, the Internet, or any combination.
 
 As such, it is easy to get started experimenting, without having to set up any radio
 transceivers or infrastructure just to try it out. Launching the programs on separate
@@ -44,7 +63,7 @@ You can install Nomad Network via pip:
 .. code::
 
    # Install ...
-   pip3 install nomadnet
+   pip install nomadnet
 
    # ... and run
    nomadnet
@@ -61,13 +80,22 @@ If you would rather use a program with a graphical user interface, you can take
 a look at `Sideband <https://unsigned.io/sideband>`_, which is available for Android,
 Linux and macOS.
 
-.. image:: screenshots/sideband_1.png
-    :width: 400px
-    :align: center
-    :target: _images/sideband_1.png
+.. only:: html
 
-Sideband is currently in the early stages of development, but already provides basic
-communication features, and interoperates with Nomad Network, or any other LXMF client.
+  .. image:: screenshots/sideband_devices.webp
+      :align: center
+      :target: _images/sideband_devices.webp
+
+.. only:: latexpdf
+
+  .. image:: screenshots/sideband_devices.png
+      :align: center
+      :target: _images/sideband_devices.png
+
+Sideband allows you to communicate with other people or LXMF-compatible
+systems over Reticulum networks using LoRa, Packet Radio, WiFi, I2P, Encrypted QR
+Paper Messages, or anything else Reticulum supports. It also interoperates with
+the Nomad Network program.
 
 Using the Included Utilities
 =============================================
@@ -92,7 +120,7 @@ or use the interactive ``rnsconfig`` utility.
 
 When Reticulum is started for the first time, it will create a default
 configuration file, with one active interface. This default interface uses
-your existing ethernet and WiFi networks (if any), and only allows you to
+your existing Ethernet and WiFi networks (if any), and only allows you to
 communicate with other Reticulum peers within your local broadcast domains.
 
 To communicate further, you will have to add one or more interfaces. The default
@@ -103,16 +131,16 @@ With Reticulum, you only need to configure what interfaces you want to communica
 over. There is no need to configure address spaces, subnets, routing tables,
 or other things you might be used to from other network types.
 
-Once Reticulums knows which interfaces it should use, it will automatically
+Once Reticulum knows which interfaces it should use, it will automatically
 discover topography and configure transport of data to any destinations it
 knows about.
 
-In situations where you already have an established WiFi or ethernet network, and
-many devices that want to utilise the same external Reticulum network (for example over
+In situations where you already have an established WiFi or Ethernet network, and
+many devices that want to utilise the same external Reticulum network paths (for example over
 LoRa), it will often be sufficient to let one system act as a Reticulum gateway, by
-adding any external interfaces to this systems configuration, and enabling transport. Any
+adding any external interfaces to the configuration of this system, and then enabling transport on it. Any
 other device on your local WiFi will then be able to connect to this wider Reticulum
-network just using the default interface configuration.
+network just using the default (:ref:`AutoInterface<interfaces-auto>`) configuration.
 
 Possibly, the examples in the config file are enough to get you started. If
 you want more information, you can read the :ref:`Building Networks<networks-main>`
@@ -132,12 +160,12 @@ TCP connections reveal the IP address of both your instance and the server to an
 inspect the connection. Someone could use this information to determine your location or identity. Adversaries 
 inspecting your packets may be able to record packet metadata like time of transmission and packet size.
 Even though Reticulum encrypts traffic, TCP does not, so an adversary may be able to use
-packet inspection to learn that a system is running Reticulum, and what other IP adresses connect to it.
+packet inspection to learn that a system is running Reticulum, and what other IP addresses connect to it.
 Hosting a publicly reachable instance over TCP also requires a publicly reachable IP address,
 which most Internet connections don't offer anymore.
 
 The ``I2PInterface`` routes messages through the `Invisible Internet Protocol 
-(I2P) <https://geti2p.net/en/>`_. To properly use this interface, users must also run an I2P daemon in
+(I2P) <https://geti2p.net/en/>`_. To use this interface, users must also run an I2P daemon in
 parallel to ``rnsd``. For always-on I2P nodes it is recommended to use `i2pd <https://i2pd.website/>`_. 
 
 By default, I2P will encrypt and mix all traffic sent over the Internet, and 
@@ -146,12 +174,13 @@ will also relay other I2P user's encrypted packets, which will use extra
 bandwidth and compute power, but also makes timing attacks and other forms of 
 deep-packet-inspection much more difficult.
 
-I2P also allows users to host globally available Reticulum instances from non-public IPs and behind firewalls.
+I2P also allows users to host globally available Reticulum instances from non-public IP's and behind firewalls and NAT.
 
-In general it is recommended to use an I2P node if you want to host a publically accessible
+In general it is recommended to use an I2P node if you want to host a publicly accessible
 instance, while preserving anonymity. If you care more about performance, and a slightly
 easier setup, use TCP.
 
+
 Connect to the Public Testnet
 ===========================================
 
@@ -160,26 +189,61 @@ by adding one of the following interfaces to your ``.reticulum/config`` file:
 
 .. code::
 
-  # For connecting over TCP/IP:
-  [[RNS Testnet Frankfurt]]
+  # TCP/IP interface to the Dublin hub
+  [[RNS Testnet Dublin]]
     type = TCPClientInterface
-    interface_enabled = yes
-    outgoing = True
-    target_host = frankfurt.rns.unsigned.io
+    enabled = yes
+    target_host = dublin.connect.reticulum.network
     target_port = 4965
 
+  # TCP/IP interface to the Frankfurt hub
+  [[RNS Testnet Dublin]]
+    type = TCPClientInterface
+    enabled = yes
+    target_host = frankfurt.connect.reticulum.network
+    target_port = 5377
 
-  # For connecting over I2P:
-  [[RNS Testnet I2P Node A]]
+  # Interface to I2P hub A
+  [[RNS Testnet I2P Hub A]]
     type = I2PInterface
-    interface_enabled = yes
-    peers = ykzlw5ujbaqc2xkec4cpvgyxj257wcrmmgkuxqmqcur7cq3w3lha.b32.i2p
+    enabled = yes
+    peers = uxg5kubabakh3jtnvsipingbr5574dle7bubvip7llfvwx2tgrua.b32.i2p
 
 Many other Reticulum instances are connecting to this testnet, and you can also join it
 via other entry points if you know them. There is absolutely no control over the network
 topography, usage or what types of instances connect. It will also occasionally be used
 to test various failure scenarios, and there are no availability or service guarantees.
 
+
+Adding Radio Interfaces
+==============================================
+Once you have Reticulum installed and working, you can add radio interfaces with
+any compatible hardware you have available. Reticulum supports a wide range of radio
+hardware, and if you already have any available, it is very likely that it will
+work with Reticulum. For information on how to configure this, see the
+:ref:`Interfaces<interfaces-main>` section of this manual.
+
+If you do not already have transceiver hardware available, you can easily and
+cheaply build an :ref:`RNode<rnode-main>`, which is a general-purpose long-range
+digital radio transceiver, that integrates easily with Reticulum.
+
+To build one yourself requires installing a custom firmware on a supported LoRa
+development board with an auto-install script. Please see the :ref:`Communications Hardware<hardware-main>`
+chapter for a guide. If you prefer purchasing a ready-made unit, you can refer to the
+:ref:`list of suppliers<rnode-suppliers>`. For more information on RNode, you can also
+refer to these additional external resources:
+
+* `How To Make Your Own RNodes <https://unsigned.io/how-to-make-your-own-rnodes/>`_
+* `Installing RNode Firmware on Compatible LoRa Devices <https://unsigned.io/installing-rnode-firmware-on-t-beam-and-lora32-devices/>`_
+* `Private, Secure and Uncensorable Messaging Over a LoRa Mesh <https://unsigned.io/private-messaging-over-lora/>`_
+* `RNode Firmware <https://github.com/markqvist/RNode_Firmware/>`_
+
+If you have communications hardware that is not already supported by any of the
+:ref:`existing interface types<interfaces-main>`, but you think would be suitable for use with Reticulum,
+you are welcome to head over to the `GitHub discussion pages <https://github.com/markqvist/Reticulum/discussions>`_
+and propose adding an interface for the hardware.
+
+
 Develop a Program with Reticulum
 ===========================================
 If you want to develop programs that use Reticulum, the easiest way to get
@@ -237,19 +301,22 @@ don't use pip, but try this recipe:
 
     # Run the example in client mode to "ping" the server.
     # Replace the hash below with the actual destination hash of your server.
-    python3 Examples/Echo.py 3e12fc71692f8ec47bc5
+    python3 Examples/Echo.py 174a64852a75682259ad8b921b8bf416
 
     # Have a look at another example
     python3 Examples/Filetransfer.py -h
 
 When you have experimented with the basic examples, it's time to go read the
-:ref:`Understanding Reticulum<understanding-main>` chapter.
+:ref:`Understanding Reticulum<understanding-main>` chapter. Before submitting
+your first pull request, it is probably a good idea to introduce yourself on
+the `disucssion forum on GitHub <https://github.com/markqvist/Reticulum/discussions>`_,
+or ask one of the developers or maintainers for a good place to start.
 
 
 Reticulum on ARM64
 ==============================================
 On some architectures, including ARM64, not all dependencies have precompiled
-binaries. On such systems, you will need to install ``python3-dev`` before
+binaries. On such systems, you may need to install ``python3-dev`` before
 installing Reticulum or programs that depend on Reticulum.
 
 .. code::
@@ -262,6 +329,12 @@ installing Reticulum or programs that depend on Reticulum.
    python3 -m pip install rns
 
 
+Reticulum on Raspberry Pi
+==============================================
+It is currently recommended to use a 64-bit version of the Raspberry Pi OS
+if you want to run Reticulum on Raspberry Pi computers, since 32-bit versions
+don't always have packages available for some dependencies.
+
 Reticulum on Android
 ==============================================
 Reticulum can be used on Android in different ways. The easiest way to get
@@ -275,12 +348,31 @@ Termux is a terminal emulator and Linux environment for Android based devices,
 which includes the ability to use many different programs and libraries,
 including Reticulum.
 
-Since the Python cryptography.io module does not offer pre-built wheels for
-Android, the standard one-line install of Reticulum does not work on Android,
-and a few extra commands are required.
+To use Reticulum within the Termux environment, you will need to install
+``python`` and the ``python-cryptography`` library using ``pkg``, the package-manager
+build into Termux. After that, you can use ``pip`` to install Reticulum.
 
 From within Termux, execute the following:
 
+.. code::
+
+    # First, make sure indexes and packages are up to date.
+    pkg update
+    pkg upgrade
+
+    # Then install python and the cryptography library.
+    pkg install python python-cryptography
+
+    # Make sure pip is up to date, and install the wheel module.
+    pip install wheel pip --upgrade
+
+    # Install Reticulum
+    pip install rns
+
+If for some reason the ``python-cryptography`` package is not available for
+your platform via the Termux package manager, you can attempt to build it
+locally on your device using the following command:
+
 .. code::
 
     # First, make sure indexes and packages are up to date.
@@ -291,7 +383,7 @@ From within Termux, execute the following:
     pkg install python build-essential openssl libffi rust
 
     # Make sure pip is up to date, and install the wheel module.
-    pip3 install wheel pip --upgrade
+    pip install wheel pip --upgrade
 
     # To allow the installer to build the cryptography module,
     # we need to let it know what platform we are compiling for:
@@ -300,31 +392,37 @@ From within Termux, execute the following:
     # Start the install process for the cryptography module.
     # Depending on your device, this can take several minutes,
     # since the module must be compiled locally on your device.
-    pip3 install cryptography
+    pip install cryptography
 
     # If the above installation succeeds, you can now install
     # Reticulum and any related software
-    pip3 install rns
+    pip install rns
 
 It is also possible to include Reticulum in apps compiled and distributed as
 Android APKs. A detailed tutorial and example source code will be included
-here at a later point.
+here at a later point. Until then you can use the `Sideband source code <https://github.com/markqvist/sideband>`_ as an example and startig point.
 
-Adding Radio Interfaces
+Pure-Python Reticulum
 ==============================================
-Once you have Reticulum installed and working, you can add radio interfaces with
-any compatible hardware you have available. For information on how to configure
-this, see the :ref:`Interfaces<interfaces-main>` section of this manual.
+In some rare cases, and on more obscure system types, it is not possible to
+install one or more dependencies
 
-A range of common LoRa development boards and transceiver modules can be used
-as interfaces with Reticulum. You can refer to the following external resources
-for more information:
+On more unusual systems, and in some rare cases, it might not be possible to
+install or even compile one or more of the above modules. In such situations,
+you can use the ``rnspure`` package instead of the ``rns`` package, or use ``pip``
+with the ``--no-dependencies`` command-line option. The ``rnspure``
+package requires no external dependencies for installation. Please note that the
+actual contents of the ``rns`` and ``rnspure`` packages are *completely identical*.
+The only difference is that the ``rnspure`` package lists no dependencies required
+for installation.
 
-* `How To Make Your Own RNodes <https://unsigned.io/how-to-make-your-own-rnodes/>`_
-* `Installing RNode Firmware on Compatible LoRa Devices <https://unsigned.io/installing-rnode-firmware-on-t-beam-and-lora32-devices/>`_
-* `Private, Secure and Uncensorable Messaging Over a LoRa Mesh <https://unsigned.io/private-messaging-over-lora/>`_
-* `RNode Firmware <https://github.com/markqvist/RNode_Firmware/>`_
+No matter how Reticulum is installed and started, it will load external dependencies
+only if they are *needed* and *available*. If for example you want to use Reticulum
+on a system that cannot support ``pyserial``, it is perfectly possible to do so using
+the `rnspure` package, but Reticulum will not be able to use serial-based interfaces.
+All other available modules will still be loaded when needed.
 
-If you have communications hardware that you think would be suitable for use with Reticulum,
-you are welcome to head over to the `GitHub discussion pages <https://github.com/markqvist/Reticulum/discussions>`_
-and propose adding an interface for the hardware.
+**Please Note!** If you use the `rnspure` package to run Reticulum on systems that
+do not support `PyCA/cryptography <https://github.com/pyca/cryptography>`_, it is
+important that you read and understand the :ref:`Cryptographic Primitives <understanding-primitives>`
+section of this manual.

docs/manual/_sources/hardware.rst.txt

@@ -0,0 +1,241 @@
+.. _hardware-main:
+
+***********************
+Communications Hardware
+***********************
+
+One of the truly valuable aspects of Reticulum is the ability to use it over
+almost any conceivable kind of communications medium. The :ref:`interface types<interfaces-main>`
+available for configuration in Reticulum are flexible enough to cover the use
+of most wired and wireless communications hardware available, from decades-old
+packet radio modems to modern millimeter-wave backhaul systems.
+
+If you already have or operate some kind of communications hardware, there is a
+very good chance that it will work with Reticulum out of the box. In case it does
+not, it is possible to provide the necessary glue with very little effort using
+for example the :ref:`PipeInterface<interfaces-pipe>` or the :ref:`TCPClientInterface<interfaces-tcpc>`
+in combination with code like `TCP KISS Server <https://github.com/simplyequipped/tcpkissserver>`_
+by `simplyequipped <https://github.com/simplyequipped>`_.
+
+While this broad support and flexibility is very useful, an abundance of options
+can sometimes make it difficult to know where to begin, especially when you are
+starting from scratch.
+
+This chapter will outline a few different sensible starting paths to get
+real-world functional wireless communications up and running with minimal cost
+and effort. Two fundamental devices categories will be covered, *RNodes* and
+*WiFi-based radios*.
+
+While there are many other device categories that are useful in building Reticulum
+networks, knowing how to employ just these two will make it possible to build
+a wide range of useful networks with little effort.
+
+.. _rnode-main:
+
+RNode
+=====
+
+Reliable and general-purpose long-range digital radio transceiver systems are
+commonly either very expensive, difficult to set up and operate, hard to source,
+power-hungry, or all of the above at the same time. In an attempt to alleviate
+this situation, the transceiver system *RNode* was designed. It is important to
+note that RNode is not one specific device, from one particular vendor, but
+*an open plaform* that anyone can use to build interoperable digital transceivers
+suited to their needs and particular situations.
+
+An RNode is a general purpose, interoperable, low-power and long-range, reliable,
+open and flexible radio communications device. Depending on its components, it can
+operate on many different frequency bands, and use many different modulation
+schemes, but most commonly, and for the purposes of this chapter, we will limit
+the discussion to RNodes using *LoRa* modulation in common ISM bands.
+
+**Avoid Confusion!** RNodes can use LoRa as a *physical-layer modulation*, but it
+does not use, and has nothing to do with the *LoRaWAN* protocol and standard, commonly
+used for centrally controlled IoT devices. RNodes use *raw LoRa modulation*, without
+any additional protocol overhead. All high-level protocol functionality is handled
+directly by Reticulum.
+
+.. _rnode-creating:
+
+Creating RNodes
+^^^^^^^^^^^^^^^
+RNode has been designed as a system that is easy to replicate across time and
+space. You can put together a functioning transceiver using commonly available
+components, and a few open source software tools. While you can design and build RNodes
+completely from scratch, to your exact desired specifications, this chapter
+will explain the easiest possible approach to creating RNodes: Using common
+LoRa development boards. This approach can be boiled down to two simple steps:
+
+1. Obtain one or more supported development boards
+2. Install the RNode firmware with the automated installer
+
+Once the firmware has been installed and provisioned by the install script, it
+is ready to use with any software that supports RNodes, including Reticulum.
+The device can be used with Reticulum by adding an :ref:`RNodeInterface<interfaces-rnode>`
+to the configuration.
+
+.. _rnode-supported:
+
+Supported Boards
+^^^^^^^^^^^^^^^^
+To create one or more RNodes, you will need to obtain supported development
+boards. The following boards are supported by the auto-installer.
+
+LilyGO LoRa32 v2.1
+""""""""""""""""""
+.. image:: graphics/board_t3v21.png
+    :width: 46%
+    :align: center
+
+- **Supported Firmware Lines** v1.x & v2.x
+- **Transceiver IC** Semtech SX1276
+- **Device Platform** ESP32
+- **Manufacturer** `LilyGO <https://lilygo.cn>`_
+
+
+LilyGO LoRa32 v2.0
+""""""""""""""""""
+.. image:: graphics/board_t3v20.png
+    :width: 46%
+    :align: center
+
+- **Supported Firmware Lines** v1.x & v2.x
+- **Transceiver IC** Semtech SX1276
+- **Device Platform** ESP32
+- **Manufacturer** `LilyGO <https://lilygo.cn>`_
+
+
+LilyGO T-Beam
+"""""""""""""
+.. image:: graphics/board_tbeam.png
+    :width: 75%
+    :align: center
+
+- **Supported Firmware Lines** v1.x & v2.x
+- **Transceiver IC** Semtech SX1276
+- **Device Platform** ESP32
+- **Manufacturer** `LilyGO <https://lilygo.cn>`_
+
+
+Heltec LoRa32 v2.0
+""""""""""""""""""
+.. image:: graphics/board_heltec32.png
+    :width: 58%
+    :align: center
+
+- **Supported Firmware Lines** v1.x & v2.x
+- **Transceiver IC** Semtech SX1276
+- **Device Platform** ESP32
+- **Manufacturer** `Heltec Automation <https://heltec.org>`_
+
+
+Unsigned RNode v2.x
+"""""""""""""""""""
+.. image:: graphics/board_rnodev2.png
+    :width: 58%
+    :align: center
+
+- **Supported Firmware Lines** v1.x & v2.x
+- **Transceiver IC** Semtech SX1276
+- **Device Platform** ESP32
+- **Manufacturer** `unsigned.io <https://unsigned.io>`_
+
+
+Unsigned RNode v1.x
+"""""""""""""""""""
+.. image:: graphics/board_rnode.png
+    :width: 50%
+    :align: center
+
+- **Supported Firmware Lines** v1.x
+- **Transceiver IC** Semtech SX1276
+- **Device Platform** AVR ATmega1284p
+- **Manufacturer** `unsigned.io <https://unsigned.io>`_
+
+
+.. _rnode-installation:
+
+Installation
+^^^^^^^^^^^^
+
+Once you have obtained compatible boards, you can install the `RNode Firmware <https://github.com/markqvist/RNode_Firmware>`_
+using the `RNode Configuration Utility <https://github.com/markqvist/rnodeconfigutil>`_.
+If you have installed Reticulum on your system, the ``rnodeconf`` program will already be
+available. If not, make sure that ``Python3`` and ``pip`` is installed on your system, and
+then install Reticulum with with ``pip``:
+
+.. code::
+
+   pip install rns
+
+Once installation has completed, it is time to start installing the firmware on your
+devices. Run ``rnodeconf`` in auto-install mode like so:
+
+.. code::
+
+   rnodeconf --autoinstall
+
+The utility will guide you through the installation process by asking a series of
+questions about your hardware. Simply follow the guide, and the utility will
+auto-install and configure your devices.
+
+.. _rnode-usage:
+
+Usage with Reticulum
+^^^^^^^^^^^^^^^^^^^^
+When the devices have been installed and provisioned, you can use them with Reticulum
+by adding the :ref:`relevant interface section<interfaces-rnode>` to the configuration
+file of Reticulum. For v1.x firmwares, you will have to specify all interface parameters,
+such as serial port and on-air parameters. For v2.x firmwares, you just need to specify
+the Connection ID of the RNode, and Reticulum will automatically locate and connect to the
+RNode, using the parameters stored in the RNode itself.
+
+.. _rnode-suppliers:
+
+Suppliers
+^^^^^^^^^
+Get in touch if you want to have your RNode supplier listed here, or if you want help to
+get started with producing RNodes.
+
+
+WiFi-based Hardware
+===================
+
+It is possible to use all kinds of both short- and long-range WiFi-based hardware
+with Reticulum. Any kind of hardware that fully supports bridged Ethernet over the
+WiFi interface will work with the :ref:`AutoInterface<interfaces-auto>` in Reticulum.
+Most devices will behave like this by default, or allow it via configuration options.
+
+This means that you can simply configure the physical links of the WiFi based devices,
+and start communicating over them using Reticulum. It is not necessary to enable any IP
+infrastructure such as DHCP servers, DNS or similar, as long as at least Ethernet is
+available, and packets are passed transparently over the physical WiFi-based devices.
+
+.. only:: html
+
+    .. image:: graphics/radio_rblhg5.png
+        :width: 49%
+
+    .. image:: graphics/radio_is5ac.png
+        :width: 49%
+
+Below is a list of example WiFi (and similar) radios that work well for high capacity
+Reticulum links over long distances:
+
+- `Ubiquiti airMAX radios <https://store.ui.com/collections/operator-airmax-devices>`_
+- `Ubiquiti LTU radios <https://store.ui.com/collections/operator-ltu>`_
+- `MikroTik radios <https://mikrotik.com/products/group/wireless-systems>`_
+
+This list is by no means exhaustive, and only serves as a few examples of radio hardware
+that is relatively cheap while providing long range and high capacity for Reticulum
+networks. As in all other cases, it is also possible for Reticulum to co-exist with IP
+networks running concurrently on such devices.
+
+Combining Hardware Types
+========================
+
+It is useful to combine different link and hardware types when designing and
+building a network. One useful design pattern is to employ high-capacity point-to-point
+links based on WiFi or millimeter-wave radios (with high-gain directional antennas)
+for the network backbone, and using LoRa-based RNodes for covering large areas with
+connectivity for client devices.

docs/manual/_sources/index.rst.txt

@@ -5,21 +5,34 @@ This manual aims to provide you with all the information you need to
 understand Reticulum, build networks or develop programs using it, or
 to participate in the development of Reticulum itself.
 
+.. only:: html
+
+   Table Of Contents
+   =================
+
 .. toctree::
    :maxdepth: 3
 
    whatis
    gettingstartedfast
    using
-   networks
-   interfaces
    understanding
-   reference
+   hardware
+   interfaces
+   networks
    examples
+   support
+
+.. toctree::
+   :maxdepth: 2
+
+   reference
 
 
-Indices and Tables
-==================
+.. only:: html
 
-* :ref:`genindex`
-* :ref:`search`
+   Indices and Tables
+   ==================
+
+   * :ref:`genindex`
+   * :ref:`search`

docs/manual/_sources/interfaces.rst.txt

@@ -98,13 +98,13 @@ and persistent I2P address that your Reticulum instance can be reached
 at.
 
 To use the I2P interface, you must have an I2P router running
-on your system. The easiest way to acheive this is to download and
+on your system. The easiest way to achieve this is to download and
 install the `latest release <https://github.com/PurpleI2P/i2pd/releases/latest>`_
-of the ``ì2pd`` package. For more details about I2P, see the
-`geti2p.net website <https://geti2p.net/en/about/intro>`_.`
+of the ``i2pd`` package. For more details about I2P, see the
+`geti2p.net website <https://geti2p.net/en/about/intro>`_.
 
 When an I2P router is running on your system, you can simply add
-an I2P interface to reticulum:
+an I2P interface to Reticulum:
 
 .. code::
 
@@ -270,14 +270,10 @@ with all other peers on a local area network.
 
 *Please Note!* Using broadcast UDP traffic has performance implications,
 especially on WiFi. If your goal is simply to enable easy communication
-with all peers in your local ethernet broadcast domain, the
+with all peers in your local Ethernet broadcast domain, the
 :ref:`Auto Interface<interfaces-auto>` performs better, and is even
 easier to use.
 
-The below example is enabled by default on new Reticulum installations,
-as it provides an easy way to get started and to test Reticulum on a
-pre-existing LAN.
-
 .. code::
 
   # This example enables communication with other
@@ -408,7 +404,7 @@ directly over a wire-pair, or for using devices such as data radios and lasers.
 Pipe Interface
 ==============
 
-Using this interface, reticulum can use any program as an interface via `stdin` and
+Using this interface, Reticulum can use any program as an interface via `stdin` and
 `stdout`. This can be used to easily create virtual interfaces, or to interface with
 custom hardware or other systems.
 
@@ -425,7 +421,7 @@ custom hardware or other systems.
     respawn_delay = 5
 
 Reticulum will write all packets to `stdin` of the ``command`` option, and will
-continously read and scan its `stdout` for Reticulum packets. If ``EOF`` is reached,
+continuously read and scan its `stdout` for Reticulum packets. If ``EOF`` is reached,
 Reticulum will try to respawn the program after waiting for ``respawn_interval`` seconds.
 
 .. _interfaces-kiss:
@@ -634,32 +630,58 @@ Interface Modes
 The optional ``mode`` setting is available on all interfaces, and allows
 selecting the high-level behaviour of the interface from a number of modes.
 These modes affect how Reticulum selects paths in the network, how announces
-are propagated and how long paths are valid.
+are propagated, how long paths are valid and how paths are discovered.
 
-Configuring modes on interfaces is not strictly necessary, but can be useful
-when building or connecting to more complex networks. When not running a
-Transport Node, it is rarely useful to configure an interface mode.
+Configuring modes on interfaces is **not** strictly necessary, but can be useful
+when building or connecting to more complex networks. If your Reticulum
+instance is not running a Transport Node, it is rarely useful to configure
+interface modes, and in such cases interfaces should generally be left in
+the default mode.
 
- * | The default value is ``full``. In this mode, all discovery,
+ * | The default mode is ``full``. In this mode, all discovery,
      meshing and transport functionality is activated.
 
+ * | The ``gateway`` mode (or shorthand ``gw``) also has all
+     discovery, meshing and transport functionality available,
+     but will additionally try to discover unknown paths on
+     behalf of other nodes residing on the ``gateway`` interface.
+     If Reticulum receives a path request for an unknown
+     destination, from a node on a ``gateway`` interface, it
+     will try to discover this path via all other active interfaces,
+     and forward the discovered path to the requestor if one is
+     found.
+
+   | If you want to allow other nodes to widely resolve paths or connect
+     to a network via an interface, it might be useful to put it in this
+     mode. By creating a chain of ``gateway`` interfaces, other
+     nodes will be able to immediately discover paths to any
+     destination along the chain.
+
+   | *Please note!* It is the interface *facing the clients* that
+     must be put into ``gateway`` mode for this to work, not
+     the interface facing the wider network (for this, the ``boundary``
+     mode can be useful, though).
+
  * | In the ``access_point`` (or shorthand ``ap``) mode, the
      interface will operate as a network access point. In this
      mode, announces will not be automatically broadcasted on
      the interface, and paths to destinations on the interface
-     will have a much shorter expiry time. This mode is useful
-     for creating interfaces that remain quiet, unless when
-     someone is actually using them. An example of this could
-     be a radio interface serving a wide area, where users are
-     expected to connect momentarily, use the network, and then
-     disappear again.
+     will have a much shorter expiry time. In addition, path
+     requests from clients on the access point interface will
+     be handled in the same way as the ``gateway`` interface.
+
+   | This mode is useful for creating interfaces that remain
+     quiet, until someone actually starts using them. An example
+     of this could be a radio interface serving a wide area,
+     where users are expected to connect momentarily, use the
+     network, and then disappear again.
 
  * | The ``roaming`` mode should be used on interfaces that are
      roaming (physically mobile), seen from the perspective of
      other nodes in the network. As an example, if a vehicle is
      equipped with an external LoRa interface, and an internal,
      WiFi-based interface, that serves devices that are moving
-     _with_ the vehicle, the external LoRa interface should be
+     *with* the vehicle, the external LoRa interface should be
      configured as ``roaming``, and the internal interface can
      be left in the default mode. With transport enabled, such
      a setup will allow all internal devices to reach each other,

docs/manual/_sources/networks.rst.txt

@@ -9,7 +9,7 @@ Reticulum, which can often be easier than using traditional stacks, since you
 don't have to worry about coordinating addresses, subnets and routing for an
 entire network that you might not know how will evolve in the future. With
 Reticulum, you can simply add more segments to your network when it becomes
-necesarry, and Reticulum will handle the convergence of the entire network
+necessary, and Reticulum will handle the convergence of the entire network
 automatically.
 
 Concepts & Overview
@@ -18,13 +18,13 @@ Concepts & Overview
 There are important points that need to be kept in mind when building networks
 with Reticulum:
 
- * | In a Reticulum network, any node can autonomously generate as many adresses
+ * | In a Reticulum network, any node can autonomously generate as many addresses
      (called *destinations* in Reticulum terminology) as it needs, which become
      globally reachable to the rest of the network. There is no central point of
-     control over the adress space.
+     control over the address space.
 
  * | Reticulum was designed to handle both very small, and very large networks.
-     While the adress space can support billions of endpoints, Reticulum is
+     While the address space can support billions of endpoints, Reticulum is
      also very useful when just a few devices needs to communicate.
 
  * | Low-bandwidth networks, like LoRa and packet radio, can interoperate and
@@ -113,8 +113,8 @@ WiFi based radios for interconnecting the sites.
 
 At each site, a Raspberry Pi is installed to function as a gateway. A LoRa radio
 is connected to the Pi with a USB cable, and the WiFi radio is connected to the
-ethernet port of the Pi. At site B, two WiFi radios are needed to be able to reach
-both site A and site C, so an extra ethernet adapter is connected to the Pi in
+Ethernet port of the Pi. At site B, two WiFi radios are needed to be able to reach
+both site A and site C, so an extra Ethernet adapter is connected to the Pi in
 this location.
 
 Once the hardware has been installed, Reticulum is installed on all the Pis, and at

docs/manual/_sources/reference.rst.txt

@@ -1,18 +1,23 @@
+:tocdepth: 4
+
 .. _api-main:
 
 *************
 API Reference
 *************
-This reference guide lists and explains all classes exposed by the RNS API.
-
-Classes
-=========================
-Communication over a Reticulum network is achieved using a set of classes exposed by RNS.
+Communication over Reticulum networks is achieved by using a simple set of classes exposed by the RNS API.
+This chapter lists and explains all classes exposed by the Reticulum Network Stack API, along with their method signatures and usage. It can be used as a reference while writing applications that utilise Reticulum, or it can be read in entirity to gain an understanding of the complete functionality of RNS from a developers perspective.
 
 .. _api-reticulum:
 
-Reticulum
----------
+.. only:: html
+
+   |start-h3| Reticulum |end-h3|
+
+.. only:: latex
+
+   Reticulum
+   ---------
 
 .. autoclass:: RNS.Reticulum
    :members:
@@ -20,64 +25,120 @@ Reticulum
 
 .. _api-identity:
 
-Identity
---------
+.. only:: html
+
+   |start-h3| Identity |end-h3|
+
+.. only:: latex
+
+   Identity
+   --------
 
 .. autoclass:: RNS.Identity
    :members:
 
 .. _api-destination:
 
-Destination
------------
+.. only:: html
+
+   |start-h3| Destination |end-h3|
+
+.. only:: latex
+
+   Destination
+   -----------
 
 .. autoclass:: RNS.Destination
    :members:
 
 .. _api-packet:
 
-Packet
-------
+.. only:: html
+
+   |start-h3| Packet |end-h3|
+
+.. only:: latex
+
+   Packet
+   ------
 
 .. autoclass:: RNS.Packet(destination, data, create_receipt = True)
    :members:
 
 .. _api-packetreceipt:
 
-Packet Receipt
---------------
+.. only:: html
+
+   |start-h3| Packet Receipt |end-h3|
+
+.. only:: latex
+
+   Packet Receipt
+   --------------
 
 .. autoclass:: RNS.PacketReceipt()
    :members:
 
 .. _api-link:
 
-Link
-----
+.. only:: html
+
+   |start-h3| Link |end-h3|
+
+.. only:: latex
+
+   Link
+   ----
 
 .. autoclass:: RNS.Link(destination, established_callback=None, closed_callback = None)
    :members:
 
 .. _api-requestreceipt:
 
-Request Receipt
----------------
+.. only:: html
+
+   |start-h3| Request Receipt |end-h3|
+
+.. only:: latex
+
+   Request Receipt
+   ---------------
 
 .. autoclass:: RNS.RequestReceipt()
    :members:
 
 .. _api-resource:
 
-Resource
---------
+.. only:: html
+
+   |start-h3| Resource |end-h3|
+
+.. only:: latex
+
+   Resource
+   --------
 
 .. autoclass:: RNS.Resource(data, link, advertise=True, auto_compress=True, callback=None, progress_callback=None, timeout=None)
    :members:
 
 .. _api-transport:
 
-Transport
----------
+.. only:: html
+
+   |start-h3| Transport |end-h3|
+
+.. only:: latex
+
+   Transport
+   ---------
 
 .. autoclass:: RNS.Transport
-   :members:
+   :members:
+
+.. |start-h3| raw:: html
+
+     <h3>
+
+.. |end-h3| raw:: html
+
+     </h3>

docs/manual/_sources/support.rst.txt

@@ -0,0 +1,42 @@
+.. _support-main:
+
+*****************
+Support Reticulum
+*****************
+You can help support the continued development of open, free and private communications
+systems by donating, providing feedback and contributing code and learning resources.
+
+Donations
+=========
+Donations are gratefully accepted via the following channels:
+
+
+.. code:: text
+
+    Monero:
+    84FpY1QbxHcgdseePYNmhTHcrgMX4nFfBYtz2GKYToqHVVhJp8Eaw1Z1EedRnKD19b3B8NiLCGVxzKV17UMmmeEsCrPyA5w
+
+    Ethereum:
+    0x81F7B979fEa6134bA9FD5c701b3501A2e61E897a
+
+    Bitcoin:
+    3CPmacGm34qYvR6XWLVEJmi2aNe3PZqUuq
+
+    Ko-Fi:
+    https://ko-fi.com/markqvist
+
+Are certain features in the development roadmap are important to you or your
+organisation? Make them a reality quickly by sponsoring their implementation.
+
+Provide Feedback
+================
+All feedback on the usage, functioning and potential dysfunctioning of any and
+all components of the system is very valuable to the continued development and
+improvement of Reticulum. Absolutely no automated analytics, telemetry, error
+reporting or statistics is collected and reported by Reticulum under any
+circumstances, so we rely on old-fashioned human feedback.
+
+Contribute Code
+===============
+Join us on `the GitHub repository <https://github.com/markqvist/reticulum>`_ to
+report issues, suggest functionality and contribute code to Reticulum.

docs/manual/_sources/understanding.rst.txt

@@ -70,7 +70,7 @@ guide the design of Reticulum:
 * **Hardware layer agnosticism**
     Reticulum must be fully hardware agnostic, and shall be useable over a wide range of
     physical networking layers, such as data radios, serial lines, modems, handheld transceivers,
-    wired ethernet, wifi, or anything else that can carry a digital data stream. Hardware made for
+    wired Ethernet, WiFi, or anything else that can carry a digital data stream. Hardware made for
     dedicated Reticulum use shall be as cheap as possible and use off-the-shelf components, so
     it can be easily modified and replicated by anyone interested in doing so.
 * **Very low bandwidth requirements**
@@ -109,7 +109,7 @@ Introduction & Basic Functionality
 
 Reticulum is a networking stack suited for high-latency, low-bandwidth links. Reticulum is at it’s
 core a *message oriented* system. It is suited for both local point-to-point or point-to-multipoint
-scenarios where alle nodes are within range of each other, as well as scenarios where packets need
+scenarios where all nodes are within range of each other, as well as scenarios where packets need
 to be transported over multiple hops in a complex network to reach the recipient.
 
 Reticulum does away with the idea of addresses and ports known from IP, TCP and UDP. Instead
@@ -117,29 +117,30 @@ Reticulum uses the singular concept of *destinations*. Any application using Ret
 networking stack will need to create one or more destinations to receive data, and know the
 destinations it needs to send data to.
 
-All destinations in Reticulum are represented as a 10 byte hash, derived from truncating a full
+All destinations in Reticulum are _represented_ as a 16 byte hash. This hash is derived from truncating a full
 SHA-256 hash of identifying characteristics of the destination. To users, the destination addresses
-will be displayed as 10 bytes in hexadecimal representation, as in the following example: ``<80e29bf7cccaf31431b3>``.
+will be displayed as 16 hexadecimal bytes, like this example: ``<13425ec15b621c1d928589718000d814>``.
 
-The truncation size of 10 bytes (80 bits) for destinations has been choosen as a reasonable tradeoff between address space
-and packet overhead. The address space accomodated by this size can support many billions of
+The truncation size of 16 bytes (128 bits) for destinations has been chosen as a reasonable trade-off
+between address space
+and packet overhead. The address space accommodated by this size can support many billions of
 simultaneously active devices on the same network, while keeping packet overhead low, which is
 essential on low-bandwidth networks. In the very unlikely case that this address space nears
 congestion, a one-line code change can upgrade the Reticulum address space all the way up to 256
 bits, ensuring the Reticulum address space could potentially support galactic-scale networks.
-This is obviusly complete and ridiculous over-allocation, and as such, the current 80 bits should
+This is obviously complete and ridiculous over-allocation, and as such, the current 128 bits should
 be sufficient, even far into the future.
 
-By default Reticulum encrypts all data using elliptic curve cryptography. Any packet sent to a
-destination is encrypted with a derived ephemeral key. Reticulum can also set up an encrypted
-channel to a destination with *Forward Secrecy* and *Initiator Anonymity* using a elliptic
-curve cryptography and ephemeral keys derived from a Diffie Hellman exchange on Curve25519. In
-Reticulum terminology, this is called a *Link*. The multi-hop transport, coordination, verification
-and reliability layers are fully autonomous and also based on elliptic curve cryptography.
+By default Reticulum encrypts all data using elliptic curve cryptography and AES. Any packet sent to a
+destination is encrypted with a per-packet derived key. Reticulum can also set up an encrypted
+channel to a destination, called a *Link*. Both data sent over Links and single packets offer
+*Initiator Anonymity*, and links additionally offer *Forward Secrecy* by using an Elliptic Curve
+Diffie Hellman key exchange on Curve25519 to derive per-link ephemeral keys. The multi-hop transport,
+coordination, verification and reliability layers are fully autonomous and also based on elliptic
+curve cryptography.
 
 Reticulum also offers symmetric key encryption for group-oriented communications, as well as
-unencrypted packets for broadcast purposes, or situations where you need the communication to be in
-plain text.
+unencrypted packets for local broadcast purposes.
 
 Reticulum can connect to a variety of interfaces such as radio modems, data radios and serial ports,
 and offers the possibility to easily tunnel Reticulum traffic over IP links such as the Internet or
@@ -163,7 +164,7 @@ destinations. Reticulum uses three different basic destination types, and one sp
     A *plain* destination type is unencrypted, and suited for traffic that should be broadcast to a
     number of users, or should be readable by anyone. Traffic to a *plain* destination is not encrypted.
     Generally, *plain* destinations can be used for broadcast information intended to be public.
-    Plain destinations are only reachable directly, and packets adressed to plain destinations are
+    Plain destinations are only reachable directly, and packets addressed to plain destinations are
     never transported over multiple hops in the network. To be transportable over multiple hops in Reticulum, information
     *must* be encrypted, since Reticulum uses the per-packet encryption to verify routing paths and
     keep them alive.
@@ -186,7 +187,7 @@ Destination Naming
 ^^^^^^^^^^^^^^^^^^
 
 Destinations are created and named in an easy to understand dotted notation of *aspects*, and
-represented on the network as a hash of this value. The hash is a SHA-256 truncated to 80 bits. The
+represented on the network as a hash of this value. The hash is a SHA-256 truncated to 128 bits. The
 top level aspect should always be a unique identifier for the application using the destination.
 The next levels of aspects can be defined in any way by the creator of the application.
 
@@ -202,7 +203,7 @@ application name, a device type and measurement type, like this:
    aspects   : remotesensor, temperature
 
    full name : environmentlogger.remotesensor.temperature
-   hash      : fa7ddfab5213f916dea
+   hash      : 4faf1b2e0a077e6a9d92fa051f256038
 
 For the *single* destination, Reticulum will automatically append the associated public key as a
 destination aspect before hashing. This is done to ensure only the correct destination is reached,
@@ -219,10 +220,10 @@ packet.
 
 In actual use of *single* destination naming, it is advisable not to use any uniquely identifying
 features in aspect naming. Aspect names should be general terms describing what kind of destination
-is represented. The uniquely identifying aspect is always acheived by the appending the public key,
-which expands the destination into a uniquely identifyable one. Reticulum does this automatically.
+is represented. The uniquely identifying aspect is always achieved by the appending the public key,
+which expands the destination into a uniquely identifiable one. Reticulum does this automatically.
 
-Any destination on a Reticulum network can be addressed and reached simply by knowning its
+Any destination on a Reticulum network can be addressed and reached simply by knowing its
 destination hash (and public key, but if the public key is not known, it can be requested from the
 network simply by knowing the destination hash). The use of app names and aspects makes it easy to
 structure Reticulum programs and makes it possible to filter what information and data your program
@@ -349,7 +350,7 @@ Node Types
 ----------
 
 Currently, Reticulum distinguishes between two types of network nodes. All nodes on a Reticulum network
-are *Reticulum Instances*, and some are alo *Transport Nodes*. If a system running Reticulum is fixed in
+are *Reticulum Instances*, and some are also *Transport Nodes*. If a system running Reticulum is fixed in
 one place, and is intended to be kept available most of the time, it is a good contender to be a *Transport Node*.
 
 Any Reticulum Instance can become a Transport Node by enabling it in the configuration.
@@ -442,8 +443,7 @@ For exchanges of small amounts of information, Reticulum offers the *Packet* API
 * | When the destination receives the packet, it can itself perform an ECDH key exchange and decrypt the
     packet.
 
-* | A new ephemeral key is used for every packet sent in this way, and forward secrecy is guaranteed on a
-    per packet level.
+* | A new ephemeral key is used for every packet sent in this way.
 
 * | Once the packet has been received and decrypted by the addressed destination, that destination can opt
     to *prove* its receipt of the packet. It does this by calculating the SHA-256 hash of the received packet,
@@ -453,7 +453,7 @@ For exchanges of small amounts of information, Reticulum offers the *Packet* API
 
 * | In case the packet is addressed to a *group* destination type, the packet will be encrypted with the
     pre-shared AES-128 key associated with the destination. In case the packet is addressed to a *plain*
-    destination type, the payload data will not be encrypted. Neither of these two destination types offer
+    destination type, the payload data will not be encrypted. Neither of these two destination types can offer
     forward secrecy. In general, it is recommended to always use the *single* destination type, unless it is
     strictly necessary to use one of the others.
 
@@ -485,18 +485,23 @@ For exchanges of larger amounts of data, or when longer sessions of bidirectiona
     the destination using a Reticulum Identity. This authentication is happening inside the encrypted
     link, and is only revealed to the verified destination, and no intermediaries.
 
-In a moment, we will discuss the details of how this methodology is implemented, but let’s first
-recap what purposes this methodology serves. We first ensure that the node answering our request
-is actually the one we want to communicate with, and not a malicious actor pretending to be so.
-At the same time we establish an efficient encrypted channel. The setup of this is relatively cheap in
-terms of bandwidth, so it can be used just for a short exchange, and then recreated as needed, which will
-also rotate encryption keys. The link can also be kept alive for longer periods of time, if this is
-more suitable to the application. The procedure also inserts the *link id* , a hash calculated from the link request packet, into the memory of forwarding nodes, which means that the communicating nodes can thereafter reach each other simply by referring to this *link id*.
+In a moment, we will discuss the details of how this methodology is
+implemented, but let’s first recap what purposes this methodology serves. We
+first ensure that the node answering our request is actually the one we want to
+communicate with, and not a malicious actor pretending to be so.  At the same
+time we establish an efficient encrypted channel. The setup of this is
+relatively cheap in terms of bandwidth, so it can be used just for a short
+exchange, and then recreated as needed, which will also rotate encryption keys.
+The link can also be kept alive for longer periods of time, if this is more
+suitable to the application. The procedure also inserts the *link id* , a hash
+calculated from the link request packet, into the memory of forwarding nodes,
+which means that the communicating nodes can thereafter reach each other simply
+by referring to this *link id*.
 
-The combined bandwidth cost of setting up a link is 3 packets totalling 237 bytes (more info in the
+The combined bandwidth cost of setting up a link is 3 packets totalling 297 bytes (more info in the
 :ref:`Binary Packet Format<understanding-packetformat>` section). The amount of bandwidth used on keeping
-a link open is practically negligible, at 0.62 bits per second. Even on a slow 1200 bits per second packet
-radio channel, 100 concurrent links will still leave 95% channel capacity for actual data.
+a link open is practically negligible, at 0.45 bits per second. Even on a slow 1200 bits per second packet
+radio channel, 100 concurrent links will still leave 96% channel capacity for actual data.
 
 
 Link Establishment in Detail
@@ -538,14 +543,16 @@ an arbitrary number of hops, where information will be exchanged between two nod
 
 * | A *link proof* packet is now constructed and transmitted over the network. This packet is
     addressed to the *link id* of the *link*. It contains the following data: The newly generated X25519
-    public key *LKr* and an Ed25519 signature of the *link id* and *LKr* made by the signing key of
+    public key *LKr* and an Ed25519 signature of the *link id* and *LKr* made by the *original signing key* of
     the addressed destination.
    
 * | By verifying this *link proof* packet, all nodes that originally transported the *link request*
     packet to the destination from the originator can now verify that the intended destination received
     the request and accepted it, and that the path they chose for forwarding the request was valid.
-    In sucessfully carrying out this verification, the transporting nodes marks the link as active.
+    In successfully carrying out this verification, the transporting nodes marks the link as active.
     An abstract bi-directional communication channel has now been established along a path in the network.
+    Packets can now be exchanged bi-directionally from either end of the link simply by adressing the
+    packets to the *link id* of the link.
 
 * | When the source receives the *proof* , it will know unequivocally that a verified path has been
     established to the destination. It can now also use the X25519 public key contained in the
@@ -683,7 +690,7 @@ Wire Format
 
     A Reticulum packet is composed of the following fields:
 
-    [HEADER 2 bytes] [ADDRESSES 10/20 bytes] [CONTEXT 1 byte] [DATA 0-477 bytes]
+    [HEADER 2 bytes] [ADDRESSES 16/32 bytes] [CONTEXT 1 byte] [DATA 0-465 bytes]
 
     * The HEADER field is 2 bytes long.
       * Byte 1: [IFAC Flag], [Header Type], [Propagation Type], [Destination Type] and [Packet Type]
@@ -695,15 +702,15 @@ Wire Format
         capabilities and configuration.
 
     * The ADDRESSES field contains either 1 or 2 addresses.
-      * Each address is 10 bytes long.
+      * Each address is 16 bytes long.
       * The Header Type flag in the HEADER field determines
         whether the ADDRESSES field contains 1 or 2 addresses.
-      * Addresses are Reticulum hashes truncated to 10 bytes.
+      * Addresses are SHA-256 hashes truncated to 16 bytes.
 
     * The CONTEXT field is 1 byte.
       * It is used by Reticulum to determine packet context.
 
-    * The DATA field is between 0 and 477 bytes.
+    * The DATA field is between 0 and 465 bytes.
       * It contains the packets data payload.
 
     IFAC Flag
@@ -714,8 +721,8 @@ Wire Format
 
     Header Types
     -----------------
-    type 1           0  Two byte header, one 10 byte address field
-    type 2           1  Two byte header, two 10 byte address fields
+    type 1           0  Two byte header, one 16 byte address field
+    type 2           1  Two byte header, two 16 byte address fields
 
 
     Propagation Types
@@ -747,7 +754,7 @@ Wire Format
        HEADER FIELD           DESTINATION FIELDS            CONTEXT FIELD  DATA FIELD
      _______|_______   ________________|________________   ________|______   __|_
     |               | |                                 | |               | |    |
-    01010000 00000100 [HASH1, 10 bytes] [HASH2, 10 bytes] [CONTEXT, 1 byte] [DATA]
+    01010000 00000100 [HASH1, 16 bytes] [HASH2, 16 bytes] [CONTEXT, 1 byte] [DATA]
     || | | |    |
     || | | |    +-- Hops             = 4
     || | | +------- Packet Type      = DATA
@@ -762,9 +769,9 @@ Wire Format
        HEADER FIELD   DESTINATION FIELD   CONTEXT FIELD  DATA FIELD
      _______|_______   _______|_______   ________|______   __|_
     |               | |               | |               | |    |
-    00000000 00000111 [HASH1, 10 bytes] [CONTEXT, 1 byte] [DATA]
+    00000000 00000111 [HASH1, 16 bytes] [CONTEXT, 1 byte] [DATA]
     || | | |    |
-    || | | |    +-- Hops             = 0
+    || | | |    +-- Hops             = 7
     || | | +------- Packet Type      = DATA
     || | +--------- Destination Type = SINGLE
     || +----------- Propagation Type = BROADCAST
@@ -777,9 +784,9 @@ Wire Format
        HEADER FIELD     IFAC FIELD    DESTINATION FIELD   CONTEXT FIELD  DATA FIELD
      _______|_______   ______|______   _______|_______   ________|______   __|_
     |               | |             | |               | |               | |    |
-    10000000 00000111 [IFAC, N bytes] [HASH1, 10 bytes] [CONTEXT, 1 byte] [DATA]
+    10000000 00000111 [IFAC, N bytes] [HASH1, 16 bytes] [CONTEXT, 1 byte] [DATA]
     || | | |    |
-    || | | |    +-- Hops             = 0
+    || | | |    +-- Hops             = 7
     || | | +------- Packet Type      = DATA
     || | +--------- Destination Type = SINGLE
     || +----------- Propagation Type = BROADCAST
@@ -795,12 +802,12 @@ Wire Format
     wire size counting all fields including headers,
     but excluding any interface access codes.
 
-    - Path Request    :    33  bytes
-    - Announce        :    151 bytes
-    - Link Request    :    77  bytes
-    - Link Proof      :    77  bytes
-    - Link RTT packet :    83  bytes
-    - Link keepalive  :    14  bytes
+    - Path Request    :    51  bytes
+    - Announce        :    167 bytes
+    - Link Request    :    83  bytes
+    - Link Proof      :    115 bytes
+    - Link RTT packet :    99  bytes
+    - Link keepalive  :    20  bytes
 
 
 .. _understanding-announcepropagation:
@@ -809,12 +816,14 @@ Announce Propagation Rules
 --------------------------
 
 The following table illustrates the rules for automatically propagating announces
-from one interface type to another, for all possible combinations. See the
-:ref:`Interface Modes<interfaces-modes>` section for a conceptual overview of the
-different interface modes, and how they are configured.
+from one interface type to another, for all possible combinations. For the purpose
+of announce propagation, the *Full* and *Gateway* modes are identical.
 
 .. image:: graphics/if_mode_graph_b.png
 
+See the :ref:`Interface Modes<interfaces-modes>` section for a conceptual overview
+of the different interface modes, and how they are configured.
+
 .. 
       (.. code-block:: text)
       Full ────── ✓ ──┐              ┌── ✓ ── Full
@@ -837,3 +846,52 @@ different interface modes, and how they are configured.
       Boundary ── ✓ ──┤              ├── ✓ ── Boundary
       Roaming ─── ✕ ──┘              └── ✕ ── Roaming
 
+
+.. _understanding-primitives:
+
+Cryptographic Primitives
+------------------------
+
+Reticulum has been designed to use a simple suite of efficient, strong and modern
+cryptographic primitives, with widely available implementations that can be used
+both on general-purpose CPUs and on microcontrollers. The necessary primitives are:
+
+* Ed25519 for signatures
+
+* X22519 for ECDH key exchanges
+
+* HKDF for key derivation
+
+* Fernet for encrypted tokens
+
+  * AES-128 in CBC mode
+
+  * HMAC for message authentication
+
+* SHA-256
+
+* SHA-512
+
+In the default installation configuration, the ``X25519``, ``Ed25519`` and ``AES-128-CBC``
+primitives are provided by `OpenSSL <https://www.openssl.org/>`_ (via the `PyCA/cryptography <https://github.com/pyca/cryptography>`_
+package). The hashing functions ``SHA-256`` and ``SHA-512`` are provided by the standard
+Python `hashlib <https://docs.python.org/3/library/hashlib.html>`_. The ``HKDF``, ``HMAC``,
+``Fernet`` primitives, and the ``PKCS7`` padding function are always provided by the
+following internal implementations:
+
+- ``RNS/Cryptography/HKDF.py``
+- ``RNS/Cryptography/HMAC.py``
+- ``RNS/Cryptography/Fernet.py``
+- ``RNS/Cryptography/PKCS7.py``
+
+
+Reticulum also includes a complete implementation of all necessary primitives in pure Python.
+If OpenSSL & PyCA are not available on the system when Reticulum is started, Reticulum will
+instead use the internal pure-python primitives. A trivial consequence of this is performance,
+with the OpenSSL backend being *much* faster. The most important consequence however, is the
+potential loss of security by using primitives that has not seen the same amount of scrutiny,
+testing and review as those from OpenSSL.
+
+If you want to use the internal pure-python primitives, it is **highly advisable** that you
+have a good understanding of the risks that this pose, and make an informed decision on whether
+those risks are acceptable to you.

docs/manual/_sources/using.rst.txt

@@ -5,28 +5,43 @@ Using Reticulum on Your System
 ******************************
 
 Reticulum is not installed as a driver or kernel module, as one might expect
-of a networking stack. Instead, Reticulum is distributed as a Python module.
+of a networking stack. Instead, Reticulum is distributed as a Python module, 
+containing the networking core, and a set of utility and daemon programs.
+
 This means that no special privileges are required to install or use it. It
-is also very light-weight, and easy to transfer to and install on new systems.
-Any program or application that uses Reticulum will automatically load and
-initialise Reticulum when it starts.
+is also very light-weight, and easy to transfer to, and install on new systems.
+
+When you have Reticulum installed, any program or application that uses Reticulum
+will automatically load and initialise Reticulum when it starts, if it is not
+already running.
 
 In many cases, this approach is sufficient. When any program needs to use
 Reticulum, it is loaded, initialised, interfaces are brought up, and the
 program can now communicate over any Reticulum networks available. If another
-program starts up and also wants access to the same Reticulum network, the
-instance is simply shared. This works for any number of programs running
+program starts up and also wants access to the same Reticulum network, the already
+running instance is simply shared. This works for any number of programs running
 concurrently, and is very easy to use, but depending on your use case, there
 are other options.
 
 Configuration & Data
 --------------------
 
-A Reticulum stores all information that it needs to function in a single file-
-system directory. By default, this directory is ``~/.reticulum``, but you can
-use any directory you wish. You can also run multiple separate Reticulum
-instances on the same physical system, in complete isolation from each other,
-or connected together.
+Reticulum stores all information that it needs to function in a single file-system
+directory. When Reticulum is started, it will look for a valid configuration
+directory in the following places:
+
+- ``/etc/reticulum``
+- ``~/.config/reticulum``
+- ``~/.reticulum``
+
+If no existing configuration directory is found, the directory ``~/.reticulum``
+is created, and the default configuration will be automatically created here.
+You can move it to one of the other locations if you wish.
+
+It is also possible to use completely arbitrary configuration directories by
+specifying the relevant command-line parameters when running Reticulum-based
+programs. You can also run multiple separate Reticulum instances on the same
+physical system, either in isolation from each other, or connected together.
 
 In most cases, a single physical system will only need to run one Reticulum
 instance. This can either be launched at boot, as a system service, or simply
@@ -35,7 +50,7 @@ running on the same system will automatically share the same Reticulum instance,
 if the configuration allows for it, which it does by default.
 
 The entire configuration of Reticulum is found in the ``~/.reticulum/config``
-file. When Reticulum is first started on a new system, a basic, functional
+file. When Reticulum is first started on a new system, a basic, but fully functional
 configuration file is created. The default configuration looks like this:
 
 .. code::
@@ -138,6 +153,10 @@ take a look at the :ref:`Interfaces<interfaces-main>` chapter of this manual.
 Included Utility Programs
 -------------------------
 
+Reticulum includes a range of useful utilities, both for managing your Reticulum
+networks, and for carrying out common tasks over Reticulum networks, such as
+transferring files to remote systems, and executing commands and programs remotely.
+
 If you often use Reticulum from several different programs, or simply want
 Reticulum to stay available all the time, for example if you are hosting
 a transport node, you might want to run Reticulum as a separate service that
@@ -219,7 +238,7 @@ interfaces, similar to the ``ifconfig`` program.
      Traffic : 8.49 KB↑
                9.23 KB↓
 
-  Reticulum Transport Instance <5245a8efe1788c6a70e1> running
+  Reticulum Transport Instance <5245a8efe1788c6a1cd36144a270e13b> running
 
 .. code:: text
 
@@ -244,14 +263,14 @@ destinations on the Reticulum network.
 .. code:: text
 
   # Run rnpath
-  rnpath eca6f4e4dc26ae329e61
+  rnpath c89b4da064bf66d280f0e4d8abfd9806
 
   # Example output
-  Path found, destination <eca6f4e4dc26ae329e61> is 4 hops away via <56b115c30cd386cad69c> on TCPInterface[Testnet/frankfurt.rns.unsigned.io:4965]
+  Path found, destination <c89b4da064bf66d280f0e4d8abfd9806> is 4 hops away via <f53a1c4278e0726bb73fcc623d6ce763> on TCPInterface[Testnet/frankfurt.connect.reticulu.network:4965]
 
 .. code:: text
 
-  usage: rnpath.py [-h] [--config CONFIG] [--version] [-t] [-r] [-d] [-D] [-w seconds] [-v] [destination]
+  usage: rnpath [-h] [--config CONFIG] [--version] [-t] [-r] [-d] [-D] [-w seconds] [-v] [destination]
   
   Reticulum Path Discovery Utility
   
@@ -281,16 +300,16 @@ destinations will not have this option enabled, and will not be probable.
 .. code:: text
 
   # Run rnprobe
-  python3 -m RNS.Utilities.rnprobe example_utilities.echo.request 9382f334de63217a4278
+  rnprobe example_utilities.echo.request 2d03725b327348980d570f739a3a5708
 
   # Example output
-  Sent 16 byte probe to <9382f334de63217a4278>
-  Valid reply received from <9382f334de63217a4278>
+  Sent 16 byte probe to <2d03725b327348980d570f739a3a5708>
+  Valid reply received from <2d03725b327348980d570f739a3a5708>
   Round-trip time is 38.469 milliseconds over 2 hops
 
 .. code:: text
 
-  usage: rnprobe.py [-h] [--config CONFIG] [--version] [-v] [full_name] [destination_hash]
+  usage: rnprobe [-h] [--config CONFIG] [--version] [-v] [full_name] [destination_hash]
 
   Reticulum Probe Utility
 
@@ -305,6 +324,153 @@ destinations will not have this option enabled, and will not be probable.
     -v, --verbose
 
 
+The rncp Utility
+================
+
+The ``rncp`` utility is a simple file transfer tool. Using it, you can transfer
+files through Reticulum.
+
+.. code:: text
+
+  # Run rncp on the receiving system, specifying which identities
+  # are allowed to send files
+  rncp --receive -a 1726dbad538775b5bf9b0ea25a4079c8 -a c50cc4e4f7838b6c31f60ab9032cbc62
+
+  # From another system, copy a file to the receiving system
+  rncp ~/path/to/file.tgz 73cbd378bb0286ed11a707c13447bb1e
+
+You can specify as many allowed senders as needed, or complete disable authentication.
+
+.. code:: text
+
+  usage: rncp [-h] [--config path] [-v] [-q] [-p] [-r] [-b] [-a allowed_hash] [-n] [-w seconds] [--version] [file] [destination]
+
+  Reticulum File Transfer Utility
+
+  positional arguments:
+    file                  file to be transferred
+    destination           hexadecimal hash of the receiver
+
+  optional arguments:
+    -h, --help            show this help message and exit
+    --config path         path to alternative Reticulum config directory
+    -v, --verbose         increase verbosity
+    -q, --quiet           decrease verbosity
+    -p, --print-identity  print identity and destination info and exit
+    -r, --receive         wait for incoming files
+    -b, --no-announce     don't announce at program start
+    -a allowed_hash       accept from this identity
+    -n, --no-auth         accept files from anyone
+    -w seconds            sender timeout before giving up
+    --version             show program's version number and exit
+    -v, --verbose
+
+
+The rnx Utility
+================
+
+The ``rnx`` utility is a basic remote command execution program. It allows you to
+execute commands on remote systems over Reticulum, and to view returned command
+output.
+
+.. code:: text
+
+  # Run rnx on the listening system, specifying which identities
+  # are allowed to execute commands
+  rncp --listen -a 941bed5e228775e5a8079fc38b1ccf3f -a 1b03013c25f1c2ca068a4f080b844a10
+
+  # From another system, run a command 
+  rnx 7a55144adf826958a9529a3bcf08b149 "cat /proc/cpuinfo"
+
+  # Or enter the interactive mode pseudo-shell
+  rnx 7a55144adf826958a9529a3bcf08b149 -x
+
+  # The default identity file is stored in
+  # ~/.reticulum/identities/rnx, but you can use
+  # another one, which will be created if it does
+  # not already exist
+  rnx 7a55144adf826958a9529a3bcf08b149 -i /path/to/identity -x
+
+You can specify as many allowed senders as needed, or completely disable authentication.
+
+.. code:: text
+
+  usage: rnx [-h] [--config path] [-v] [-q] [-p] [-l] [-i identity] [-x] [-b] [-a allowed_hash] [-n] [-N] [-d] [-m] [-w seconds] [-W seconds] [--stdin STDIN] [--stdout STDOUT] [--stderr STDERR] [--version]
+             [destination] [command]
+
+  Reticulum Remote Execution Utility
+
+  positional arguments:
+    destination           hexadecimal hash of the listener
+    command               command to be execute
+
+  optional arguments:
+    -h, --help            show this help message and exit
+    --config path         path to alternative Reticulum config directory
+    -v, --verbose         increase verbosity
+    -q, --quiet           decrease verbosity
+    -p, --print-identity  print identity and destination info and exit
+    -l, --listen          listen for incoming commands
+    -i identity           path to identity to use
+    -x, --interactive     enter interactive mode
+    -b, --no-announce     don't announce at program start
+    -a allowed_hash       accept from this identity
+    -n, --noauth          accept files from anyone
+    -N, --noid            don't identify to listener
+    -d, --detailed        show detailed result output
+    -m                    mirror exit code of remote command
+    -w seconds            connect and request timeout before giving up
+    -W seconds            max result download time
+    --stdin STDIN         pass input to stdin
+    --stdout STDOUT       max size in bytes of returned stdout
+    --stderr STDERR       max size in bytes of returned stderr
+    --version             show program's version number and exit
+
+
+The rnodeconf Utility
+=====================
+
+The ``rnodeconf`` utility allows you to inspect and configure existing :ref:`RNodes<rnode-main>`, and
+to create and provision new :ref:`RNodes<rnode-main>` from any supported hardware devices.
+
+.. code:: text
+
+  usage: rnodeconf [-h] [-i] [-a] [-u] [-U] [--fw-version version] [--nocheck] [-C] [-N] [-T] [-b] [-B] [-p] [--freq Hz] [--bw Hz] [--txp dBm] [--sf factor] [--cr rate] [--eeprom-backup] [--eeprom-dump] [--eeprom-wipe] [--version] [port]
+
+  RNode Configuration and firmware utility. This program allows you to change various settings and startup modes of RNode. It can also install, flash and update the firmware on supported devices.
+
+  positional arguments:
+    port                  serial port where RNode is attached
+
+  options:
+    -h, --help            show this help message and exit
+    -i, --info            Show device info
+    -a, --autoinstall     Automatic installation on various supported devices
+    -u, --update          Update firmware to the latest version
+    -U, --force-update    Update to specified firmware even if version matches or is older than installed version
+    --fw-version version  Use a specific firmware version for update or autoinstall
+    --nocheck             Don't check for firmware updates online
+    -e, --extract         Extract firmware from connected RNode for later use
+    -E, --use-extracted   Use the extracted firmware for autoinstallation or update
+    -C, --clear-cache     Clear locally cached firmware files
+    -N, --normal          Switch device to normal mode
+    -T, --tnc             Switch device to TNC mode
+    -b, --bluetooth-on    Turn device bluetooth on
+    -B, --bluetooth-off   Turn device bluetooth off
+    -p, --bluetooth-pair  Put device into bluetooth pairing mode
+    --freq Hz             Frequency in Hz for TNC mode
+    --bw Hz               Bandwidth in Hz for TNC mode
+    --txp dBm             TX power in dBm for TNC mode
+    --sf factor           Spreading factor for TNC mode (7 - 12)
+    --cr rate             Coding rate for TNC mode (5 - 8)
+    --eeprom-backup       Backup EEPROM to file
+    --eeprom-dump         Dump EEPROM to console
+    --eeprom-wipe         Unlock and wipe EEPROM
+    --version             Print program version and exit
+
+For more information on how to create your own RNodes, please read the :ref:`Creating RNodes<rnode-creating>`
+section of this manual.
+
 Improving System Configuration
 ------------------------------
 

docs/manual/_sources/whatis.rst.txt

@@ -2,23 +2,48 @@
 What is Reticulum?
 ******************
 
-Reticulum is a cryptography-based networking stack for building wide-area networks with readily available hardware, that can continue to operate even with extremely low bandwidth and very high latency.
+Reticulum is a cryptography-based networking stack for building both local and
+wide-area networks with readily available hardware, that can continue to operate
+under adverse conditions, such as extremely low bandwidth and very high latency.
 
-Reticulum allows you to build wide-area networks with off-the-shelf tools, and offers end-to-end encryption, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, unforgeable packet acknowledgements and more.
+Reticulum allows you to build wide-area networks with off-the-shelf tools, and
+offers end-to-end encryption, forward secrecy, autoconfiguring cryptographically
+backed multi-hop transport, efficient addressing, unforgeable packet
+acknowledgements and more.
 
-Reticulum is a complete networking stack, and does not need IP or higher layers, although it is easy to utilise IP (with TCP or UDP) as the underlying carrier for Reticulum. It is therefore trivial to tunnel Reticulum over the Internet or private IP networks. Reticulum is built directly on cryptographic principles, allowing resilience and stable functionality in open and trustless networks.
+From a users perspective, Reticulum allows the creation of applications that
+respect and empower the autonomy and sovereignty of communities and individuals.
+Reticulum enables secure digital communication that cannot be subjected to
+outside control, manipulation or censorship.
 
-No kernel modules or drivers are required. Reticulum runs completely in userland, and can run on practically any system that runs Python 3. Reticulum runs well even on small single-board computers like the Pi Zero.
+Reticulum enables the construction of both small and potentially planetary-scale
+networks, without any need for hierarchical or beaureucratic structures to control
+or manage them, while ensuring individuals and communities full sovereignty
+over their own network segments.
+
+Reticulum is a complete networking stack, and does not need IP or higher
+layers, although it is easy to utilise IP (with TCP or UDP) as the underlying
+carrier for Reticulum. It is therefore trivial to tunnel Reticulum over the
+Internet or private IP networks. Reticulum is built directly on cryptographic
+principles, allowing resilience and stable functionality in open and trustless
+networks.
+
+No kernel modules or drivers are required. Reticulum runs completely in
+userland, and can run on practically any system that runs Python 3. Reticulum
+runs well even on small single-board computers like the Pi Zero.
 
 
 Current Status
 ==============
-Reticulum should currently be considered beta software. All core protocol features are implemented and functioning, but additions will probably occur as real-world use is explored. There will be bugs. The API and wire-format can be considered stable at the moment, but could change if absolutely warranted.
+**Please know!** Reticulum should currently be considered beta software. All core protocol
+features are implemented and functioning, but additions will probably occur as
+real-world use is explored. *There will be bugs*. The API and wire-format can be
+considered stable at the moment, but could change if absolutely warranted.
 
 
 What does Reticulum Offer?
 ==========================
-* Coordination-less globally unique adressing and identification
+* Coordination-less globally unique addressing and identification
 
 * Fully self-configuring multi-hop routing
 
@@ -26,7 +51,7 @@ What does Reticulum Offer?
 
 * Asymmetric encryption based on X25519, and Ed25519 signatures as a basis for all communication
 
-* Forward Secrecy by using ephemereal Elliptic Curve Diffie-Hellman keys on Curve25519
+* Forward Secrecy by using ephemeral Elliptic Curve Diffie-Hellman keys on Curve25519
 
 * Reticulum uses the `Fernet <https://github.com/fernet/spec/blob/master/Spec.md>`_ specification for on-the-wire / over-the-air encryption
 
@@ -46,11 +71,11 @@ What does Reticulum Offer?
 
 * Efficient link establishment
 
-  * Total bandwidth cost of setting up a link is only 3 packets, totalling 237 bytes
+  * Total bandwidth cost of setting up a link is only 3 packets, totalling 297 bytes
 
-  * Low cost of keeping links open at only 0.62 bits per second
+  * Low cost of keeping links open at only 0.44 bits per second
 
-* Reliable and efficient transfer of arbritrary amounts of data
+* Reliable and efficient transfer of arbitrary amounts of data
 
   * Reticulum can handle a few bytes of data or files of many gigabytes
 
@@ -77,7 +102,7 @@ Reticulum. It is possible to build it yourself, to transform a common LoRa
 development board into one, or it can be purchased as a complete transceiver.
 
 Reticulum can also be encapsulated over existing IP networks, so there's
-nothing stopping you from using it over wired ethernet or your local WiFi
+nothing stopping you from using it over wired Ethernet or your local WiFi
 network, where it'll work just as well. In fact, one of the strengths of
 Reticulum is how easily it allows you to connect different mediums into a
 self-configuring, resilient and encrypted mesh.
@@ -92,15 +117,15 @@ Interface Types and Devices
 ===========================
 Reticulum implements a range of generalised interface types that covers the communications hardware that Reticulum can run over. If your hardware is not supported, it's relatively simple to implement an interface class. Currently, Reticulum can use the following devices and communication mediums:
 
-* Any ethernet device
+* Any Ethernet device
 
   * WiFi devices
 
-  * Wired ethernet devices
+  * Wired Ethernet devices
 
   * Fibre-optic transceivers
 
-  * Data radios with ethernet ports
+  * Data radios with Ethernet ports
 
 * LoRa using `RNode <https://unsigned.io/rnode>`_
 
@@ -135,4 +160,9 @@ For a full list and more details, see the :ref:`Supported Interfaces<interfaces-
 
 Caveat Emptor
 ==============
-Reticulum is an experimental networking stack, and should be considered as such. While it has been built with cryptography best-practices very foremost in mind, it has not been externally security audited, and there could very well be privacy-breaking bugs. To be considered secure, Reticulum needs a thourough security review by independt cryptographers and security researchers. If you want to help out, or help sponsor an audit, please do get in touch.
+Reticulum is an experimental networking stack, and should be considered as
+such. While it has been built with cryptography best-practices very foremost in
+mind, it has not been externally security audited, and there could very well be
+privacy-breaking bugs. To be considered secure, Reticulum needs a thorough
+security review by independent cryptographers and security researchers. If you
+want to help out, or help sponsor an audit, please do get in touch.

docs/manual/_static/_sphinx_javascript_frameworks_compat.js

@@ -0,0 +1,134 @@
+/*
+ * _sphinx_javascript_frameworks_compat.js
+ * ~~~~~~~~~~
+ *
+ * Compatability shim for jQuery and underscores.js.
+ *
+ * WILL BE REMOVED IN Sphinx 6.0
+ * xref RemovedInSphinx60Warning
+ *
+ */
+
+/**
+ * select a different prefix for underscore
+ */
+$u = _.noConflict();
+
+
+/**
+ * small helper function to urldecode strings
+ *
+ * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#Decoding_query_parameters_from_a_URL
+ */
+jQuery.urldecode = function(x) {
+    if (!x) {
+        return x
+    }
+    return decodeURIComponent(x.replace(/\+/g, ' '));
+};
+
+/**
+ * small helper function to urlencode strings
+ */
+jQuery.urlencode = encodeURIComponent;
+
+/**
+ * This function returns the parsed url parameters of the
+ * current request. Multiple values per key are supported,
+ * it will always return arrays of strings for the value parts.
+ */
+jQuery.getQueryParameters = function(s) {
+    if (typeof s === 'undefined')
+        s = document.location.search;
+    var parts = s.substr(s.indexOf('?') + 1).split('&');
+    var result = {};
+    for (var i = 0; i < parts.length; i++) {
+        var tmp = parts[i].split('=', 2);
+        var key = jQuery.urldecode(tmp[0]);
+        var value = jQuery.urldecode(tmp[1]);
+        if (key in result)
+            result[key].push(value);
+        else
+            result[key] = [value];
+    }
+    return result;
+};
+
+/**
+ * highlight a given string on a jquery object by wrapping it in
+ * span elements with the given class name.
+ */
+jQuery.fn.highlightText = function(text, className) {
+    function highlight(node, addItems) {
+        if (node.nodeType === 3) {
+            var val = node.nodeValue;
+            var pos = val.toLowerCase().indexOf(text);
+            if (pos >= 0 &&
+                !jQuery(node.parentNode).hasClass(className) &&
+                !jQuery(node.parentNode).hasClass("nohighlight")) {
+                var span;
+                var isInSVG = jQuery(node).closest("body, svg, foreignObject").is("svg");
+                if (isInSVG) {
+                    span = document.createElementNS("http://www.w3.org/2000/svg", "tspan");
+                } else {
+                    span = document.createElement("span");
+                    span.className = className;
+                }
+                span.appendChild(document.createTextNode(val.substr(pos, text.length)));
+                node.parentNode.insertBefore(span, node.parentNode.insertBefore(
+                    document.createTextNode(val.substr(pos + text.length)),
+                    node.nextSibling));
+                node.nodeValue = val.substr(0, pos);
+                if (isInSVG) {
+                    var rect = document.createElementNS("http://www.w3.org/2000/svg", "rect");
+                    var bbox = node.parentElement.getBBox();
+                    rect.x.baseVal.value = bbox.x;
+                    rect.y.baseVal.value = bbox.y;
+                    rect.width.baseVal.value = bbox.width;
+                    rect.height.baseVal.value = bbox.height;
+                    rect.setAttribute('class', className);
+                    addItems.push({
+                        "parent": node.parentNode,
+                        "target": rect});
+                }
+            }
+        }
+        else if (!jQuery(node).is("button, select, textarea")) {
+            jQuery.each(node.childNodes, function() {
+                highlight(this, addItems);
+            });
+        }
+    }
+    var addItems = [];
+    var result = this.each(function() {
+        highlight(this, addItems);
+    });
+    for (var i = 0; i < addItems.length; ++i) {
+        jQuery(addItems[i].parent).before(addItems[i].target);
+    }
+    return result;
+};
+
+/*
+ * backward compatibility for jQuery.browser
+ * This will be supported until firefox bug is fixed.
+ */
+if (!jQuery.browser) {
+    jQuery.uaMatch = function(ua) {
+        ua = ua.toLowerCase();
+
+        var match = /(chrome)[ \/]([\w.]+)/.exec(ua) ||
+            /(webkit)[ \/]([\w.]+)/.exec(ua) ||
+            /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) ||
+            /(msie) ([\w.]+)/.exec(ua) ||
+            ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) ||
+            [];
+
+        return {
+            browser: match[ 1 ] || "",
+            version: match[ 2 ] || "0"
+        };
+    };
+    jQuery.browser = {};
+    jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true;
+}

docs/manual/_static/basic.css

@@ -4,7 +4,7 @@
  *
  * Sphinx stylesheet -- basic theme.
  *
- * :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
+ * :copyright: Copyright 2007-2022 by the Sphinx team, see AUTHORS.
  * :license: BSD, see LICENSE for details.
  *
  */
@@ -222,7 +222,7 @@ table.modindextable td {
 /* -- general body styles --------------------------------------------------- */
 
 div.body {
-    min-width: 450px;
+    min-width: 360px;
     max-width: 800px;
 }
 
@@ -236,7 +236,6 @@ div.body p, div.body dd, div.body li, div.body blockquote {
 a.headerlink {
     visibility: hidden;
 }
-
 a.brackets:before,
 span.brackets > a:before{
     content: "[";
@@ -247,6 +246,7 @@ span.brackets > a:after {
     content: "]";
 }
 
+
 h1:hover > a.headerlink,
 h2:hover > a.headerlink,
 h3:hover > a.headerlink,
@@ -334,13 +334,11 @@ aside.sidebar {
 p.sidebar-title {
     font-weight: bold;
 }
-
 div.admonition, div.topic, blockquote {
     clear: left;
 }
 
 /* -- topics ---------------------------------------------------------------- */
-
 div.topic {
     border: 1px solid #ccc;
     padding: 7px;
@@ -428,10 +426,6 @@ table.docutils td, table.docutils th {
     border-bottom: 1px solid #aaa;
 }
 
-table.footnote td, table.footnote th {
-    border: 0 !important;
-}
-
 th {
     text-align: left;
     padding-right: 5px;
@@ -614,7 +608,6 @@ ol.simple p,
 ul.simple p {
     margin-bottom: 0;
 }
-
 dl.footnote > dt,
 dl.citation > dt {
     float: left;
@@ -643,11 +636,11 @@ dl.field-list > dt {
     padding-left: 0.5em;
     padding-right: 5px;
 }
-
 dl.field-list > dt:after {
     content: ":";
 }
 
+
 dl.field-list > dd {
     padding-left: 0.5em;
     margin-top: 0em;
@@ -731,8 +724,9 @@ dl.glossary dt {
 
 .classifier:before {
     font-style: normal;
-    margin: 0.5em;
+    margin: 0 0.5em;
     content: ":";
+    display: inline-block;
 }
 
 abbr, acronym {
@@ -756,6 +750,7 @@ span.pre {
     -ms-hyphens: none;
     -webkit-hyphens: none;
     hyphens: none;
+    white-space: nowrap;
 }
 
 div[class*="highlight-"] {
@@ -819,7 +814,7 @@ div.code-block-caption code {
 
 table.highlighttable td.linenos,
 span.linenos,
-div.doctest > div.highlight span.gp {  /* gp: Generic.Prompt */
+div.highlight span.gp {  /* gp: Generic.Prompt */
   user-select: none;
   -webkit-user-select: text; /* Safari fallback only */
   -webkit-user-select: none; /* Chrome/Safari */

docs/manual/_static/check-solid.svg

@@ -0,0 +1,4 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-check" width="44" height="44" viewBox="0 0 24 24" stroke-width="2" stroke="#22863a" fill="none" stroke-linecap="round" stroke-linejoin="round">
+  <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+  <path d="M5 12l5 5l10 -10" />
+</svg>

docs/manual/_static/classic.css

@@ -1,266 +0,0 @@
-/*
- * classic.css_t
- * ~~~~~~~~~~~~~
- *
- * Sphinx stylesheet -- classic theme.
- *
- * :copyright: Copyright 2007-2021 by the Sphinx team, see AUTHORS.
- * :license: BSD, see LICENSE for details.
- *
- */
-
-@import url("basic.css");
-
-/* -- page layout ----------------------------------------------------------- */
-
-html {
-    /* CSS hack for macOS's scrollbar (see #1125) */
-    background-color: #FFFFFF;
-}
-
-body {
-    font-family: sans-serif;
-    font-size: 100%;
-    background-color: #11303d;
-    color: #000;
-    margin: 0;
-    padding: 0;
-}
-
-div.document {
-    background-color: #1c4e63;
-}
-
-div.documentwrapper {
-    float: left;
-    width: 100%;
-}
-
-div.bodywrapper {
-    margin: 0 0 0 230px;
-}
-
-div.body {
-    background-color: #ffffff;
-    color: #000000;
-    padding: 0 20px 30px 20px;
-}
-
-div.footer {
-    color: #ffffff;
-    width: 100%;
-    padding: 9px 0 9px 0;
-    text-align: center;
-    font-size: 75%;
-}
-
-div.footer a {
-    color: #ffffff;
-    text-decoration: underline;
-}
-
-div.related {
-    background-color: #133f52;
-    line-height: 30px;
-    color: #ffffff;
-}
-
-div.related a {
-    color: #ffffff;
-}
-
-div.sphinxsidebar {
-}
-
-div.sphinxsidebar h3 {
-    font-family: 'Trebuchet MS', sans-serif;
-    color: #ffffff;
-    font-size: 1.4em;
-    font-weight: normal;
-    margin: 0;
-    padding: 0;
-}
-
-div.sphinxsidebar h3 a {
-    color: #ffffff;
-}
-
-div.sphinxsidebar h4 {
-    font-family: 'Trebuchet MS', sans-serif;
-    color: #ffffff;
-    font-size: 1.3em;
-    font-weight: normal;
-    margin: 5px 0 0 0;
-    padding: 0;
-}
-
-div.sphinxsidebar p {
-    color: #ffffff;
-}
-
-div.sphinxsidebar p.topless {
-    margin: 5px 10px 10px 10px;
-}
-
-div.sphinxsidebar ul {
-    margin: 10px;
-    padding: 0;
-    color: #ffffff;
-}
-
-div.sphinxsidebar a {
-    color: #98dbcc;
-}
-
-div.sphinxsidebar input {
-    border: 1px solid #98dbcc;
-    font-family: sans-serif;
-    font-size: 1em;
-}
-
-
-
-/* -- hyperlink styles ------------------------------------------------------ */
-
-a {
-    color: #355f7c;
-    text-decoration: none;
-}
-
-a:visited {
-    color: #355f7c;
-    text-decoration: none;
-}
-
-a:hover {
-    text-decoration: underline;
-}
-
-
-
-/* -- body styles ----------------------------------------------------------- */
-
-div.body h1,
-div.body h2,
-div.body h3,
-div.body h4,
-div.body h5,
-div.body h6 {
-    font-family: 'Trebuchet MS', sans-serif;
-    background-color: #f2f2f2;
-    font-weight: normal;
-    color: #20435c;
-    border-bottom: 1px solid #ccc;
-    margin: 20px -20px 10px -20px;
-    padding: 3px 0 3px 10px;
-}
-
-div.body h1 { margin-top: 0; font-size: 200%; }
-div.body h2 { font-size: 160%; }
-div.body h3 { font-size: 140%; }
-div.body h4 { font-size: 120%; }
-div.body h5 { font-size: 110%; }
-div.body h6 { font-size: 100%; }
-
-a.headerlink {
-    color: #c60f0f;
-    font-size: 0.8em;
-    padding: 0 4px 0 4px;
-    text-decoration: none;
-}
-
-a.headerlink:hover {
-    background-color: #c60f0f;
-    color: white;
-}
-
-div.body p, div.body dd, div.body li, div.body blockquote {
-    text-align: justify;
-    line-height: 130%;
-}
-
-div.admonition p.admonition-title + p {
-    display: inline;
-}
-
-div.admonition p {
-    margin-bottom: 5px;
-}
-
-div.admonition pre {
-    margin-bottom: 5px;
-}
-
-div.admonition ul, div.admonition ol {
-    margin-bottom: 5px;
-}
-
-div.note {
-    background-color: #eee;
-    border: 1px solid #ccc;
-}
-
-div.seealso {
-    background-color: #ffc;
-    border: 1px solid #ff6;
-}
-
-div.topic {
-    background-color: #eee;
-}
-
-div.warning {
-    background-color: #ffe4e4;
-    border: 1px solid #f66;
-}
-
-p.admonition-title {
-    display: inline;
-}
-
-p.admonition-title:after {
-    content: ":";
-}
-
-pre {
-    padding: 5px;
-    background-color: unset;
-    color: unset;
-    line-height: 120%;
-    border: 1px solid #ac9;
-    border-left: none;
-    border-right: none;
-}
-
-code {
-    background-color: #ecf0f3;
-    padding: 0 1px 0 1px;
-    font-size: 0.95em;
-}
-
-th, dl.field-list > dt {
-    background-color: #ede;
-}
-
-.warning code {
-    background: #efc2c2;
-}
-
-.note code {
-    background: #d6d6d6;
-}
-
-.viewcode-back {
-    font-family: sans-serif;
-}
-
-div.viewcode-block:target {
-    background-color: #f4debf;
-    border-top: 1px solid #ac9;
-    border-bottom: 1px solid #ac9;
-}
-
-div.code-block-caption {
-    color: #efefef;
-    background-color: #1c4e63;
-}

docs/manual/_static/copy-button.svg

@@ -0,0 +1,5 @@
+<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-copy" width="44" height="44" viewBox="0 0 24 24" stroke-width="1.5" stroke="#000000" fill="none" stroke-linecap="round" stroke-linejoin="round">
+  <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+  <rect x="8" y="8" width="12" height="12" rx="2" />
+  <path d="M16 8v-2a2 2 0 0 0 -2 -2h-8a2 2 0 0 0 -2 2v8a2 2 0 0 0 2 2h2" />
+</svg>

docs/manual/_static/copybutton.css

@@ -0,0 +1,93 @@
+/* Copy buttons */
+button.copybtn {
+    position: absolute;
+    display: flex;
+    top: .3em;
+    right: .3em;
+    width: 1.7em;
+    height: 1.7em;
+	opacity: 0;
+    transition: opacity 0.3s, border .3s, background-color .3s;
+    user-select: none;
+    padding: 0;
+    border: none;
+    outline: none;
+    border-radius: 0.4em;
+    /* The colors that GitHub uses */
+    border: #1b1f2426 1px solid;
+    background-color: #f6f8fa;
+    color: #57606a;
+}
+
+button.copybtn.success {
+    border-color: #22863a;
+    color: #22863a;
+}
+
+button.copybtn svg {
+    stroke: currentColor;
+    width: 1.5em;
+    height: 1.5em;
+    padding: 0.1em;
+}
+
+div.highlight  {
+    position: relative;
+}
+
+.highlight:hover button.copybtn {
+	opacity: 1;
+}
+
+.highlight button.copybtn:hover {
+    background-color: rgb(235, 235, 235);
+}
+
+.highlight button.copybtn:active {
+    background-color: rgb(187, 187, 187);
+}
+
+/**
+ * A minimal CSS-only tooltip copied from:
+ *   https://codepen.io/mildrenben/pen/rVBrpK
+ *
+ * To use, write HTML like the following:
+ *
+ * <p class="o-tooltip--left" data-tooltip="Hey">Short</p>
+ */
+ .o-tooltip--left {
+  position: relative;
+ }
+
+ .o-tooltip--left:after {
+    opacity: 0;
+    visibility: hidden;
+    position: absolute;
+    content: attr(data-tooltip);
+    padding: .2em;
+    font-size: .8em;
+    left: -.2em;
+    background: grey;
+    color: white;
+    white-space: nowrap;
+    z-index: 2;
+    border-radius: 2px;
+    transform: translateX(-102%) translateY(0);
+    transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1);
+}
+
+.o-tooltip--left:hover:after {
+    display: block;
+    opacity: 1;
+    visibility: visible;
+    transform: translateX(-100%) translateY(0);
+    transition: opacity 0.2s cubic-bezier(0.64, 0.09, 0.08, 1), transform 0.2s cubic-bezier(0.64, 0.09, 0.08, 1);
+    transition-delay: .5s;
+}
+
+/* By default the copy button shouldn't show up when printing a page */
+@media print {
+    button.copybtn {
+        display: none;
+    }
+}

docs/manual/_static/copybutton.js

@@ -0,0 +1,220 @@
+// Localization support
+const messages = {
+  'en': {
+    'copy': 'Copy',
+    'copy_to_clipboard': 'Copy to clipboard',
+    'copy_success': 'Copied!',
+    'copy_failure': 'Failed to copy',
+  },
+  'es' : {
+    'copy': 'Copiar',
+    'copy_to_clipboard': 'Copiar al portapapeles',
+    'copy_success': '¡Copiado!',
+    'copy_failure': 'Error al copiar',
+  },
+  'de' : {
+    'copy': 'Kopieren',
+    'copy_to_clipboard': 'In die Zwischenablage kopieren',
+    'copy_success': 'Kopiert!',
+    'copy_failure': 'Fehler beim Kopieren',
+  },
+  'fr' : {
+    'copy': 'Copier',
+    'copy_to_clipboard': 'Copié dans le presse-papier',
+    'copy_success': 'Copié !',
+    'copy_failure': 'Échec de la copie',
+  },
+  'ru': {
+    'copy': 'Скопировать',
+    'copy_to_clipboard': 'Скопировать в буфер',
+    'copy_success': 'Скопировано!',
+    'copy_failure': 'Не удалось скопировать',
+  },
+  'zh-CN': {
+    'copy': '复制',
+    'copy_to_clipboard': '复制到剪贴板',
+    'copy_success': '复制成功!',
+    'copy_failure': '复制失败',
+  },
+  'it' : {
+    'copy': 'Copiare',
+    'copy_to_clipboard': 'Copiato negli appunti',
+    'copy_success': 'Copiato!',
+    'copy_failure': 'Errore durante la copia',
+  }
+}
+
+let locale = 'en'
+if( document.documentElement.lang !== undefined
+    && messages[document.documentElement.lang] !== undefined ) {
+  locale = document.documentElement.lang
+}
+
+let doc_url_root = DOCUMENTATION_OPTIONS.URL_ROOT;
+if (doc_url_root == '#') {
+    doc_url_root = '';
+}
+
+/**
+ * SVG files for our copy buttons
+ */
+let iconCheck = `<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-check" width="44" height="44" viewBox="0 0 24 24" stroke-width="2" stroke="#22863a" fill="none" stroke-linecap="round" stroke-linejoin="round">
+  <title>${messages[locale]['copy_success']}</title>
+  <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+  <path d="M5 12l5 5l10 -10" />
+</svg>`
+
+// If the user specified their own SVG use that, otherwise use the default
+let iconCopy = ``;
+if (!iconCopy) {
+  iconCopy = `<svg xmlns="http://www.w3.org/2000/svg" class="icon icon-tabler icon-tabler-copy" width="44" height="44" viewBox="0 0 24 24" stroke-width="1.5" stroke="#000000" fill="none" stroke-linecap="round" stroke-linejoin="round">
+  <title>${messages[locale]['copy_to_clipboard']}</title>
+  <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
+  <rect x="8" y="8" width="12" height="12" rx="2" />
+  <path d="M16 8v-2a2 2 0 0 0 -2 -2h-8a2 2 0 0 0 -2 2v8a2 2 0 0 0 2 2h2" />
+</svg>`
+}
+
+/**
+ * Set up copy/paste for code blocks
+ */
+
+const runWhenDOMLoaded = cb => {
+  if (document.readyState != 'loading') {
+    cb()
+  } else if (document.addEventListener) {
+    document.addEventListener('DOMContentLoaded', cb)
+  } else {
+    document.attachEvent('onreadystatechange', function() {
+      if (document.readyState == 'complete') cb()
+    })
+  }
+}
+
+const codeCellId = index => `codecell${index}`
+
+// Clears selected text since ClipboardJS will select the text when copying
+const clearSelection = () => {
+  if (window.getSelection) {
+    window.getSelection().removeAllRanges()
+  } else if (document.selection) {
+    document.selection.empty()
+  }
+}
+
+// Changes tooltip text for two seconds, then changes it back
+const temporarilyChangeTooltip = (el, oldText, newText) => {
+  el.setAttribute('data-tooltip', newText)
+  el.classList.add('success')
+  setTimeout(() => el.setAttribute('data-tooltip', oldText), 2000)
+  setTimeout(() => el.classList.remove('success'), 2000)
+}
+
+// Changes the copy button icon for two seconds, then changes it back
+const temporarilyChangeIcon = (el) => {
+  el.innerHTML = iconCheck;
+  setTimeout(() => {el.innerHTML = iconCopy}, 2000)
+}
+
+const addCopyButtonToCodeCells = () => {
+  // If ClipboardJS hasn't loaded, wait a bit and try again. This
+  // happens because we load ClipboardJS asynchronously.
+  if (window.ClipboardJS === undefined) {
+    setTimeout(addCopyButtonToCodeCells, 250)
+    return
+  }
+
+  // Add copybuttons to all of our code cells
+  const codeCells = document.querySelectorAll('div.highlight pre')
+  codeCells.forEach((codeCell, index) => {
+    const id = codeCellId(index)
+    codeCell.setAttribute('id', id)
+
+    const clipboardButton = id =>
+    `<button class="copybtn o-tooltip--left" data-tooltip="${messages[locale]['copy']}" data-clipboard-target="#${id}">
+      ${iconCopy}
+    </button>`
+    codeCell.insertAdjacentHTML('afterend', clipboardButton(id))
+  })
+
+function escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+}
+
+// Callback when a copy button is clicked. Will be passed the node that was clicked
+// should then grab the text and replace pieces of text that shouldn't be used in output
+function formatCopyText(textContent, copybuttonPromptText, isRegexp = false, onlyCopyPromptLines = true, removePrompts = true, copyEmptyLines = true, lineContinuationChar = "", hereDocDelim = "") {
+
+    var regexp;
+    var match;
+
+    // Do we check for line continuation characters and "HERE-documents"?
+    var useLineCont = !!lineContinuationChar
+    var useHereDoc = !!hereDocDelim
+
+    // create regexp to capture prompt and remaining line
+    if (isRegexp) {
+        regexp = new RegExp('^(' + copybuttonPromptText + ')(.*)')
+    } else {
+        regexp = new RegExp('^(' + escapeRegExp(copybuttonPromptText) + ')(.*)')
+    }
+
+    const outputLines = [];
+    var promptFound = false;
+    var gotLineCont = false;
+    var gotHereDoc = false;
+    const lineGotPrompt = [];
+    for (const line of textContent.split('\n')) {
+        match = line.match(regexp)
+        if (match || gotLineCont || gotHereDoc) {
+            promptFound = regexp.test(line)
+            lineGotPrompt.push(promptFound)
+            if (removePrompts && promptFound) {
+                outputLines.push(match[2])
+            } else {
+                outputLines.push(line)
+            }
+            gotLineCont = line.endsWith(lineContinuationChar) & useLineCont
+            if (line.includes(hereDocDelim) & useHereDoc)
+                gotHereDoc = !gotHereDoc
+        } else if (!onlyCopyPromptLines) {
+            outputLines.push(line)
+        } else if (copyEmptyLines && line.trim() === '') {
+            outputLines.push(line)
+        }
+    }
+
+    // If no lines with the prompt were found then just use original lines
+    if (lineGotPrompt.some(v => v === true)) {
+        textContent = outputLines.join('\n');
+    }
+
+    // Remove a trailing newline to avoid auto-running when pasting
+    if (textContent.endsWith("\n")) {
+        textContent = textContent.slice(0, -1)
+    }
+    return textContent
+}
+
+
+var copyTargetText = (trigger) => {
+  var target = document.querySelector(trigger.attributes['data-clipboard-target'].value);
+  return formatCopyText(target.innerText, '', false, true, true, true, '', '')
+}
+
+  // Initialize with a callback so we can modify the text before copy
+  const clipboard = new ClipboardJS('.copybtn', {text: copyTargetText})
+
+  // Update UI with error/success messages
+  clipboard.on('success', event => {
+    clearSelection()
+    temporarilyChangeTooltip(event.trigger, messages[locale]['copy'], messages[locale]['copy_success'])
+    temporarilyChangeIcon(event.trigger)
+  })
+
+  clipboard.on('error', event => {
+    temporarilyChangeTooltip(event.trigger, messages[locale]['copy'], messages[locale]['copy_failure'])
+  })
+}
+
+runWhenDOMLoaded(addCopyButtonToCodeCells)

docs/manual/_static/copybutton_funcs.js

@@ -0,0 +1,58 @@
+function escapeRegExp(string) {
+    return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
+}
+
+// Callback when a copy button is clicked. Will be passed the node that was clicked
+// should then grab the text and replace pieces of text that shouldn't be used in output
+export function formatCopyText(textContent, copybuttonPromptText, isRegexp = false, onlyCopyPromptLines = true, removePrompts = true, copyEmptyLines = true, lineContinuationChar = "", hereDocDelim = "") {
+
+    var regexp;
+    var match;
+
+    // Do we check for line continuation characters and "HERE-documents"?
+    var useLineCont = !!lineContinuationChar
+    var useHereDoc = !!hereDocDelim
+
+    // create regexp to capture prompt and remaining line
+    if (isRegexp) {
+        regexp = new RegExp('^(' + copybuttonPromptText + ')(.*)')
+    } else {
+        regexp = new RegExp('^(' + escapeRegExp(copybuttonPromptText) + ')(.*)')
+    }
+
+    const outputLines = [];
+    var promptFound = false;
+    var gotLineCont = false;
+    var gotHereDoc = false;
+    const lineGotPrompt = [];
+    for (const line of textContent.split('\n')) {
+        match = line.match(regexp)
+        if (match || gotLineCont || gotHereDoc) {
+            promptFound = regexp.test(line)
+            lineGotPrompt.push(promptFound)
+            if (removePrompts && promptFound) {
+                outputLines.push(match[2])
+            } else {
+                outputLines.push(line)
+            }
+            gotLineCont = line.endsWith(lineContinuationChar) & useLineCont
+            if (line.includes(hereDocDelim) & useHereDoc)
+                gotHereDoc = !gotHereDoc
+        } else if (!onlyCopyPromptLines) {
+            outputLines.push(line)
+        } else if (copyEmptyLines && line.trim() === '') {
+            outputLines.push(line)
+        }
+    }
+
+    // If no lines with the prompt were found then just use original lines
+    if (lineGotPrompt.some(v => v === true)) {
+        textContent = outputLines.join('\n');
+    }
+
+    // Remove a trailing newline to avoid auto-running when pasting
+    if (textContent.endsWith("\n")) {
+        textContent = textContent.slice(0, -1)
+    }
+    return textContent
+}