Better comments on outbound transport

Examples/Broadcast.py

@@ -0,0 +1,93 @@
+##########################################################
+# This RNS example demonstrates broadcasting unencrypted #
+# information to any listening destinations.             #
+##########################################################
+
+import sys
+import argparse
+import RNS
+
+# Let's define an app name. We'll use this for all
+# destinations we create. Since this basic example
+# is part of a range of example utilities, we'll put
+# them all within the app namespace "example_utilities"
+APP_NAME = "example_utilitites"
+
+# This initialisation is executed when the program is started
+def program_setup(configpath, channel=None):
+	# We must first initialise Reticulum
+	reticulum = RNS.Reticulum(configpath)
+	
+	# If the user did not select a "channel" we use
+	# a default one called "public_information".
+	# This "channel" is added to the destination name-
+	# space, so the user can select different broadcast
+	# channels.
+	if channel == None:
+		channel = "public_information"
+
+	# We create a PLAIN destination. This is an uncencrypted endpoint
+	# that anyone can listen to and send information to.
+	broadcast_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, APP_NAME, "broadcast", channel)
+
+	# We specify a callback that will get called every time
+	# the destination receives data.
+	broadcast_destination.packet_callback(packet_callback)
+	
+	# Everything's ready!
+	# Let's hand over control to the main loop
+	broadcastLoop(broadcast_destination)
+
+def packet_callback(data, packet):
+	# Simply print out the received data
+	print("")
+	print("Received data: "+data.decode("utf-8")+"\r\n> ", end="")
+	sys.stdout.flush()
+
+def broadcastLoop(destination):
+	# Let the user know that everything is ready
+	RNS.log("Broadcast example "+RNS.prettyhexrep(destination.hash)+" running, enter text and hit enter to broadcast (Ctrl-C to quit)")
+
+	# We enter a loop that runs until the users exits.
+	# If the user hits enter, we will send the information
+	# that the user entered into the prompt.
+	while True:
+		print("> ", end="")
+		entered = input()
+
+		if entered != "":
+			data    = entered.encode("utf-8")
+			packet  = RNS.Packet(destination, data)
+			packet.send()
+
+		
+
+##########################################################
+#### Program Startup #####################################
+##########################################################
+
+# This part of the program gets run at startup,
+# and parses input from the user, and then starts
+# the program.
+if __name__ == "__main__":
+	try:
+		parser = argparse.ArgumentParser(description="Reticulum example that demonstrates sending and receiving unencrypted broadcasts")
+		parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
+		parser.add_argument("--channel", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
+		args = parser.parse_args()
+
+		if args.config:
+			configarg = args.config
+		else:
+			configarg = None
+
+		if args.channel:
+			channelarg = args.channel
+		else:
+			channelarg = None
+
+		program_setup(configarg, channelarg)
+
+	except KeyboardInterrupt:
+		print("")
+		exit()

Examples/Filetransfer.py

@@ -55,7 +55,8 @@ def server(configpath, path):
 
 def announceLoop(destination):
 	# Let the user know that everything is ready
-	RNS.log("File server "+RNS.prettyhexrep(destination.hash)+" running, hit enter to manually send an announce (Ctrl-C to quit)")
+	RNS.log("File server "+RNS.prettyhexrep(destination.hash)+" running")
+	RNS.log("Hit enter to manually send an announce (Ctrl-C to quit)")
 
 	# We enter a loop that runs until the users exits.
 	# If the user hits enter, we will announce our server
@@ -238,8 +239,9 @@ def download(filename):
 
 	# We just create a packet containing the
 	# requested filename, and send it down the
-	# link.
-	request_packet = RNS.Packet(server_link, filename.encode("utf-8"))
+	# link. We also specify we don't need a
+	# packet receipt.
+	request_packet = RNS.Packet(server_link, filename.encode("utf-8"), create_receipt=False)
 	request_packet.send()
 	
 	print("")

Examples/Link.py

@@ -75,10 +75,13 @@ def client_disconnected(link):
 def server_packet_received(message, packet):
 	global latest_client_link
 
+	# When data is received over any active link,
+	# it will all be directed to the last client
+	# that connected.
 	text = message.decode("utf-8")
 	RNS.log("Received data on the link: "+text)
 	
-	reply_text = "I got \""+text+"\" from you"
+	reply_text = "I received \""+text+"\" over the link"
 	reply_data = reply_text.encode("utf-8")
 	RNS.Packet(latest_client_link, reply_data).send()
 
@@ -149,19 +152,23 @@ def client_loop():
 
 	should_quit = False
 	while not should_quit:
-		print("> ", end=" ")
-		text = input()
+		try:
+			print("> ", end=" ")
+			text = input()
 
-		# Check if we should quit the example
-		if text == "quit" or text == "q" or text == "exit":
+			# Check if we should quit the example
+			if text == "quit" or text == "q" or text == "exit":
+				should_quit = True
+				server_link.teardown()
+
+			# If not, send the entered text over the link
+			if text != "":
+				data = text.encode("utf-8")
+				RNS.Packet(server_link, data).send()
+		except Exception as e:
 			should_quit = True
 			server_link.teardown()
 
-		# If not, send the entered text over the link
-		if text != "":
-			data = text.encode("utf-8")
-			RNS.Packet(server_link, data).send()
-
 # This function is called when a link
 # has been established with the server
 def link_established(link):

Notes/Header format

@@ -1,41 +0,0 @@
-header types
------------------
-type 1			00	Two byte header, one 10 byte address field
-type 2			01	Two byte header, two 10 byte address fields
-type 3			10	Reserved
-type 4			11	Reserved for extended header format
-
-
-propagation types
------------------
-broadcast		00
-transport		01
-relay			10
-tunnel			11
-
-
-destination types
------------------
-single			00
-group			01
-plain			10
-link			11
-
-
-packet types
------------------
-data			00
-announce		01
-link request	10
-proof			11
-
-
-+- Header example -+
-
-01010000 00000100 [ADDR 1, 10 bytes] [ADDR 2, 10 bytes] [CONTEXT]
- | | | |    |
- | | | |    +-- Context = RESOURCE_HMU
- | | | +------- DATA packet
- | | +--------- SINGLE destination
- | +----------- TRANSPORT propagation type
- +------------- HEADER_2, two byte header, two address fields

Notes/Wire Format

@@ -0,0 +1,54 @@
+Reticulum Wire Format
+
+Header Types
+-----------------
+type 1          00  Two byte header, one 10 byte address field
+type 2          01  Two byte header, two 10 byte address fields
+type 3          10  Reserved
+type 4          11  Reserved for extended header format
+
+
+Propagation Types
+-----------------
+broadcast       00
+transport       01
+relay           10
+tunnel          11
+
+
+Destination Types
+-----------------
+single          00
+group           01
+plain           10
+link            11
+
+
+Packet Types
+-----------------
+data            00
+announce        01
+link request    10
+proof           11
+
+
++- Packet Example -+
+
+01010000 00000100 [ADDR1, 10 bytes] [ADDR2, 10 bytes] [CONTEXT, 1 byte] [DATA]
+ | | | |    |
+ | | | |    +-- Hops = 4
+ | | | +------- DATA packet
+ | | +--------- SINGLE destination
+ | +----------- TRANSPORT propagation type
+ +------------- HEADER_2, two byte header, two address fields
+
+
+ +- Packet Example -+
+
+00000000 00000111 [ADDR1, 10 bytes] [CONTEXT, 1 byte] [DATA]
+ | | | |    |
+ | | | |    +-- Hops = 7
+ | | | +------- DATA packet
+ | | +--------- SINGLE destination
+ | +----------- BROADCAST propagation type
+ +------------- HEADER_1, two byte header, one address field

README.html

@@ -732,16 +732,16 @@ pre {
 }
 </style><title>README</title></head><body><article class="markdown-body"><h1>
 <a id="user-content-reticulum-network-stack-α" class="anchor" href="#reticulum-network-stack-%CE%B1" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Reticulum Network Stack α</h1>
-<p>Reticulum is a cryptography-based networking stack for low-bandwidth, high-latency, wide-area networks built on cheap and readily available hardware. Reticulum allows you to build very wide-area networks with cheap off-the-shelf tools, and offers end-to-end encryption, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, resource caching, unforgeable packet acknowledgements and much more.</p>
-<p>Reticulum is a complete networking stack, and does not use IP or higher layers, although it can be easily tunnelled through conventional IP networks. This frees up a lot of overhead, that has been utilised to implement a networking stack built directly on cryptographic principles, allowing resilience and stable functionality in open and trustless networks.</p>
-<p>Reticulum runs completely in userland, and can run on practically any system that runs Python 3.</p>
+<p>Reticulum is a cryptography-based networking stack for high-latency, wide-area networks built on readily available hardware. Reticulum allows you to build very wide-area networks with off-the-shelf tools, and offers end-to-end encryption, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, resource caching, unforgeable packet acknowledgements and much more.</p>
+<p>Reticulum is a complete networking stack, and does not use IP or higher layers, although it can be easily tunnelled through conventional IP networks. This frees up overhead, that has been utilised to implement a networking stack built directly on cryptographic principles, allowing resilience and stable functionality in open and trustless networks.</p>
+<p>No kernel modules or drivers are required. Reticulum runs completely in userland, and can run on practically any system that runs Python 3.</p>
 <p>For more info, see <a href="https://unsigned.io/projects/reticulum/" rel="nofollow">unsigned.io/projects/reticulum</a></p>
 <h2>
-<a id="user-content-what-hardware-does-reticulum-work-with" class="anchor" href="#what-hardware-does-reticulum-work-with" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>What hardware does Reticulum work with?</h2>
-<p>Practically any hardware that can support at least a half-duplex channel with 1.000 bits per second throughput, and an MTU of 500 bytes. Data radios, modems, LoRa radios, serial lines, AX.25 TNCs, HAM radio digital modes, free-space optical links and similar systems are all examples of the types of interfaces Reticulum was designed for.</p>
-<p>An open-source LoRa-based interface called <a href="https://unsigned.io/projects/rnode/" rel="nofollow">RNode</a> has been designed specifically for use with Reticulum. It is easy to build yourself, or can be purchased as a complete radio that just needs a USB connection to the host.</p>
-<p>Reticulum can also be tunneled over existing IP networks, so there's nothing stopping you from using it over gigabit fiber or your local WiFi network, where it'll work just as well. In fact, one of the strengths of Reticulum is how easily it allows you to connect different mediums into a self-configuring, resilient and encrypted mesh.</p>
-<p>As an example, it's possible to set up a Raspberry Pi connected to both a LoRa radio, a packet radio TNC and your home WiFi. Once the interfaces are configured, Reticulum will take care of the rest, and any device on your home WiFi can communicate with nodes on the LoRa and packet radio sides of the network.</p>
+<a id="user-content-where-can-reticulum-be-used" class="anchor" href="#where-can-reticulum-be-used" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Where can Reticulum be used?</h2>
+<p>On practically any hardware that can support at least a half-duplex channel with 1.000 bits per second throughput, and an MTU of 500 bytes. Data radios, modems, LoRa radios, serial lines, AX.25 TNCs, amateur radio digital modes, free-space optical links and similar systems are all examples of the types of interfaces Reticulum was designed for.</p>
+<p>An open-source LoRa-based interface called <a href="https://unsigned.io/projects/rnode/" rel="nofollow">RNode</a> has been designed specifically for use with Reticulum. It is possible to build yourself, or can be purchased as a complete transceiver that just needs a USB connection to the host.</p>
+<p>Reticulum can also be encapsulated over existing IP networks, so there's nothing stopping you from using it over wired ethernet or your local WiFi network, where it'll work just as well. In fact, one of the strengths of Reticulum is how easily it allows you to connect different mediums into a self-configuring, resilient and encrypted mesh.</p>
+<p>As an example, it's possible to set up a Raspberry Pi connected to both a LoRa radio, a packet radio TNC and a WiFi network. Once the interfaces are configured, Reticulum will take care of the rest, and any device on the WiFi network can communicate with nodes on the LoRa and packet radio sides of the network, and vice versa.</p>
 <h2>
 <a id="user-content-current-status" class="anchor" href="#current-status" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Current Status</h2>
 <p>Consider Reticulum experimental at this stage. Most features are implemented and working, but at this point the protocol may still change significantly, and is made publicly available for development collaboration, previewing and testing.</p>
@@ -749,29 +749,36 @@ pre {
 <h2>
 <a id="user-content-what-is-implemented-at-this-point" class="anchor" href="#what-is-implemented-at-this-point" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>What is implemented at this point?</h2>
 <ul>
-<li>All basic adressing and identification features</li>
+<li>Adressing and identification</li>
+<li>Fully self-configuring multi-hop routing</li>
 <li>RSA assymetric encryption and signatures as basis for all communication</li>
 <li>AES-128 symmetric encryption for group destinations</li>
 <li>Elliptic curve encryption for links (on the SECP256R1 curve)</li>
 <li>Perfect Forward Secrecy on links with ephemereal ECDH keys</li>
 <li>Unforgeable packet delivery confirmations</li>
-<li>Fully self-configuring multi-hop routing</li>
 <li>A variety of supported interface types</li>
 <li>Efficient and easy resource transfers</li>
 <li>A simple and easy-to-use API</li>
-<li>A few basic examples</li>
+<li>Some basic programming examples</li>
 </ul>
 <h2>
-<a id="user-content-what-features-are-still-missing" class="anchor" href="#what-features-are-still-missing" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>What features are still missing?</h2>
+<a id="user-content-supported-interface-types-and-devices" class="anchor" href="#supported-interface-types-and-devices" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>Supported interface types and devices</h2>
+<p>Reticulum implements a range of generalised interface types that covers most of the communications hardware that Reticulum can run over. If your hardware is not supported, it's relatively simple to implement an interface class. Currently, the following interfaces are supported:</p>
 <ul>
-<li>On-network caching and cache queries</li>
+<li>Any ethernet device</li>
+<li>LoRa using <a href="https://unsigned.io/projects/rnode/" rel="nofollow">RNode</a>
+</li>
+<li>Packet Radio TNCs (with or without AX.25)</li>
+<li>Any device with a serial port</li>
+<li>TCP over IP networks</li>
+<li>UDP over IP networks</li>
 </ul>
 <h2>
 <a id="user-content-what-is-currently-being-worked-on" class="anchor" href="#what-is-currently-being-worked-on" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>What is currently being worked on?</h2>
 <ul>
+<li>Delay/disruption tolerant bundle transfers</li>
 <li>Useful example programs and utilities</li>
 <li>API documentation</li>
-<li>Cleanup and code commenting</li>
 <li>A messaging protocol built on Reticulum, see <a href="https://github.com/markqvist/lxmf">LXMF</a>
 </li>
 <li>A few useful-in-the-real-world apps built with Reticulum</li>
@@ -788,7 +795,7 @@ pre {
 </ul>
 <h2>
 <a id="user-content-how-do-i-get-started" class="anchor" href="#how-do-i-get-started" aria-hidden="true"><span aria-hidden="true" class="octicon octicon-link"></span></a>How do I get started?</h2>
-<p>Full documentation and video tutorials are coming with the stable alpha release. Until then, you are mostly on your own. If you really want to experiment already, you could take a look in the "Examples" folder, for some well-documented example programs. The default configuration file created by Reticulum on the first run is also worth reading. Be sure to also read the <a href="http://unsigned.io/wp-content/uploads/2018/04/Reticulum_Overview_v0.4.pdf" rel="nofollow">Reticulum Overview Document</a>.</p>
+<p>Full documentation and tutorials are coming with the stable alpha release. Until then, you are mostly on your own. If you want to experiment already, you could take a look in the "Examples" folder, for some well-documented example programs. The default configuration file created by Reticulum on the first run is also worth reading. Be sure to also read the <a href="http://unsigned.io/wp-content/uploads/2018/04/Reticulum_Overview_v0.4.pdf" rel="nofollow">Reticulum Overview Document</a>.</p>
 <p>If you just need Reticulum as a dependency for another application, the easiest way is probably via pip:</p>
 <div class="highlight highlight-source-shell"><pre>pip3 install rns</pre></div>
 <p>For development, you might want to get the latest source from GitHub. In that case, don't use pip, but try this recipe:</p>

README.md

@@ -1,22 +1,22 @@
 Reticulum Network Stack α
 ==========
 
-Reticulum is a cryptography-based networking stack for low-bandwidth, high-latency, wide-area networks built on cheap and readily available hardware. Reticulum allows you to build very wide-area networks with cheap off-the-shelf tools, and offers end-to-end encryption, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, resource caching, unforgeable packet acknowledgements and much more.
+Reticulum is a cryptography-based networking stack for high-latency, wide-area networks built on readily available hardware. Reticulum allows you to build very wide-area networks with off-the-shelf tools, and offers end-to-end encryption, autoconfiguring cryptographically backed multi-hop transport, efficient addressing, resource caching, unforgeable packet acknowledgements and much more.
 
-Reticulum is a complete networking stack, and does not use IP or higher layers, although it can be easily tunnelled through conventional IP networks. This frees up a lot of overhead, that has been utilised to implement a networking stack built directly on cryptographic principles, allowing resilience and stable functionality in open and trustless networks.
+Reticulum is a complete networking stack, and does not use IP or higher layers, although it can be easily tunnelled through conventional IP networks. This frees up overhead, that has been utilised to implement a networking stack built directly on cryptographic principles, allowing resilience and stable functionality in open and trustless networks.
 
-Reticulum runs completely in userland, and can run on practically any system that runs Python 3.
+No kernel modules or drivers are required. Reticulum runs completely in userland, and can run on practically any system that runs Python 3.
 
 For more info, see [unsigned.io/projects/reticulum](https://unsigned.io/projects/reticulum/)
 
-## What hardware does Reticulum work with?
-Practically any hardware that can support at least a half-duplex channel with 1.000 bits per second throughput, and an MTU of 500 bytes. Data radios, modems, LoRa radios, serial lines, AX.25 TNCs, HAM radio digital modes, free-space optical links and similar systems are all examples of the types of interfaces Reticulum was designed for.
+## Where can Reticulum be used?
+On practically any hardware that can support at least a half-duplex channel with 1.000 bits per second throughput, and an MTU of 500 bytes. Data radios, modems, LoRa radios, serial lines, AX.25 TNCs, amateur radio digital modes, free-space optical links and similar systems are all examples of the types of interfaces Reticulum was designed for.
 
-An open-source LoRa-based interface called [RNode](https://unsigned.io/projects/rnode/) has been designed specifically for use with Reticulum. It is easy to build yourself, or can be purchased as a complete radio that just needs a USB connection to the host.
+An open-source LoRa-based interface called [RNode](https://unsigned.io/projects/rnode/) has been designed specifically for use with Reticulum. It is possible to build yourself, or can be purchased as a complete transceiver that just needs a USB connection to the host.
 
-Reticulum can also be tunneled over existing IP networks, so there's nothing stopping you from using it over gigabit fiber or your local WiFi network, where it'll work just as well. In fact, one of the strengths of Reticulum is how easily it allows you to connect different mediums into a self-configuring, resilient and encrypted mesh.
+Reticulum can also be encapsulated over existing IP networks, so there's nothing stopping you from using it over wired ethernet or your local WiFi network, where it'll work just as well. In fact, one of the strengths of Reticulum is how easily it allows you to connect different mediums into a self-configuring, resilient and encrypted mesh.
 
-As an example, it's possible to set up a Raspberry Pi connected to both a LoRa radio, a packet radio TNC and your home WiFi. Once the interfaces are configured, Reticulum will take care of the rest, and any device on your home WiFi can communicate with nodes on the LoRa and packet radio sides of the network.
+As an example, it's possible to set up a Raspberry Pi connected to both a LoRa radio, a packet radio TNC and a WiFi network. Once the interfaces are configured, Reticulum will take care of the rest, and any device on the WiFi network can communicate with nodes on the LoRa and packet radio sides of the network, and vice versa.
 
 ## Current Status
 Consider Reticulum experimental at this stage. Most features are implemented and working, but at this point the protocol may still change significantly, and is made publicly available for development collaboration, previewing and testing.
@@ -24,25 +24,33 @@ Consider Reticulum experimental at this stage. Most features are implemented and
 An API- and wireformat-stable alpha release is coming in the near future. Until then expect things to change unexpectedly if something warrants it.
 
 ## What is implemented at this point?
- - All basic adressing and identification features
+ - Adressing and identification
+ - Fully self-configuring multi-hop routing
  - RSA assymetric encryption and signatures as basis for all communication
  - AES-128 symmetric encryption for group destinations
  - Elliptic curve encryption for links (on the SECP256R1 curve)
  - Perfect Forward Secrecy on links with ephemereal ECDH keys
  - Unforgeable packet delivery confirmations
- - Fully self-configuring multi-hop routing
  - A variety of supported interface types
  - Efficient and easy resource transfers
  - A simple and easy-to-use API
- - A few basic examples
+ - Some basic programming examples
 
-## What features are still missing?
- - On-network caching and cache queries
+## Supported interface types and devices
+
+Reticulum implements a range of generalised interface types that covers most of the communications hardware that Reticulum can run over. If your hardware is not supported, it's relatively simple to implement an interface class. Currently, the following interfaces are supported:
+
+ - Any ethernet device
+ - LoRa using [RNode](https://unsigned.io/projects/rnode/)
+ - Packet Radio TNCs (with or without AX.25)
+ - Any device with a serial port
+ - TCP over IP networks
+ - UDP over IP networks
 
 ## What is currently being worked on?
+ - Delay/disruption tolerant bundle transfers
  - Useful example programs and utilities
  - API documentation
- - Cleanup and code commenting
  - A messaging protocol built on Reticulum, see [LXMF](https://github.com/markqvist/lxmf)
  - A few useful-in-the-real-world apps built with Reticulum
 
@@ -55,7 +63,7 @@ Some countries still ban the use of encryption when operating under an amateur r
  - pyserial
 
 ## How do I get started?
-Full documentation and video tutorials are coming with the stable alpha release. Until then, you are mostly on your own. If you really want to experiment already, you could take a look in the "Examples" folder, for some well-documented example programs. The default configuration file created by Reticulum on the first run is also worth reading. Be sure to also read the [Reticulum Overview Document](http://unsigned.io/wp-content/uploads/2018/04/Reticulum_Overview_v0.4.pdf).
+Full documentation and tutorials are coming with the stable alpha release. Until then, you are mostly on your own. If you want to experiment already, you could take a look in the "Examples" folder, for some well-documented example programs. The default configuration file created by Reticulum on the first run is also worth reading. Be sure to also read the [Reticulum Overview Document](http://unsigned.io/wp-content/uploads/2018/04/Reticulum_Overview_v0.4.pdf).
 
 If you just need Reticulum as a dependency for another application, the easiest way is probably via pip:
 

RNS/Destination.py

@@ -171,11 +171,16 @@ class Destination:
 		if self.type == Destination.SINGLE and self.identity != None:
 			return self.identity.encrypt(plaintext)
 
-		if self.type == Destination.GROUP and self.prv != None:
-			try:
-				return base64.urlsafe_b64decode(self.prv.encrypt(plaintext))
-			except:
-				return None
+		if self.type == Destination.GROUP:
+			if hasattr(self, "prv") and self.prv != None:
+				try:
+					return base64.urlsafe_b64decode(self.prv.encrypt(plaintext))
+				except Exception as e:
+					RNS.log("The GROUP destination could not encrypt data", RNS.LOG_ERROR)
+					RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+			else:
+				raise ValueError("No private key held by GROUP destination. Did you create or load one?")
+
 
 
 	def decrypt(self, ciphertext):
@@ -186,7 +191,14 @@ class Destination:
 			return self.identity.decrypt(ciphertext)
 
 		if self.type == Destination.GROUP:
-			return self.prv.decrypt(base64.urlsafe_b64encode(ciphertext))
+			if hasattr(self, "prv") and self.prv != None:
+				try:
+					return self.prv.decrypt(base64.urlsafe_b64encode(ciphertext))
+				except Exception as e:
+					RNS.log("The GROUP destination could not decrypt data", RNS.LOG_ERROR)
+					RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+			else:
+				raise ValueError("No private key held by GROUP destination. Did you create or load one?")
 
 
 	def sign(self, message):

RNS/Identity.py

@@ -33,7 +33,6 @@ class Identity:
 
 	@staticmethod
 	def remember(packet_hash, destination_hash, public_key, app_data = None):
-		RNS.log("Remembering "+RNS.prettyhexrep(destination_hash), RNS.LOG_VERBOSE)
 		Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data]
 
 
@@ -108,7 +107,7 @@ class Identity:
 
 			if announced_identity.pub != None and announced_identity.validate(signature, signed_data):
 				RNS.Identity.remember(packet.getHash(), destination_hash, public_key)
-				RNS.log("Stored valid announce from "+RNS.prettyhexrep(destination_hash), RNS.LOG_INFO)
+				RNS.log("Stored valid announce from "+RNS.prettyhexrep(destination_hash), RNS.LOG_DEBUG)
 				del announced_identity
 				return True
 			else:

RNS/Interfaces/LocalInterface.py

@@ -0,0 +1,188 @@
+from .Interface import Interface
+import socketserver
+import threading
+import socket
+import time
+import sys
+import os
+import RNS
+
+class HDLC():
+    FLAG              = 0x7E
+    ESC               = 0x7D
+    ESC_MASK          = 0x20
+
+    @staticmethod
+    def escape(data):
+        data = data.replace(bytes([HDLC.ESC]), bytes([HDLC.ESC, HDLC.ESC^HDLC.ESC_MASK]))
+        data = data.replace(bytes([HDLC.FLAG]), bytes([HDLC.ESC, HDLC.FLAG^HDLC.ESC_MASK]))
+        return data
+
+class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
+    pass
+
+class LocalClientInterface(Interface):
+
+    def __init__(self, owner, name, target_port = None, connected_socket=None):
+        self.IN               = True
+        self.OUT              = False
+        self.socket           = None
+        self.parent_interface = None
+        self.name             = name
+
+        if connected_socket != None:
+            self.receives    = True
+            self.target_ip   = None
+            self.target_port = None
+            self.socket      = connected_socket
+
+        elif target_port != None:
+            self.receives    = True
+            self.target_ip   = "127.0.0.1"
+            self.target_port = target_port
+
+            self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.socket.connect((self.target_ip, self.target_port))
+
+            self.is_connected_to_shared_instance = True
+
+        self.owner   = owner
+        self.online  = True
+        self.writing = False
+
+        if connected_socket == None:
+            thread = threading.Thread(target=self.read_loop)
+            thread.setDaemon(True)
+            thread.start()
+
+    def processIncoming(self, data):
+        self.owner.inbound(data, self)
+
+    def processOutgoing(self, data):
+        if self.online:
+            while self.writing:
+                time.sleep(0.01)
+
+            try:
+                self.writing = True
+                data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
+                self.socket.sendall(data)
+                self.writing = False
+            except Exception as e:
+                RNS.log("Exception occurred while transmitting via "+str(self)+", tearing down interface", RNS.LOG_ERROR)
+                RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+                self.teardown()
+
+
+    def read_loop(self):
+        try:
+            in_frame = False
+            escape = False
+            data_buffer = b""
+
+            while True:
+                data_in = self.socket.recv(4096)
+                if len(data_in) > 0:
+                    pointer = 0
+                    while pointer < len(data_in):
+                        byte = data_in[pointer]
+                        pointer += 1
+                        if (in_frame and byte == HDLC.FLAG):
+                            in_frame = False
+                            self.processIncoming(data_buffer)
+                        elif (byte == HDLC.FLAG):
+                            in_frame = True
+                            data_buffer = b""
+                        elif (in_frame and len(data_buffer) < RNS.Reticulum.MTU):
+                            if (byte == HDLC.ESC):
+                                escape = True
+                            else:
+                                if (escape):
+                                    if (byte == HDLC.FLAG ^ HDLC.ESC_MASK):
+                                        byte = HDLC.FLAG
+                                    if (byte == HDLC.ESC  ^ HDLC.ESC_MASK):
+                                        byte = HDLC.ESC
+                                    escape = False
+                                data_buffer = data_buffer+bytes([byte])
+                else:
+                    RNS.log("Socket for "+str(self)+" was closed, tearing down interface", RNS.LOG_VERBOSE)
+                    self.teardown()
+                    break
+
+                
+        except Exception as e:
+            self.online = False
+            RNS.log("An interface error occurred, the contained exception was: "+str(e), RNS.LOG_ERROR)
+            RNS.log("Tearing down "+str(self), RNS.LOG_ERROR)
+            self.teardown()
+
+    def teardown(self):
+        self.online = False
+        self.OUT = False
+        self.IN = False
+
+        if self in RNS.Transport.interfaces:
+            RNS.Transport.interfaces.remove(self)
+
+        if self in RNS.Transport.local_client_interfaces:
+            RNS.Transport.local_client_interfaces.remove(self)
+
+
+    def __str__(self):
+        return "LocalInterface["+str(self.target_port)+"]"
+
+
+class LocalServerInterface(Interface):
+
+    def __init__(self, owner, bindport=None):
+        self.IN  = True
+        self.OUT = False
+        self.name = "Reticulum"
+
+        if (bindport != None):
+            self.receives = True
+            self.bind_ip = "127.0.0.1"
+            self.bind_port = bindport
+
+            def handlerFactory(callback):
+                def createHandler(*args, **keys):
+                    return LocalInterfaceHandler(callback, *args, **keys)
+                return createHandler
+
+            self.owner = owner
+            self.is_local_shared_instance = True
+
+            address = (self.bind_ip, self.bind_port)
+            self.server = ThreadingTCPServer(address, handlerFactory(self.incoming_connection))
+
+            thread = threading.Thread(target=self.server.serve_forever)
+            thread.setDaemon(True)
+            thread.start()
+
+
+    def incoming_connection(self, handler):
+        interface_name = str(str(handler.client_address[1]))
+        spawned_interface = LocalClientInterface(self.owner, name=interface_name, connected_socket=handler.request)
+        spawned_interface.OUT = self.OUT
+        spawned_interface.IN  = self.IN
+        spawned_interface.target_ip = handler.client_address[0]
+        spawned_interface.target_port = str(handler.client_address[1])
+        spawned_interface.parent_interface = self
+        RNS.log("Accepting new connection to shared instance: "+str(spawned_interface), RNS.LOG_VERBOSE)
+        RNS.Transport.interfaces.append(spawned_interface)
+        RNS.Transport.local_client_interfaces.append(spawned_interface)
+        spawned_interface.read_loop()
+
+    def processOutgoing(self, data):
+        pass
+
+    def __str__(self):
+        return "Shared Instance ["+str(self.bind_port)+"]"
+
+class LocalInterfaceHandler(socketserver.BaseRequestHandler):
+    def __init__(self, callback, *args, **keys):
+        self.callback = callback
+        socketserver.BaseRequestHandler.__init__(self, *args, **keys)
+
+    def handle(self):
+        self.callback(handler=self)

RNS/Interfaces/TCPInterface.py

@@ -24,32 +24,29 @@ class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
 class TCPClientInterface(Interface):
 
     def __init__(self, owner, name, target_ip=None, target_port=None, connected_socket=None):
-        self.IN  = True
-        self.OUT = False
-        self.socket = None
+        self.IN               = True
+        self.OUT              = False
+        self.socket           = None
         self.parent_interface = None
-        self.name = name
-
-        # TODO: Optimise so this is not needed
-        self.transmit_delay = 0.001
+        self.name             = name
 
         if connected_socket != None:
-            self.receives = True
-            self.target_ip = None
+            self.receives    = True
+            self.target_ip   = None
             self.target_port = None
-            self.socket = connected_socket
+            self.socket      = connected_socket
 
         elif target_ip != None and target_port != None:
-            self.receives = True
-            self.target_ip = target_ip
+            self.receives    = True
+            self.target_ip   = target_ip
             self.target_port = target_port
 
-            RNS.log("Client init: "+str(self))
             self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
             self.socket.connect((self.target_ip, self.target_port))
 
-        self.owner = owner
-        self.online = True
+        self.owner   = owner
+        self.online  = True
+        self.writing = False
 
         if connected_socket == None:
             thread = threading.Thread(target=self.read_loop)
@@ -61,10 +58,14 @@ class TCPClientInterface(Interface):
 
     def processOutgoing(self, data):
         if self.online:
+            while self.writing:
+                time.sleep(0.01)
+
             try:
-                time.sleep(self.transmit_delay)
+                self.writing = True
                 data = bytes([HDLC.FLAG])+HDLC.escape(data)+bytes([HDLC.FLAG])
                 self.socket.sendall(data)
+                self.writing = False
             except Exception as e:
                 RNS.log("Exception occurred while transmitting via "+str(self)+", tearing down interface", RNS.LOG_ERROR)
                 RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
@@ -78,8 +79,7 @@ class TCPClientInterface(Interface):
             data_buffer = b""
 
             while True:
-                data_in = self.socket.recv(1024)
-
+                data_in = self.socket.recv(4096)
                 if len(data_in) > 0:
                     pointer = 0
                     while pointer < len(data_in):

RNS/Packet.py

@@ -1,3 +1,4 @@
+import threading
 import struct
 import math
 import time
@@ -55,7 +56,7 @@ class Packet:
 	# Default packet timeout
 	TIMEOUT 	 = 60
 
-	def __init__(self, destination, data, packet_type = DATA, context = NONE, transport_type = RNS.Transport.BROADCAST, header_type = HEADER_1, transport_id = None, attached_interface = None):
+	def __init__(self, destination, data, packet_type = DATA, context = NONE, transport_type = RNS.Transport.BROADCAST, header_type = HEADER_1, transport_id = None, attached_interface = None, create_receipt = True):
 		if destination != None:
 			if transport_type == None:
 				transport_type = RNS.Transport.BROADCAST
@@ -65,21 +66,23 @@ class Packet:
 			self.transport_type = transport_type
 			self.context        = context
 
-			self.hops		    = 0;
+			self.hops           = 0;
 			self.destination    = destination
 			self.transport_id   = transport_id
-			self.data 		    = data
-			self.flags	 	    = self.getPackedFlags()
+			self.data           = data
+			self.flags          = self.getPackedFlags()
 
-			self.raw    		= None
-			self.packed 		= False
-			self.sent   		= False
-			self.receipt 		= None
-			self.fromPacked		= False
+			self.raw            = None
+			self.packed         = False
+			self.sent           = False
+			self.create_receipt = create_receipt
+			self.receipt        = None
+			self.fromPacked     = False
 		else:
 			self.raw            = data
 			self.packed         = True
 			self.fromPacked     = True
+			self.create_receipt = False
 
 		self.MTU         = RNS.Reticulum.MTU
 		self.sent_at     = None
@@ -173,6 +176,9 @@ class Packet:
 		self.packed = False
 		self.updateHash()
 
+	# Sends the packet. Returns a receipt if one is generated,
+	# or None if no receipt is available. Returns False if the
+	# packet could not be sent.
 	def send(self):
 		if not self.sent:
 			if self.destination.type == RNS.Destination.LINK:
@@ -189,9 +195,11 @@ class Packet:
 			if RNS.Transport.outbound(self):
 				return self.receipt
 			else:
-				# TODO: Decide whether this failure should simply
-				# return none, or raise an error
-				raise IOError("No interfaces could process the outbound packet")
+				RNS.log("No interfaces could process the outbound packet", RNS.LOG_ERROR)
+				self.sent = False
+				self.receipt = None
+				return False
+				
 		else:
 			raise IOError("Packet was already sent")
 
@@ -200,8 +208,10 @@ class Packet:
 			if RNS.Transport.outbound(self):
 				return self.receipt
 			else:
-				# TODO: Don't raise error here, handle gracefully
-				raise IOError("Packet could not be sent! Do you have any outbound interfaces configured?")
+				RNS.log("No interfaces could process the outbound packet", RNS.LOG_ERROR)
+				self.sent = False
+				self.receipt = None
+				return False
 		else:
 			raise IOError("Packet was not sent yet")
 
@@ -257,6 +267,7 @@ class PacketReceipt:
 	FAILED    = 0x00
 	SENT	  = 0x01
 	DELIVERED = 0x02
+	CULLED    = 0xFF
 
 
 	EXPL_LENGTH = RNS.Identity.HASHLENGTH//8+RNS.Identity.SIGLENGTH//8
@@ -366,10 +377,18 @@ class PacketReceipt:
 
 	def check_timeout(self):
 		if self.is_timed_out():
-			self.status = PacketReceipt.FAILED
+			if self.timeout == -1:
+				self.status = PacketReceipt.CULLED
+			else:
+				self.status = PacketReceipt.FAILED
+
 			self.concluded_at = time.time()
+
 			if self.callbacks.timeout:
-				self.callbacks.timeout(self)
+				thread = threading.Thread(target=self.callbacks.timeout, args=(self,))
+				thread.setDaemon(True)
+				thread.start()
+				#self.callbacks.timeout(self)
 
 
 	# Set the timeout in seconds

RNS/Resource.py

@@ -344,6 +344,8 @@ class Resource:
 				sleep(sleep_time)
 
 	def assemble(self):
+		# TODO: Optimise assembly. It's way too
+		# slow for larger files
 		if not self.status == Resource.FAILED:
 			try:
 				self.status = Resource.ASSEMBLING
@@ -546,8 +548,10 @@ class Resource:
 			if wants_more_hashmap:
 				last_map_hash = request_data[1:Resource.MAPHASH_LEN+1]
 				
-				part_index = self.receiver_min_consecutive_height
-				for part in self.parts[self.receiver_min_consecutive_height:]:
+				part_index   = self.receiver_min_consecutive_height
+				search_start = part_index
+				search_end   = self.receiver_min_consecutive_height+ResourceAdvertisement.COLLISION_GUARD_SIZE
+				for part in self.parts[search_start:search_end]:
 					part_index += 1
 					if part.map_hash == last_map_hash:
 						break

RNS/Reticulum.py

@@ -41,6 +41,13 @@ class Reticulum:
 		Reticulum.__transport_enabled = False
 		Reticulum.__use_implicit_proof = True
 
+		self.local_interface_port = 37428
+		self.share_instance = True
+
+		self.is_shared_instance = False
+		self.is_connected_to_shared_instance = False
+		self.is_standalone_instance = False
+
 		if not os.path.isdir(Reticulum.storagepath):
 			os.makedirs(Reticulum.storagepath)
 
@@ -65,10 +72,45 @@ class Reticulum:
 		self.applyConfig()
 		RNS.Identity.loadKnownDestinations()
 
-		RNS.Transport.start()
+		RNS.Transport.start(self)
 
 		atexit.register(Reticulum.exit_handler)
 
+	def start_local_interface(self):
+		if self.share_instance:
+			try:
+				interface = LocalInterface.LocalServerInterface(
+					RNS.Transport,
+					self.local_interface_port
+				)
+				interface.OUT = True
+				RNS.Transport.interfaces.append(interface)
+				self.is_shared_instance = True
+				RNS.log("Started shared instance interface: "+str(interface), RNS.LOG_DEBUG)
+			except Exception as e:
+				try:
+					interface = LocalInterface.LocalClientInterface(
+						RNS.Transport,
+						"Local shared instance",
+						self.local_interface_port)
+					interface.target_port = self.local_interface_port
+					interface.OUT = True
+					RNS.Transport.interfaces.append(interface)
+					self.is_shared_instance = False
+					self.is_standalone_instance = False
+					self.is_connected_to_shared_instance = True
+					RNS.log("Connected to local shared instance via: "+str(interface), RNS.LOG_DEBUG)
+				except Exception as e:
+					RNS.log("Local shared instance appears to be running, but it could not be connected", RNS.LOG_ERROR)
+					RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+					self.is_shared_instance = False
+					self.is_standalone_instance = True
+					self.is_connected_to_shared_instance = False
+		else:
+			self.is_shared_instance = False
+			self.is_standalone_instance = True
+			self.is_connected_to_shared_instance = False
+
 	def applyConfig(self):
 		if "logging" in self.config:
 			for option in self.config["logging"]:
@@ -83,6 +125,12 @@ class Reticulum:
 		if "reticulum" in self.config:
 			for option in self.config["reticulum"]:
 				value = self.config["reticulum"][option]
+				if option == "share_instance":
+					value = self.config["reticulum"].as_bool(option)
+					self.share_instance = value
+				if option == "shared_instance_port":
+					value = int(self.config["reticulum"][option])
+					self.local_interface_port = value
 				if option == "enable_transport":
 					v = self.config["reticulum"].as_bool(option)
 					if v == True:
@@ -108,211 +156,214 @@ class Reticulum:
 						RNS.log("", RNS.LOG_CRITICAL)
 						Reticulum.__allow_unencrypted = True
 
-		interface_names = []
-		for name in self.config["interfaces"]:
-			if not name in interface_names:
-				c = self.config["interfaces"][name]
+		self.start_local_interface()
 
-				try:
-					if ("interface_enabled" in c) and c.as_bool("interface_enabled") == True:
-						if c["type"] == "UdpInterface":
-							interface = UdpInterface.UdpInterface(
-								RNS.Transport,
-								name,
-								c["listen_ip"],
-								int(c["listen_port"]),
-								c["forward_ip"],
-								int(c["forward_port"])
-							)
+		if self.is_shared_instance or self.is_standalone_instance:
+			interface_names = []
+			for name in self.config["interfaces"]:
+				if not name in interface_names:
+					c = self.config["interfaces"][name]
 
-							if "outgoing" in c and c.as_bool("outgoing") == True:
-								interface.OUT = True
-							else:
-								interface.OUT = False
+					try:
+						if ("interface_enabled" in c) and c.as_bool("interface_enabled") == True:
+							if c["type"] == "UdpInterface":
+								interface = UdpInterface.UdpInterface(
+									RNS.Transport,
+									name,
+									c["listen_ip"],
+									int(c["listen_port"]),
+									c["forward_ip"],
+									int(c["forward_port"])
+								)
 
-							RNS.Transport.interfaces.append(interface)
+								if "outgoing" in c and c.as_bool("outgoing") == True:
+									interface.OUT = True
+								else:
+									interface.OUT = False
+
+								RNS.Transport.interfaces.append(interface)
 
 
-						if c["type"] == "TCPServerInterface":
-							interface = TCPInterface.TCPServerInterface(
-								RNS.Transport,
-								name,
-								c["listen_ip"],
-								int(c["listen_port"])
-							)
+							if c["type"] == "TCPServerInterface":
+								interface = TCPInterface.TCPServerInterface(
+									RNS.Transport,
+									name,
+									c["listen_ip"],
+									int(c["listen_port"])
+								)
 
-							if "outgoing" in c and c.as_bool("outgoing") == True:
-								interface.OUT = True
-							else:
-								interface.OUT = False
+								if "outgoing" in c and c.as_bool("outgoing") == True:
+									interface.OUT = True
+								else:
+									interface.OUT = False
 
-							RNS.Transport.interfaces.append(interface)
+								RNS.Transport.interfaces.append(interface)
 
 
-						if c["type"] == "TCPClientInterface":
-							interface = TCPInterface.TCPClientInterface(
-								RNS.Transport,
-								name,
-								c["target_host"],
-								int(c["target_port"])
-							)
+							if c["type"] == "TCPClientInterface":
+								interface = TCPInterface.TCPClientInterface(
+									RNS.Transport,
+									name,
+									c["target_host"],
+									int(c["target_port"])
+								)
 
-							if "outgoing" in c and c.as_bool("outgoing") == True:
-								interface.OUT = True
-							else:
-								interface.OUT = False
+								if "outgoing" in c and c.as_bool("outgoing") == True:
+									interface.OUT = True
+								else:
+									interface.OUT = False
 
-							RNS.Transport.interfaces.append(interface)
+								RNS.Transport.interfaces.append(interface)
 
 
-						if c["type"] == "SerialInterface":
-							port = c["port"] if "port" in c else None
-							speed = int(c["speed"]) if "speed" in c else 9600
-							databits = int(c["databits"]) if "databits" in c else 8
-							parity = c["parity"] if "parity" in c else "N"
-							stopbits = int(c["stopbits"]) if "stopbits" in c else 1
+							if c["type"] == "SerialInterface":
+								port = c["port"] if "port" in c else None
+								speed = int(c["speed"]) if "speed" in c else 9600
+								databits = int(c["databits"]) if "databits" in c else 8
+								parity = c["parity"] if "parity" in c else "N"
+								stopbits = int(c["stopbits"]) if "stopbits" in c else 1
 
-							if port == None:
-								raise ValueError("No port specified for serial interface")
+								if port == None:
+									raise ValueError("No port specified for serial interface")
 
-							interface = SerialInterface.SerialInterface(
-								RNS.Transport,
-								name,
-								port,
-								speed,
-								databits,
-								parity,
-								stopbits
-							)
+								interface = SerialInterface.SerialInterface(
+									RNS.Transport,
+									name,
+									port,
+									speed,
+									databits,
+									parity,
+									stopbits
+								)
 
-							if "outgoing" in c and c["outgoing"].lower() == "true":
-								interface.OUT = True
-							else:
-								interface.OUT = False
+								if "outgoing" in c and c["outgoing"].lower() == "true":
+									interface.OUT = True
+								else:
+									interface.OUT = False
 
-							RNS.Transport.interfaces.append(interface)
+								RNS.Transport.interfaces.append(interface)
 
-						if c["type"] == "KISSInterface":
-							preamble = int(c["preamble"]) if "preamble" in c else None
-							txtail = int(c["txtail"]) if "txtail" in c else None
-							persistence = int(c["persistence"]) if "persistence" in c else None
-							slottime = int(c["slottime"]) if "slottime" in c else None
-							flow_control = (True if c["flow_control"] == "true" else False) if "flow_control" in c else False
+							if c["type"] == "KISSInterface":
+								preamble = int(c["preamble"]) if "preamble" in c else None
+								txtail = int(c["txtail"]) if "txtail" in c else None
+								persistence = int(c["persistence"]) if "persistence" in c else None
+								slottime = int(c["slottime"]) if "slottime" in c else None
+								flow_control = (True if c["flow_control"] == "true" else False) if "flow_control" in c else False
 
-							port = c["port"] if "port" in c else None
-							speed = int(c["speed"]) if "speed" in c else 9600
-							databits = int(c["databits"]) if "databits" in c else 8
-							parity = c["parity"] if "parity" in c else "N"
-							stopbits = int(c["stopbits"]) if "stopbits" in c else 1
+								port = c["port"] if "port" in c else None
+								speed = int(c["speed"]) if "speed" in c else 9600
+								databits = int(c["databits"]) if "databits" in c else 8
+								parity = c["parity"] if "parity" in c else "N"
+								stopbits = int(c["stopbits"]) if "stopbits" in c else 1
 
-							if port == None:
-								raise ValueError("No port specified for serial interface")
+								if port == None:
+									raise ValueError("No port specified for serial interface")
 
-							interface = KISSInterface.KISSInterface(
-								RNS.Transport,
-								name,
-								port,
-								speed,
-								databits,
-								parity,
-								stopbits,
-								preamble,
-								txtail,
-								persistence,
-								slottime,
-								flow_control
-							)
+								interface = KISSInterface.KISSInterface(
+									RNS.Transport,
+									name,
+									port,
+									speed,
+									databits,
+									parity,
+									stopbits,
+									preamble,
+									txtail,
+									persistence,
+									slottime,
+									flow_control
+								)
 
-							if "outgoing" in c and c["outgoing"].lower() == "true":
-								interface.OUT = True
-							else:
-								interface.OUT = False
+								if "outgoing" in c and c["outgoing"].lower() == "true":
+									interface.OUT = True
+								else:
+									interface.OUT = False
 
-							RNS.Transport.interfaces.append(interface)
+								RNS.Transport.interfaces.append(interface)
 
-						if c["type"] == "AX25KISSInterface":
-							preamble = int(c["preamble"]) if "preamble" in c else None
-							txtail = int(c["txtail"]) if "txtail" in c else None
-							persistence = int(c["persistence"]) if "persistence" in c else None
-							slottime = int(c["slottime"]) if "slottime" in c else None
-							flow_control = (True if c["flow_control"] == "true" else False) if "flow_control" in c else False
+							if c["type"] == "AX25KISSInterface":
+								preamble = int(c["preamble"]) if "preamble" in c else None
+								txtail = int(c["txtail"]) if "txtail" in c else None
+								persistence = int(c["persistence"]) if "persistence" in c else None
+								slottime = int(c["slottime"]) if "slottime" in c else None
+								flow_control = (True if c["flow_control"] == "true" else False) if "flow_control" in c else False
 
-							port = c["port"] if "port" in c else None
-							speed = int(c["speed"]) if "speed" in c else 9600
-							databits = int(c["databits"]) if "databits" in c else 8
-							parity = c["parity"] if "parity" in c else "N"
-							stopbits = int(c["stopbits"]) if "stopbits" in c else 1
+								port = c["port"] if "port" in c else None
+								speed = int(c["speed"]) if "speed" in c else 9600
+								databits = int(c["databits"]) if "databits" in c else 8
+								parity = c["parity"] if "parity" in c else "N"
+								stopbits = int(c["stopbits"]) if "stopbits" in c else 1
 
-							callsign = c["callsign"] if "callsign" in c else ""
-							ssid = int(c["ssid"]) if "ssid" in c else -1
+								callsign = c["callsign"] if "callsign" in c else ""
+								ssid = int(c["ssid"]) if "ssid" in c else -1
 
-							if port == None:
-								raise ValueError("No port specified for serial interface")
+								if port == None:
+									raise ValueError("No port specified for serial interface")
 
-							interface = AX25KISSInterface.AX25KISSInterface(
-								RNS.Transport,
-								name,
-								callsign,
-								ssid,
-								port,
-								speed,
-								databits,
-								parity,
-								stopbits,
-								preamble,
-								txtail,
-								persistence,
-								slottime,
-								flow_control
-							)
+								interface = AX25KISSInterface.AX25KISSInterface(
+									RNS.Transport,
+									name,
+									callsign,
+									ssid,
+									port,
+									speed,
+									databits,
+									parity,
+									stopbits,
+									preamble,
+									txtail,
+									persistence,
+									slottime,
+									flow_control
+								)
 
-							if "outgoing" in c and c["outgoing"].lower() == "true":
-								interface.OUT = True
-							else:
-								interface.OUT = False
+								if "outgoing" in c and c["outgoing"].lower() == "true":
+									interface.OUT = True
+								else:
+									interface.OUT = False
 
-							RNS.Transport.interfaces.append(interface)
+								RNS.Transport.interfaces.append(interface)
 
-						if c["type"] == "RNodeInterface":
-							frequency = int(c["frequency"]) if "frequency" in c else None
-							bandwidth = int(c["bandwidth"]) if "bandwidth" in c else None
-							txpower = int(c["txpower"]) if "txpower" in c else None
-							spreadingfactor = int(c["spreadingfactor"]) if "spreadingfactor" in c else None
-							codingrate = int(c["codingrate"]) if "codingrate" in c else None
-							flow_control = (True if c["flow_control"] == "true" else False) if "flow_control" in c else False
+							if c["type"] == "RNodeInterface":
+								frequency = int(c["frequency"]) if "frequency" in c else None
+								bandwidth = int(c["bandwidth"]) if "bandwidth" in c else None
+								txpower = int(c["txpower"]) if "txpower" in c else None
+								spreadingfactor = int(c["spreadingfactor"]) if "spreadingfactor" in c else None
+								codingrate = int(c["codingrate"]) if "codingrate" in c else None
+								flow_control = (True if c["flow_control"] == "true" else False) if "flow_control" in c else False
 
-							port = c["port"] if "port" in c else None
-							
-							if port == None:
-								raise ValueError("No port specified for RNode interface")
+								port = c["port"] if "port" in c else None
+								
+								if port == None:
+									raise ValueError("No port specified for RNode interface")
 
-							interface = RNodeInterface.RNodeInterface(
-								RNS.Transport,
-								name,
-								port,
-								frequency,
-								bandwidth,
-								txpower,
-								spreadingfactor,
-								flow_control
-							)
+								interface = RNodeInterface.RNodeInterface(
+									RNS.Transport,
+									name,
+									port,
+									frequency,
+									bandwidth,
+									txpower,
+									spreadingfactor,
+									flow_control
+								)
 
-							if "outgoing" in c and c["outgoing"].lower() == "true":
-								interface.OUT = True
-							else:
-								interface.OUT = False
+								if "outgoing" in c and c["outgoing"].lower() == "true":
+									interface.OUT = True
+								else:
+									interface.OUT = False
 
-							RNS.Transport.interfaces.append(interface)
-					else:
-						RNS.log("Skipping disabled interface \""+name+"\"", RNS.LOG_VERBOSE)
+								RNS.Transport.interfaces.append(interface)
+						else:
+							RNS.log("Skipping disabled interface \""+name+"\"", RNS.LOG_VERBOSE)
 
-				except Exception as e:
-					RNS.log("The interface \""+name+"\" could not be created. Check your configuration file for errors!", RNS.LOG_ERROR)
-					RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+					except Exception as e:
+						RNS.log("The interface \""+name+"\" could not be created. Check your configuration file for errors!", RNS.LOG_ERROR)
+						RNS.log("The contained exception was: "+str(e), RNS.LOG_ERROR)
+						RNS.panic()
+				else:
+					RNS.log("The interface name \""+name+"\" was already used. Check your configuration file for errors!", RNS.LOG_ERROR)
 					RNS.panic()
-			else:
-				RNS.log("The interface name \""+name+"\" was already used. Check your configuration file for errors!", RNS.LOG_ERROR)
-				RNS.panic()
 				
 
 	def createDefaultConfig(self):
@@ -346,16 +397,41 @@ __default_rns_config__ = '''# This is the default Reticulum config file.
 # Don't allow unencrypted links by default.
 # If you REALLY need to allow unencrypted links, for example
 # for debug or regulatory purposes, this can be set to true.
+# This directive is optional and can be removed for brevity.
+
 allow_unencrypted = False
 
+
 # If you enable Transport, your system will route traffic
 # for other peers, pass announces and serve path requests.
 # Unless you really know what you're doing, this should be
 # done only for systems that are suited to act as transport
-# nodes, ie. if they are stationary and always-on.
+# nodes, ie. if they are stationary and always-on. This
+# directive is optional and can be removed for brevity.
+
 enable_transport = False
 
 
+# By default, the first program to launch the Reticulum
+# Network Stack will create a shared instance, that other
+# programs can communicate with. Only the shared instance
+# opens all the configured interfaces directly, and other
+# local programs communicate with the shared instance over
+# a local socket. This is completely transparent to the
+# user, and should generally be turned on. This directive
+# is optional and can be removed for brevity.
+
+share_instance = Yes
+
+
+# If you want to run multiple *different* shared instances
+# on the same system, you will need to specify a different
+# shared instance port for each. The default is given below,
+# and again, this option is optional and can be left out.
+
+shared_instance_port = 37428
+
+
 [logging]
 # Valid log levels are 0 through 7:
 #   0: Log only critical information

RNS/Transport.py

@@ -35,24 +35,37 @@ class Transport:
 	# various situations
 	LOCAL_REBROADCASTS_MAX = 2	# How many local rebroadcasts of an announce is allowed
 
-	PATH_REQUEST_GRACE  = 0.25       # Grace time before a path announcement is made, allows directly reachable peers to respond first
+	PATH_REQUEST_GRACE  = 0.35       # Grace time before a path announcement is made, allows directly reachable peers to respond first
 	PATH_REQUEST_RW     = 2 		 # Path request random window
 
-	LINK_TIMEOUT        = RNS.Link.KEEPALIVE * 2
-	REVERSE_TIMEOUT     = 30*60		 # Reverse table entries are removed after max 30 minutes
-	DESTINATION_TIMEOUT = 60*60*24*7 # Destination table entries are removed if unused for one week
+	LINK_TIMEOUT         = RNS.Link.KEEPALIVE * 2
+	REVERSE_TIMEOUT      = 30*60		 # Reverse table entries are removed after max 30 minutes
+	DESTINATION_TIMEOUT  = 60*60*24*7 # Destination table entries are removed if unused for one week
+	MAX_RECEIPTS         = 1024       # Maximum number of receipts to keep track of
 
-	interfaces	 	    = []		 # All active interfaces
-	destinations        = []		 # All active destinations
-	pending_links       = []		 # Links that are being established
-	active_links	    = []		 # Links that are active
-	packet_hashlist     = []		 # A list of packet hashes for duplicate detection
-	receipts		    = []		 # Receipts of all outgoing packets for proof processing
+	interfaces	 	     = []		 # All active interfaces
+	destinations         = []		 # All active destinations
+	pending_links        = []		 # Links that are being established
+	active_links	     = []		 # Links that are active
+	packet_hashlist      = []		 # A list of packet hashes for duplicate detection
+	receipts		     = []		 # Receipts of all outgoing packets for proof processing
 
-	announce_table      = {}		 # A table for storing announces currently waiting to be retransmitted
-	destination_table   = {}		 # A lookup table containing the next hop to a given destination
-	reverse_table	    = {}		 # A lookup table for storing packet hashes used to return proofs and replies
-	link_table          = {}		 # A lookup table containing hops for links
+	# TODO: "destination_table" should really be renamed to "path_table"
+	announce_table       = {}         # A table for storing announces currently waiting to be retransmitted
+	destination_table    = {}         # A lookup table containing the next hop to a given destination
+	reverse_table	     = {}         # A lookup table for storing packet hashes used to return proofs and replies
+	link_table           = {}         # A lookup table containing hops for links
+	held_announces       = {}         # A table containing temporarily held announce-table entries
+
+	# Transport control destinations are used
+	# for control purposes like path requests
+	control_destinations = []
+	control_hashes       = []
+
+	# Interfaces for communicating with
+	# local clients connected to a shared
+	# Reticulum instance
+	local_client_interfaces = []
 
 	jobs_locked = False
 	jobs_running = False
@@ -68,7 +81,9 @@ class Transport:
 	identity = None
 
 	@staticmethod
-	def start():
+	def start(reticulum_instance):
+		Transport.owner = reticulum_instance
+
 		if Transport.identity == None:
 			transport_identity_path = RNS.Reticulum.storagepath+"/transport_identity"
 			if os.path.isfile(transport_identity_path):
@@ -91,8 +106,10 @@ class Transport:
 				RNS.log("Could not load packet hashlist from storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
 
 		# Create transport-specific destinations
-		path_request_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, Transport.APP_NAME, "path", "request")
-		path_request_destination.packet_callback(Transport.pathRequestHandler)
+		Transport.path_request_destination = RNS.Destination(None, RNS.Destination.IN, RNS.Destination.PLAIN, Transport.APP_NAME, "path", "request")
+		Transport.path_request_destination.packet_callback(Transport.pathRequestHandler)
+		Transport.control_destinations.append(Transport.path_request_destination)
+		Transport.control_hashes.append(Transport.path_request_destination.hash)
 		
 		thread = threading.Thread(target=Transport.jobloop)
 		thread.setDaemon(True)
@@ -100,7 +117,7 @@ class Transport:
 
 		if RNS.Reticulum.transport_enabled():
 			destination_table_path = RNS.Reticulum.storagepath+"/destination_table"
-			if os.path.isfile(destination_table_path):
+			if os.path.isfile(destination_table_path) and not Transport.owner.is_connected_to_shared_instance:
 				serialised_destinations = []
 				try:
 					file = open(destination_table_path, "rb")
@@ -109,16 +126,21 @@ class Transport:
 
 					for serialised_entry in serialised_destinations:
 						destination_hash = serialised_entry[0]
+						timestamp = serialised_entry[1]
+						received_from = serialised_entry[2]
+						hops = serialised_entry[3]
+						expires = serialised_entry[4]
+						random_blobs = serialised_entry[5]
 						receiving_interface = Transport.find_interface_from_hash(serialised_entry[6])
 						announce_packet = Transport.get_cached_packet(serialised_entry[7])
 
 						if announce_packet != None and receiving_interface != None:
 							announce_packet.unpack()
-							timestamp = serialised_entry[1]
-							received_from = serialised_entry[2]
-							hops = serialised_entry[3]
-							expires = serialised_entry[4]
-							random_blobs = serialised_entry[5]
+							# We increase the hops, since reading a packet
+							# from cache is equivalent to receiving it again
+							# over an interface. It is cached with it's non-
+							# increased hop-count.
+							announce_packet.hops += 1
 							Transport.destination_table[destination_hash] = [timestamp, received_from, hops, expires, random_blobs, receiving_interface, announce_packet]
 							RNS.log("Loaded path table entry for "+RNS.prettyhexrep(destination_hash)+" from storage", RNS.LOG_DEBUG)
 						else:
@@ -154,44 +176,72 @@ class Transport:
 			if not Transport.jobs_locked:
 				# Process receipts list for timed-out packets
 				if time.time() > Transport.receipts_last_checked+Transport.receipts_check_interval:
+					while len(Transport.receipts) > Transport.MAX_RECEIPTS:
+						culled_receipt = Transport.receipts.pop(0)
+						culled_receipt.timeout = -1
+						receipt.check_timeout()
+
 					for receipt in Transport.receipts:
-						thread = threading.Thread(target=receipt.check_timeout)
-						thread.setDaemon(True)
-						thread.start()
+						receipt.check_timeout()
 						if receipt.status != RNS.PacketReceipt.SENT:
 							Transport.receipts.remove(receipt)
 
 					Transport.receipts_last_checked = time.time()
 
-				if RNS.Reticulum.transport_enabled():
-					# Process announces needing retransmission
-					if time.time() > Transport.announces_last_checked+Transport.announces_check_interval:
-						for destination_hash in Transport.announce_table:
-							announce_entry = Transport.announce_table[destination_hash]
-							if announce_entry[2] > Transport.PATHFINDER_R:
-								RNS.log("Dropping announce for "+RNS.prettyhexrep(destination_hash)+", retries exceeded", RNS.LOG_DEBUG)
-								Transport.announce_table.pop(destination_hash)
-								break
-							else:
-								if time.time() > announce_entry[1]:
-									announce_entry[1] = time.time() + math.pow(Transport.PATHFINDER_C, announce_entry[4]) + Transport.PATHFINDER_T + Transport.PATHFINDER_RW
-									announce_entry[2] += 1
-									packet = announce_entry[5]
-									block_rebroadcasts = announce_entry[7]
-									announce_context = RNS.Packet.NONE
-									if block_rebroadcasts:
-										announce_context = RNS.Packet.PATH_RESPONSE
-									announce_data = packet.data
-									announce_identity = RNS.Identity.recall(packet.destination_hash)
-									announce_destination = RNS.Destination(announce_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, "unknown", "unknown");
-									announce_destination.hash = packet.destination_hash
-									announce_destination.hexhash = announce_destination.hash.hex()
-									new_packet = RNS.Packet(announce_destination, announce_data, RNS.Packet.ANNOUNCE, context = announce_context, header_type = RNS.Packet.HEADER_2, transport_type = Transport.TRANSPORT, transport_id = Transport.identity.hash)
-									new_packet.hops = announce_entry[4]
-									RNS.log("Rebroadcasting announce for "+RNS.prettyhexrep(announce_destination.hash)+" with hop count "+str(new_packet.hops), RNS.LOG_DEBUG)
-									outgoing.append(new_packet)
+				# Process announces needing retransmission
+				if time.time() > Transport.announces_last_checked+Transport.announces_check_interval:
+					for destination_hash in Transport.announce_table:
+						announce_entry = Transport.announce_table[destination_hash]
+						if announce_entry[2] > Transport.PATHFINDER_R:
+							RNS.log("Dropping announce for "+RNS.prettyhexrep(destination_hash)+", retries exceeded", RNS.LOG_DEBUG)
+							Transport.announce_table.pop(destination_hash)
+							break
+						else:
+							if time.time() > announce_entry[1]:
+								announce_entry[1] = time.time() + math.pow(Transport.PATHFINDER_C, announce_entry[4]) + Transport.PATHFINDER_T + Transport.PATHFINDER_RW
+								announce_entry[2] += 1
+								packet = announce_entry[5]
+								block_rebroadcasts = announce_entry[7]
+								attached_interface = announce_entry[8]
+								announce_context = RNS.Packet.NONE
+								if block_rebroadcasts:
+									announce_context = RNS.Packet.PATH_RESPONSE
+								announce_data = packet.data
+								announce_identity = RNS.Identity.recall(packet.destination_hash)
+								announce_destination = RNS.Destination(announce_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, "unknown", "unknown");
+								announce_destination.hash = packet.destination_hash
+								announce_destination.hexhash = announce_destination.hash.hex()
+								
+								new_packet = RNS.Packet(
+									announce_destination,
+									announce_data,
+									RNS.Packet.ANNOUNCE,
+									context = announce_context,
+									header_type = RNS.Packet.HEADER_2,
+									transport_type = Transport.TRANSPORT,
+									transport_id = Transport.identity.hash,
+									attached_interface = attached_interface
+								)
 
-						Transport.announces_last_checked = time.time()
+								new_packet.hops = announce_entry[4]
+								if block_rebroadcasts:
+									RNS.log("Rebroadcasting announce as path response for "+RNS.prettyhexrep(announce_destination.hash)+" with hop count "+str(new_packet.hops), RNS.LOG_DEBUG)
+								else:
+									RNS.log("Rebroadcasting announce for "+RNS.prettyhexrep(announce_destination.hash)+" with hop count "+str(new_packet.hops), RNS.LOG_DEBUG)
+								outgoing.append(new_packet)
+
+								# This handles an edge case where a peer sends a past
+								# request for a destination just after an announce for
+								# said destination has arrived, but before it has been
+								# rebroadcast locally. In such a case the actual announce
+								# is temporarily held, and then reinserted when the path
+								# request has been served to the peer.
+								if destination_hash in Transport.held_announces:
+									held_entry = Transport.held_announces.pop(destination_hash)
+									Transport.announce_table[destination_hash] = held_entry
+									RNS.log("Reinserting held announce into table", RNS.LOG_DEBUG)
+
+					Transport.announces_last_checked = time.time()
 
 
 				# Cull the packet hashlist if it has reached max size
@@ -206,16 +256,47 @@ class Transport:
 							Transport.reverse_table.pop(truncated_packet_hash)
 
 					# Cull the link table according to timeout
+					stale_links = []
 					for link_id in Transport.link_table:
 						link_entry = Transport.link_table[link_id]
 						if time.time() > link_entry[0] + Transport.LINK_TIMEOUT:
-							Transport.link_table.pop(link_id)
+							stale_links.append(link_id)
 
-					# Cull the destination table
+					# Cull the path table
+					stale_paths = []
 					for destination_hash in Transport.destination_table:
 						destination_entry = Transport.destination_table[destination_hash]
+						attached_interface = destination_entry[5]
+
 						if time.time() > destination_entry[0] + Transport.DESTINATION_TIMEOUT:
-							Transport.destination_table.pop(destination_hash)
+							stale_paths.append(destination_hash)
+							RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" timed out and was removed", RNS.LOG_DEBUG)
+
+						if not attached_interface in Transport.interfaces:
+							stale_paths.append(destination_hash)
+							RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" was removed since the attached interface no longer exists", RNS.LOG_DEBUG)
+
+					i = 0
+					for link_id in stale_links:
+						Transport.link_table.pop(link_id)
+						i += 1
+
+					if i > 0:
+						if i == 1:
+							RNS.log("Dropped "+str(i)+" link", RNS.LOG_DEBUG)
+						else:
+							RNS.log("Dropped "+str(i)+" links", RNS.LOG_DEBUG)
+
+					i = 0
+					for destination_hash in stale_paths:
+						Transport.destination_table.pop(destination_hash)
+						i += 1
+
+					if i > 0:
+						if i == 1:
+							RNS.log("Removed "+str(i)+" path", RNS.LOG_DEBUG)
+						else:
+							RNS.log("Removed "+str(i)+" paths", RNS.LOG_DEBUG)
 
 					Transport.tables_last_culled = time.time()
 
@@ -239,34 +320,62 @@ class Transport:
 		packet.updateHash()
 		sent = False
 
-		# Check if we have a known path for the destination
-		# in the destination table
+		# Check if we have a known path for the destination in the path table
 		if packet.packet_type != RNS.Packet.ANNOUNCE and packet.destination_hash in Transport.destination_table:
 			outbound_interface = Transport.destination_table[packet.destination_hash][5]
 
+			# If there's more than one hop to the destination, and we know
+			# a path, we insert the packet into transport by adding the next
+			# transport nodes address to the header, and modifying the flags.
+			# This rule applies both for "normal" transport, and when connected
+			# to a local shared Reticulum instance.
 			if Transport.destination_table[packet.destination_hash][2] > 1:
-				# Insert packet into transport
-				new_flags = (RNS.Packet.HEADER_2) << 6 | (Transport.TRANSPORT) << 4 | (packet.flags & 0b00001111)
-				new_raw = struct.pack("!B", new_flags)
-				new_raw += packet.raw[1:2]
-				new_raw += Transport.destination_table[packet.destination_hash][1]
-				new_raw += packet.raw[2:]
-				RNS.log("Packet was inserted into transport via "+RNS.prettyhexrep(Transport.destination_table[packet.destination_hash][1])+" on: "+str(outbound_interface), RNS.LOG_DEBUG)
-				outbound_interface.processOutgoing(new_raw)
-				Transport.destination_table[packet.destination_hash][0] = time.time()
-				sent = True
-			else:
-				# Destination is directly reachable, and we know on
-				# what interface, so transmit only on that one
+				if packet.header_type == RNS.Packet.HEADER_1:
+					# Insert packet into transport
+					new_flags = (RNS.Packet.HEADER_2) << 6 | (Transport.TRANSPORT) << 4 | (packet.flags & 0b00001111)
+					new_raw = struct.pack("!B", new_flags)
+					new_raw += packet.raw[1:2]
+					new_raw += Transport.destination_table[packet.destination_hash][1]
+					new_raw += packet.raw[2:]
+					# TODO: Remove at some point
+					RNS.log("Packet was inserted into transport via "+RNS.prettyhexrep(Transport.destination_table[packet.destination_hash][1])+" on: "+str(outbound_interface), RNS.LOG_EXTREME)
+					outbound_interface.processOutgoing(new_raw)
+					Transport.destination_table[packet.destination_hash][0] = time.time()
+					sent = True
 
-				RNS.log("Transmitting "+str(len(packet.raw))+" bytes on: "+str(outbound_interface), RNS.LOG_EXTREME)
-				RNS.log("Hash is "+RNS.prettyhexrep(packet.packet_hash), RNS.LOG_EXTREME)
+			# In the special case where we are connected to a local shared
+			# Reticulum instance, and the destination is one hop away, we
+			# also add transport headers to inject the packet into transport
+			# via the shared instance. Normally a packet for a destination
+			# one hop away would just be broadcast directly, but since we
+			# are "behind" a shared instance, we need to get that instance
+			# to transport it onto the network.
+			elif Transport.destination_table[packet.destination_hash][2] == 1 and Transport.owner.is_connected_to_shared_instance:
+				if packet.header_type == RNS.Packet.HEADER_1:
+					# Insert packet into transport
+					new_flags = (RNS.Packet.HEADER_2) << 6 | (Transport.TRANSPORT) << 4 | (packet.flags & 0b00001111)
+					new_raw = struct.pack("!B", new_flags)
+					new_raw += packet.raw[1:2]
+					new_raw += Transport.destination_table[packet.destination_hash][1]
+					new_raw += packet.raw[2:]
+					# TODO: Remove at some point
+					RNS.log("Packet was inserted into transport via "+RNS.prettyhexrep(Transport.destination_table[packet.destination_hash][1])+" on: "+str(outbound_interface), RNS.LOG_EXTREME)
+					outbound_interface.processOutgoing(new_raw)
+					Transport.destination_table[packet.destination_hash][0] = time.time()
+					sent = True
+
+			# If none of the above applies, we know the destination is
+			# directly reachable, and also on which interface, so we
+			# simply transmit the packet directly on that one.
+			else:
 				outbound_interface.processOutgoing(packet.raw)
 				sent = True
 
+		# If we don't have a known path for the destination, we'll
+		# broadcast the packet on all outgoing interfaces, or the
+		# just the relevant interface if the packet has an attached
+		# interface, or belongs to a link.
 		else:
-			# Broadcast packet on all outgoing interfaces, or relevant
-			# interface, if packet is for a link or has an attachede interface
 			for interface in Transport.interfaces:
 				if interface.OUT:
 					should_transmit = True
@@ -288,7 +397,17 @@ class Transport:
 			packet.sent = True
 			packet.sent_at = time.time()
 
-			if (packet.packet_type == RNS.Packet.DATA and packet.destination.type != RNS.Destination.PLAIN):
+			    # Don't generate receipt if it has been explicitly disabled
+			if (packet.create_receipt == True and
+				# Only generate receipts for DATA packets
+				packet.packet_type == RNS.Packet.DATA and
+				# Don't generate receipts for PLAIN destinations
+				packet.destination.type != RNS.Destination.PLAIN and
+				# Don't generate receipts for link-related packets
+				not (packet.context >= RNS.Packet.KEEPALIVE and packet.context <= RNS.Packet.LRPROOF) and
+				# Don't generate receipts for resource packets
+				not (packet.context >= RNS.Packet.RESOURCE and packet.context <= RNS.Packet.RESOURCE_RCL)):
+
 				packet.receipt = RNS.PacketReceipt(packet)
 				Transport.receipts.append(packet.receipt)
 			
@@ -299,7 +418,9 @@ class Transport:
 
 	@staticmethod
 	def packet_filter(packet):
-		# TODO: Think long and hard about this
+		# TODO: Think long and hard about this.
+		# Is it even strictly necessary with the current
+		# transport rules?
 		if packet.context == RNS.Packet.KEEPALIVE:
 			return True
 		if packet.context == RNS.Packet.RESOURCE_REQ:
@@ -308,6 +429,8 @@ class Transport:
 			return True
 		if packet.context == RNS.Packet.RESOURCE:
 			return True
+		if packet.destination_type == RNS.Destination.PLAIN:
+			return True
 		if not packet.packet_hash in Transport.packet_hashlist:
 			return True
 		else:
@@ -331,14 +454,62 @@ class Transport:
 
 		RNS.log(str(interface)+" received packet with hash "+RNS.prettyhexrep(packet.packet_hash), RNS.LOG_EXTREME)
 
+		if len(Transport.local_client_interfaces) > 0:
+			new_raw  = packet.raw[0:1]
+			new_raw += struct.pack("!B", packet.hops)
+			new_raw += packet.raw[2:]
+
+			if Transport.is_local_client_interface(interface):
+				packet.hops -= 1
+		elif Transport.interface_to_shared_instance(interface):
+			packet.hops -= 1
+
+
 		if Transport.packet_filter(packet):
 			Transport.packet_hashlist.append(packet.packet_hash)
 			Transport.cache(packet)
 			
+			# Check special conditions for local clients connected
+			# through a shared Reticulum instance
+			from_local_client         = (packet.receiving_interface in Transport.local_client_interfaces)
+			for_local_client          = (packet.packet_type != RNS.Packet.ANNOUNCE) and (packet.destination_hash in Transport.destination_table and Transport.destination_table[packet.destination_hash][2] == 0)
+			for_local_client_link     = (packet.packet_type != RNS.Packet.ANNOUNCE) and (packet.destination_hash in Transport.link_table and Transport.link_table[packet.destination_hash][4] in Transport.local_client_interfaces)
+			for_local_client_link    |= (packet.packet_type != RNS.Packet.ANNOUNCE) and (packet.destination_hash in Transport.link_table and Transport.link_table[packet.destination_hash][2] in Transport.local_client_interfaces)
+			proof_for_local_client    = (packet.destination_hash in Transport.reverse_table) and (Transport.reverse_table[packet.destination_hash][0] in Transport.local_client_interfaces)
+
+			# Plain broadcast packets from local clients are sent
+			# directly on all attached interfaces, since they are
+			# never injected into transport.
+			if not packet.destination_hash in Transport.control_hashes:
+				if packet.destination_type == RNS.Destination.PLAIN and packet.transport_type == Transport.BROADCAST:
+					# Send to all interfaces except the originator
+					if from_local_client:
+						for interface in Transport.interfaces:
+							if interface != packet.receiving_interface:
+								interface.processOutgoing(packet.raw)
+					# If the packet was not from a local client, send
+					# it directly to all local clients
+					else:
+						for interface in Transport.local_client_interfaces:
+							interface.processOutgoing(packet.raw)
+
+
 			# General transport handling. Takes care of directing
 			# packets according to transport tables and recording
 			# entries in reverse and link tables.
-			if RNS.Reticulum.transport_enabled():
+			if RNS.Reticulum.transport_enabled() or from_local_client or for_local_client or for_local_client_link:
+
+				# If there is no transport id, but the packet is
+				# for a local client, we generate the transport
+				# id (it was stripped on the previous hop, since
+				# we "spoof" the hop count for clients behind a
+				# shared instance, so they look directly reach-
+				# able), and reinsert, so the normal transport
+				# implementation can handle the packet.
+
+				if packet.transport_id == None and for_local_client:
+					packet.transport_id = Transport.identity.hash
+
 				if packet.transport_id != None and packet.packet_type != RNS.Packet.ANNOUNCE:
 					if packet.transport_id == Transport.identity.hash:
 						RNS.log("Received packet in transport for "+RNS.prettyhexrep(packet.destination_hash)+" with matching transport ID, transporting it...", RNS.LOG_DEBUG)
@@ -348,16 +519,21 @@ class Transport:
 							RNS.log("Next hop to destination is "+RNS.prettyhexrep(next_hop)+" with "+str(remaining_hops)+" hops remaining, transporting it.", RNS.LOG_DEBUG)
 							if remaining_hops > 1:
 								# Just increase hop count and transmit
-								new_raw = packet.raw[0:1]
+								new_raw  = packet.raw[0:1]
 								new_raw += struct.pack("!B", packet.hops)
 								new_raw += next_hop
 								new_raw += packet.raw[12:]
-							else:
+							elif remaining_hops == 1:
 								# Strip transport headers and transmit
 								new_flags = (RNS.Packet.HEADER_1) << 6 | (Transport.BROADCAST) << 4 | (packet.flags & 0b00001111)
 								new_raw = struct.pack("!B", new_flags)
 								new_raw += struct.pack("!B", packet.hops)
 								new_raw += packet.raw[12:]
+							elif remaining_hops == 0:
+								# Just increase hop count and transmit
+								new_raw  = packet.raw[0:1]
+								new_raw += struct.pack("!B", packet.hops)
+								new_raw += packet.raw[2:]
 
 							outbound_interface = Transport.destination_table[packet.destination_hash][5]
 							outbound_interface.processOutgoing(new_raw)
@@ -394,7 +570,7 @@ class Transport:
 
 				# Link transport handling. Directs packets according
 				# to entries in the link tables
-				if packet.packet_type != RNS.Packet.ANNOUNCE and packet.packet_type != RNS.Packet.LINKREQUEST:
+				if packet.packet_type != RNS.Packet.ANNOUNCE and packet.packet_type != RNS.Packet.LINKREQUEST and packet.context != RNS.Packet.LRPROOF:
 					if packet.destination_hash in Transport.link_table:
 						link_entry = Transport.link_table[packet.destination_hash]
 						# If receiving and outbound interface is
@@ -508,26 +684,72 @@ class Transport:
 							should_add = True
 
 						if should_add:
-							now = time.time()
-							retries = 0
-							expires = now + Transport.PATHFINDER_E
+							now                = time.time()
+							retries            = 0
+							expires            = now + Transport.PATHFINDER_E
+							announce_hops      = packet.hops
 							local_rebroadcasts = 0
 							block_rebroadcasts = False
-							random_blobs.append(random_blob)
+							attached_interface = None
 							retransmit_timeout = now + math.pow(Transport.PATHFINDER_C, packet.hops) + (RNS.rand() * Transport.PATHFINDER_RW)
+							
+							random_blobs.append(random_blob)
 
-							if RNS.Reticulum.transport_enabled() and packet.context != RNS.Packet.PATH_RESPONSE:
-								Transport.announce_table[packet.destination_hash] = [now, retransmit_timeout, retries, received_from, packet.hops, packet, local_rebroadcasts, block_rebroadcasts]
+							if (RNS.Reticulum.transport_enabled() or Transport.from_local_client(packet)) and packet.context != RNS.Packet.PATH_RESPONSE:
+								# If the announce is from a local client,
+								# we announce it immediately, but only one
+								# time.
+								if Transport.from_local_client(packet):
+									retransmit_timeout = now
+									retries = Transport.PATHFINDER_R
 
-							Transport.destination_table[packet.destination_hash] = [now, received_from, packet.hops, expires, random_blobs, packet.receiving_interface, packet]
-							RNS.log("Path to "+RNS.prettyhexrep(packet.destination_hash)+" is now "+str(packet.hops)+" hops away via "+RNS.prettyhexrep(received_from)+" on "+str(packet.receiving_interface), RNS.LOG_DEBUG)
+								Transport.announce_table[packet.destination_hash] = [
+									now,
+									retransmit_timeout,
+									retries,
+									received_from,
+									announce_hops,
+									packet,
+									local_rebroadcasts,
+									block_rebroadcasts,
+									attached_interface
+								]
+
+							# If we have any local clients connected, we re-
+							# transmit the announce to them immediately
+							if (len(Transport.local_client_interfaces)):
+								announce_identity = RNS.Identity.recall(packet.destination_hash)
+								announce_destination = RNS.Destination(announce_identity, RNS.Destination.OUT, RNS.Destination.SINGLE, "unknown", "unknown");
+								announce_destination.hash = packet.destination_hash
+								announce_destination.hexhash = announce_destination.hash.hex()
+								announce_context = RNS.Packet.NONE
+								announce_data = packet.data
+
+								new_announce = RNS.Packet(
+									announce_destination,
+									announce_data,
+									RNS.Packet.ANNOUNCE,
+									context = announce_context,
+									header_type = RNS.Packet.HEADER_2,
+									transport_type = Transport.TRANSPORT,
+									transport_id = Transport.identity.hash,
+									attached_interface = attached_interface
+								)
+
+								new_announce.hops = packet.hops
+								new_announce.send()
+
+							Transport.destination_table[packet.destination_hash] = [now, received_from, announce_hops, expires, random_blobs, packet.receiving_interface, packet]
+							RNS.log("Path to "+RNS.prettyhexrep(packet.destination_hash)+" is now "+str(announce_hops)+" hops away via "+RNS.prettyhexrep(received_from)+" on "+str(packet.receiving_interface), RNS.LOG_VERBOSE)
 			
+			# Handling for linkrequests to local destinations
 			elif packet.packet_type == RNS.Packet.LINKREQUEST:
 				for destination in Transport.destinations:
 					if destination.hash == packet.destination_hash and destination.type == packet.destination_type:
 						packet.destination = destination
 						destination.receive(packet)
 			
+			# Handling for local data packets
 			elif packet.packet_type == RNS.Packet.DATA:
 				if packet.destination_type == RNS.Destination.LINK:
 					for link in Transport.active_links:
@@ -548,12 +770,12 @@ class Transport:
 									if destination.callbacks.proof_requested(packet):
 										packet.prove()
 
+			# Handling for proofs and link-request proofs
 			elif packet.packet_type == RNS.Packet.PROOF:
 				if packet.context == RNS.Packet.LRPROOF:
 					# This is a link request proof, check if it
 					# needs to be transported
-
-					if RNS.Reticulum.transport_enabled() and packet.destination_hash in Transport.link_table:
+					if (RNS.Reticulum.transport_enabled() or for_local_client_link or from_local_client) and packet.destination_hash in Transport.link_table:
 						link_entry = Transport.link_table[packet.destination_hash]
 						if packet.receiving_interface == link_entry[2]:
 							# TODO: Should we validate the LR proof at each transport
@@ -589,7 +811,7 @@ class Transport:
 						proof_hash = None
 
 					# Check if this proof neds to be transported
-					if RNS.Reticulum.transport_enabled() and packet.destination_hash in Transport.reverse_table:
+					if (RNS.Reticulum.transport_enabled() or from_local_client or proof_for_local_client) and packet.destination_hash in Transport.reverse_table:
 						reverse_entry = Transport.reverse_table.pop(packet.destination_hash)
 						if packet.receiving_interface == reverse_entry[1]:
 							RNS.log("Proof received on correct interface, transporting it via "+str(reverse_entry[0]), RNS.LOG_DEBUG)
@@ -657,6 +879,11 @@ class Transport:
 
 		return False
 
+	# When caching packets to storage, they are written
+	# exactly as they arrived over their interface. This
+	# means that they have not had their hop count
+	# increased yet! Take note of this when reading from
+	# the packet cache.
 	@staticmethod
 	def cache(packet, force_cache=False):
 		if RNS.Transport.shouldCache(packet) or force_cache:
@@ -730,12 +957,23 @@ class Transport:
 		packet.send()
 
 	@staticmethod
-	def pathRequestHandler(data, packet):
-		if len(data) >= RNS.Identity.TRUNCATED_HASHLENGTH//8:
-			Transport.pathRequest(data[:RNS.Identity.TRUNCATED_HASHLENGTH//8])
+	def requestPathOnInterface(destination_hash, interface):
+		path_request_data = destination_hash + RNS.Identity.getRandomHash()
+		path_request_dst = RNS.Destination(None, RNS.Destination.OUT, RNS.Destination.PLAIN, Transport.APP_NAME, "path", "request")
+		packet = RNS.Packet(path_request_dst, path_request_data, packet_type = RNS.Packet.DATA, transport_type = RNS.Transport.BROADCAST, header_type = RNS.Packet.HEADER_1, attached_interface = interface)
+		packet.send()
 
 	@staticmethod
-	def pathRequest(destination_hash):
+	def pathRequestHandler(data, packet):
+		if len(data) >= RNS.Identity.TRUNCATED_HASHLENGTH//8:
+			Transport.pathRequest(
+				data[:RNS.Identity.TRUNCATED_HASHLENGTH//8],
+				Transport.from_local_client(packet),
+				packet.receiving_interface
+			)
+
+	@staticmethod
+	def pathRequest(destination_hash, is_from_local_client, attached_interface):
 		RNS.log("Path request for "+RNS.prettyhexrep(destination_hash), RNS.LOG_DEBUG)
 		
 		local_destination = next((d for d in Transport.destinations if d.hash == destination_hash), None)
@@ -743,7 +981,7 @@ class Transport:
 			RNS.log("Destination is local to this system, announcing", RNS.LOG_DEBUG)
 			local_destination.announce(path_response=True)
 
-		elif RNS.Reticulum.transport_enabled() and destination_hash in Transport.destination_table:
+		elif (RNS.Reticulum.transport_enabled() or is_from_local_client) and destination_hash in Transport.destination_table:
 			RNS.log("Path found, inserting announce for transmission", RNS.LOG_DEBUG)
 			packet = Transport.destination_table[destination_hash][6]
 			received_from = Transport.destination_table[destination_hash][5]
@@ -752,9 +990,38 @@ class Transport:
 			retries = Transport.PATHFINDER_R
 			local_rebroadcasts = 0
 			block_rebroadcasts = True
-			retransmit_timeout = now + Transport.PATH_REQUEST_GRACE # + (RNS.rand() * Transport.PATHFINDER_RW)
+			announce_hops      = packet.hops
 
-			Transport.announce_table[packet.destination_hash] = [now, retransmit_timeout, retries, received_from, packet.hops, packet, local_rebroadcasts, block_rebroadcasts]
+			if is_from_local_client:
+				retransmit_timeout = now
+			else:
+				# TODO: Look at this timing
+				retransmit_timeout = now + Transport.PATH_REQUEST_GRACE # + (RNS.rand() * Transport.PATHFINDER_RW)
+
+			# This handles an edge case where a peer sends a past
+			# request for a destination just after an announce for
+			# said destination has arrived, but before it has been
+			# rebroadcast locally. In such a case the actual announce
+			# is temporarily held, and then reinserted when the path
+			# request has been served to the peer.
+			if packet.destination_hash in Transport.announce_table:
+				held_entry = Transport.announce_table[packet.destination_hash]
+				Transport.held_announces[packet.destination_hash] = held_entry
+			
+			Transport.announce_table[packet.destination_hash] = [now, retransmit_timeout, retries, received_from, announce_hops, packet, local_rebroadcasts, block_rebroadcasts, attached_interface]
+
+		elif is_from_local_client:
+			# Forward path request on all interfaces
+			# except the local client
+			for interface in Transport.interfaces:
+				if not interface == attached_interface:
+					Transport.requestPathOnInterface(destination_hash, interface)
+
+		elif not is_from_local_client and len(Transport.local_client_interfaces) > 0:
+			# Forward the path request on all local
+			# client interfaces
+			for interface in Transport.local_client_interfaces:
+				Transport.requestPathOnInterface(destination_hash, interface)
 
 		else:
 			RNS.log("No known path to requested destination, ignoring request", RNS.LOG_DEBUG)
@@ -766,6 +1033,30 @@ class Transport:
 		# TODO: implement this
 		pass
 
+	@staticmethod
+	def from_local_client(packet):
+		if hasattr(packet.receiving_interface, "parent_interface"):
+			return Transport.is_local_client_interface(packet.receiving_interface)
+		else:
+			return False
+
+	@staticmethod
+	def is_local_client_interface(interface):
+		if hasattr(interface, "parent_interface"):
+			if hasattr(interface.parent_interface, "is_local_shared_instance"):
+				return True
+			else:
+				return False
+		else:
+			return False
+
+	@staticmethod
+	def interface_to_shared_instance(interface):
+		if hasattr(interface, "is_connected_to_shared_instance"):
+			return True
+		else:
+			return False
+
 	@staticmethod
 	def exitHandler():
 		RNS.log("Saving packet hashlist to storage...", RNS.LOG_VERBOSE)
@@ -778,27 +1069,47 @@ class Transport:
 		except Exception as e:
 			RNS.log("Could not save packet hashlist to storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
 
-		RNS.log("Saving path table to storage...", RNS.LOG_VERBOSE)
-		try:
-			serialised_destinations = []
-			for destination_hash in Transport.destination_table:
-				# Get the destination entry from the destination table
-				de = Transport.destination_table[destination_hash]
-				interface_hash = de[5].get_hash()
+		if not Transport.owner.is_connected_to_shared_instance:
+			RNS.log("Saving path table to storage...", RNS.LOG_VERBOSE)
+			try:
+				serialised_destinations = []
+				for destination_hash in Transport.destination_table:
+					# Get the destination entry from the destination table
+					de = Transport.destination_table[destination_hash]
+					interface_hash = de[5].get_hash()
 
-				# Only store destination tablee entry if the associated
-				# interface is still active
-				interface = Transport.find_interface_from_hash(interface_hash)
-				if interface != None:
-					Transport.cache(de[6], force_cache=True)
-					packet_hash = de[6].getHash()
-					serialised_entry = [destination_hash, de[0], de[1], de[2], de[3], de[4], interface_hash, packet_hash]
-					serialised_destinations.append(serialised_entry)
+					# Only store destination table entry if the associated
+					# interface is still active
+					interface = Transport.find_interface_from_hash(interface_hash)
+					if interface != None:
+						# Get the destination entry from the destination table
+						de = Transport.destination_table[destination_hash]
+						timestamp = de[0]
+						received_from = de[1]
+						hops = de[2]
+						expires = de[3]
+						random_blobs = de[4]
+						packet_hash = de[6].getHash()
 
-			destination_table_path = RNS.Reticulum.storagepath+"/destination_table"
-			file = open(destination_table_path, "wb")
-			file.write(umsgpack.packb(serialised_destinations))
-			file.close()
-			RNS.log("Done saving path table to storage", RNS.LOG_VERBOSE)
-		except Exception as e:
-			RNS.log("Could not save path table to storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
+						serialised_entry = [
+							destination_hash,
+							timestamp,
+							received_from,
+							hops,
+							expires,
+							random_blobs,
+							interface_hash,
+							packet_hash
+						]
+
+						serialised_destinations.append(serialised_entry)
+
+						Transport.cache(de[6], force_cache=True)
+
+				destination_table_path = RNS.Reticulum.storagepath+"/destination_table"
+				file = open(destination_table_path, "wb")
+				file.write(umsgpack.packb(serialised_destinations))
+				file.close()
+				RNS.log("Done saving path table to storage", RNS.LOG_VERBOSE)
+			except Exception as e:
+				RNS.log("Could not save path table to storage, the contained exception was: "+str(e), RNS.LOG_ERROR)

setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r") as fh:
 
 setuptools.setup(
     name="rns",
-    version="0.1.0",
+    version="0.1.1",
     author="Mark Qvist",
     author_email="mark@unsigned.io",
     description="Self-configuring, encrypted and resilient mesh networking stack for LoRa, packet radio, WiFi and everything in between",