Mirror/Reticulum
1
0
Fork 0
mirror of https://github.com/markqvist/Reticulum.git synced 2026-06-08 14:11:53 -07:00
Code Issues Packages Projects Releases Wiki Activity

Added scaffolding for periodic upstream mirror sync and manual fork/mirror sync

Browse Source
This commit is contained in:
Mark Qvist
2026-05-16 22:06:16 +02:00
parent 0c68f6491a
commit b76beb602d
1 changed files with 184 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
RNS/Utilities/rngit/server.py
+184 -37
View File
@@ -113,6 +113,10 @@ def program_setup(configdir, rnsconfigdir=None, verbosity=0, quietness=0, servic
git_client = ReticulumGitClient(configdir=configdir, verbosity=targetverbosity, identitypath=identity)
git_client.fork_repository(source=task["source"], target=task["target"])
elif command == "sync":
git_client = ReticulumGitClient(configdir=configdir, verbosity=targetverbosity, identitypath=identity)
git_client.sync_repository(remote=task["remote"])
elif command == "mirror":
git_client = ReticulumGitClient(configdir=configdir, verbosity=targetverbosity, identitypath=identity)
git_client.mirror_repository(source=task["source"], target=task["target"])
@@ -122,7 +126,7 @@ def program_setup(configdir, rnsconfigdir=None, verbosity=0, quietness=0, servic
exit(0)
def main():
subcommands = ["node", "release", "work", "create", "fork", "mirror"]
subcommands = ["node", "release", "work", "create", "fork", "sync", "mirror"]
try:
if len(sys.argv) < 2 or sys.argv[1] not in subcommands: subcommand = "node"
else: subcommand = sys.argv[1]; sys.argv.pop(1)
@@ -171,6 +175,13 @@ def main():
parser.add_argument("source", default=None, help="URL of source repository", type=str)
parser.add_argument("target", default=None, help="URL of target repository", type=str)
elif subcommand == "sync":
parser = argparse.ArgumentParser(description="Reticulum Git Repository Syncer")
parser.add_argument("--config", action="store", default=None, help="path to alternative config directory", type=str)
parser.add_argument("--rnsconfig", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
parser.add_argument("-i", "--identity", action="store", metavar="PATH", default=None, help="path to identity", type=str)
parser.add_argument("repository", default=None, help="URL of repository", type=str)
elif subcommand == "mirror":
parser = argparse.ArgumentParser(description="Reticulum Git Mirror Management")
parser.add_argument("--config", action="store", default=None, help="path to alternative config directory", type=str)
@@ -219,6 +230,11 @@ def main():
program_setup(configdir=configarg, rnsconfigdir=rnsconfigarg, verbosity=args.verbose, quietness=args.quiet,
task=task, identity=args.identity)
elif subcommand == "sync":
task = {"command": subcommand, "operation": "sync", "remote": args.repository}
program_setup(configdir=configarg, rnsconfigdir=rnsconfigarg, verbosity=args.verbose, quietness=args.quiet,
task=task, identity=args.identity)
elif subcommand == "mirror":
task = {"command": subcommand, "operation": "fork", "source": args.source, "target": args.target}
program_setup(configdir=configarg, rnsconfigdir=rnsconfigarg, verbosity=args.verbose, quietness=args.quiet,
@@ -237,6 +253,7 @@ class ReticulumGitClient():
PATH_DELETE = "/git/delete"
PATH_CREATE = "/git/create"
PATH_FORK = "/git/fork"
PATH_SYNC = "/git/sync"
PATH_MIRROR = "/git/mirror"
PATH_RELEASE = "/mgmt/release"
PATH_WORK = "/mgmt/work"
@@ -252,6 +269,7 @@ class ReticulumGitClient():
PATH_TIMEOUT = 15
LINK_TIMEOUT = 15
WAIT_SLEEP = 0.2
def __init__(self, configdir=None, verbosity=None, identitypath=None):
self.identity = None
@@ -260,6 +278,7 @@ class ReticulumGitClient():
self.verbosity = verbosity or 0
self.path_timeout = self.PATH_TIMEOUT
self.link_timeout = self.LINK_TIMEOUT
self.wait_sleep = self.WAIT_SLEEP
self._should_run = True
self.link_ready = False
@@ -408,8 +427,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.1)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Link establishment failed")
print("\r \r", end="")
@@ -458,8 +477,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.1)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Link establishment failed")
print("\r \r", end="")
@@ -496,6 +515,50 @@ class ReticulumGitClient():
finally:
if self.link: self.link.teardown()
def sync_repository(self, remote=None):
if not remote: print(f"No remote specified"); exit(1)
self.connect_remote(remote)
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Link establishment failed")
print("\r \r", end="")
try:
destination_hash, group, repo = self.parse_remote_url(remote)
repo_path = f"{group}/{repo}"
request_data = {self.IDX_REPOSITORY: repo_path}
print(f"Remote is syncing repository...")
response, metadata = self.send_request(self.PATH_SYNC, request_data, timeout=900)
if not response or not isinstance(response, bytes): self.abort("No response from remote")
status_byte = response[0]
if status_byte == 0: print(f"Repository synced")
elif status_byte == self.RES_NOT_FOUND:
error_msg = response[1:].decode("utf-8", errors="ignore") if len(response) > 1 else "Not found"
self.abort(f"{error_msg}")
elif status_byte == self.RES_INVALID_REQ:
error_msg = response[1:].decode("utf-8", errors="ignore") if len(response) > 1 else "Invalid request"
self.abort(f"{error_msg}")
elif status_byte == self.RES_DISALLOWED:
error_msg = response[1:].decode("utf-8", errors="ignore") if len(response) > 1 else "Not allowed"
self.abort(f"{error_msg}")
else:
error_msg = response[1:].decode("utf-8", errors="ignore") if len(response) > 1 else "Unknown error"
self.abort(f"Server error: {error_msg}")
except Exception as e: self.abort(f"Error syncing repository: {e}")
finally:
if self.link: self.link.teardown()
######################
# Release Management #
######################
@@ -548,8 +611,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.1)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Link establishment failed")
@@ -607,8 +670,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.5)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Link establishment failed")
@@ -666,8 +729,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.1)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -764,8 +827,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.5)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -807,8 +870,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.5)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -853,8 +916,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.1)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Link establishment failed")
@@ -911,8 +974,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.5)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Link establishment failed")
@@ -986,8 +1049,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.1)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1031,8 +1094,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.1)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1076,8 +1139,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.2)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1130,8 +1193,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.2)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1170,8 +1233,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.2)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1213,8 +1276,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.2)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1252,8 +1315,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.2)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1291,8 +1354,8 @@ class ReticulumGitClient():
timeout = self.link_timeout
while not self.link_ready and not self.link_failed and timeout > 0:
time.sleep(0.2)
timeout -= 1
time.sleep(self.wait_sleep)
timeout -= self.wait_sleep
if not self.link_ready: self.abort("Failed to establish link")
print("\r \r", end="")
@@ -1454,6 +1517,7 @@ class ReticulumGitNode():
PATH_DELETE = "/git/delete"
PATH_CREATE = "/git/create"
PATH_FORK = "/git/fork"
PATH_SYNC = "/git/sync"
PATH_MIRROR = "/git/mirror"
PATH_RELEASE = "/mgmt/release"
PATH_WORK = "/mgmt/work"
@@ -1484,8 +1548,12 @@ class ReticulumGitNode():
self.last_stats_job = time.time()
self.link_clean_interval = 5
self.last_link_clean = 0
self.mirror_interval = 24*60*60
self.sync_check_interval = 15*60
self.last_sync_check = time.time()
self.active_links_lock = Lock()
self.stats_lock = Lock()
self.sync_lock = Lock()
self.stats_ignored = {}
self.node_name = "Anonymous Git Node"
@@ -1581,6 +1649,28 @@ class ReticulumGitNode():
os.rename(tmp_path, self.statspath)
except Exception as e: RNS.log(f"Could not write stats file to {self.statspath}: {e}", RNS.LOG_ERROR)
def __sync_mirrors(self):
if self.sync_lock.locked(): return
with self.sync_lock:
try:
for group_name in self.groups.copy():
for repository_name in self.groups[group_name]["repositories"].copy():
repo = self.groups[group_name]["repositories"][repository_name]
if repo["mirror"] and time.time() > self.__mirror_synced(repo["path"]) + self.mirror_interval:
self.__sync_mirror(group_name, repository_name)
except Exception as e: RNS.log(f"Could not sync mirrors: {e}", RNS.LOG_ERROR)
def __sync_mirror(self, group_name, repository_name):
RNS.log(f"Syncing mirror {group_name}/{repository_name}", RNS.LOG_INFO)
# TODO: Implement
pass
def __sync_fork(self, group_name, repository_name):
RNS.log(f"Syncing fork {group_name}/{repository_name} with upstream", RNS.LOG_INFO)
# TODO: Implement
pass
def __apply_config(self):
if not os.path.isfile(self.identitypath):
identity = RNS.Identity()
@@ -1601,6 +1691,7 @@ class ReticulumGitNode():
section = self.config["rngit"]
if "node_name" in section: self.node_name = section["node_name"]
if "announce_interval" in section: self.announce_interval = section.as_int("announce_interval")*60
if "mirror_interval" in section: self.mirror_interval = section.as_int("mirror_interval")*60*60
if "record_stats" in section: self.stats_enabled = section.as_bool("record_stats")
if "stats_ignore_identities" in section:
ignored = section.as_list("stats_ignore_identities")
@@ -2003,6 +2094,10 @@ class ReticulumGitNode():
self.__persist_stats()
self.last_stats_job = time.time()
if time.time() > self.last_sync_check + self.sync_check_interval:
self.__sync_mirrors()
self.last_sync_check = time.time()
if time.time() > self.last_link_clean + self.link_clean_interval:
stale_links = []
with self.active_links_lock:
@@ -2085,6 +2180,25 @@ class ReticulumGitNode():
except: return False
def __mirror_synced(self, path):
try:
result = subprocess.run(["git", "config", "repository.rngit.upstream.sync"], cwd=path, check=True, capture_output=True, text=True)
if not result: return 0
else: synced = int(result.stdout.strip())
return synced
except: return False
def __set_mirror_synced(self, path):
try:
result = subprocess.run(["git", "config", "repository.rngit.upstream.sync", str(int(time.time()))], cwd=path, check=True, capture_output=True, text=True)
if result: return True
else:
RNS.log(f"Could not set mirror sync time for {path}: {result.stderr}", RNS.LOG_ERROR)
return False
except: return False
def register_request_handlers(self):
ga_list = self.global_allowed_list if self.global_allow == RNS.Destination.ALLOW_LIST else None
self.destination.register_request_handler(self.PATH_LIST, self.handle_list, allow=self.global_allow, allowed_list=ga_list)
@@ -2092,6 +2206,7 @@ class ReticulumGitNode():
self.destination.register_request_handler(self.PATH_PUSH, self.handle_push, allow=self.global_allow, allowed_list=ga_list)
self.destination.register_request_handler(self.PATH_CREATE, self.handle_create, allow=self.global_allow, allowed_list=ga_list)
self.destination.register_request_handler(self.PATH_FORK, self.handle_fork, allow=self.global_allow, allowed_list=ga_list)
self.destination.register_request_handler(self.PATH_SYNC, self.handle_sync, allow=self.global_allow, allowed_list=ga_list)
self.destination.register_request_handler(self.PATH_MIRROR, self.handle_mirror, allow=self.global_allow, allowed_list=ga_list)
self.destination.register_request_handler(self.PATH_DELETE, self.handle_delete, allow=self.global_allow, allowed_list=ga_list)
self.destination.register_request_handler(self.PATH_RELEASE, self.handle_release, allow=self.global_allow, allowed_list=ga_list)
@@ -2454,6 +2569,7 @@ class ReticulumGitNode():
return self._handle_remote_clone(path, data, request_id, link_id, remote_identity, requested_at, "mirror")
def _handle_remote_clone(self, path, data, request_id, link_id, remote_identity, requested_at, repo_type):
if repo_type not in ["mirror", "fork"]: return self.RES_INVALID_REQ.to_bytes(1, "big") + b"Invalid request"
with self.active_links_lock:
if not link_id in self.active_links: return self.RES_DISALLOWED.to_bytes(1, "big") + b"Not identified"
else: link = self.active_links[link_id]
@@ -2511,14 +2627,19 @@ class ReticulumGitNode():
result = subprocess.run(["git", "config", "repository.rngit.type", repo_type], cwd=repo_temp_path, capture_output=True, text=True, check=False)
if result.returncode != 0:
RNS.log(f"Failed to set rngit_type config: {result.stderr}", RNS.LOG_WARNING)
RNS.log(f"Failed to set rngit.type config: {result.stderr}", RNS.LOG_WARNING)
return self.RES_REMOTE_FAIL.to_bytes(1, "big") + f"Failed to configure repository type: {result.stderr}".encode("utf-8")
result = subprocess.run(["git", "config", "repository.rngit.upstream.source", source_url], cwd=repo_temp_path, capture_output=True, text=True, check=False)
if result.returncode != 0:
RNS.log(f"Failed to set rngit_upstream_source config: {result.stderr}", RNS.LOG_WARNING)
RNS.log(f"Failed to set rngit.upstream.source config: {result.stderr}", RNS.LOG_WARNING)
return self.RES_REMOTE_FAIL.to_bytes(1, "big") + f"Failed to configure repository upstream source: {result.stderr}".encode("utf-8")
if repo_type == "mirror":
if not self.__set_mirror_synced(repo_temp_path):
RNS.log(f"Failed to set rngit.upstream.sync config: {result.stderr}", RNS.LOG_WARNING)
return self.RES_REMOTE_FAIL.to_bytes(1, "big") + f"Failed to configure repository type: {result.stderr}".encode("utf-8")
try:
allowed_path = final_repository_path + ".allowed"
tmp_allowed = allowed_path + ".tmp"
@@ -2553,6 +2674,26 @@ class ReticulumGitNode():
except: pass
return self.RES_REMOTE_FAIL.to_bytes(1, "big") + b"Remote error"
def handle_sync(self, path, data, request_id, remote_identity, requested_at):
RNS.log(f"Upstream sync request from remote {remote_identity}", RNS.LOG_DEBUG)
if not remote_identity: return self.RES_DISALLOWED.to_bytes(1, "big") + b"Not identified"
if not type(data) == dict: return self.RES_INVALID_REQ.to_bytes(1, "big") + b"Invalid request"
if not self.IDX_REPOSITORY in data: return self.RES_INVALID_REQ.to_bytes(1, "big") + b"No repository specified"
group_name, repository_name = self.parse_request_repository_path(data[self.IDX_REPOSITORY])
read_access = self.resolve_permission(remote_identity, group_name, repository_name, self.PERM_READ)
write_access = self.resolve_permission(remote_identity, group_name, repository_name, self.PERM_WRITE)
if not read_access: return self.RES_NOT_FOUND.to_bytes(1, "big") + b"Not found"
if not write_access: return self.RES_DISALLOWED.to_bytes(1, "big") + b"Not allowed"
else:
repo = self.groups[group_name]["repositories"][repository_name]
repository_path = repo["path"]
if not repo["mirror"] and not repo["fork"]: return self.RES_INVALID_REQ.to_bytes(1, "big") + b"Repository is neither fork nor mirror"
# TODO: Implement
return b"\x00"
def handle_release(self, path, data, request_id, remote_identity, requested_at):
RNS.log(f"Release request from remote {remote_identity}", RNS.LOG_DEBUG)
if not remote_identity: return self.RES_DISALLOWED.to_bytes(1, "big") + b"Not identified"
@@ -3888,6 +4029,12 @@ showcase = /another/path/to/directory/with/git/repositories
# directory as the repository folder, or set it in the bare
# repository with `git config repository.description`.
# If you have mirrored repositories with the "rngit mirror"
# command, you can configure the global mirroring interval
# in hours.
# mirror_interval = 24
[access]