diff --git a/tools/hub-clone.py b/tools/hub-clone.py index a37f532..f1d3615 100644 --- a/tools/hub-clone.py +++ b/tools/hub-clone.py @@ -1,94 +1,238 @@ import os +import stat import sys +import time import requests -if len(sys.argv) < 3: - print("") - print(f" Usage: {sys.argv[0]} [remote-host] [local-directory]") - print("") - print(" Example:") - print(f" {sys.argv[0]} https://hub.grid.tf /tmp/users") - print("") +class HubFlistSyncer: + def __init__(self, baseurl, localdir): + self.baseurl = baseurl + self.localdir = localdir - sys.exit(1) + self.officials = [] -host = sys.argv[1] -target = sys.argv[2] -officials = [] -root = os.getcwd() -download = 0 -files = 0 + self.root = os.getcwd() + self.downloaded = 0 + self.files = 0 -# -# fetching repositories -# + # + # remote helpers + # + def remote_repositories(self): + r = requests.get(f"{self.baseurl}/api/repositories") + repositories = r.json() -r = requests.get(f"{host}/api/repositories") -repositories = r.json() + return repositories -for repo in repositories: - userpath = f"{target}/{repo['name']}" + def remote_repository(self, username): + sys.stdout.write(f"\r[+] fetching user informations: {username} \033\x5bK") - if not os.path.exists(userpath): - os.mkdir(userpath) + r = requests.get(f"{self.baseurl}/api/flist/{username}") + entries = r.json() - if repo['official']: - officials.append(repo['name']) + return entries -print(f"[+] created: {len(repositories)} repositories") + # + # local helpers + # + def local_sync_repositories(self, repositories): + updated = [] -# -# fetching flist for each repositories -# + for repo in repositories: + userpath = f"{self.localdir}/{repo['name']}" -for repo in repositories: - sys.stdout.write(f"\r[+] fetching user informations: {repo['name']} \033[K") + if repo['official']: + self.officials.append(repo['name']) - r = requests.get(f"{host}/api/flist/{repo['name']}") - entries = r.json() - for entry in entries: - targetfile = f"{target}/{repo['name']}/{entry['name']}" + if not os.path.exists(userpath): + os.mkdir(userpath) + os.utime(userpath, (int(time.time()), repo['updated'])) + updated.append(repo) + continue - # skip if local file exists - # FIXME: should be updated if different - if os.path.exists(targetfile): - continue + dirstat = os.stat(userpath) + if repo['updated'] > int(dirstat.st_mtime): + updated.append(repo) - if entry['type'] == 'regular': - url = f"{host}/{repo['name']}/{entry['name']}" - sys.stdout.write(f"\r[+] downloading: {url} \033[K") + else: + # FIXME: force update for legacy purpose + os.utime(userpath, (int(time.time()), repo['updated'])) + + print(f"[+] {len(updated)} / {len(repositories)} local repositories to update") + + return updated - r = requests.get(url) - with open(targetfile, "wb") as f: - f.write(r.content) + def local_sync_repository(self, username, entries, updated): + userpath = f"{self.localdir}/{username}" - download += len(r.content) - files += 1 + for entry in entries: + targetfile = f"{self.localdir}/{username}/{entry['name']}" + self.local_sync_entryfile(username, entry, targetfile) + + # update last modification time + os.utime(userpath, (int(time.time()), updated)) + + def local_sync_entryfile(self, username, entry, targetfile): + # FIXME: support deleted entries + # (need to compare extra local entries) + + if entry['type'] == 'regular': + return self.local_sync_regular_file(username, entry, targetfile) if entry['type'] == 'symlink': - os.chdir(f"{target}/{repo['name']}") + return self.local_sync_symlink(username, entry, targetfile) + + if entry['type'] == 'tag': + return self.local_sync_tag(username, entry, targetfile) + + if entry['type'] == 'taglink': + return self.local_sync_taglink(username, entry, targetfile) + + raise RuntimeError(f"Unexpected entry type: {entry['type']}") - if "/" in entry['target']: - os.symlink(f"../{entry['target']}", entry['name']) + # + # entry type specific handlers + # + def local_sync_regular_file(self, username, entry, targetfile): + now = int(time.time()) + + if os.path.lexists(targetfile): + filestat = os.lstat(targetfile) + + # checking if local is a regular file as well + if stat.S_ISREG(filestat.st_mode): + # checking if remote file is newer + if entry['updated'] <= int(filestat.st_mtime): + return None else: - os.symlink(entry['target'], entry['name']) + # local file is not a regular file and remote + # file is a regular file, removing local file and + # updating it + os.remove(targetfile) - os.chdir(root) + url = f"{self.baseurl}/{username}/{entry['name']}" + sys.stdout.write(f"\r[+] downloading: {url} \033\x5bK") - if entry['type'] == 'tag': - targetfile = f"{target}/{repo['name']}/.tag-{entry['name']}" + r = requests.get(url) + with open(targetfile, "wb") as f: + f.write(r.content) - if not os.path.exists(targetfile): - os.mkdir(targetfile) + # apply same modification time on symlink than remote host + os.utime(targetfile, (now, entry['updated'])) - if entry['type'] == 'taglink': - items = entry['target'].split("/") + self.downloaded += len(r.content) + self.files += 1 + + return True + + def local_sync_symlink(self, username, entry, targetfile): + now = int(time.time()) + + if os.path.lexists(targetfile): + filestat = os.lstat(targetfile) + + # checking if local is a symlink as well + if stat.S_ISLNK(filestat.st_mode): + # checking if symlink is newer + if entry['linktime'] <= int(filestat.st_mtime): + return None + + # update required, removing local file + os.remove(targetfile) + + else: + # local file is not a symlink and remote file + # is a symlink, updating + os.remove(targetfile) + + os.chdir(f"{self.localdir}/{username}") + target = entry['target'] + + # checking for crosslink + if "/" in entry['target']: + target = f"../{entry['target']}" + + os.symlink(target, entry['name']) + os.chdir(self.root) + + # apply same modification time on the tag directory than remote host + os.utime(targetfile, (now, entry['linktime']), follow_symlinks=False) + + return True + + def local_sync_tag(self, username, entry, targetfile): + now = int(time.time()) + + # update targetfile with tag syntax + targetfile = f"{self.localdir}/{username}/.tag-{entry['name']}" + + # ignoring last modification time and updating anyway + if not os.path.exists(targetfile): + os.mkdir(targetfile) + + # apply same modification time than remote host + os.utime(targetfile, (now, entry['updated'])) + + return True + + def local_sync_taglink(self, username, entry, targetfile): + now = int(time.time()) + items = entry['target'].split("/") + + if os.path.lexists(targetfile): + os.remove(targetfile) + + # ignoring last modification time and updating anyway + os.chdir(f"{self.localdir}/{username}") + os.symlink(f"../{items[0]}/.tag-{items[2]}", entry['name']) + os.chdir(self.root) + + # apply same modification on the symlink time than remote host + os.utime(targetfile, (now, entry['linktime']), follow_symlinks=False) + + return True + + # + # sync statistics + # + def statistics(self): + print("[+]") + print("[+] remote official repositories configuration:") + print("[+] ------------------------------------------") + print(f"[+] {self.officials}") + print("[+] ------------------------------------------") + + mbsize = self.downloaded / (1024 * 1024) + print(f"[+] downloaded: {mbsize:.2f} MB ({self.files} files)") + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("") + print(f" Usage: {sys.argv[0]} [remote-host] [local-directory]") + print("") + print(" Example:") + print(f" {sys.argv[0]} https://hub.grid.tf /tmp/users") + print("") + + sys.exit(1) + + host = sys.argv[1] + target = sys.argv[2] + + sync = HubFlistSyncer(host, target) + + repositories = sync.remote_repositories() + updating = sync.local_sync_repositories(repositories) + + if len(updating) == 0: + print("[+] nothing to update") + + for repo in updating: + username = repo['name'] - os.chdir(f"{target}/{repo['name']}") - os.symlink(f"../{items[0]}/.tag-{items[2]}", entry['name']) - os.chdir(root) + userdata = sync.remote_repository(username) + sync.local_sync_repository(username, userdata, repo['updated']) -print("") -print(f"[+] official repos: {officials}") -print(f"[+] downloaded: {download / (1024 * 1024)} MB ({files} files)") + sync.statistics()