circinus: improved tarball repo sync script for debugging

4 files changed, 60 insertions, 30 deletions

diff --git a/new/lib/git.py b/new/lib/git.py
index ba6f02d..94ce6bd 100644
--- a/new/lib/git.py
+++ b/new/lib/git.py
@@ -1,3 +1,4 @@
+import logging
 import os.path
 import re
 
@@ -5,45 +6,46 @@ from lib.helpers import execute, quote_all, ProcessException
 
 
 class Git:
-    def __init__(self, repo_path):
+    def __init__(self, repo_path, debug=False):
         self.repo_path = repo_path
+        self.debug = debug
 
     def exists(self):
         return os.path.exists(self.repo_path)
 
     def clone(self, git_url, branch=None):
         if branch is not None:
-            execute("git clone -b %s --single-branch %s %s" % quote_all(
+            self.execute("git clone -b %s --single-branch %s %s" % quote_all(
                 branch, git_url, self.repo_path
             ))
         else:
-            execute("git clone %s %s" % quote_all(
+            self.execute("git clone %s %s" % quote_all(
                 git_url, self.repo_path
             ))
 
     def checkout(self, pathspec, branch=None):
         if branch is not None:
-            execute("git -C %s checkout -b %s %s" % quote_all(self.repo_path, branch, pathspec))
+            self.execute("git -C %s checkout -b %s %s" % quote_all(self.repo_path, branch, pathspec))
         else:
-            execute("git -C %s checkout %s" % quote_all(self.repo_path, pathspec))
+            self.execute("git -C %s checkout %s" % quote_all(self.repo_path, pathspec))
 
     def add_remote(self, git_url, remote_name):
-        execute("git -C %s remote add %s %s" % quote_all(self.repo_path, remote_name, git_url))
+        self.execute("git -C %s remote add %s %s" % quote_all(self.repo_path, remote_name, git_url))
 
     def rm_remote(self, remote_name):
-        execute("git -C %s remote rm %s" % quote_all(self.repo_path, remote_name))
+        self.execute("git -C %s remote rm %s" % quote_all(self.repo_path, remote_name))
 
     def get_remote_url(self, remote_name):
-        return execute("git -C %s config --get remote.%s.url" % quote_all(self.repo_path, remote_name)).strip()
+        return self.execute("git -C %s config --get remote.%s.url" % quote_all(self.repo_path, remote_name)).strip()
 
     def set_remote_url(self, remote_name, git_url):
-        execute("git -C %s remote set-url %s %s" % quote_all(self.repo_path, remote_name, git_url))
+        self.execute("git -C %s remote set-url %s %s" % quote_all(self.repo_path, remote_name, git_url))
 
     def fetch(self):
-        execute("git -C %s fetch --all" % quote_all(self.repo_path))
+        self.execute("git -C %s fetch --all" % quote_all(self.repo_path))
 
     def pull(self, remote=None, branch=None, ff_only=False):
-        execute("git -C %s reset --hard" % quote_all(self.repo_path))
+        self.execute("git -C %s reset --hard" % quote_all(self.repo_path))
 
         extra = ""
         if remote:
@@ -53,23 +55,23 @@ class Git:
         if ff_only:
             extra += " --ff-only"
 
-        execute("git -C %s pull%s" % tuple(quote_all(self.repo_path) + (extra,)))
+        self.execute("git -C %s pull%s" % tuple(quote_all(self.repo_path) + (extra,)))
 
     def push(self, remote):
-        return execute("git -C %s push %s" % quote_all(self.repo_path, remote))
+        return self.execute("git -C %s push %s" % quote_all(self.repo_path, remote))
 
     def add(self):
-        execute("git -C %s add --all" % quote_all(self.repo_path))
+        self.execute("git -C %s add --all" % quote_all(self.repo_path))
 
     def commit(self, message):
-        execute("git -C %s commit -m %s" % quote_all(self.repo_path, message))
+        self.execute("git -C %s commit -m %s" % quote_all(self.repo_path, message))
 
     def get_last_commit_hash(self):
-        return execute("git -C %s rev-parse HEAD" % quote_all(self.repo_path)).strip()
+        return self.execute("git -C %s rev-parse HEAD" % quote_all(self.repo_path)).strip()
 
     def get_changed_files(self, ref1, ref2):
         try:
-            return execute("git diff --name-only %s %s" % quote_all(ref1, ref2)).strip()
+            return self.execute("git diff --name-only %s %s" % quote_all(ref1, ref2)).strip()
         except ProcessException as e:
             if e.exit_code == 1 and "Could not access" in e.output:
                 return ""  # ignore non-existing commits (caused by repo changed or force-push)
@@ -111,3 +113,11 @@ class Git:
             changed = False
 
         return changed
+
+    def execute(self, command, timeout: int = None, passthrough=False, passthrough_prefix=None, **kwargs):
+        if self.debug:
+            logging.info("GIT command: '%s'" % command)
+            if not passthrough:
+                passthrough = True
+                passthrough_prefix = "GIT: "
+        return execute(command, timeout, passthrough, passthrough_prefix, passthrough_output=self.debug,**kwargs)
diff --git a/new/lib/helpers.py b/new/lib/helpers.py
index 7adab63..e6b5521 100644
--- a/new/lib/helpers.py
+++ b/new/lib/helpers.py
@@ -30,7 +30,8 @@ def quote_all(*args):
     return tuple(quoted)
 
 
-def execute(command, timeout: int = None, passthrough=False, passthrough_prefix=None, **kwargs):
+def execute(command, timeout: int = None, passthrough=False, passthrough_prefix=None, passthrough_output=False,
+            **kwargs):
     if passthrough:
         kwargs["stdout"] = subprocess.PIPE
         kwargs["stderr"] = subprocess.STDOUT
@@ -43,9 +44,11 @@ def execute(command, timeout: int = None, passthrough=False, passthrough_prefix=
         kwargs["shell"] = True
 
     process = subprocess.Popen(command, **kwargs)
+    buffer = None
     if passthrough:
         file_log_handler = find_file_log_handler()
-        buffer = TerminalLineBuffer()
+        buffer = bytearray()
+        terminal_buffer = TerminalLineBuffer()
         stdout = process.stdout
 
         deadline = monotonic() + timeout if timeout is not None else None
@@ -53,22 +56,24 @@ def execute(command, timeout: int = None, passthrough=False, passthrough_prefix=
             # noinspection PyTypeChecker
             value: bytes = stdout.read(1)
             sys.stdout.buffer.write(value)
+            buffer.extend(value)
 
             if file_log_handler is not None:
-                buffer.feed(value)
-                if buffer.is_complete():
+                terminal_buffer.feed(value)
+                if terminal_buffer.is_complete():
                     sys.stdout.buffer.flush()
-                    line = buffer.get_line()
+                    line = terminal_buffer.get_line()
                     file_log_handler.handle(create_stdout_log_record(line, passthrough_prefix))
 
         # noinspection PyTypeChecker
         rest: bytes = stdout.read()
         sys.stdout.buffer.write(rest)
         sys.stdout.buffer.flush()
+        buffer.extend(rest)
 
         if file_log_handler is not None:
-            buffer.feed(rest)
-            line = buffer.get_line()
+            terminal_buffer.feed(rest)
+            line = terminal_buffer.get_line()
             if line:
                 file_log_handler.handle(create_stdout_log_record(line, passthrough_prefix))
 
@@ -82,13 +87,18 @@ def execute(command, timeout: int = None, passthrough=False, passthrough_prefix=
     if exit_code != 0:
         message = "Command '%s' failed, exit code: %s" % (command, exit_code)
         output = None
-        if not passthrough:
-            # noinspection PyUnresolvedReferences
-            output = process.stdout.read().decode("utf-8")
+        if not passthrough or passthrough_output:
+            if passthrough_output:
+                output = buffer.decode("utf-8")
+            else:
+                # noinspection PyUnresolvedReferences
+                output = process.stdout.read().decode("utf-8")
             message += ", output: %s" % output
         raise ProcessException(message, exit_code, output)
 
     if passthrough:
+        if passthrough_output:
+            return buffer.decode("utf-8")
         return exit_code
     else:
         # noinspection PyUnresolvedReferences
diff --git a/new/tools/.gitignore b/new/tools/.gitignore
index 587b17a..5663e7e 100644
--- a/new/tools/.gitignore
+++ b/new/tools/.gitignore
@@ -1,4 +1,4 @@
-sources/*
+sources*
 work/
 
 !.gitignore
diff --git a/new/tools/tarball-repo-sync.py b/new/tools/tarball-repo-sync.py
index 2ebb9a3..f8c40bc 100644
--- a/new/tools/tarball-repo-sync.py
+++ b/new/tools/tarball-repo-sync.py
@@ -14,12 +14,14 @@ from lib.helpers import setup_logging, ProcessException, execute, quote_all
 
 
 class TarballRepoSync:
-    def __init__(self, branch, source_org, target_org, skip_analyze, single_package):
+    def __init__(self, branch, source_org, target_org, skip_analyze, single_package, skip_until, debug=False):
         self.branch = branch
         self.source_org = source_org
         self.target_org = target_org
         self.skip_analyze = skip_analyze
         self.single_package = single_package
+        self.skip_until = skip_until
+        self.debug = debug
         self.source_dir = os.path.realpath("./sources")
         self.working_dir = os.path.realpath("./work")
         self.package_aliases = {
@@ -117,16 +119,21 @@ class TarballRepoSync:
         return found
 
     def sync_repositories(self, matched):
+        skipping = self.skip_until is not None
         for info in matched:
             if self.single_package is not None and info["name"] != self.single_package:
                 continue
 
+            if skipping and info["name"] != self.skip_until:
+                continue
+            skipping = False
+
             logging.info("processing %s - https://github.com/%s/%s/tree/%s - %s" % (
                 info["name"], self.target_org, info["name"], self.branch, info["path"]
             ))
 
             repo_path = os.path.join(self.working_dir, info["name"])
-            git = Git(repo_path)
+            git = Git(repo_path, debug=self.debug)
             if git.exists():
                 shutil.rmtree(repo_path)
 
@@ -194,6 +201,7 @@ class TarballRepoSync:
         for parent, directories, files in os.walk(path):
             if len(directories) > 1 or len(files) > 0:
                 return parent
+        raise Exception("unable to find root directory")
 
     def destroy_path(self, path):
         if os.path.isdir(path):
@@ -218,6 +226,8 @@ if __name__ == "__main__":
         parser.add_argument("--target-org", default="NOTvyos")
         parser.add_argument("--skip-analyze", action="store_true")
         parser.add_argument("--single-package")
+        parser.add_argument("--skip-until", help="skip packages until this one")
+        parser.add_argument("--debug", action="store_true")
 
         args = parser.parse_args()
         values = vars(args)