From cd43a431a9979b6e1f261d60a1d75a3d2e3e3598 Mon Sep 17 00:00:00 2001 From: Lephenixnoir Date: Fri, 8 Jan 2021 14:30:53 +0100 Subject: [PATCH] improve install/update semantics --- giteapc.py | 28 ++++--- giteapc/repo.py | 218 +++++++++++++++++++++++++++--------------------- 2 files changed, 137 insertions(+), 109 deletions(-) diff --git a/giteapc.py b/giteapc.py index 25841ff..a81bcfd 100755 --- a/giteapc.py +++ b/giteapc.py @@ -36,6 +36,16 @@ is either a full name like "Lephenixnoir/sh-elf-gcc", or a short name like {R}giteapc list{_} [{R}-r{_}] [{g}{i}PATTERN{_}] List all repositories on this computer. With -r, list repositories on the forge. A wildcard pattern can be specified to filter the results. +{R}giteapc install{_} [{R}--https{_}|{R}--ssh{_}] [{R}-u{_}] [{R}-y{_}] [{R}-n{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_} + Fetch repositories and their dependencies, then build and install them. + With -u, pulls local repositories (update mode). With -y, do not ask for + interactive confirmation. With -n, don't build or install (dry run). +{R}giteapc uninstall{_} [{R}-k{_}] {g}{i}REPOSITORY...{_} + Uninstall the build products of the specified repositories and remove the + source files. With -k, keep the source files. + +Advanced commands: + {R}giteapc fetch{_} [{R}--https{_}|{R}--ssh{_}] [{R}-u{_}] [{R}-f{_}] [{g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}]{g}{i}...{_}] Clone, fetch, or pull a repository. If no repository is specified, fetches all local repositories. HTTPS or SSH can be selected when cloning (HTTPS by @@ -44,18 +54,10 @@ is either a full name like "Lephenixnoir/sh-elf-gcc", or a short name like Show the branches and tags (versions) for the specified local repositories. With -r, show information for remote repositories on the forge. With -p, just print the path of local repositories (useful in scripts). -{R}giteapc build{_} [{R}-i{_}] [{R}-u{_}] [{R}--skip-configure{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_} +{R}giteapc build{_} [{R}-i{_}] [{R}--skip-configure{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_} Configure and build a local repository. A specific configuration can be requested. With -i, also install if build is successful. --skip-configure - builds without configuring (useful for rebuilds). With -u, pull the current - branch before building (update mode). -{R}giteapc install{_} [{R}--https{_}|{R}--ssh{_}] [{R}-u{_}] [{R}-y{_}] {g}{i}REPOSITORY{_}[{R}@{_}{g}{i}VERSION{_}][{R}:{_}{g}{i}CONFIG{_}]{g}{i}...{_} - Fetch repositories and their dependencies, then build and install them. - With -u, pulls local repositories (update mode). With -yes, do not ask for - interactive confirmation. -{R}giteapc uninstall{_} [{R}-k{_}] {g}{i}REPOSITORY...{_} - Uninstall the build products of the specified repositories and remove the - source files. With -k, keep the source files. + builds without configuring (useful for rebuilds). {W}Important folders{_} @@ -88,12 +90,12 @@ commands = { }, "build": { "function": giteapc.repo.build, - "args": "install:-i,--install skip_configure:--skip-configure "+\ - "update:-u,--update", + "args": "install:-i,--install skip_configure:--skip-configure", }, "install": { "function": giteapc.repo.install, - "args": "use_ssh:--ssh use_https:--https update:-u,--update", + "args": "use_ssh:--ssh use_https:--https update:-u,--update "+\ + "yes:-y,--yes dry_run:-n,--dry-run", }, "uninstall": { "function": giteapc.repo.uninstall, diff --git a/giteapc/repo.py b/giteapc/repo.py index f85d105..c6a0727 100644 --- a/giteapc/repo.py +++ b/giteapc/repo.py @@ -99,20 +99,47 @@ def pretty_repo(r): color = "ARGYBMW"[sum(map(ord,r.owner[:5])) % 7] return colors()[color] + "{}{_}/{W}{}{_}".format(r.owner,r.name,**colors()) -def split_config(name): - """Splits REPOSITORY[@VERSION][:CONFIGURATION] into components.""" - RE_CONFIG = re.compile(r'^([^@:]+)(?:@([^@:]+))?(?:[:]([^@:]+))?') - m = re.match(RE_CONFIG, name) - if m is None: - return None +# +# Repository specifications +# - repo, version, config = m[1], m[2] or "", m[3] or "" - return repo, version, config +class Spec: + # A spec in REPOSITORY[@VERSION][:CONFIGURATION] + RE_SPEC = re.compile(r'^([^@:]+)(?:@([^@:]+))?(?:[:]([^@:]+))?') -def make_config(name, version, config): - version = f"@{version}" if version else "" - config = f":{config}" if config else "" - return name + version + config + def __init__(self, string): + m = re.match(Spec.RE_SPEC, string) + if m is None: + raise Error(f"wrong format in specification {string}") + + self.name = m[1] + self.version = m[2] + self.config = m[3] + self.repo = None + + def resolve(self, local_only=False, remote_only=False): + self.repo = resolve(self.name, local_only, remote_only) + return self.repo + + def str(self, pretty=False): + if self.repo and pretty: + name = pretty_repo(self.repo) + elif self.repo: + name = self.repo.fullname + else: + name = self.name + + version = f"@{self.version}" if self.version else "" + config = f":{self.config}" if self.config else "" + return name + version + config + + def __str__(self): + return self.str(pretty=False) + def __repr__(self): + return self.str(pretty=False) + + def pretty_str(self): + return self.str(pretty=True) # # repo list command @@ -174,33 +201,33 @@ def fetch(*args, use_ssh=False, use_https=False, force=False, update=False): r.pull() return 0 - for spec in args: - name, version, config = split_config(spec) - r = resolve(name) + for arg in args: + s = Spec(arg) + r = s.resolve() # If this is a local repository, just git fetch if not r.remote: - if version: - msg("Checking out {W}{}{_}".format(version, **colors())) - r.checkout(version) - msg(f"Fetching {pretty_repo(r)}...") r.fetch() - if update: - r.pull() - continue + # If this is a remote repository, clone it + else: + msg(f"Cloning {pretty_repo(r)}...") + has_tag = "giteapc" in gitea.repo_topics(r) - msg(f"Cloning {pretty_repo(r)}...") + if has_tag or force: + LocalRepo.clone(r, protocol) + if not has_tag and force: + warn(f"{r.fullname} doesn't have the [giteapc] tag") + if not has_tag and not force: + return fatal(f"{r.fullname} doesn't have the [giteapc] tag, "+\ + "use -f to force") - # For remote repositories, make sure the repository supports GiteaPC - has_tag = "giteapc" in gitea.repo_topics(r) - - if has_tag or force: - LocalRepo.clone(r, protocol) - if not has_tag and force: - warn(f"{r.fullname} doesn't have the [giteapc] tag") - if not has_tag and not force: - fatal(f"{r.fullname} doesn't have the [giteapc] tag, use -f to force") + # Checkout requested version, if any + if s.version: + msg("Checking out {W}{}{_}".format(s.version, **colors())) + r.checkout(s.version) + if update: + r.pull() # # repo show command @@ -234,38 +261,31 @@ def show(*args, remote=False, path=False): # repo build command # -def build(*args, install=False, skip_configure=False, update=False): +def build(*args, install=False, skip_configure=False): if len(args) < 1: return fatal("repo build: specify at least one repository") specs = [] - for spec in args: - repo, version, config = split_config(spec) - repo = resolve(repo, local_only=True) - specs.append((repo, version, config)) + for arg in args: + s = arg if isinstance(arg, Spec) else Spec(arg) + s.resolve(local_only=True) + specs.append(s) - for (r, version, config) in specs: + for s in specs: + r = s.repo pretty = pretty_repo(r) - config_string = f" for {config}" if config else "" + config_string = f" for {s.config}" if s.config else "" - if version != "": - msg("{}: Checking out {W}{}{_}".format(pretty, version, **colors())) - r.checkout(version) - if update: - previous = r.describe() - r.pull() - new = r.describe() - if new == previous: - msg("{}: Still at {W}{}{_}, skipping build".format(pretty, - previous, **colors())) - continue + if s.version: + msg("{}: Checking out {W}{}{_}".format(pretty,s.version,**colors())) + r.checkout(s.version) # Check that the project has a Makefile if not os.path.exists(r.makefile): raise Error(f"{r.fullname} has no giteapc.make") env = os.environ.copy() - if config: + if s.config: env["GITEAPC_CONFIG"] = config env["GITEAPC_PREFIX"] = PREFIX_FOLDER @@ -287,59 +307,65 @@ def build(*args, install=False, skip_configure=False, update=False): # repo install command # -def install(*args, use_https=False, use_ssh=False, update=False, yes=False): - if args == (): +def search_dependencies(names, fetched, plan, **kwargs): + for name in names: + s = Spec(name) + r = s.resolve() + + if r.fullname not in fetched: + fetch(r.fullname, **kwargs) + fetched.add(r.fullname) + # Schedule dependencies before r + search_dependencies(r.dependencies(), fetched, plan, **kwargs) + plan.append(s) + +def install(*args, use_https=False, use_ssh=False, update=False, yes=False, + dry_run=False): + + # Update all repositories + if args == () and update == True: + args = [ r.fullname for r in LocalRepo.all() ] + + # Fetch every repository and determine its dependencies to form a basic + # plan of what to build in what order + + basic_plan = [] + search_dependencies(args, set(), basic_plan, use_https=use_https, + use_ssh=use_ssh, update=update) + + # Sanitize the build plan by checking occurrences of the same repository + # are consistent and eliminating duplicates + + # Build plan organized by name + named = dict() + # Final plan + plan = [] + + for s in basic_plan: + r = s.repo + if r.fullname not in named: + named[r.fullname] = s + plan.append(s) + continue + s2 = named[r.fullname] + if not s2.compatible_with(s): + return fatal(f"repo install: cannot install both {s} and {s2}") + + # Plan review and confirmation + + msg("Will install:", ", ".join(s.pretty_str() for s in plan)) + + if dry_run: return 0 - def recursive_fetch(spec, fetched_so_far): - repos_to_build = [] - repo, version, config = split_config(spec) - r = resolve(repo, local_only=True) - - if r.fullname not in fetched_so_far: - fetched_so_far.add(r.fullname) - fetch(repo, use_https=use_https, use_ssh=use_ssh, update=update) - - for dep_spec in r.dependencies(): - rtb, fsf = recursive_fetch(dep_spec, fetched_so_far) - repos_to_build += rtb - fetched_so_far = fetched_so_far.union(fsf) - - return repos_to_build + [(r, version, config)], fetched_so_far - - # First download every repository, and only then build - repos_to_build = [] - fetched_so_far = set() - - for spec in args: - rtb, fsf = recursive_fetch(spec, fetched_so_far) - repos_to_build += rtb - fetched_so_far = fetched_so_far.union(fsf) - - # Eliminate duplicates and look for version collisions - rd = dict() - - for (r, version, config) in repos_to_build: - name = r.fullname - if name not in rd: - rd[name] = (version, config) - elif rd[name] != (version, config): - s1 = make_config(name, *rd[name]) - s2 = make_config(name, version, config) - return fatal(f"repo install: cannot install both {s1} and {s2}") - - msg("Will install:", ", ".join(make_config(pretty_repo(r), version, config) - for (r, version, config) in repos_to_build)) - if not yes: msg("Is that okay (Y/n)? ", end="") confirm = input().strip() if confirm not in ["Y", "y", ""]: return 1 - for (r, version, config) in repos_to_build: - build(make_config(r.fullname, version, config), install=True, - update=update) + # Final build + build(*plan, install=True) # # repo uninstall command