giteapc: improve install plan logic (#5)

Fixes: #7

Makefile

@@ -1,12 +1,13 @@
 PREFIX ?= $(HOME)/.local
-VERSION = 1.0
+DESTDIR ?= ""
+VERSION = 1.1
 
 install: $(bin)
-	install -d $(PREFIX)/bin
-	sed -e 's*%PREFIX%*$(PREFIX)*; s*%VERSION%*$(VERSION)*' giteapc.py > $(PREFIX)/bin/giteapc
-	chmod +x $(PREFIX)/bin/giteapc
-	install -d $(PREFIX)/lib/giteapc/giteapc
-	install giteapc/*.py $(PREFIX)/lib/giteapc/giteapc
+	install -d $(DESTDIR)$(PREFIX)/bin
+	sed -e 's*%PREFIX%*$(PREFIX)*; s*%VERSION%*$(VERSION)*' giteapc.py > $(DESTDIR)$(PREFIX)/bin/giteapc
+	chmod +x $(DESTDIR)$(PREFIX)/bin/giteapc
+	install -d $(DESTDIR)$(PREFIX)/lib/giteapc/giteapc
+	install giteapc/*.py $(DESTDIR)$(PREFIX)/lib/giteapc/giteapc
 
 uninstall:
 	rm -f $(PREFIX)/bin/giteapc

README.md

@@ -0,0 +1,165 @@
+# GiteaPC: A build/install automation tool
+
+GiteaPC ("Git-Enabled Applications of Planète Casio") is a Python tool to automate the installation of programs and libraries from the [Planète Casio Git forge](https://git.planet-casio.com/). It basically clones, pulls and checks out repositories then runs configure/make/install in them. It is sometimes called a package manager for the fxSDK, since it is the recommended way to install the SDK and its libraries.
+
+## Getting GiteaPC
+
+**Mac OS note**: there are some adjustments, please look at [Choukas' guide](https://www.planet-casio.com/Fr/forums/topic16614-8-giteapc-installer-et-mettre-a-jour-automatiquement-des-projets-gitea.html#186763).
+
+In order to use GiteaPC you will need Python, git and some common building tools. The following commands will install them with the package manager for common distributions:
+
+```bash
+# For users of Debian, Ubuntu, WSL on Microsft Windows, and other Debian-based:
+% sudo apt install curl git python3 build-essential pkg-config
+
+# For users of Arch Linux, Manjaro, and other Arch-based:
+% sudo pacman -S curl git python3 gcc make pkgconf
+```
+
+You can then use the following one-liner to install GiteaPC with itself:
+
+```bash
+% curl "https://git.planet-casio.com/Lephenixnoir/GiteaPC/raw/branch/master/install.sh" -o /tmp/giteapc-install.sh && bash /tmp/giteapc-install.sh
+```
+
+Feel invited to [read the script first](https://git.planet-casio.com/Lephenixnoir/GiteaPC/src/branch/master/install.sh) or even install manually.
+
+GiteaPC might ask you to add a folder to your PATH in these terms:
+
+```
+<giteapc> In order to use programs installed by GiteaPC, you will need to add their
+<giteapc> install folder to your PATH. This can be done automatically when you log
+<giteapc> in by adding the following command to your startup file:
+<giteapc>
+<giteapc>   export PATH="$PATH:/home/el/.local/bin"
+<giteapc>
+<giteapc> -> Press Enter to add this command to /home/el/.profile, or
+<giteapc> -> Type another file name to add this command to, or
+<giteapc> -> Type "-" to skip setting the PATH entirely.
+```
+
+If you don't know what it is then press Enter then log out of your computer and in again to update the PATH.
+
+To check that the tool is properly installed, type `giteapc` in a terminal; you should get a help message. If you type `giteapc list` you will get a list of installed repositories, which should contain just GiteaPC itself. Since GiteaPC can update itself you only need to do this section once.
+
+## Installing the fxSDK with GiteaPC
+
+Once you have GiteaPC, you can install the fxSDK with it. There are some more system dependencies detailed below:
+
+```bash
+# For users of WSL on Microsoft Windows:
+% sudo apt install cmake python3-pil libusb-1.0-0-dev libsdl2-dev libpng-dev libncurses-dev
+
+# For users of Debian, Ubuntu, and other Debian-based:
+% sudo apt install cmake python3-pil libusb-1.0-0-dev libsdl2-dev libudisks2-dev libglib2.0-dev libpng-dev
+
+# For users of Arch Linux, Manjaro, and other Arch-based:
+% sudo pacman -S cmake python-pillow libusb sdl2 udisks2 libpng ncurses
+```
+
+* CMake: This is the canonical build system for fxSDK tools and add-ins.
+* PIL/Pillow: This image processing library is used for asset conversion in fxconv.
+* libusb: This is used in fxlink to communicate with the calculator over USB.
+* SDL2: This is used in fxlink to show real-time video captures.
+* ncurses: For the TUI in `fxlink -t`.
+* UDisks2: This is used in fxlink to copy files to the calculator on the command-line. (optional)
+* GLib: This is needed to communicate with UDisks2 on the DBus interface. (optional)
+
+On Linux you can skip UDisks2/GLib if it doesn't fit your system; in this case write `Lephenixnoir/fxsdk:noudisks2` instead of `Lephenixnoir/fxsdk` below.
+
+First use GiteaPC to install to install the [command-line fxSDK tools](https://git.planet-casio.com/Lephenixnoir/fxsdk) and a cross-compiler ([binutils](https://git.planet-casio.com/Lephenixnoir/sh-elf-binutils/src/branch/dev) then [GCC](https://git.planet-casio.com/Lephenixnoir/sh-elf-gcc)). This will take quite long, usually about 30 minutes to build binutils and GCC. This is also a good time to compile [gdb](https://git.planet-casio.com/Lephenixnoir/sh-elf-gdb) for remote debugging.
+
+```bash
+% giteapc install Lephenixnoir/fxsdk Lephenixnoir/sh-elf-binutils Lephenixnoir/sh-elf-gcc Lephenixnoir/sh-elf-gdb
+```
+
+At this stage the cross-compiler will be installed, but not the C++ library, because the C library is not yet available. So we continue by installing the [math library](https://git.planet-casio.com/Lephenixnoir/OpenLibm) and the [C library](https://git.planet-casio.com/Vhex-Kernel-Core/fxlibc/), before reinstalling the GCC repo which will pick up where it left off and build the C++ standard library.
+
+```bash
+% giteapc install Lephenixnoir/OpenLibm Vhex-Kernel-Core/fxlibc
+% giteapc install Lephenixnoir/sh-elf-gcc
+```
+
+Finally, we can install the gint kernel and any libraries we need such as [libprof](https://git.planet-casio.com/Lephenixnoir/libprof). Any repository with the `giteapc` tag ([list here](https://git.planet-casio.com/explore/repos?q=giteapc&topic=1) or with `giteapc list -r`) can be installed.
+
+```bash
+% giteapc install Lephenixnoir/gint Lephenixnoir/libprof
+```
+
+## Updating repositories
+
+When there is a new version of a tool or library, you can update your install by typing `giteapc install -u`, which will pull and reinstall all repositories. Expensive repositories like binutils and gcc will *not* rebuild unless they change versions (which is rare and always announced) so updates should only take a few seconds.
+
+## Detailed use
+
+**List and search repositories**
+
+Use `giteapc list -r` to list the online repos that can be installed with GiteaPC, and `giteapc list` to list all the repositories you have on your computer. If an additional argument is given, it will be used to filter by name and description. For example, `giteapc list -r gcc`.
+
+**Install and update repositories**
+
+Use `giteapc install` to install a repository and `giteapc install -u` to pull before installing (update).
+
+**Install specific versions**
+
+Repository names in install and build commands accept two suffixes: `@version` and `:config` (in that order). The first allows you to select a branch or a tag (with git checkout). The second allows you to customize the build with repository-specific configuration files.
+
+For example, to install the development version of gint, one can write:
+
+```bash
+% giteapc install Lephenixnoir/gint@dev
+```
+
+Note that the repository will then remain on the `dev` branch until you explicitly install `Lephenixnoir/gint@master` to go back to the main branch.
+
+As another example, [Lephenixnoir/sh-elf-gcc](/Lephenixnoir/sh-elf-gcc) provides a configuration called `:any` which skips version upgrades if the cross-compiler is already installed and you don't want to update it. To use that configuration, write:
+
+```bash
+% giteapc install Lephenixnoir/sh-elf-gcc:any
+```
+
+**Low-level commands**
+
+* `giteapc fetch` allows you to clone or update (git fetch) a repository without touching anything.
+* `giteapc build` allows to configure/build a repository without installing it or to recompile without reconfiguring.
+* `giteapc show` allows you to see the available versions of a local or remote repository.
+
+See the help (`giteapc --help`) for the details of the options.
+
+**Uninstall a repository**
+
+`gitapc uninstall` uninstalls a repository and removes the local clone; `giteapc uninstall -k` uninstalls the repository but keeps the local clone. Dependencies are not checked during an uninstall so keep an eye out for them.
+
+## Support GiteaPC in your project
+
+Any repository on this forge can be installed with GiteaPC provided it satisfies a few requirements:
+
+* Have the "giteapc" topic on the web interface (which can be added by clicking the "Manage topics" link on the repo's page): this is how compatibility is announced.
+* Provide a `giteapc.make` which contains some metadata, includes `giteapc-config.make` and provides four targets `configure`, `build`, `install` and `uninstall` (details below).
+* Have `giteapc-config.make` in `.gitignore`. This file is a GiteaPC-managed symbolic link to the current configuration `giteapc-config-*.make` so we don't want it to dirty the clone.
+
+The `giteapc.make` file should look like this:
+
+```makefile
+# giteapc: version=1
+# giteapc: depends=Lephenixnoir/sh-elf-gcc
+
+-include giteapc-config.make
+
+configure:
+    ...
+build:
+    ...
+install:
+    ...
+uninstall:
+    ...
+
+.PHONY: configure build install uninstall
+```
+
+First the metadata; there are two for now: `version` (must be `1`) and `depends` (a list of repo dependencies). Then the inclusion of `giteapc-config.make` if it exists. And finally, the `configure`, `build`, `install`, and `uninstall` rules, where you can run whatever code you need.
+
+You can provide configurations by adding `giteapc-config-*.make` files. Usually these would export a variable for the rest of the Makefile to use; see [sh-elf-gcc](/Lephenixnoir/sh-elf-gcc) for an example.
+
+See also the [Lephenixnoir/Template-gint-library](https://git.planet-casio.com/Lephenixnoir/Template-gint-library) repository for a template of an fxSDK library, which comes with a decently-featured build system and supports GiteaPC.

giteapc.py

@@ -2,8 +2,8 @@
 
 """
 GiteaPC is an automated installer/updated for repositories of the Planète Casio
-Gitea forge (https://gitea.planet-casio.com/). It is mainly used to set up
-installs of the add-in development fxSDK.
+Git forge (https://git.planet-casio.com/). It is mainly used to set up installs
+of the add-in development fxSDK.
 """
 
 # Install prefix (inserted at compile-time)
@@ -29,7 +29,7 @@ usage_string = """
 usage: {R}giteapc{_} [{R}list{_}|{R}fetch{_}|{R}show{_}|{R}build{_}|{R}install{_}|{R}uninstall{_}] [{g}{i}ARGS...{_}]
 
 GiteaPC is a tool to automatically clone, install and update repositories from
-the Planète Casio Gitea forge. In the following commands, each {g}{i}REPOSITORY{_}
+the Planète Casio Git forge. In the following commands, each {g}{i}REPOSITORY{_}
 is either a full name like "Lephenixnoir/sh-elf-gcc", or a short name like
 "sh-elf-gcc" when there is no ambiguity.
 

giteapc/config.py

@@ -1,7 +1,7 @@
 import os
 
-# URL to the Gitea forge supplying the resources
-GITEA_URL = "https://gitea.planet-casio.com"
+# URL to the Git forge supplying the resources
+GIT_URL = "https://git.planet-casio.com"
 # Data folder to store repositores
 XDG_DATA_HOME = os.getenv("XDG_DATA_HOME", os.getenv("HOME")+"/.local/share")
 REPO_FOLDER = os.getenv("GITEAPC_HOME") or XDG_DATA_HOME + "/giteapc"

giteapc/gitea.py

@@ -26,7 +26,7 @@ if _http_get is None:
 def _get(url, params=None):
     if _http_get is None:
         raise Error("cannot access forge (network is disabled)")
-    return _http_get(giteapc.config.GITEA_URL + "/api/v1" + url, params)
+    return _http_get(giteapc.config.GIT_URL + "/api/v1" + url, params)
 
 # Search for repositories
 def repo_search(keyword, **kwargs):

giteapc/repo.py

@@ -105,7 +105,7 @@ def pretty_repo(r):
 
 class Spec:
     # A spec in REPOSITORY[@VERSION][:CONFIGURATION]
-    RE_SPEC = re.compile(r'^([^@:]+)(?:@([^@:]+))?(?:[:]([^@:]+))?')
+    RE_SPEC = re.compile(r'^([^@:]+)(?:@([^@:]+))?(?:([:])([^@:]*))?')
 
     def __init__(self, string):
         m = re.match(Spec.RE_SPEC, string)
@@ -114,13 +114,16 @@ class Spec:
 
         self.name    = m[1]
         self.version = m[2]
-        self.config  = m[3]
+        self.config  = m[4] or (m[3] and "")
         self.repo    = None
 
     def resolve(self, local_only=False, remote_only=False):
         self.repo = resolve(self.name, local_only, remote_only)
         return self.repo
 
+    def is_blank(self):
+        return self.version is None and self.config is None
+
     def str(self, pretty=False):
         if self.repo and pretty:
             name = pretty_repo(self.repo)
@@ -133,6 +136,9 @@ class Spec:
         config  = f":{self.config}"  if self.config  else ""
         return name + version + config
 
+    def same_version_config_as(self, other):
+        return self.version == other.version and self.config == other.config
+
     def __str__(self):
         return self.str(pretty=False)
     def __repr__(self):
@@ -286,7 +292,7 @@ def build(*args, install=False, skip_configure=False):
             raise Error(f"{r.fullname} has no giteapc.make")
 
         env = os.environ.copy()
-        if s.config:
+        if s.config is not None:
             env["GITEAPC_CONFIG"] = s.config
             r.set_config(s.config)
         env["GITEAPC_PREFIX"] = PREFIX_FOLDER
@@ -329,36 +335,42 @@ def install(*args, use_https=False, use_ssh=False, update=False, yes=False,
     dry_run=False):
 
     # Update all repositories
-    if args == () and update == True:
+    if args == () and not update:
+        return fatal(f"repo install: need one argument (unless -u is given)")
+    if args == () and update:
         args = [ r.fullname for r in LocalRepo.all() ]
 
     # Fetch every repository and determine its dependencies to form a basic
-    # plan of what to build in what order
+    # plan of what repo to build in what order, but without version/config info
 
-    basic_plan = []
-    search_dependencies(args, set(), basic_plan, use_https=use_https,
+    dep_order = []
+    search_dependencies(args, set(), dep_order, use_https=use_https,
         use_ssh=use_ssh, update=update)
 
-    # Sanitize the build plan by checking occurrences of the same repository
-    # are consistent and eliminating duplicates
-
-    # Build plan organized by name
-    named = dict()
-    # Final plan
-    plan = []
-
-    for s in basic_plan:
-        r = s.repo
-        if r.fullname not in named:
-            named[r.fullname] = s
-            plan.append(s)
+    # Apply configurations on everyone and make sure they are no contradictions
+    # and all configs exist
+    spec_by_repo = { spec.repo.fullname: spec for spec in dep_order }
+    for arg in args:
+        s = Spec(arg)
+        if s.is_blank():
             continue
-        s2 = named[r.fullname]
-        if not s2.compatible_with(s):
-            return fatal(f"repo install: cannot install both {s} and {s2}")
+        r = s.resolve(local_only=True)
+
+        name = s.repo.fullname
+        if not s.same_version_config_as(spec_by_repo[name]):
+            return fatal(f"repo install: multiple specs for {name}: {s}, " +
+                f"{spec_by_repo[name]}")
+
+        if s.config not in ["", None] and s.config not in r.configs():
+            return fatal(f"repo install: no config {s.config} for {name}" +
+                " (configs: " + ", ".join(r.configs()) + ")")
+
+        spec_by_repo[name] = s
+
+    # Final plan
+    plan = [ spec_by_repo[s.repo.fullname] for s in dep_order ]
 
     # Plan review and confirmation
-
     msg("Will install:", ", ".join(s.pretty_str() for s in plan))
 
     if dry_run:

giteapc/repos.py

@@ -101,10 +101,24 @@ class LocalRepo:
                 return False
             raise e
 
+    def updateRemotes(self):
+        # Automatically switch old gitea URLs to Forgejo
+        for r in self._git(["remote"], stdout=PIPE).stdout.split():
+            url = self._git(["remote", "get-url", r], stdout=PIPE).stdout.rstrip(b"\n")
+            new = url
+            if url.startswith(b"gitea@gitea.planet-casio.com:"):
+                new = b"forgejo@git" + url[11:]
+            if url.startswith(b"https://gitea.planet-casio.com/"):
+                new = b"https://git." + url[14:]
+            if new != url:
+                self._git(["remote", "set-url", r, new])
+
     def fetch(self):
+        self.updateRemotes()
         self._git(["fetch"])
 
     def pull(self):
+        self.updateRemotes()
         if self.is_on_branch():
             self._git(["pull"])
 
@@ -138,15 +152,37 @@ class LocalRepo:
     # Make commands
 
     def make(self, target, env=None):
+        # Use GNU Make even on OpenBSD
+        if shutil.which("gmake") is not None:
+            command = "gmake"
+        else:
+            command = "make"
+
         with ChangeDirectory(self.folder):
-            return run(["make", "-f", "giteapc.make", target], env=env)
+            return run([command, "-f", "giteapc.make", target], env=env)
+
+    def configs(self):
+        RE_NAME = re.compile(r'giteapc-config-(.*)\.make')
+        files = glob.glob(self.folder + "/giteapc-config-*.make")
+        return sorted(RE_NAME.match(os.path.basename(x))[1] for x in files)
 
     def set_config(self, config):
         source = self.folder + f"/giteapc-config.make"
         target = self.folder + f"/giteapc-config-{config}.make"
 
-        if os.path.exists(source):
+        if config == "":
+            if os.path.islink(source):
+                os.remove(source)
+            return
+
+        if not os.path.exists(target):
+            raise Error(f"config '{config}' does not exist")
+
+        if os.path.islink(source):
             os.remove(source)
+        elif os.path.exists(source):
+            raise Error("giteapc-config.make is not a link, was it altered?")
+
         os.symlink(target, source)
 
     # Metadata

install.sh

@@ -2,15 +2,14 @@
 
 TAG=$(printf "\x1b[36m<giteapc>\x1b[0m")
 PREFIX=${GITEAPC_PREFIX:-$HOME/.local}
-URL="https://gitea.planet-casio.com/Lephenixnoir/GiteaPC/archive/master.tar.gz"
+URL="https://git.planet-casio.com/Lephenixnoir/GiteaPC"
 
 # Download the source code
 
 cd "$(mktemp -d)"
-curl "$URL" -o giteapc-master.tar.gz
-tar -xzf giteapc-master.tar.gz && cd giteapc
+git clone --depth=1 "$URL" giteapc && cd giteapc
 
-# Bootstrap the program by installing it with iself (allowing updates)
+# Bootstrap the program by installing it with itself (allowing updates)
 
 python3 giteapc.py install Lephenixnoir/GiteaPC