diff --git a/.gitignore b/.gitignore index 6e4266f..e856212 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ __pycache__/ *.py[cod] *$py.class +bkp/ diff --git a/config.yaml b/config.yaml deleted file mode 100644 index abbd89c..0000000 --- a/config.yaml +++ /dev/null @@ -1,11 +0,0 @@ -repo: - base_url: "https://arch.middleearth.fr" - db_name: "casio.db" - arch: "x86_64" - -chroot: - pacman: "/etc/pacman.conf" - makepkg: "/etc/makepkg.conf" - -templates: "./pkgtmpl" -pkgbuilds: "/home/eldeberen/tmp/pkgbuild" diff --git a/config/config.yaml b/config/config.yaml new file mode 100644 index 0000000..ac34d2c --- /dev/null +++ b/config/config.yaml @@ -0,0 +1,6 @@ +repo: + base_url: "https://arch.middleearth.fr" + db_name: "casio.db" + arch: "x86_64" + +output: "/home/eldeberen/tmp/pkgbuild" diff --git a/packages.yaml b/config/packages.yaml similarity index 98% rename from packages.yaml rename to config/packages.yaml index cba6d80..1cd7757 100644 --- a/packages.yaml +++ b/config/packages.yaml @@ -49,11 +49,13 @@ repo: "https://gitea.planet-casio.com/Lephenixnoir/libimg.git" maintainers: - "Eldeberen " + force: True - name: "libprof" repo: "https://gitea.planet-casio.com/Lephenixnoir/libprof.git" maintainers: - "Eldeberen " + force: True - name: "justui" repo: "https://gitea.planet-casio.com/Lephenixnoir/justui.git" diff --git a/makepkg.conf b/config/pacman/makepkg.conf similarity index 80% rename from makepkg.conf rename to config/pacman/makepkg.conf index 5df7a79..fc94185 100644 --- a/makepkg.conf +++ b/config/pacman/makepkg.conf @@ -1,4 +1,7 @@ #!/hint/bash +# +# /etc/makepkg.conf +# ######################################################################### # SOURCE ACQUISITION @@ -6,10 +9,10 @@ # #-- The download utilities that makepkg should use to acquire sources # Format: 'protocol::agent' -DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u' - 'ftp::/usr/bin/curl -gqfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' - 'http::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' - 'https::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' +DLAGENTS=('file::/usr/bin/curl -qgC - -o %o %u' + 'ftp::/usr/bin/curl -qgfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' + 'http::/usr/bin/curl -qgb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'https::/usr/bin/curl -qgb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' 'rsync::/usr/bin/rsync --no-motd -z %u %o' 'scp::/usr/bin/scp -C %u %o') @@ -21,6 +24,7 @@ DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u' #-- The package required by makepkg to download VCS sources # Format: 'protocol::package' VCSCLIENTS=('bzr::bzr' + 'fossil::fossil' 'git::git' 'hg::mercurial' 'svn::subversion') @@ -28,26 +32,30 @@ VCSCLIENTS=('bzr::bzr' ######################################################################### # ARCHITECTURE, COMPILE FLAGS ######################################################################### - +# CARCH="x86_64" CHOST="x86_64-pc-linux-gnu" #-- Compiler and Linker Flags -CPPFLAGS="-D_FORTIFY_SOURCE=2" -CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt" -CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt" +#CPPFLAGS="" +CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt -fexceptions \ + -Wp,-D_FORTIFY_SOURCE=2 -Wformat -Werror=format-security \ + -fstack-clash-protection -fcf-protection" +CXXFLAGS="$CFLAGS -Wp,-D_GLIBCXX_ASSERTIONS" LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now" +RUSTFLAGS="-C opt-level=2" #-- Make Flags: change this for DistCC/SMP systems -MAKEFLAGS="-j16" +#MAKEFLAGS="-j2" #-- Debugging flags DEBUG_CFLAGS="-g -fvar-tracking-assignments" DEBUG_CXXFLAGS="-g -fvar-tracking-assignments" +DEBUG_RUSTFLAGS="-C debuginfo=2" ######################################################################### # BUILD ENVIRONMENT ######################################################################### # -# Defaults: BUILDENV=(!distcc !color !ccache check !sign) +# Makepkg defaults: BUILDENV=(!distcc !color !ccache check !sign) # A negated environment option will do the opposite of the comments below. # #-- distcc: Use the Distributed C/C++/ObjC compiler @@ -70,7 +78,7 @@ BUILDENV=(!distcc color !ccache check !sign) # These are default values for the options=() settings ######################################################################### # -# Default: OPTIONS=(!strip docs libtool staticlibs emptydirs !zipman !purge !debug) +# Makepkg defaults: OPTIONS=(!strip docs libtool staticlibs emptydirs !zipman !purge !debug !lto) # A negated option will do the opposite of the comments below. # #-- strip: Strip symbols from binaries/libraries @@ -81,11 +89,12 @@ BUILDENV=(!distcc color !ccache check !sign) #-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip #-- purge: Remove files specified by PURGE_TARGETS #-- debug: Add debugging flags as specified in DEBUG_* variables +#-- lto: Add compile flags for building with link time optimization # -OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !debug) +OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !debug !lto) -#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 -INTEGRITY_CHECK=(md5) +#-- File integrity checks to use. Valid: md5, sha1, sha224, sha256, sha384, sha512, b2 +INTEGRITY_CHECK=(sha256) #-- Options to be used when stripping binaries. See `man strip' for details. STRIP_BINARIES="--strip-all" #-- Options to be used when stripping shared libraries. See `man strip' for details. @@ -138,5 +147,12 @@ COMPRESSLZ=(lzip -c -f) # EXTENSION DEFAULTS ######################################################################### # -PKGEXT='.pkg.tar.xz' +PKGEXT='.pkg.tar.zst' SRCEXT='.src.tar.gz' + +######################################################################### +# OTHER +######################################################################### +# +#-- Command used to run pacman as root, instead of trying sudo and su +#PACMAN_AUTH=() diff --git a/config/pacman/pacman.conf b/config/pacman/pacman.conf new file mode 100644 index 0000000..8aaed6d --- /dev/null +++ b/config/pacman/pacman.conf @@ -0,0 +1,53 @@ +# +# GENERAL OPTIONS +# + +[options] +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +#IgnorePkg = +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +Color +#TotalDownload +CheckSpace +#VerbosePkgLists + + +# +# REPOSITORIES +# + +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra] +Server = https://mirrors.kernel.org/archlinux/$repo/os/$arch +Include = /etc/pacman.d/mirrorlist + +[community] +Include = /etc/pacman.d/mirrorlist + +[casio] +SigLevel = Optional TrustAll +Server = https://arch.middleearth.fr/$arch diff --git a/pkgtmpl/fxlibc.j2 b/config/pkgbuilds/fxlibc.j2 similarity index 97% rename from pkgtmpl/fxlibc.j2 rename to config/pkgbuilds/fxlibc.j2 index f349df1..56e4cb5 100644 --- a/pkgtmpl/fxlibc.j2 +++ b/config/pkgbuilds/fxlibc.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="A C standard library for fx Casio calculators, built for gint" arch=('i686' 'x86_64') url="https://gitea.planet-casio.com/Lephenixnoir/${pkgname}" diff --git a/pkgtmpl/fxsdk.j2 b/config/pkgbuilds/fxsdk.j2 similarity index 98% rename from pkgtmpl/fxsdk.j2 rename to config/pkgbuilds/fxsdk.j2 index f42d191..b8bb5c6 100644 --- a/pkgtmpl/fxsdk.j2 +++ b/config/pkgbuilds/fxsdk.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc='Tools to program for the Casio fx9860 calculators' arch=('i686' 'x86_64') url="https://gitea.planet-casio.com/Lephenixnoir/fxsdk" diff --git a/pkgtmpl/gint.j2 b/config/pkgbuilds/gint.j2 similarity index 98% rename from pkgtmpl/gint.j2 rename to config/pkgbuilds/gint.j2 index 48e9507..dc57b62 100644 --- a/pkgtmpl/gint.j2 +++ b/config/pkgbuilds/gint.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc='Alternative library and kernel for add-in development on fx-9860G and fx-CG50 under Linux' arch=('i686' 'x86_64') url="https://gitea.planet-casio.com/Lephenixnoir/gint" diff --git a/pkgtmpl/isl.j2 b/config/pkgbuilds/isl.j2 similarity index 97% rename from pkgtmpl/isl.j2 rename to config/pkgbuilds/isl.j2 index 23cbddf..b9f532a 100644 --- a/pkgtmpl/isl.j2 +++ b/config/pkgbuilds/isl.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="Library for manipulating sets and relations of integer points bounded by linear constraints" arch=('i686' 'x86_64') url="http://isl.gforge.inria.fr/" diff --git a/pkgtmpl/justui.j2 b/config/pkgbuilds/justui.j2 similarity index 97% rename from pkgtmpl/justui.j2 rename to config/pkgbuilds/justui.j2 index 37449b0..0e68235 100644 --- a/pkgtmpl/justui.j2 +++ b/config/pkgbuilds/justui.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=2 +pkgrel={{ rel }} pkgdesc="Lephe's GUI toolkit for gint" arch=('i686' 'x86_64') url="https://gitea.planet-casio.com/Lephenixnoir/${pkgname}" diff --git a/pkgtmpl/libimg.j2 b/config/pkgbuilds/libimg.j2 similarity index 97% rename from pkgtmpl/libimg.j2 rename to config/pkgbuilds/libimg.j2 index 84e5719..809ed7b 100644 --- a/pkgtmpl/libimg.j2 +++ b/config/pkgbuilds/libimg.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=2 +pkgrel={{ rel }} pkgdesc="Lephe's GUI toolkit for gint" arch=('i686' 'x86_64') url="https://gitea.planet-casio.com/Lephenixnoir/${pkgname}" diff --git a/pkgtmpl/libp7.j2 b/config/pkgbuilds/libp7.j2 similarity index 97% rename from pkgtmpl/libp7.j2 rename to config/pkgbuilds/libp7.j2 index 696bf66..1136525 100644 --- a/pkgtmpl/libp7.j2 +++ b/config/pkgbuilds/libp7.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="Casio Communication Protocol 7.00 implementation" arch=('i686' 'x86_64') url="https://p7.planet-casio.com" diff --git a/pkgtmpl/libprof.j2 b/config/pkgbuilds/libprof.j2 similarity index 97% rename from pkgtmpl/libprof.j2 rename to config/pkgbuilds/libprof.j2 index 6154788..b41ead0 100644 --- a/pkgtmpl/libprof.j2 +++ b/config/pkgbuilds/libprof.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=2 +pkgrel={{ rel }} pkgdesc="A microsecond-level performance profiling library for gint" arch=('i686' 'x86_64') url="https://gitea.planet-casio.com/Lephenixnoir/${pkgname}" diff --git a/pkgtmpl/mkg3a.j2 b/config/pkgbuilds/mkg3a.j2 similarity index 97% rename from pkgtmpl/mkg3a.j2 rename to config/pkgbuilds/mkg3a.j2 index 89206a3..0321557 100644 --- a/pkgtmpl/mkg3a.j2 +++ b/config/pkgbuilds/mkg3a.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="A tool to create Casio FX-CG addon (.g3a) files." arch=('i686' 'x86_64') url="https://gitlab.com/taricorp/mkg3a/" diff --git a/pkgtmpl/openlibm-casio.j2 b/config/pkgbuilds/openlibm-casio.j2 similarity index 98% rename from pkgtmpl/openlibm-casio.j2 rename to config/pkgbuilds/openlibm-casio.j2 index c7165e0..b6a2851 100644 --- a/pkgtmpl/openlibm-casio.j2 +++ b/config/pkgbuilds/openlibm-casio.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=3 +pkgrel={{ rel }} pkgdesc='Fork of the OpenLibm math library with support for fx-9860G and fx-CG 50' arch=('i686' 'x86_64') url="https://gitea.planet-casio.com/Lephenixnoir/OpenLibm" diff --git a/pkgtmpl/p7.j2 b/config/pkgbuilds/p7.j2 similarity index 97% rename from pkgtmpl/p7.j2 rename to config/pkgbuilds/p7.j2 index 7f96536..145cc5d 100644 --- a/pkgtmpl/p7.j2 +++ b/config/pkgbuilds/p7.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="Casio Communication Protocol 7.00 implementation" arch=('i686' 'x86_64') url="https://p7.planet-casio.com/" diff --git a/pkgtmpl/p7screen.j2 b/config/pkgbuilds/p7screen.j2 similarity index 96% rename from pkgtmpl/p7screen.j2 rename to config/pkgbuilds/p7screen.j2 index dcef9e9..0e1db89 100644 --- a/pkgtmpl/p7screen.j2 +++ b/config/pkgbuilds/p7screen.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="Casio Communication Protocol 7.00 implementation" arch=('i686' 'x86_64') url="https://p7.planet-casio.com/" diff --git a/pkgtmpl/sh-elf-binutils-casio.j2 b/config/pkgbuilds/sh-elf-binutils-casio.j2 similarity index 98% rename from pkgtmpl/sh-elf-binutils-casio.j2 rename to config/pkgbuilds/sh-elf-binutils-casio.j2 index 6731202..5a887b2 100644 --- a/pkgtmpl/sh-elf-binutils-casio.j2 +++ b/config/pkgbuilds/sh-elf-binutils-casio.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="GNU binary utilities for the Casio calculators SuperH processors." arch=('i686' 'x86_64') url="https://www.gnu.org/software/binutils/" diff --git a/pkgtmpl/sh-elf-gcc-casio.j2 b/config/pkgbuilds/sh-elf-gcc-casio.j2 similarity index 98% rename from pkgtmpl/sh-elf-gcc-casio.j2 rename to config/pkgbuilds/sh-elf-gcc-casio.j2 index 6cc5d23..cf6ed4b 100644 --- a/pkgtmpl/sh-elf-gcc-casio.j2 +++ b/config/pkgbuilds/sh-elf-gcc-casio.j2 @@ -1,6 +1,6 @@ pkgname={{ name }} pkgver={{ tag }} -pkgrel=1 +pkgrel={{ rel }} pkgdesc="The GNU Compiler Collection for the Casio calculators SuperH processors." arch=(i686 x86_64) license=('GPL' 'LGPL') diff --git a/main.py b/main.py deleted file mode 100755 index 83bf76f..0000000 --- a/main.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import logging, sys, os -import requests as r -from datetime import date -from utils import * - -def main(arch): - packages = load_config() - init_directories() - - commands = [] - diffs = 0 - - # Manage each package - for p in packages: - # Retreive tag to apply - if 'tag' in p: - logging.debug(f"{p['name']} forced to {p['tag']}") - elif 'repo' in p: - p['tag'] = get_tags_from_git(p['repo']) - else: - p['tag'] = get_package_version(p['name'], arch) - - # Retreive current package version - current_tag = get_package_version(p['name'], arch) - - # Check if update is required - if p['force'] == True: - logging.info(f"{p['name']} will be rebuilt: {p['tag']}") - render_pkgbuild(p['name'], p['tag']) - elif p['tag'] != current_tag: - logging.info(f"{p['name']} will be updated: {current_tag} → {p['tag']}") - render_pkgbuild(p['name'], p['tag']) - else: - tarball, url = get_package_tarball(p['name'], arch) - logging.info(f"{p['name']} will be retreived from {url}") - content = r.get(url).content - with open(f"packages/{tarball}", "wb") as file: - file.write(content) - commands.append(f"pacman -U --noconfirm /tmp/packages/{tarball}") - - if p['tag'] != current_tag: - diffs += 1 - - # Write the installation script - with open("scripts/run.sh", "w") as file: - file.write("#!/usr/bin/env bash\n") - file.write("useradd user\n") - file.write("echo 'user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers\n") - file.write(" && \\\n".join(commands)) - - if diffs == 0: - logging.info("no updates planned") - sys.exit(0) - - # docker_build() - - - os.popen(f"rsync output/*.pkg.* khazad-dum:/var/lib/nginx/http/arch.middleearth.fr/{arch}/") - os.popen(f"ssh khazad-dum 'cd /var/lib/nginx/http/arch.middleearth.fr/{arch} && repose -z casio'") - - # TODO: - # - create a systemd timer to run this script - # - setup a web server to publish logs in case of failure - # - clean the 2 lines above - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Build some packages.') - parser.add_argument('-c', '--config-dir', default="/home/eldeberen/Programmation/MiddleArch/") - parser.add_argument('--arch', default="x86_64", help='target architecture') - parser.add_argument('--logs', default="/tmp", help='logs destination') # TODO: add clean path filter - - args = parser.parse_args() - - logging.basicConfig(format="[%(asctime)s] %(levelname)s (%(filename)s.%(funcName)s): %(message)s", - filename=os.path.join(args.logs, date.today().isoformat()), level=logging.INFO) - - main(args.arch) diff --git a/middlearch/__main__.py b/middlearch/__main__.py index 92a1663..fae7893 100644 --- a/middlearch/__main__.py +++ b/middlearch/__main__.py @@ -3,10 +3,10 @@ import argparse, logging import os.path as path -from chroot import Chroot -from config import Config -from package import Package -from repository import Repository +from .chroot import Chroot +from .config import Config +from .package import Package +from .repository import Repository parser = argparse.ArgumentParser(description='Build some packages.') @@ -23,10 +23,11 @@ logging.basicConfig(format="[%(asctime)s] %(levelname)s (%(filename)s.%(funcName config = Config(args.config_dir) repository = Repository(**config.get('repo')) -packages = Package.load(args.config_dir, repository) -#chroot = Chroot(config) +packages = Package.load(config, repository) + +chroot = Chroot(config, True) for package in packages: - package.render_pkgbuild(config) -# if package.build: -# chroot.makepkg(package, config) + if package.build: + package.render_pkgbuild(config) + chroot.makepkg(package) diff --git a/middlearch/chroot.py b/middlearch/chroot.py index 146e6b7..795d572 100644 --- a/middlearch/chroot.py +++ b/middlearch/chroot.py @@ -1,31 +1,50 @@ import logging, os, tempfile +import os import os.path as path +import shutil class Chroot(object): - def __init__(self, config): + def __init__(self, config, static=False): + self.config = config + # Create directories - self.basedir = tempfile.mkdtemp() - rootdir = path.join(self.basedir, "root") - logging.info(f"created temporary directory in {self.basedir}") + if static: + self.basedir = "/home/eldeberen/tmp/middlearch-chroot" + logging.debug(f"use static directory in {self.basedir}") + os.makedirs(self.basedir, exist_ok=True) + else: + logging.debug(f"create temporary directory in {self.basedir}") + self.basedir = tempfile.mkdtemp() + + self.rootdir = path.join(self.basedir, "root") + self.flags = f"-C {path.join(config.dir, 'pacman', 'pacman.conf')} " \ + f"-M {path.join(config.dir, 'pacman', 'makepkg.conf')}" # Initialize base-devel packages - pacman = f"-C {p}" if (p := config.get('chroot.pacman')) else "" - makepkg = f"-M {p}" if (p := config.get('chroot.makepkg')) else "" - _exec(f"mkarchroot {pacman} {makepkg} {rootdir} base-devel") + if not path.exists(self.rootdir): + logging.info(f"create chroot in {self.rootdir}") + _exec(f"mkarchroot {self.flags} {self.rootdir} base-devel") # Make sure everything is up-to-date - _exec(f"arch-nspawn {rootdir} pacman -Syu") + _exec(f"arch-nspawn {self.rootdir} pacman -Syu") - logging.info("chroot initialized") + logging.info(f"chroot ready ({self.rootdir})") - def makepkg(self, package, config): - current_dir = os.getcwd() - os.chdir(path.join(config.get('pkgbuilds'), package.name)) - _exec(f"makechrootpkg -c -r {self.basedir}") - os.chdir(current_dir) + def install(self, package): + logging.info(f"install {package.name} from remote repository") + _exec(f"arch-nspawn {self.rootdir} pacman -S {package.name}") -def _exec(cmd): - logging.info(cmd) + def makepkg(self, package): + logging.info(f"build {package.name}") + ref_dir = os.getcwd() + os.chdir(path.join(self.config.get('output'), package.name)) + _exec(f"makechrootpkg -r {self.basedir}") + os.chdir(ref_dir) + +def _exec(cmd, sudo=False): + if sudo: + cmd = "sudo " + cmd + logging.debug(cmd) r = os.popen(cmd) logging.debug(r.read()) diff --git a/middlearch/config.py b/middlearch/config.py index 961bd73..010c1ce 100644 --- a/middlearch/config.py +++ b/middlearch/config.py @@ -6,9 +6,12 @@ import yaml class Config(object): def __init__(self, config_dir): + self.dir = path.abspath(config_dir) + with open(path.join(config_dir, "config.yaml")) as file: self.conf = yaml.load(file, Loader=yaml.SafeLoader) - logging.info(f"{path.join(config_dir, 'config.yaml')} loaded") + logging.info(f"{path.join(self.dir, 'config.yaml')} loaded") + logging.debug('configuration:\n'+yaml.dump(self.conf)) def get(self, key, default=None): diff --git a/middlearch/git.py b/middlearch/git.py index a551506..6d88c64 100644 --- a/middlearch/git.py +++ b/middlearch/git.py @@ -16,13 +16,7 @@ def get_tag_from_git(remote): return "" raw_tags = os.popen(f"git ls-remote --tags {remote}").read().split() - tags = sorted( - list( - filter(''.__ne__, - map(extract_tag, raw_tags) - ), - ) - ) + tags = sorted(list(filter(''.__ne__, map(extract_tag, raw_tags)))) logging.debug(f"{remote}: {', '.join(tags)}") return tags[-1] diff --git a/templates/maintainers.j2 b/middlearch/maintainers.j2 similarity index 100% rename from templates/maintainers.j2 rename to middlearch/maintainers.j2 diff --git a/middlearch/package.py b/middlearch/package.py index 15e0709..38dd768 100644 --- a/middlearch/package.py +++ b/middlearch/package.py @@ -4,7 +4,7 @@ import os.path as path import requests as r import yaml -from git import get_tag_from_git +from .git import get_tag_from_git class Package(object): def __init__(self, name, **params): @@ -18,26 +18,36 @@ class Package(object): self.build = False logging.debug(self) + @property + def template(self): + return self.name + ".j2" + def __repr__(self): return f"{self.name} [repo: {self.repo}, tag: {self.tag}, force: {self.force}]" - def load(config_dir, repository): - with open(path.join(config_dir, "packages.yaml")) as file: + def load(config, repository): + with open(path.join(config.dir, "packages.yaml")) as file: p_list = yaml.load(file, Loader=yaml.SafeLoader) packages = [] for p in p_list: package = Package(**p) + package.pkgbuild = path.join(config.dir, f"pkgbuilds/{package.name}.j2") packages.append(package) - current_tag = repository.get_package_version(package.name) + current_version, current_rev = repository.get_package_version(package.name) - if package.tag > current_tag: - logging.info(f"{package.name} will be updated ({current_tag} → {package.tag})") + if package.tag > current_version: + package.rel = 1 package.build = True elif package.force: - logging.info(f"{package.name} will be updated ({package.tag})") + package.rel = current_rev + 1 package.build = True + else: + package.rel = current_rev + + if package.build: + logging.info(f"{package.name} will be updated ({current_version}-{current_rev} → {package.tag}-{package.rel})") else: logging.info(f"{package.name} will be skipped") @@ -45,43 +55,49 @@ class Package(object): return packages + def get_source_hash(self, config): + regex = re.compile("^source=.+(https?://.+\.tar\.[a-z]+)", re.MULTILINE) + with open(path.join(config.dir, "pkgbuilds", self.template)) as file: + content = file.read() + try: + url = regex.search(content).group(1) + except: + logging.warning(f"cannot find valid source for {self.name} (is it a git version?)") + return "SKIP" + url = url.replace("${pkgver}", self.tag) + url = url.replace("${pkgname}", self.name) + + try: + archive = r.get(url, timeout=10) + except Exception as e: + logging.warning(f"cannot retreive {url}") + logging.debug(e) + return "SKIP" + + hash = hashlib.sha256(archive.content).hexdigest() + return hash + def render_pkgbuild(self, config): - def get_hash(name, tag): - regex = re.compile("^source=.+(https://.+\.tar\.[a-z]+)", re.MULTILINE) - with open(f"{config.get('templates', 'templates')}/{name}.j2") as file: - content = file.read() - try: - url = regex.search(content).group(1) - except: - logging.warning(f"cannot find valid source for {name} (is it a git version?)") - return "SKIP" - url = url.replace("${pkgver}", tag) - url = url.replace("${pkgname}", name) - - archive = r.get(url) - hash = hashlib.sha256(archive.content).hexdigest() - - return hash - env = j2.Environment( - loader=j2.FileSystemLoader(config.get('templates', 'templates')), + loader=j2.FileSystemLoader(path.join(config.dir, "pkgbuilds")), autoescape=j2.select_autoescape([]) ) - template = env.get_template(f"{self.name}.j2") - hash = get_hash(self.name, self.tag) + template = env.get_template(self.template) + hash = self.get_source_hash(config) sysenv = j2.Environment( - loader=j2.FileSystemLoader('templates'), + loader=j2.FileSystemLoader('middlearch'), autoescape=j2.select_autoescape([]) ) maintainers_template = sysenv.get_template("maintainers.j2") - dest_dir = path.join(config.get('pkgbuilds', "pkgbuilds"), self.name) + dest_dir = path.join(config.get('output'), self.name) + dest_file = path.join(dest_dir, "PKGBUILD") os.makedirs(dest_dir, exist_ok=True) - with open(f"{config.get('pkgbuilds')}/{self.name}/PKGBUILD", "w") as file: + with open(dest_file, "w") as file: file.write(maintainers_template.render( maintainers=self.maintainers, contributors=self.contributors )) - file.write(template.render(name=self.name, tag=self.tag, hash=hash)) + file.write(template.render(name=self.name, tag=self.tag, rel=self.rel, hash=hash)) diff --git a/middlearch/repository.py b/middlearch/repository.py index 9189491..6060798 100644 --- a/middlearch/repository.py +++ b/middlearch/repository.py @@ -30,8 +30,8 @@ class Repository(object): logging.warning(f"{pkgname} is not online") version = "0.0.0" else: - regex = re.compile(".+-(\d+\.\d+(?:\.\d+)?)-\d+") - version = regex.search(name).group(1) - logging.debug(f"{pkgname} is {version} online") + regex = re.compile(".+-(\d+\.\d+(?:\.\d+)?)-(\d+)") + version, rev = regex.search(name).groups() + logging.debug(f"{pkgname} is {version}-{rev} online") - return version + return version, int(rev) diff --git a/pacman.conf b/pacman.conf deleted file mode 100644 index 673daaf..0000000 --- a/pacman.conf +++ /dev/null @@ -1,101 +0,0 @@ -# -# /etc/pacman.conf -# -# See the pacman.conf(5) manpage for option and repository directives - -# -# GENERAL OPTIONS -# -[options] -# The following paths are commented out with their default values listed. -# If you wish to use different paths, uncomment and update the paths. -#RootDir = / -#DBPath = /var/lib/pacman/ -#CacheDir = /var/cache/pacman/pkg/ -#LogFile = /var/log/pacman.log -#GPGDir = /etc/pacman.d/gnupg/ -#HookDir = /etc/pacman.d/hooks/ -HoldPkg = pacman glibc -#XferCommand = /usr/bin/curl -L -C - -f -o %o %u -#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u -#CleanMethod = KeepInstalled -Architecture = auto - -# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup -#IgnorePkg = -#IgnorePkg = -#IgnoreGroup = - -#NoUpgrade = -#NoExtract = - -# Misc options -#UseSyslog -Color -#TotalDownload -CheckSpace -#VerbosePkgLists - -# By default, pacman accepts packages signed by keys that its local keyring -# trusts (see pacman-key and its man page), as well as unsigned packages. -SigLevel = Required DatabaseOptional -LocalFileSigLevel = Optional -#RemoteFileSigLevel = Required - -# NOTE: You must run `pacman-key --init` before first using pacman; the local -# keyring can then be populated with the keys of all official Arch Linux -# packagers with `pacman-key --populate archlinux`. - -# -# REPOSITORIES -# - can be defined here or included from another file -# - pacman will search repositories in the order defined here -# - local/custom mirrors can be added here or in separate files -# - repositories listed first will take precedence when packages -# have identical names, regardless of version number -# - URLs will have $repo replaced by the name of the current repo -# - URLs will have $arch replaced by the name of the architecture -# -# Repository entries are of the format: -# [repo-name] -# Server = ServerName -# Include = IncludePath -# -# The header [repo-name] is crucial - it must be present and -# uncommented to enable the repo. -# - -# The testing repositories are disabled by default. To enable, uncomment the -# repo name header and Include lines. You can add preferred servers immediately -# after the header, and they will be used before the default mirrors. - -#[testing] -#Include = /etc/pacman.d/mirrorlist - -[core] -Include = /etc/pacman.d/mirrorlist - -[extra] -Server = https://mirrors.kernel.org/archlinux/$repo/os/$arch -Include = /etc/pacman.d/mirrorlist - -#[community-testing] -#Include = /etc/pacman.d/mirrorlist - -[community] -Include = /etc/pacman.d/mirrorlist - -# If you want to run 32 bit applications on your x86_64 system, -# enable the multilib repositories as required here. - -#[multilib-testing] -#Include = /etc/pacman.d/mirrorlist - -#[multilib] -#Include = /etc/pacman.d/mirrorlist - -# An example of a custom package repository. See the pacman manpage for -# tips on creating your own repositories. -[casio] -SigLevel = Optional TrustAll -Server = https://arch.middleearth.fr/$arch diff --git a/utils/__init__.py b/utils/__init__.py deleted file mode 100644 index c7dbc8d..0000000 --- a/utils/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .config import load_config, init_directories -from .git import get_tags_from_git -from .middlearch import get_package_version, get_package_tarball -from .pkgbuild import render_pkgbuild diff --git a/utils/config.py b/utils/config.py deleted file mode 100644 index afd59b8..0000000 --- a/utils/config.py +++ /dev/null @@ -1,21 +0,0 @@ -import logging -import os, shutil -import yaml - - -def load_config(path="packages.yaml"): - with open(path) as file: - packages = yaml.load(file, Loader=yaml.SafeLoader) - - logging.info(f"{len(packages)} packages loaded") - logging.debug(", ".join([p['name'] for p in packages])) - - return packages - -def clean_directories(): - for i in ["pkgbuilds", "packages", "output"]: - try: - shutil.rmtree(i) - except FileNotFoundError: - pass - os.mkdir(i) diff --git a/utils/git.py b/utils/git.py deleted file mode 100644 index eeaf7a8..0000000 --- a/utils/git.py +++ /dev/null @@ -1,24 +0,0 @@ -import logging -import os -import re - - -def get_tags_from_git(remote): - regex = re.compile("^refs/tags/v?(\d+\.\d+\.\d+)$") - def extract_tag(tag): - m = regex.search(tag) - if m: - return m.groups()[0] - return "" - - raw_tags = os.popen(f"git ls-remote --tags {remote}").read().split() - tags = sorted( - list( - filter(''.__ne__, - map(extract_tag, raw_tags) - ), - ) - ) - logging.debug(f"{remote}: {', '.join(tags)}") - - return tags[-1] diff --git a/utils/middlearch.py b/utils/middlearch.py deleted file mode 100644 index 362e398..0000000 --- a/utils/middlearch.py +++ /dev/null @@ -1,36 +0,0 @@ -from io import BytesIO -import logging -import re -import requests as r -import tarfile - - -BASE_URL = "https://arch.middleearth.fr" -DB_NAME = "casio.db" - -def get_from_regex(regex, package, arch): - file = BytesIO(r.get(f"{BASE_URL}/{arch}/{DB_NAME}").content) - db = tarfile.open(fileobj=file) - - name = list(filter( - lambda item: item.isdir() and package in item.name, - db.getmembers() - ))[0].name - - return regex.search(name).group(1) - -def get_package_version(package, arch): - regex = re.compile(".+-(\d+\.\d+(?:\.\d+)?)-\d+") - - version = get_from_regex(regex, package, arch) - logging.debug(f"{package}: found {version}") - - return version - -def get_package_tarball(package, arch): - regex = re.compile("(.+)") - - tarball = get_from_regex(regex, package, arch) + f"-{arch}.pkg.tar.xz" - logging.debug(f"{package}: found {tarball}") - - return tarball, f"{BASE_URL}/{arch}/{tarball}" diff --git a/utils/pkgbuild.py b/utils/pkgbuild.py deleted file mode 100644 index 4cf9ae2..0000000 --- a/utils/pkgbuild.py +++ /dev/null @@ -1,40 +0,0 @@ -import hashlib -import jinja2 as j2 -import logging -import os -import re -import requests as r - - -MAINTAINER = "Eldeberen " - -env = j2.Environment( - loader=j2.FileSystemLoader('templates'), - autoescape=j2.select_autoescape([]) -) - -def get_hash(name, tag): - regex = re.compile("^source=.+(https://.+\.tar\.[a-z]+)", re.MULTILINE) - with open(f"templates/{name}.j2") as file: - content = file.read() - try: - url = regex.search(content).group(1) - except: - logging.warning(f"cannot find valid source for {name} (is it a git version?)") - return "SKIP" - url = url.replace("${pkgver}", tag) - url = url.replace("${pkgname}", name) - - archive = r.get(url) - hash = hashlib.sha256(archive.content).hexdigest() - - return hash - -def render_pkgbuild(name, tag): - template = env.get_template(f"{name}.j2") - hash = get_hash(name, tag) - - os.mkdir(f"pkgbuilds/{name}") - - with open(f"pkgbuilds/{name}/PKGBUILD", "w") as file: - file.write(template.render(tag=tag, hash=hash, maintainer=MAINTAINER))