From 3c0d8d5e669686e722fe6417ddfd7e6c26487f6c Mon Sep 17 00:00:00 2001 From: Darks Date: Wed, 7 Jul 2021 19:36:08 +0200 Subject: [PATCH] WIP: I don't know the status of this, but I want it online --- config.yaml | 6 ++ main.py | 10 ++- makepkg.conf | 142 ++++++++++++++++++++++++++++++++++ middlearch/__main__.py | 19 +++-- middlearch/build/template.py | 35 +++++++++ middlearch/config/__init__.py | 2 + middlearch/config/chroot.py | 21 +++++ middlearch/config/config.py | 19 +++-- packages.yaml | 8 ++ pacman.conf | 101 ++++++++++++++++++++++++ 10 files changed, 347 insertions(+), 16 deletions(-) create mode 100644 config.yaml create mode 100644 makepkg.conf create mode 100644 middlearch/build/template.py create mode 100644 middlearch/config/__init__.py create mode 100644 pacman.conf diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..0072186 --- /dev/null +++ b/config.yaml @@ -0,0 +1,6 @@ +repo: + base_url: "https://arch.middleearth.fr" + db_name: "casio.db" + arch: "x86_64" + +maintainer: "Eldeberen " diff --git a/main.py b/main.py index e0c3213..83bf76f 100755 --- a/main.py +++ b/main.py @@ -27,11 +27,12 @@ def main(arch): current_tag = get_package_version(p['name'], arch) # Check if update is required - if p['tag'] != current_tag: + if p['force'] == True: + logging.info(f"{p['name']} will be rebuilt: {p['tag']}") + render_pkgbuild(p['name'], p['tag']) + elif p['tag'] != current_tag: logging.info(f"{p['name']} will be updated: {current_tag} → {p['tag']}") render_pkgbuild(p['name'], p['tag']) - commands.append(f"su -c '/tmp/build.sh /tmp/pkgbuilds/{p['name']}' user") - diffs += 1 else: tarball, url = get_package_tarball(p['name'], arch) logging.info(f"{p['name']} will be retreived from {url}") @@ -40,6 +41,9 @@ def main(arch): file.write(content) commands.append(f"pacman -U --noconfirm /tmp/packages/{tarball}") + if p['tag'] != current_tag: + diffs += 1 + # Write the installation script with open("scripts/run.sh", "w") as file: file.write("#!/usr/bin/env bash\n") diff --git a/makepkg.conf b/makepkg.conf new file mode 100644 index 0000000..5df7a79 --- /dev/null +++ b/makepkg.conf @@ -0,0 +1,142 @@ +#!/hint/bash + +######################################################################### +# SOURCE ACQUISITION +######################################################################### +# +#-- The download utilities that makepkg should use to acquire sources +# Format: 'protocol::agent' +DLAGENTS=('file::/usr/bin/curl -gqC - -o %o %u' + 'ftp::/usr/bin/curl -gqfC - --ftp-pasv --retry 3 --retry-delay 3 -o %o %u' + 'http::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'https::/usr/bin/curl -gqb "" -fLC - --retry 3 --retry-delay 3 -o %o %u' + 'rsync::/usr/bin/rsync --no-motd -z %u %o' + 'scp::/usr/bin/scp -C %u %o') + +# Other common tools: +# /usr/bin/snarf +# /usr/bin/lftpget -c +# /usr/bin/wget + +#-- The package required by makepkg to download VCS sources +# Format: 'protocol::package' +VCSCLIENTS=('bzr::bzr' + 'git::git' + 'hg::mercurial' + 'svn::subversion') + +######################################################################### +# ARCHITECTURE, COMPILE FLAGS +######################################################################### + +CARCH="x86_64" +CHOST="x86_64-pc-linux-gnu" + +#-- Compiler and Linker Flags +CPPFLAGS="-D_FORTIFY_SOURCE=2" +CFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt" +CXXFLAGS="-march=x86-64 -mtune=generic -O2 -pipe -fno-plt" +LDFLAGS="-Wl,-O1,--sort-common,--as-needed,-z,relro,-z,now" +#-- Make Flags: change this for DistCC/SMP systems +MAKEFLAGS="-j16" +#-- Debugging flags +DEBUG_CFLAGS="-g -fvar-tracking-assignments" +DEBUG_CXXFLAGS="-g -fvar-tracking-assignments" + +######################################################################### +# BUILD ENVIRONMENT +######################################################################### +# +# Defaults: BUILDENV=(!distcc !color !ccache check !sign) +# A negated environment option will do the opposite of the comments below. +# +#-- distcc: Use the Distributed C/C++/ObjC compiler +#-- color: Colorize output messages +#-- ccache: Use ccache to cache compilation +#-- check: Run the check() function if present in the PKGBUILD +#-- sign: Generate PGP signature file +# +BUILDENV=(!distcc color !ccache check !sign) +# +#-- If using DistCC, your MAKEFLAGS will also need modification. In addition, +#-- specify a space-delimited list of hosts running in the DistCC cluster. +#DISTCC_HOSTS="" +# +#-- Specify a directory for package building. +#BUILDDIR=/tmp/makepkg + +######################################################################### +# GLOBAL PACKAGE OPTIONS +# These are default values for the options=() settings +######################################################################### +# +# Default: OPTIONS=(!strip docs libtool staticlibs emptydirs !zipman !purge !debug) +# A negated option will do the opposite of the comments below. +# +#-- strip: Strip symbols from binaries/libraries +#-- docs: Save doc directories specified by DOC_DIRS +#-- libtool: Leave libtool (.la) files in packages +#-- staticlibs: Leave static library (.a) files in packages +#-- emptydirs: Leave empty directories in packages +#-- zipman: Compress manual (man and info) pages in MAN_DIRS with gzip +#-- purge: Remove files specified by PURGE_TARGETS +#-- debug: Add debugging flags as specified in DEBUG_* variables +# +OPTIONS=(strip docs !libtool !staticlibs emptydirs zipman purge !debug) + +#-- File integrity checks to use. Valid: md5, sha1, sha256, sha384, sha512 +INTEGRITY_CHECK=(md5) +#-- Options to be used when stripping binaries. See `man strip' for details. +STRIP_BINARIES="--strip-all" +#-- Options to be used when stripping shared libraries. See `man strip' for details. +STRIP_SHARED="--strip-unneeded" +#-- Options to be used when stripping static libraries. See `man strip' for details. +STRIP_STATIC="--strip-debug" +#-- Manual (man and info) directories to compress (if zipman is specified) +MAN_DIRS=({usr{,/local}{,/share},opt/*}/{man,info}) +#-- Doc directories to remove (if !docs is specified) +DOC_DIRS=(usr/{,local/}{,share/}{doc,gtk-doc} opt/*/{doc,gtk-doc}) +#-- Files to be removed from all packages (if purge is specified) +PURGE_TARGETS=(usr/{,share}/info/dir .packlist *.pod) +#-- Directory to store source code in for debug packages +DBGSRCDIR="/usr/src/debug" + +######################################################################### +# PACKAGE OUTPUT +######################################################################### +# +# Default: put built package and cached source in build directory +# +#-- Destination: specify a fixed directory where all packages will be placed +#PKGDEST=/home/packages +#-- Source cache: specify a fixed directory where source files will be cached +#SRCDEST=/home/sources +#-- Source packages: specify a fixed directory where all src packages will be placed +#SRCPKGDEST=/home/srcpackages +#-- Log files: specify a fixed directory where all log files will be placed +#LOGDEST=/home/makepkglogs +#-- Packager: name/email of the person or organization building packages +PACKAGER="Eldeberen " +#-- Specify a key to use for package signing +#GPGKEY="" + +######################################################################### +# COMPRESSION DEFAULTS +######################################################################### +# +COMPRESSGZ=(gzip -c -f -n) +COMPRESSBZ2=(bzip2 -c -f) +COMPRESSXZ=(xz -c -z -) +COMPRESSZST=(zstd -c -z -q -) +COMPRESSLRZ=(lrzip -q) +COMPRESSLZO=(lzop -q) +COMPRESSZ=(compress -c -f) +COMPRESSLZ4=(lz4 -q) +COMPRESSLZ=(lzip -c -f) + +######################################################################### +# EXTENSION DEFAULTS +######################################################################### +# +PKGEXT='.pkg.tar.xz' +SRCEXT='.src.tar.gz' diff --git a/middlearch/__main__.py b/middlearch/__main__.py index 1ae6f15..eeefefd 100644 --- a/middlearch/__main__.py +++ b/middlearch/__main__.py @@ -1,13 +1,22 @@ #!/usr/bin/env python3 -import argparse +import argparse, logging +import os.path as path +from config import Config, make_chroot parser = argparse.ArgumentParser(description='Build some packages.') -parser.add_argument('-c', '--config-dir', - default="/home/eldeberen/Programmation/MiddleArch/", +parser.add_argument('-c', metavar='config', dest="config", + default="/etc/middlearch", help="configuration directory") -parser.add_argument('--arch', default="x86_64", help='target architecture') -parser.add_argument('--logs', default="/tmp", help='logs destination') # TODO: add clean path filter +parser.add_argument('--verbose', '-v', action='count', default=0) args = parser.parse_args() + +verbosities = [logging.WARNING, logging.INFO, logging.DEBUG] +logging.basicConfig(format="[%(asctime)s] %(levelname)s (%(filename)s.%(funcName)s): %(message)s", + level=verbosities[min(args.verbose, len(verbosities)-1)]) + +config = Config(args.config) + +make_chroot(args.config) diff --git a/middlearch/build/template.py b/middlearch/build/template.py new file mode 100644 index 0000000..9dddb22 --- /dev/null +++ b/middlearch/build/template.py @@ -0,0 +1,35 @@ +import hashlib, logging, os, re +import jinja2 as j2 +import requests as r + + +env = j2.Environment( + loader=j2.FileSystemLoader('templates'), + autoescape=j2.select_autoescape([]) +) + +def get_hash(name, tag): + regex = re.compile("^source=.+(https://.+\.tar\.[a-z]+)", re.MULTILINE) + with open(f"templates/{name}.j2") as file: + content = file.read() + try: + url = regex.search(content).group(1) + except: + logging.warning(f"cannot find valid source for {name} (is it a git version?)") + return "SKIP" + url = url.replace("${pkgver}", tag) + url = url.replace("${pkgname}", name) + + archive = r.get(url) + hash = hashlib.sha256(archive.content).hexdigest() + + return hash + +def render_pkgbuild(name, tag): + template = env.get_template(f"{name}.j2") + hash = get_hash(name, tag) + + os.mkdir(f"pkgbuilds/{name}") + + with open(f"pkgbuilds/{name}/PKGBUILD", "w") as file: + file.write(template.render(tag=tag, hash=hash, maintainer=MAINTAINER)) diff --git a/middlearch/config/__init__.py b/middlearch/config/__init__.py new file mode 100644 index 0000000..7df46ac --- /dev/null +++ b/middlearch/config/__init__.py @@ -0,0 +1,2 @@ +from .config import Config +from .chroot import make_chroot diff --git a/middlearch/config/chroot.py b/middlearch/config/chroot.py index e69de29..df75bc1 100644 --- a/middlearch/config/chroot.py +++ b/middlearch/config/chroot.py @@ -0,0 +1,21 @@ +import logging, os, tempfile +import os.path as path + + +def make_chroot(configdir): + # Create directories + basedir = tempfile.mkdtemp() + rootdir = path.join(basedir, "root") + logging.info(f"created temporary directory in {basedir}") + + # Initialize base-devel packages + pacman = path.join(configdir, "pacman.conf") + makepkg = path.join(configdir, "makepkg.conf") + r = os.popen(f"mkarchroot -C {pacman} -M {makepkg} {rootdir} base-devel") + logging.debug(r.read()) + + # Make sure everything is up-to-date + r = os.popen(f"arch-nspawn {rootdir} pacman -Syu") + logging.debug(r.read()) + + logging.info("chroot initialized") diff --git a/middlearch/config/config.py b/middlearch/config/config.py index ff72a45..298204f 100644 --- a/middlearch/config/config.py +++ b/middlearch/config/config.py @@ -1,13 +1,16 @@ import logging import os, shutil import yaml +import os.path as path -def load_config(path="packages.yaml"): - with open(path) as file: - packages = yaml.load(file, Loader=yaml.SafeLoader) - - logging.info(f"{len(packages)} packages loaded") - logging.debug(", ".join([p['name'] for p in packages])) - - return packages +class Config(): + def __init__(self, config_dir): + with open(path.join(config_dir, "config.yaml")) as file: + self.params = yaml.load(file, Loader=yaml.SafeLoader) + logging.info(f"{path.join(config_dir, 'config.yaml')} loaded") + logging.debug('\n'+yaml.dump(self.params)) + with open(path.join(config_dir, "packages.yaml")) as file: + self.packages = yaml.load(file, Loader=yaml.SafeLoader) + logging.info(f"{len(self.packages)} packages loaded") + logging.debug('\n'+yaml.dump(self.packages)) diff --git a/packages.yaml b/packages.yaml index 9909e4d..21e3904 100644 --- a/packages.yaml +++ b/packages.yaml @@ -4,27 +4,35 @@ - name: openlibm-casio repo: https://gitea.planet-casio.com/Lephenixnoir/OpenLibm.git + force: true - name: mkg3a repo: https://github.com/tari/mkg3a.git + force: true - name: fxsdk repo: https://gitea.planet-casio.com/Lephenixnoir/fxsdk.git + force: true - name: fxlibc repo: https://gitea.planet-casio.com/Vhex-Kernel-Core/fxlibc.git + force: true - name: gint repo: https://gitea.planet-casio.com/Lephenixnoir/gint.git + force: true - name: libimg repo: https://gitea.planet-casio.com/Lephenixnoir/libimg.git + force: true - name: libprof repo: https://gitea.planet-casio.com/Lephenixnoir/libprof.git + force: true - name: justui repo: https://gitea.planet-casio.com/Lephenixnoir/justui.git + force: true - name: libp7 tag: "3.0" diff --git a/pacman.conf b/pacman.conf new file mode 100644 index 0000000..673daaf --- /dev/null +++ b/pacman.conf @@ -0,0 +1,101 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +Color +#TotalDownload +CheckSpace +#VerbosePkgLists + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +#[testing] +#Include = /etc/pacman.d/mirrorlist + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra] +Server = https://mirrors.kernel.org/archlinux/$repo/os/$arch +Include = /etc/pacman.d/mirrorlist + +#[community-testing] +#Include = /etc/pacman.d/mirrorlist + +[community] +Include = /etc/pacman.d/mirrorlist + +# If you want to run 32 bit applications on your x86_64 system, +# enable the multilib repositories as required here. + +#[multilib-testing] +#Include = /etc/pacman.d/mirrorlist + +#[multilib] +#Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +[casio] +SigLevel = Optional TrustAll +Server = https://arch.middleearth.fr/$arch