From b708cf6cc1aa4beffe3b20dbd4a2e4eda9704f75 Mon Sep 17 00:00:00 2001 From: Yann MAGNIN Date: Sun, 27 Nov 2022 11:24:40 +0100 Subject: [PATCH] VxSDK - 0.12.0-27 : fix configuration file + preparation for CMake build-system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @update > [common] ¦ rename all log.warning() in log.warn() > [core/build] ¦ [compile] support explicit extra environment variables setup ¦ [rules] support explicit extra environment variables setup @fix > [__main__] ¦ fix error log print > [core/config] ¦ fix exception raising if the configuration file does not exists --- README.md | 71 +++++ assets/project/.gitignore | 13 + assets/project/Makefile | 92 ++++++ assets/project/src/main.c | 14 + assets/project/vxsdk.toml | 8 + install.sh | 140 +++++++++ requirements.txt | 3 + vxsdk/__main__.py | 102 +++++++ vxsdk/cli/__init__.py | 0 vxsdk/cli/build/__init__.py | 156 ++++++++++ vxsdk/cli/build/default.py | 49 +++ vxsdk/cli/build/doctor.py | 20 ++ vxsdk/cli/config.py | 46 +++ vxsdk/cli/conv/__init__.py | 61 ++++ vxsdk/cli/conv/addin.py | 47 +++ vxsdk/cli/conv/asset.py | 108 +++++++ vxsdk/cli/conv/doctor.py | 13 + vxsdk/cli/pkg/__init__.py | 50 ++++ vxsdk/cli/pkg/clone.py | 89 ++++++ vxsdk/cli/pkg/search.py | 161 ++++++++++ vxsdk/cli/pkg/update.py | 15 + vxsdk/cli/project.py | 34 +++ vxsdk/core/__init__.py | 0 vxsdk/core/build/__init__.py | 0 vxsdk/core/build/compile.py | 140 +++++++++ vxsdk/core/build/dependency.py | 150 ++++++++++ vxsdk/core/build/meta.py | 288 ++++++++++++++++++ vxsdk/core/build/project.py | 195 ++++++++++++ vxsdk/core/build/rules.py | 160 ++++++++++ vxsdk/core/config.py | 168 +++++++++++ vxsdk/core/conv/__init__.py | 68 +++++ vxsdk/core/conv/addin.py | 67 +++++ vxsdk/core/conv/asset.py | 126 ++++++++ vxsdk/core/conv/pixel.py | 45 +++ vxsdk/core/conv/type/__init__.py | 0 vxsdk/core/conv/type/font.py | 466 +++++++++++++++++++++++++++++ vxsdk/core/conv/type/image.py | 397 ++++++++++++++++++++++++ vxsdk/core/logger.py | 115 +++++++ vxsdk/core/pkg/__init__.py | 59 ++++ vxsdk/core/pkg/backend/__init__.py | 57 ++++ vxsdk/core/pkg/backend/core.py | 134 +++++++++ vxsdk/core/pkg/backend/gitea.py | 170 +++++++++++ vxsdk/core/pkg/backend/local.py | 89 ++++++ vxsdk/core/pkg/clone.py | 89 ++++++ vxsdk/core/pkg/find.py | 126 ++++++++ vxsdk/core/pkg/version.py | 216 +++++++++++++ vxsdk/core/project.py | 20 ++ 47 files changed, 4637 insertions(+) create mode 100644 README.md create mode 100644 assets/project/.gitignore create mode 100644 assets/project/Makefile create mode 100644 assets/project/src/main.c create mode 100644 assets/project/vxsdk.toml create mode 100755 install.sh create mode 100644 requirements.txt create mode 100644 vxsdk/__main__.py create mode 100644 vxsdk/cli/__init__.py create mode 100644 vxsdk/cli/build/__init__.py create mode 100644 vxsdk/cli/build/default.py create mode 100644 vxsdk/cli/build/doctor.py create mode 100644 vxsdk/cli/config.py create mode 100644 vxsdk/cli/conv/__init__.py create mode 100644 vxsdk/cli/conv/addin.py create mode 100644 vxsdk/cli/conv/asset.py create mode 100644 vxsdk/cli/conv/doctor.py create mode 100644 vxsdk/cli/pkg/__init__.py create mode 100644 vxsdk/cli/pkg/clone.py create mode 100644 vxsdk/cli/pkg/search.py create mode 100644 vxsdk/cli/pkg/update.py create mode 100644 vxsdk/cli/project.py create mode 100644 vxsdk/core/__init__.py create mode 100644 vxsdk/core/build/__init__.py create mode 100644 vxsdk/core/build/compile.py create mode 100644 vxsdk/core/build/dependency.py create mode 100644 vxsdk/core/build/meta.py create mode 100644 vxsdk/core/build/project.py create mode 100644 vxsdk/core/build/rules.py create mode 100644 vxsdk/core/config.py create mode 100644 vxsdk/core/conv/__init__.py create mode 100644 vxsdk/core/conv/addin.py create mode 100644 vxsdk/core/conv/asset.py create mode 100644 vxsdk/core/conv/pixel.py create mode 100644 vxsdk/core/conv/type/__init__.py create mode 100644 vxsdk/core/conv/type/font.py create mode 100644 vxsdk/core/conv/type/image.py create mode 100644 vxsdk/core/logger.py create mode 100644 vxsdk/core/pkg/__init__.py create mode 100644 vxsdk/core/pkg/backend/__init__.py create mode 100644 vxsdk/core/pkg/backend/core.py create mode 100644 vxsdk/core/pkg/backend/gitea.py create mode 100644 vxsdk/core/pkg/backend/local.py create mode 100644 vxsdk/core/pkg/clone.py create mode 100644 vxsdk/core/pkg/find.py create mode 100644 vxsdk/core/pkg/version.py create mode 100644 vxsdk/core/project.py diff --git a/README.md b/README.md new file mode 100644 index 0000000..43cbca4 --- /dev/null +++ b/README.md @@ -0,0 +1,71 @@ +# vxSDK + +## Description + +vxSDK is a tool which aims to help the user to work well by using multiple commands. + +### Usage + + vxsdk [+toolchain] [OPTIONS] [SUBCOMMAND] + +### Options + -v, --version Print version info and exit + --list List installed command + -h, --help Print helps information + +Default sub-commands used: + p, project project abstraction + g, gitea package manager for Vhex's forge Gitea + l, link USB abstraction + d, doctor vxSDK inspector + +# Commands + +## vxsdk project + +### Description + +Abstract project manipulation + +### Usage + + vxsdk project [OPTIONS] + +### Options + + --list List installed command + -h, --help Print helps information + +### Common used commands: + + n, new Create a new project + b, build Build a project + d, doctor Display all project information + a, add Add project dependency + +## vxsdk gitea + +### Description + +Package manager for the Vhex's forge Gitea + +### Usage + + vxsdk gitea [OPTIONS] + +### Common used commands: + + s, search [NAME] Search project + i, install [NAME] Try to install a project + r, remove Remove a project + u, update [NAME] Try to update a project + +## vxsdk link + +### Description + +### Usage + +### Options + +### Common used commands: \ No newline at end of file diff --git a/assets/project/.gitignore b/assets/project/.gitignore new file mode 100644 index 0000000..2c4f84b --- /dev/null +++ b/assets/project/.gitignore @@ -0,0 +1,13 @@ +# Build files +/build-fx +/build-cg +/*.g1a +/*.g3a + +# Python bytecode + __pycache__/ + +# Common IDE files +*.sublime-project +*.sublime-workspace +.vscode diff --git a/assets/project/Makefile b/assets/project/Makefile new file mode 100644 index 0000000..0e25e74 --- /dev/null +++ b/assets/project/Makefile @@ -0,0 +1,92 @@ +#! /usr/bin/make -f + + +# +# variable definition +# + +# color definition, for swagg :D +red := \033[1;31m +green := \033[1;32m +blue := \033[1;34m +white := \033[1;37m +nocolor := \033[1;0m + +src := $(foreach path,\ + $(shell find src -not -path "*/\.*" -type d), \ + $(wildcard $(path)/*.c) \ + $(wildcard $(path)/*.S) \ + $(wildcard $(path)/*.s)) + +obj := $(patsubst src_%,$(VXSDK_PREFIX_BUILD)/%.o,$(subst /,_,$(src))) +obj += $(patsubst \ + $(VXSDK_ASSETS_SRC)/%,\ + $(VXSDK_ASSETS_BUILD)/%.o,\ + $(wildcard $(VXSDK_ASSETS_SRC)/*.c) \ + ) + +cflags := -ffreestanding -nostdlib -m4-nofpu -fPIE -O1 +cflags += -mb -fstrict-volatile-bitfields +cflags += $(VXSDK_CFLAGS_INCLUDE) -I.. -Iinclude + +# debug vars +VERBOSE ?= false + +# +# build rules +# + +vxaddin: $(obj) + @ printf "$(blue)Create $(red)$@$(nocolor)\n" + sh-elf-vhex-gcc \ + -T $(VXSDK_PREFIX_LIB)/fxcg50-dynamic.ld -Wl,-q -Wl,-M \ + $(VXSDK_CFLAGS_LINK) \ + -o $@ $^ \ + -lvhex-fxcg50 -lc -lgcc \ + > $(VXSDK_PREFIX_BUILD)/map.txt + vxsdk conv addin -b $@ -n vxaddin -o /tmp/vxaddin + +version: + @echo "$(VXSDK_PKG_VERSION)" + +help: + @ echo 'Rules listing:' + @ echo '... all the default, if no target is provided' + @ echo '... clean remove build object' + @ echo '... fclean remove all generated object' + @ echo '... re same as `make fclean all`' + @ echo '... version display version' + @ echo '... install install the library' + @ echo '... uninstall uninstall the library' + +.PHONY: help version + +# +# Object rules +# + +$(VXSDK_PREFIX_BUILD)%.o: +ifeq ($(VERBOSE),true) + @ mkdir -p $(dir $@) + sh-elf-vhex-gcc \ + $(cflags) -D FXCG50 \ + -o $@ \ + -c $(addprefix src/,$(subst _,/,$(notdir $(basename $@)))) +else + @ mkdir -p $(dir $@) + @ printf "$(green)>$(nocolor) $(white)$@$(nocolor)\n" + @ sh-elf-vhex-gcc \ + $(cflags) -D FXCG50 \ + -o $@ \ + -c $(addprefix src/,$(subst _,/,$(notdir $(basename $@)))) +endif + +$(VXSDK_ASSETS_BUILD)%.o: $(VXSDK_ASSETS_SRC)/% +ifeq ($(VERBOSE),true) + @ mkdir -p $(dir $@) + sh-elf-vhex-gcc $(cflags) -D FXCG50 -o $@ -c $< +else + @ mkdir -p $(dir $@) + @ printf "$(green)>$(nocolor) $(white)$@$(nocolor)\n" + @ sh-elf-vhex-gcc $(cflags) -D FXCG50 -o $@ -c $< +endif diff --git a/assets/project/src/main.c b/assets/project/src/main.c new file mode 100644 index 0000000..d0b7f93 --- /dev/null +++ b/assets/project/src/main.c @@ -0,0 +1,14 @@ +#include +#include + +int main(void) +{ + dclear(C_WHITE); + dtext(1, 1, C_BLACK, "Sample fxSDK add-in."); + dupdate(); + + while (1) { __asm__("sleep"); } + + //getkey(); + return 1; +} diff --git a/assets/project/vxsdk.toml b/assets/project/vxsdk.toml new file mode 100644 index 0000000..22cf04b --- /dev/null +++ b/assets/project/vxsdk.toml @@ -0,0 +1,8 @@ +[project] +name = 'vxaddin' + +[dependencies] +vxKernel = 'dev' + +[build] +build = 'make' diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..a8257b8 --- /dev/null +++ b/install.sh @@ -0,0 +1,140 @@ +#! /usr/bin/env bash + +prefix="$HOME/.local" +VERSION='0.12.0' + +# +# Help screen +# +help() { + cat << OEF +Script used to manipulate the VxSDK, a set of tools for the Vhex Operating +System project. + +Usage $0 [ACTION] + +Actions: + bootstrap Try to bootstrap the vxSDK itself [default] + install Try to install the VsSDK + uninstall Try to uninstall the VsSDK + +Options: + --prefix= Installation prefix (default is ~/.local) + -h, --help Display this help +OEF + exit 0 +} + + + +# +# Parse arguments +# + +target='install' + +for arg; do case "$arg" in + --help | -h) + help;; + -v | --version) + echo "$VERSION" + exit 0;; + + install) + target='install';; + update) + target='update';; + uninstall) + target='uninstall';; + *) + echo "error: unreconized argument '$arg', giving up." >&2 + exit 1 +esac; done + + + +# +# Process +# + +if [[ "$target" = "install" ]] +then + if [[ -d "$prefix/lib/vxsdk/vxsdk" ]]; then + echo 'warning : vxsdk is already installed !' >&2 + read -n 1 -p 'Do you whant to re-install the vxSDK (package will not be removed) [y/N] ? ' reinstall + [[ "$reinstall" != 'y' ]] && exit 1 + echo '' + ./install.sh uninstall + fi + + install -d $prefix/lib/vxsdk/vxsdk + cp -r requirements.txt assets vxsdk $prefix/lib/vxsdk/vxsdk + + install -d $prefix/bin + echo '#! /usr/bin/env bash' > $prefix/bin/vxsdk + echo '' >> $prefix/bin/vxsdk + echo "source $prefix/lib/vxsdk/vxsdk/venv/bin/activate" >> $prefix/bin/vxsdk + echo "python3 $prefix/lib/vxsdk/vxsdk/vxsdk \$@" >> $prefix/bin/vxsdk + echo 'deactivate' >> $prefix/bin/vxsdk + chmod +x $prefix/bin/vxsdk + + build_date=$(date '+%Y-%m-%d') + build_hash=$(git rev-parse --short HEAD) + f="$prefix/lib/vxsdk/vxsdk/vxsdk/__main__.py" + sed -e "s*%VERSION%*$VERSION*; s*%BUILD_HASH%*$build_hash*; s*%BUILD_DATE%*$build_date*" vxsdk/__main__.py > $f + + mkdir -p $prefix/share/vxsdk + + cd $prefix/lib/vxsdk/vxsdk + python3 -m venv venv + source venv/bin/activate + pip install --upgrade pip 2>&1 > /dev/null + pip install -r requirements.txt + deactivate + exit 0 +fi + + + +if [[ "$target" = "update" ]] +then + + git clone git@github.com:Vhex-org/vxSDK.git --depth=1 /tmp/vxSDK > /dev/null 2>&1 || exit 84 + cd /tmp/vxSDK + + if [[ "$(./install.sh --version)" == "$VERSION" ]] + then + rm -rf /tmp/vxSDK + echo 'already up to date !' + exit 0 + fi + + _check=$(echo -e "$(./install.sh --version)\n$VERSION" | sort -V | head -n1) + + if [[ "$_check" != "$VERSION" ]]; then + rm -rf /tmp/vxSDK + echo 'already up to date !' + exit 0 + fi + + echo "update $VERSION -> $(./install.sh --version)" + + ./install.sh uninstall + ./install.sh install + + rm -rf /tmp/vxSDK + +fi + + + +if [[ "$target" = "uninstall" ]] +then + + rm $prefix/bin/vxsdk + rm -rf $prefix/lib/vxsdk + rmdir $prefix/share/vxsdk 2>/dev/null || exit 0 + echo 'note: repositories cloned by vxSDK have not been removed' + exit 0 + +fi diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6b2cf3e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +requests +toml +pillow diff --git a/vxsdk/__main__.py b/vxsdk/__main__.py new file mode 100644 index 0000000..28d315b --- /dev/null +++ b/vxsdk/__main__.py @@ -0,0 +1,102 @@ +""" +vxSDK is a suitable of tools used in conjunction with the Vhex Operating System +to develop game, add-in, abstract build step with dependencies resolver and +more. +""" + +import sys +import os + +from core.logger import log + + +# Program version (inserted at compile-time) +__VXSDK_VERSION__ = "%VERSION%" +__VXSDK_BUILD_HASH__ = "%BUILD_HASH%" +__VXSDK_BUILD_DATE__ = "%BUILD_DATE%" + + +__VXSDK_HELP__ = r""" +Vhex' Software Developement Kit + +USAGE: + vxsdk [+toolchain] [OPTIONS] [SUBCOMMAND] + +OPTIONS: + -v, --version Print version info and exit + --list List installed command + --update Try to update the ymtools + -h, --help Print helps information + +Default sub-commands used: + project Project abstraction + pkg Package manager + build Build abstraction + +See `vxsdk --help` for more information on a specific command +""".strip() + +def _list_modules(): + for name in os.listdir(f"{os.path.dirname(__file__)}/cli"): + try: + mod = __import__(f"cli.{name.split('.')[0]}", fromlist=[ + '__VXSDK_MODULE_META__', + 'cli_parse' + ]) + if not hasattr(mod, 'cli_validate'): + continue + if not hasattr(mod, 'cli_parse'): + continue + if not hasattr(mod, '__VXSDK_MODULE_META__'): + continue + yield mod + except ImportError as err: + log.warn(f"[vxsdk] module '{name}' cannot be imported") + log.warn(f"{err}") + +def _subcommand_parse(argv): + for mod in _list_modules(): + if mod.cli_validate(argv[0]): + return mod.cli_parse(argv) + log.emergency(f"vxsdk: '{argv[0]}' command not found :(") + return 84 + +def _subcommand_list(): + for mod in _list_modules(): + mod_info = mod.__VXSDK_MODULE_META__[1] + args = str(mod.__VXSDK_MODULE_META__[0]).strip('[]') + log.user(f" {args}".ljust(32) + f"{mod_info}") + +def _main(argv): + if not argv: + log.error(__VXSDK_HELP__) + sys.exit(84) + + if argv[0] in ['-h', '--help']: + log.user(__VXSDK_HELP__) + sys.exit(0) + + if argv[0] == '--version': + _ver = __VXSDK_VERSION__ + _hash = __VXSDK_BUILD_HASH__ + _date = __VXSDK_BUILD_DATE__ + log.user(f"vxSDK {_ver} ({_hash} {_date})") + sys.exit(0) + + if argv[0] == '--update': + return os.system( + os.path.dirname(__file__) + '/../install.sh update' + ) + + if argv[0] in ['-v', '-vv', '-vvv']: + log.level += len(argv[0]) - 1 + argv = argv[1:] + if not argv: + log.error(__VXSDK_HELP__) + sys.exit(84) + + if len(argv) == 1 and argv[0] == '--list': + return _subcommand_list() + return _subcommand_parse(argv) + +_main(sys.argv[1:]) diff --git a/vxsdk/cli/__init__.py b/vxsdk/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vxsdk/cli/build/__init__.py b/vxsdk/cli/build/__init__.py new file mode 100644 index 0000000..ff3178e --- /dev/null +++ b/vxsdk/cli/build/__init__.py @@ -0,0 +1,156 @@ +"""vxsdk-build modules + +This module provides package abstraction and build core function for the Vhex +Operating system project. +""" +from core.logger import log + +from cli.build.default import build_default_cli +from cli.build.doctor import build_doctor_cli + + +__all__ = [ + '__VXSDK_MODULE_META__', + 'cli_validate', + 'cli_parse', +] + + +__VXSDK_MODULE_META__ = ( + ['build'], + "Build a project", + r"""vxsdk-build +Build System Abstraction + +USAGE: + vxsdk build(-) [OPTIONS] + +DESCRIPTION: + Compile Vhex project. + +NOTES: + The Vex build system is extremely powerful and polyvalent. It allows the + user to totally ignore the build part and the dependencies management. + + All Vhex projects use a <.vxsdk.toml> file wich should be stored at the root + of the project directory (you cam generate a project template using the + `vxsdk project new `). This file uses the TOML language to control + the build step of a project. + + ``` + # Default meta-data information for the pacakge manager. + [project] + name = 'myaddin' # required + version = '1.0.0' # required + type = 'addin' # optional ('addin' or 'app') + description = ''' # optional + An old-school demo-scene ! + Written by Yatis :) + ''' + + # Dependencies information for the build manager. (optional) + # + # The version can target or branch name. + # + # Note that you can use a powerfull operations for managing version of + # dependencies if the 'version' is detected to respect the correct + # semantic versioning like caret (^), tilde (~) and wildcard (*): + # + # Caret requirements (^) + # ^1.2.3 := >=1.2.3, <2.0.0 + # ^1.2 := >=1.2.0, <2.0.0 + # ^1 := >=1.0.0, <2.0.0 + # ^0.2.3 := >=0.2.3, <0.3.0 + # ^0.2 := >=0.2.0, <0.3.0 + # ^0.0.3 := >=0.0.3, <0.0.4 + # ^0.0 := >=0.0.0, <0.1.0 + # ^0 := >=0.0.0, <1.0.0 + # + # Tilde requirements (~) + # ~1.2.3 := >=1.2.3, <1.3.0 + # ~1.2 := >=1.2.0, <1.3.0 + # ~1 := >=1.0.0, <2.0.0 + # + # Wildcard requirements (*) + # * := >=0.0.0 + # 1.* := >=1.0.0, <2.0.0 + # 1.2.* := >=1.2.0, <1.3.0 + # + # Note that it's possible that a package can have two same versions (one + # for branch name and another for a tag), by default, the tag is always + # selected, but here the display information will explicitly describe if + # the version is a tag and / or a branch. + [dependencies] + vxkernel = 'master' # recommanded + sh-elf-vhex = 'master' # recommanded + custom-lib = '^1.5' # exemple of carret (^) operation + + # Manual build indication (option) + # + # Note that if this section is specified then only this build indication + # will be executed, no default step will be used + # + # All building steop are, in order: + # > configure + # > build + # > install + # And during all of theses building step, the vxSDk setup some + # environment variable: + # > VXSDK_PKG_NAME - project name + # > VXSDK_PKG_VERSION - project version + # > VXSDK_PREFIX_BUILD - project build prefix (for object files) + # > VXSDK_PREFIX_INSTALL - project installation prefix + # > VXSDK_PREFIX_LIB - prefix for all stored librairy + # > VXSDK_CFLAGS_INCLUDE - Include flags for GCC + # > VXSDK_CFLAGS_LINK - Linker flags for GCC + # > VXSDK_ASSETS_SRC - Assets sources file directory + # > VXSDK_ASSETS_BUILD - Assets build directory (for object) + [build] + configure = 'mkdir -p build && cd build && ../configure --verbose' + build = 'cd build && make' + + # Dependencies Hook (optional) + # + # You need to create a section wich is named like [extra.] + # and you can "hook" some step of particular dependencies (here + # vxkernel for exemple). + # + # A hook is simply additional string that will be send to the + # appropriate step of the build. + [extra.vxkernel] + configure = '--static --verbose' + ``` + + Above is a exemple of a defaut project configuration file that can be used + for a new project. + + +OPTIONS: + -v, --verbose Disable threading and display log information + -r, --rebuild Force rebuild the project + -u, --update Update dependencies + --extra-conf [ARG]... Add extra configuration flags + --extra-build [ARG]... Add extra build flags + +ACTIONS: + doctor Display all information about one package +""" +) + +#--- +# Public +#--- + +def cli_validate(name): + """ validate the module name """ + return name.find('build') == 0 + +def cli_parse(argv): + """ Build subcommand entry """ + if len(argv) > 2: + if '--help' in argv or '-h' in argv: + log.user(__VXSDK_MODULE_META__[2]) + return 0 + if argv[1] == 'doctor': + return build_doctor_cli(argv[2:]) + return build_default_cli(argv) diff --git a/vxsdk/cli/build/default.py b/vxsdk/cli/build/default.py new file mode 100644 index 0000000..cb51b2f --- /dev/null +++ b/vxsdk/cli/build/default.py @@ -0,0 +1,49 @@ +""" +Vhex default build management +""" +import sys + +from core.build.project import VxProject +from core.logger import log + +__all__ = [ + 'build_default_cli' +] + +def build_default_cli(argv): + """Parse CLI arguments""" + board_target = None + if argv[0].find('build-') == 0: + board_target = argv[0][6:] + + path = None + verbose = False + extra_conf = { + 'configure' : '', + 'build' : '', + } + for arg in argv[1:]: +# if arg in ['-r', '--rebuild']: +# force_rebuild = True +# continue +# if arg in ['-u', '--update']: +# update_dependencies = True +# continue + if arg in ['-v', '--verbose']: + verbose = True + continue + if arg.find('--extra-conf=') == 0: + extra_conf['configure'] += ' ' + arg[13:] + continue + if arg.find('--extra-build=') == 0: + extra_conf['build'] += ' ' + arg[14:] + continue + if path: + log.error(f"argument '{arg}' not recognized") + sys.exit(85) + path = arg + + return VxProject(path, extra_conf=extra_conf).build( + board_target, + verbose + ) diff --git a/vxsdk/cli/build/doctor.py b/vxsdk/cli/build/doctor.py new file mode 100644 index 0000000..601d38f --- /dev/null +++ b/vxsdk/cli/build/doctor.py @@ -0,0 +1,20 @@ +""" +Display pacakge build information +""" + +from core.logger import log +from core.build.project import VxProject + +__all__ = [ + 'build_doctor_cli' +] + +def build_doctor_cli(argv): + r""" Package doctor + + This function will display all package information (path, name, ...) and it + will try to display the dependencies information (if dependencies have been + found, dependencies graph, ...). + """ + log.user(VxProject(None if len(argv) <= 0 else argv[0])) + return 0 diff --git a/vxsdk/cli/config.py b/vxsdk/cli/config.py new file mode 100644 index 0000000..826210e --- /dev/null +++ b/vxsdk/cli/config.py @@ -0,0 +1,46 @@ +from core.config import config_set, config_get + +__all__ = [ + '__VXSDK_MODULE_META__', + 'cli_validate', + 'cli_parse', +] + +__VXSDK_MODULE_META__ = ( + ['config'], + "vxSDK configuration module", + r"""vxsdk-configuration +VxSDK configuration and options + +USAGE: + vxsdk config [] + +DESCRIPTION: + Change / customise the vxSDK behaviour +""" +) + +# TODO: list all key available +# TODO: level selection : 'local', 'global' + +def cli_validate(name): + return name in __VXSDK_MODULE_META__[0] + +def cli_parse(argv): + """ Config subcommand entry """ + + if '--help' in argv or '-h' in argv: + logger(LOG_USER, __VXSDK_MODULE_META__[2]) + return 0 + + if len(argv) == 2: + logger(LOG_USER, config_get(argv[1])) + return 0 + + if len(argv) == 3: + if old := config_set(argv[1], argv[2]): + logger(LOG_USER, f"previous value = {old}") + return 0 + + logger(LOG_EMERG, __VXSDK_MODULE_META__[2]) + return 84 diff --git a/vxsdk/cli/conv/__init__.py b/vxsdk/cli/conv/__init__.py new file mode 100644 index 0000000..d17368c --- /dev/null +++ b/vxsdk/cli/conv/__init__.py @@ -0,0 +1,61 @@ +"""vxsdk-converter modules + +This package provides conversion abstraction (image -> source code, ELF -> +addin, ...) for the Vhex project. +""" + +from core.logger import log + +from cli.conv.doctor import conv_doctor_cli_parse +from cli.conv.asset import conv_asset_cli_parse +from cli.conv.addin import conv_addin_cli_parse + + +__all__ = [ + '__VXSDK_MODULE_META__', + 'cli_validate', + 'cli_parse', +] + +__VXSDK_MODULE_META__ = ( + ['conv'], + 'assets converter', + r"""vxsdk-conv +Project assets conv + +USAGE: + vxsdk conv(-) [OPTIONS] ... + +DESCRIPTION: + Convert vhex project assets (or binary) into various form. By default, if no + action is specified, the "asset" conversion is selected. + +ACTIONS: + doctor try to display assets and addin information (debug) + asset convert asset into source file or binary file + addin convert binary into addin file for vxOS + +See `vxsdk conv --help` for more information on a specific action +""" +) + +def cli_validate(name): + """ validate the module name """ + return name.find('conv') == 0 + +def cli_parse(argv): + """ Build subcommand entry """ + if '--help' in argv or '-h' in argv: + log.user(__VXSDK_MODULE_META__[2]) + return 0 + if argv[0].find('conv-') != 0: + argv[0] = 'conv-asset' + action = argv[0][5:] + if action == 'doctor': + return conv_doctor_cli_parse(argv[1:]) + if action == 'asset': + return conv_asset_cli_parse(argv[1:]) + if action == 'addin': + return conv_addin_cli_parse(argv[1:]) + log.error(f"unable to find action '{action}'") + return 84 diff --git a/vxsdk/cli/conv/addin.py b/vxsdk/cli/conv/addin.py new file mode 100644 index 0000000..27bf05e --- /dev/null +++ b/vxsdk/cli/conv/addin.py @@ -0,0 +1,47 @@ +from core.conv.addin import generate_addin + +__all__ = [ + 'conv_addin_cli_parse' +] + +__HELP__ = r"""vxsdk-converter-addin +Converte binary file into Vhex OS addin. + +USAGE: + vxsdk conv addin -b BINARY ... + +DESCRIPTION: + Convert a binary file into an application for the Vhex operating system. + +OPTIONS: + -b ELF binary file (no check is performed in this file) + -i 92x62 pixel image path + -o output path for the generated addin + -n internal addin name + -v internal addin version +""" + +def conv_addin_cli_parse(argv): + """Process CLI arguments""" + if '-h' in argv or '--help' in argv: + logger(LOG_USER, __HELP__, exit=0) + + action = None + info = [None, None, None, None, None] + for arg in argv: + if action == '-b': info[0] = arg + if action == '-i': info[1] = arg + if action == '-n': info[2] = arg + if action == '-o': info[3] = arg + if action == '-v': info[4] = arg + if action: + action = None + continue + if arg in ['-b', '-i', '-n', '-o', '-v']: + action = arg + continue + + if info[0] == None: + logger(LOG_ERR, 'converter: need binary path !', exit=84) + + return generate_addin(info[0], info[1], info[2], info[3], info[4]) diff --git a/vxsdk/cli/conv/asset.py b/vxsdk/cli/conv/asset.py new file mode 100644 index 0000000..94cf6c5 --- /dev/null +++ b/vxsdk/cli/conv/asset.py @@ -0,0 +1,108 @@ +""" +Vhex asset converter user interface +""" +import os + +from core.logger import log +from core.conv import assets_generate + + +__all__ = [ + 'conv_asset_cli_parse' +] + + +__HELP__ = r"""vxsdk-converter-asset +Convert all assets file in the project directory. + +USAGE: + vxsdk conv-asset [project path] [OPTIONS] + +DESCRIPTION: + Convert all assets file in the asset directory. This part of the converter + module will scan the provided folder (or the current working directory) and + will try to find all `vxconv.txt` file, which describe all assets that + should be converted. + + If no argument is provided, then the current working directory is used as + asset prefix and a storag for all generated source file. You can modify this + behaviour using OPTIONS. + + The vxconv.txt file is structured like basic key/value file: + + ``` + : + type: (font, bitmap) - required + path: - required + ... + + : + ... + ``` + + Each asset file description should have at least type and name information, + and each type have potentially its own requierements. + + type = bitmap: + ================================== ========================================= + Keys name and value type Description + ================================== ========================================= + profile: Select the bitmap pixel profile + | rgb4 | RGB 4 (indexed) + | rgb4a | RGBA 4 (indexed) + | rgb8 | RGB 8 (indexed) + | rgb8a | RGBA 8 (indexed) + | rgb16 | RGB 16 (5:R, 6:G, 5:B) + | rgb16a | RGBA 16 (5:R, 5:G, 5:B, 1:A) + ================================== ========================================= + + type = font: + ================================== ========================================= + Keys name and value type Description + ================================== ========================================= + grid_size: 8x9 (widthxheight) caracter size in pixel + grid_padding: space between caracter + grig_border: space around grid + proportional: caracter are cropped + line_height: caracter line alignement + charset: charset specification + char_spacing space between character + ================================== ========================================= + +OPTIONS: + -o The prefix for source file that will be generated + -h, --help Display this help +""" + +def conv_asset_cli_parse(argv): + """Process CLI arguments""" + # check obvious flags + if '-h' in argv or '--help' in argv: + log.user(__HELP__) + return 0 + + # fetch user indication + manual_output = False + prefix_output = None + prefix_asset = None + for arg in argv: + if arg == '-o': + manual_output = True + continue + if manual_output: + prefix_output = arg + continue + if prefix_asset: + log.warn(f"warning: previous path ({prefix_asset}) dropped") + prefix_asset = arg + + # check indication + if not prefix_asset: + prefix_asset = os.getcwd() + if not prefix_output: + prefix_output = os.getcwd() + prefix_asset = os.path.abspath(prefix_asset) + prefix_output = os.path.abspath(prefix_output) + + # generate asset information + return assets_generate(prefix_asset, prefix_output) diff --git a/vxsdk/cli/conv/doctor.py b/vxsdk/cli/conv/doctor.py new file mode 100644 index 0000000..748bcf9 --- /dev/null +++ b/vxsdk/cli/conv/doctor.py @@ -0,0 +1,13 @@ +__all__ = [ + 'conv_doctor_cli_parse' +] + +def conv_doctor_cli_parse(argv): + """Process CLI handling + + TODO: + > give asset file description to check error + > try to display asset and addin information based on the project type + """ + logger(LOG_WARN, 'conv: doctor action not implemented yet') + return 0 diff --git a/vxsdk/cli/pkg/__init__.py b/vxsdk/cli/pkg/__init__.py new file mode 100644 index 0000000..2d36af8 --- /dev/null +++ b/vxsdk/cli/pkg/__init__.py @@ -0,0 +1,50 @@ +"""vxsdk-pkg modules + +This module provides package management utilities that track installed Vhex +packages. It features : dependency support, package groups, install and +uninstall scripts, and syncs your local Vhex instance with remote repositories +to automatically upgrade packages. +""" +from core.logger import log + +from cli.pkg.search import pkg_search_cli_parse +from cli.pkg.update import pkg_update_cli_parse +from cli.pkg.clone import pkg_clone_cli_parse + + +__VXSDK_MODULE_META__ = ( + ['pkg'], + "package manager for Vhex's project", + r"""vxsdk package +Package manager for Vhex + +SYNOPSIS: + vxsdk pkg [ COMMAND ] [ OPTIONS ]... [ ARGS ]... + +COMMAND: + search [ ARGS ]... Search project + clone [ ARGS ]... Try to clone project + update [ ARGS ]... Try to update project + +For more information about a specific command, try 'vxsdk pkg -h' +""" +) + +def cli_validate(name): + """ validate or not the subcommand name """ + return name in __VXSDK_MODULE_META__[0] + +def cli_parse(argv): + """ Vhex pacakge CLI parser entry """ + if len(argv) > 2: + if argv[1] == 'search': + return pkg_search_cli_parse(argv[2:]) + if argv[1] == 'clone': + return pkg_clone_cli_parse(argv[2:]) + if argv[1] == 'update': + return pkg_update_cli_parse(argv[2:]) + if '-h' in argv or '--help' in argv: + log.user(__VXSDK_MODULE_META__[2]) + return 0 + log.error(__VXSDK_MODULE_META__[2]) + return 84 diff --git a/vxsdk/cli/pkg/clone.py b/vxsdk/cli/pkg/clone.py new file mode 100644 index 0000000..c49de88 --- /dev/null +++ b/vxsdk/cli/pkg/clone.py @@ -0,0 +1,89 @@ +""" +Vhex package cloning user interface +""" +import os +import sys + +from core.logger import log +import core.pkg + + +__all__ = [ + 'pkg_clone_cli_parse' +] + + +__HELP__ = r""" +vxsdk-pkg-clone +Package manager cloning part + +USAGE: + vxsdk pkg clone [ OPTIONS ] [ targets[, ...]] + +DESCRIPTION: + The installation part is simple, you juste need to refer the repository + name and, optionally, the version information. Like this: + + (ex: vxKernel@1.2.3) + + This will allow the searching to try to match package with specific version + information. This part is very powerfull because If the 'version' is + detected to respect the correct semantic versioning, you can perform version + operations in the version target: + + Caret requirements (^) + ^1.2.3 := >=1.2.3, <2.0.0 + ^1.2 := >=1.2.0, <2.0.0 + ^1 := >=1.0.0, <2.0.0 + ^0.2.3 := >=0.2.3, <0.3.0 + ^0.2 := >=0.2.0, <0.3.0 + ^0.0.3 := >=0.0.3, <0.0.4 + ^0.0 := >=0.0.0, <0.1.0 + ^0 := >=0.0.0, <1.0.0 + + Tilde requirements (~) + ~1.2.3 := >=1.2.3, <1.3.0 + ~1.2 := >=1.2.0, <1.3.0 + ~1 := >=1.0.0, <2.0.0 + + Wildcard requirements (*) + * := >=0.0.0 + 1.* := >=1.0.0, <2.0.0 + 1.2.* := >=1.2.0, <1.3.0 + + Note that it's possible that a package can have two same version (one for + branch name and another for a tag), by default, the tag is always selected, + but here the display information will explicitly describe if the version is + a tag and / or a branch. + +OPTIONS: + -y, --yes Do not ask for interactive confirmation + -n, --no-build Do not build the project, just clone + -h, --help Print this help and exit +""".strip() + +#--- +# Entry point of the module +#--- + +def pkg_clone_cli_parse(argv): + """ Clone a particular package """ + if not argv: + log.error(__HELP__) + sys.exit(84) + if '-h' in argv or '--help' in argv: + log.user(__HELP__) + sys.exit(0) + + confirm = True + for arg in argv: + if arg in ['-y', '--yes', '-c', '--confirm']: + confirm = arg in ['-c', '--confirm'] + continue + core.pkg.clone( + arg.split('@')[0], + None if len(arg.split('@')) != 2 else arg.split('@')[1], + os.getcwd(), + confirm + ) + return 0 diff --git a/vxsdk/cli/pkg/search.py b/vxsdk/cli/pkg/search.py new file mode 100644 index 0000000..fab70a4 --- /dev/null +++ b/vxsdk/cli/pkg/search.py @@ -0,0 +1,161 @@ +""" +Vhex package searching user interface +""" +import sys + +from core.logger import log +import core.pkg + + +__all__ = [ + 'pkg_search_cli_parse' +] + + +__HELP__ = r""" +vxsdk-pkg-search +Package manager : search commands + +SYNOPSIS: + vxsdk pkg search [ OPTIONS ] [ targets[, ...] ] + +DESCRIPTION: + This part of the vxSDK will performs search queries in remote and local + arena (see to change this default behaviour). Each "target" can + be written specificaly like this: + + (ex: vxKernel@1.2.3) + + This will allow the searching to try to match package with specific version + information. This part is very powerfull because If the 'version' is + detected to respect the correct semantic versioning, you can perform version + operations in the version target: + + Caret requirements (^) + ^1.2.3 := >=1.2.3, <2.0.0 + ^1.2 := >=1.2.0, <2.0.0 + ^1 := >=1.0.0, <2.0.0 + ^0.2.3 := >=0.2.3, <0.3.0 + ^0.2 := >=0.2.0, <0.3.0 + ^0.0.3 := >=0.0.3, <0.0.4 + ^0.0 := >=0.0.0, <0.1.0 + ^0 := >=0.0.0, <1.0.0 + + Tilde requirements (~) + ~1.2.3 := >=1.2.3, <1.3.0 + ~1.2 := >=1.2.0, <1.3.0 + ~1 := >=1.0.0, <2.0.0 + + Wildcard requirements (*) + * := >=0.0.0 + 1.* := >=1.0.0, <2.0.0 + 1.2.* := >=1.2.0, <1.3.0 + + Note that it's possible that a package can have two same version (one for + branch name and another for a tag), by default, the tag is always selected, + but here the display information will explicitly describe if the version is + a tag and / or a branch. + +OPTIONS: + -a, --all Display all package found + -l, --local Performs search in "local" and local + -L, --local-only Performs search only in "local" + -r, --remote Performs search only in "remote" + -i, --info Print extra information for each repositories + -s, --short Print short information for each repositories (default) + -h, --help Print this help and exit +""".strip() + +#--- +# Internals +#--- + +def _pkg_list_display(pkg_list, version, display_extra_info=False): + for pkg in pkg_list: + indent = ' ' + log.user(f"{pkg['full_name']}") + + # handle special version request + version_available = pkg['versions'] + if version: + version_available = [] + for ver in pkg['versions']: + if ver.validate(version): + version_available.append(ver) + + if display_extra_info: + log.user( + f"- Description : {pkg['description']}\n" + f"- Created at {pkg['created']}\n" + f"- Updated at {pkg['updated']}\n" + f"- Authors : {pkg['author']}\n" + f"- URL : {pkg['url']}\n" + "- Versions:" + ) + + if not version_available: + log.user(f"{indent}No version available for '{version}'") + return + + for ver in version_available: + ind = ' ' + if len(ver.sources) == 1: + ind = '(r-) ' if ver.sources[0] == 'remote' else '(-l) ' + if len(ver.sources) > 1: + ind = '(rl) ' + content = f"{indent}{ind}{pkg['name']}@{ver.name}".ljust(32) + content += f"({ver.type})" + log.user(content) + log.user(" ") + +#--- +# Public +#--- + +def pkg_search_cli_parse(argv): + """ Search command handling """ + if not argv: + log.notice(__HELP__) + sys.exit(84) + if '-h' in argv or '--help' in argv: + log.user(__HELP__) + sys.exit(0) + + local = True + remote = True + display_extra_info = False + for arg in argv: + # handle search arena + if arg in ['-l', '--local', '-r', '--remote']: + local = arg in ['-l', '--local'] + remote = arg in ['-r', '--remote'] + continue + # handle display mode + if arg in ['-i', '--info', '-s', '--short']: + display_extra_info = arg in ['-i', '--info'] + continue + + # performs searching operation + # @note + # - handle search exception when all package a requested + if arg in ['-a', '--all']: + pkg_list = core.pkg.find(None, None, local, remote) + else: + pkg_list = core.pkg.find( + arg.split('@')[0], + None, + local, + remote + ) + if not pkg_list: + log.warn(f"{arg}: package not found, skipped") + continue + + # display package information + _pkg_list_display( + pkg_list, + None if len(arg.split('@')) != 2 else arg.split('@')[1], + display_extra_info + ) + + return 0 diff --git a/vxsdk/cli/pkg/update.py b/vxsdk/cli/pkg/update.py new file mode 100644 index 0000000..dbde200 --- /dev/null +++ b/vxsdk/cli/pkg/update.py @@ -0,0 +1,15 @@ +""" +Vhex's packages updater subcommand +""" +import sys + +from core.logger import log + +__all__ = [ + 'pkg_update_cli_parse' +] + +def pkg_update_cli_parse(_): + """ Vhex pacakge CLI parser entry """ + log.critical("pacakge updater not implemented yet o(x_x)o") + sys.exit(85) diff --git a/vxsdk/cli/project.py b/vxsdk/cli/project.py new file mode 100644 index 0000000..62716ee --- /dev/null +++ b/vxsdk/cli/project.py @@ -0,0 +1,34 @@ +import sys + +from core.project import project_new + +__VXSDK_MODULE_META__ = ( + ['p', 'project'], + "project abstraction", + r"""vxsdk project +Abstract project manipulation + +USAGE: + vxsdk project [OPTIONS] + +OPTIONS: + --list List installed command + -h, --help Print helps information + +Common used commands: + n, new Create a new project + +See `vxsdk project help ` for more information on a specific command +""" +) + +def cli_parse(_, argv): + if argv: + if argv[0] == 'n' or argv[0] == 'new': + for path in argv[1:]: + project_new(path) + sys.exit(0) + if '-h' in argv or '--help' in argv: + logger(LOG_USER, __VXSDK_MODULE_META__[2]) + sys.exit(0) + logger(LOG_EMERG, __VXSDK_MODULE_META__[2]) diff --git a/vxsdk/core/__init__.py b/vxsdk/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vxsdk/core/build/__init__.py b/vxsdk/core/build/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vxsdk/core/build/compile.py b/vxsdk/core/build/compile.py new file mode 100644 index 0000000..91120d0 --- /dev/null +++ b/vxsdk/core/build/compile.py @@ -0,0 +1,140 @@ +""" +Compilation hadling using the dependency DAG graph +""" + +from core.logger import log +from core.build.rules import project_rules_exec +import core.conv + +__all__ = [ + 'project_compile' +] + +#--- +# Internals +#--- + +def __dep_generate_assets(dep_info, _, __): + log.user(f"[{dep_info['meta'].name}] generate assets...") + core.conv.assets_generate( + f"{dep_info['meta'].path}/assets/", + f"{dep_info['meta'].parent_path}/.vxsdk/converter/{dep_info['meta'].name}/src" + ) + return 0 + +def __dep_build_sources(dep_info, env_extra, verbose): + log.user(f"[{dep_info['meta'].name}] build sources...") + return project_rules_exec( + dep_info['meta'], + dep_info['target'], + ['configure', 'build'], + verbose, + env_extra + ) + +def __dep_install(dep_info, env_extra, verbose): + log.user(f"[{dep_info['meta'].name}] install...") + return project_rules_exec( + dep_info['meta'], + dep_info['target'], + ['intall'], + verbose, + env_extra + ) + +def __compile_dependency(dep, env_extra, verbose): + """ Compile dependency + + @args + > dep (dict) - dependency information + > verbose (bool) - display extra verbose information + + @return + > 0 on success, negative value otherwise + """ + if __dep_generate_assets(dep['info'], env_extra, verbose) != 0: + log.error(f"[{dep['info']['meta'].name}] error during asset generation") + return -1 + if __dep_build_sources(dep['info'], env_extra, verbose) != 0: + log.error(f"[{dep['info']['meta'].name}] error during source build") + return -2 + if __dep_install(dep['info'], env_extra, verbose) != 0: + log.error(f"[{dep['info']['meta'].name}] error during installation") + return -3 + return 0 + +def __env_extra_fetch(dep_graph): + """ Generate extra environement information + + @args + > dep_graph (list) : DAG graph + + @return + > a dictionary with all common env export + """ + env_extra = {} + for dep in dep_graph: + dep_meta = dep['info']['meta'] + dep_env_extra = dep_meta.get_env_extra(dep['info']['target']) + for key in dep_env_extra: + if key.upper() != key: + log.warn(f"[{dep_meta.name}] : {key} : env key must be upper") + if key in env_extra: + log.warn(f"[{dep_meta.name}] : {key} : already set, overrided") + env_extra[key] = dep_env_extra[key] + return env_extra + +#--- +# Public +#--- + +def project_compile(dep_graph, verbose=False): + r""" Build the entire project + + @args + > dep_graph (list) : DAG graph + > verbose (bool) : enable or disable verbose mode during build steps + + Return: + > 0 for succes, negative vale otherwise + """ + # generate "extra" environement configuration + env_extra = __env_extra_fetch(dep_graph) + + # main build loop + while True: + completed = True + for dep in dep_graph: + # check if the package as been completed + if dep['completed']: + continue + completed = False + + # check that all of its dependencies as been completed + can_run = True + for dep_idx in dep['dependencies']: + if not dep_graph[dep_idx]['completed']: + can_run = False + break + if not can_run: + continue + + # handle verbosity during its building + enable_verbose = verbose + if dep['info']['meta'].type == 'app': + enable_verbose = True + + # build the package + error = __compile_dependency(dep, env_extra, enable_verbose) + if error != 0: + return error + + # mark as completed + dep['completed'] = True + + # if all package as been completed, quit + if completed: + break + + # no error, exit + return 0 diff --git a/vxsdk/core/build/dependency.py b/vxsdk/core/build/dependency.py new file mode 100644 index 0000000..c0498dd --- /dev/null +++ b/vxsdk/core/build/dependency.py @@ -0,0 +1,150 @@ +""" +Dependencies resolver +""" +import os + +from core.logger import log +from core.build.meta import VxProjectMeta +import core.pkg + +__all__ = [ + 'project_dependency_clone' +] + +#--- +# Internals +#--- + +def __dep_graph_update(dep_graph, dep_parent_id, pkg_info): + """ + Update dependency graph + """ + # inject new pkg information and gues its ID + dep_id = len(dep_graph) + dep_graph.append({ + 'info' : pkg_info, + 'completed' : False, + 'dependencies' : [] + }) + + # update internal parent ID + if dep_parent_id >= 0: + dep_graph[dep_parent_id]['dependencies'].append(dep_id) + return dep_id + +def __recurs_clone(parent_path, dep_info, pkg_info, prefix, dep_stack): + """Clone all dependency and generate dependency graph + + @args + > dep_graph (list) - list dependencies + > dep_parent_id (int) - parent index in `dep_graph` + > pkg_info (dict) - package information + > prefix (str) - prefix for package cloning + + @return + > 0 if success, negative value otherwise + """ + # fetch info + dep_graph = dep_info[0] + dep_parent_id = dep_info[1] + + # check circular dependency and update the stack + for dep in dep_stack: + if dep != pkg_info: + continue + log.error(f"circular dependency with '{pkg_info['name']}' detected") + return -1 + dep_stack.append(pkg_info) + + # try to clone the package + pkg_path = core.pkg.clone( + pkg_info['name'], + pkg_info['version'], + prefix + ) + + # check pacakge validity + # @todo + # Find a way to not be dependent of VxProjectMeta to avoid spaghetti code + target = pkg_info['target'] + pkg_meta = VxProjectMeta(pkg_path, parent_path, pkg_info['extra_conf']) + if target not in pkg_meta.target_support: + log.error(f"[{pkg_meta.name}] target '{target}' not supported") + return -2 + + # generate dependency information + pkg_dep_id = __dep_graph_update( + dep_graph, + dep_parent_id, + { + 'meta' : pkg_meta, + 'target' : pkg_info['target'] + } + ) + for dep in pkg_meta.get_dependencies(target): + __recurs_clone( + parent_path, + (dep_graph, pkg_dep_id), + dep, + prefix, + dep_stack.copy() + ) + return 0 + +#--- +# Public +#--- + +def project_dependency_clone(pkg, target): + r""" Clone dependencies of package and generate a DAG graph + + This function will clone all dependency of a package optimised for a + particular target a `Direct Acyclic Graph` (DAG) graph will be generated to + facilitate project compilation later. + + @args + > pkg (dict) - first pacakge information + > target (str) - build target + + @return + > A list with package and dependency information : [ + { + 'meta' : <:obj:VxProjectMeta> + 'extra_conf' : + 'target' : build target + }, + ... + ] + """ + # create the package "storage" folder + # @note + # All dependencies will be cloned in "global" path, only symbolic link will + # be generated here + prefix = f"{pkg.parent_path}/.vxsdk/dependencies" + if not os.path.exists(prefix): + os.makedirs(prefix) + + # clone all dependencies and generate a DAG graph + # @note + # we need to manualy bootstrap the current project as a dependency to + # facilitate graph generation with a unified data structure + dep_graph = [] + dep_origin_id = __dep_graph_update( + dep_graph, + -1, + { + 'meta' : pkg, + 'target' : target, + } + ) + for dep in pkg.get_dependencies(target): + __recurs_clone( + pkg.parent_path, + (dep_graph, dep_origin_id), + dep, + prefix, + [] + ) + + # return the DAG + return dep_graph diff --git a/vxsdk/core/build/meta.py b/vxsdk/core/build/meta.py new file mode 100644 index 0000000..e9ded25 --- /dev/null +++ b/vxsdk/core/build/meta.py @@ -0,0 +1,288 @@ +""" +vxsdk.toml document parser +""" +import os +import toml + +from core.logger import log +from core.pkg.version import version_get + +__all__ = [ + 'VxProjectMeta' +] + +class VxProjectMeta(): + r""" Represente the project meta-information + + All meta-information is stored in a file named `vxsdk.toml` at the root + of the project directory. + + This class exposes: + + ================================== ========================================= + Property Description + ================================== ========================================= + name (str) Project name + version (str) Project versions + path (str) Project path + description (str) Project description + type (str) Project type + target_support (list,str) List of all supported target + ================================== ========================================= + Methods Description + ================================== ========================================= + get_dependencies (list,dict) List of all dependencies information + get_build_rules (dict) Building rules information + ================================== ========================================= + + For the file to be valid, it should expose *at least* the 'project' section + with the name of the project. Other information like the project type have + default value: + + ================================== ======================================= + Information Default value + ================================== ======================================= + name (str) Required. + version (str) auto. (see core.pkg.version.version_get) + type (str) 'bin' + description (str) Project description + target_support (list,str) [] (support all) + ================================== ======================================= + + Also, this class will perform any vxSDK configuration needed described in + the "special" section 'config' which can expose "key/value" which will be + added (if not exist) in the vxSDK's configuration file + (~/.config/vxsdk/config.toml). + + Example of a valid 'vxsdk.toml' file: + ``` + [project] + name = 'project' + type = 'addin' + + [dependencies] + vxkernel = '^1.0.0' + + [vxkernel.extra] + build = '--static --verbose' + + [config] + 'superh.path.sysroot' = '{path.sysroot}/superh' + 'superh.path.sysroot_test' = '{superh.path.sysroot}/test/yes' + ``` + """ + def __init__(self, path, parent_path=None, extra_conf=None): + """ Open and parse the vxsdk.toml file + + @args + > path (str) : project path which we can find `vxsdk.toml` file + > parent_path (str) : project parent path + > extra_conf (dict) : extra flags information about particular steps + + @todo + > proper handle exception + """ + # try to open the file + self._path = os.path.abspath(path) + with open(self._path + '/vxsdk.toml', encoding='utf-8') as file: + self._toml = toml.loads(file.read()) + + # check mandatory information + # @notes + # - 'project' section + # - project name information + if 'project' not in self._toml or 'name' not in self._toml['project']: + raise Exception(f"'{self._path}': missing project name information") + + # setup information and cache + self._type = None + self._target_support = None + self._parent_path = parent_path + self._extra_conf = extra_conf + + def __str__(self): + """ Display project information """ + content = f"project '{self.name}' - {self.version}\n" + content += '\n' + content += 'project meta\n' + content += ' - path:'.ljust(16) + f'{self.path}\n' + content += ' - parent:'.ljust(16) + f'{self.parent_path}\n' + content += ' - type:'.ljust(16) + f'{self.type}\n' + content += '\n' + content += 'Build information\n' + #if not self.build: + # content += ' default building rules used\n' + # content += '\n' + #else: + # rule_list = self.get_build_rules() + # for rule in rule_list: + # content += ' - ' + # content += f'{rule}:'.ljust(16) + # content += f'{rule_list[rule]}\n' + # content += '\n' + #content += 'Dependencies list\n' + #if not self.dependencies: + # content += ' No dependencies for this project\n' + #else: + # for dep in self.dependencies: + # content += ' - ' + f'{dep.name}'.ljust(16) + f'({dep.version})\n' + return content + + def __repr__(self): + return f'{self.name}<{self.version}>({self.type})' + + #--- + # Properties + #--- + + @property + def name(self): + """ get project name""" + return self._toml['project']['name'] + + @property + def path(self): + """ get project path""" + return self._path + + @property + def description(self): + """ get project description""" + if 'description' in self._toml['project']: + return self._toml['project']['description'] + return '' + + @property + def parent_path(self): + """ Return project parent path """ + return self._parent_path if self._parent_path else self._path + + @property + def is_original(self): + """ Return if the project is the target or a dependency""" + return not self._parent_path + + @property + def extra_conf(self): + """ Return extra pacakge information """ + return self._extra_conf + + @property + def type(self): + """ get project build rules""" + if self._type: + return self._type + if 'type' not in self._toml['project']: + return 'bin' + self._type = self._toml['project']['type'] + if self._type not in ['app', 'bin', 'lib', 'addin']: + log.warn( + f"{self.path}: project type '{self._type}' is unknown !" + f" Handled like a 'bin' type project" + ) + return self._type + + @property + def target_support(self): + """ get supported target list""" + if not self._target_support: + self._target_support = [] + if 'target' in self._toml['project']: + self._target_support = self._toml['project']['target'] + return self._target_support + + @property + def version(self): + """ get current package version""" + return version_get(self.path) + + #--- + # Methods + #--- + + def get_dependencies(self, target=None): + """ Get project dependency list for a particular board target + + @args + > target (str) - target name + + @return + > Return a list of dependencies information : [ + { + 'name' : , + 'version' : , + 'target' : , + 'extra_conf' : + }, + ... + ] + """ + # check mandatory target requirement + if self.target_support: + if not target or target not in self.target_support: + log.warn(f"[{self.name}] target '{target}' not supported") + return [] + + # help particular section dump + def section_dep_fetch(section, target): + if 'dependencies' not in section: + return [] + dep_list = [] + for dep_name in section['dependencies']: + extra = None + if 'extra' in section and dep_name in section['extra']: + extra = section['extra'][dep_name] + pkg_target = target + pkg_version = section['dependencies'][dep_name].split('@') + if len(pkg_version) >= 2: + pkg_target = pkg_version[1] + dep_list.append({ + 'name' : dep_name, + 'version' : pkg_version[0], + 'target' : pkg_target, + 'extra_conf' : extra + }) + return dep_list + + # fetch dependencies information in common and target-specific section + dep_list = section_dep_fetch(self._toml, target) + if target and target in self._toml: + dep_list += section_dep_fetch(self._toml[target], target) + return dep_list + + def get_build_rules(self, target=None): + """ Get project building rules + + @args + > target (str) - target name + + @return + > a dictionary with all available building step : { + 'configure' : , + 'build' : , + 'install' : , + 'uninstall' : , + } + """ + def section_rules_fetch(section): + if 'build' not in section: + log.warn(f"[{self.name}] no building rules") + return {} + rules = {} + for rule in section['build']: + if rule in ['configure', 'build', 'install', 'uninstall']: + rules[rule] = section['build'][rule] + continue + log.warn( + f"[{self.name}] building rule '{rule}' doesn't exists" + ) + return rules + + # check if we need to fetch common rules or target-specific one + if target: + if self.target_support and target not in self.target_support: + log.error(f"[{self.name}] not supported target '{target}'") + return {} + if target in self._toml: + return section_rules_fetch(self._toml[target]) + return section_rules_fetch(self._toml) diff --git a/vxsdk/core/build/project.py b/vxsdk/core/build/project.py new file mode 100644 index 0000000..9ef5191 --- /dev/null +++ b/vxsdk/core/build/project.py @@ -0,0 +1,195 @@ +r""" build.core - Core building function + +This module exports: + +VxProject - a class representing a "Vhex" project +""" +import os + +from core.logger import log +from core.build.meta import VxProjectMeta +from core.build.dependency import project_dependency_clone +from core.build.compile import project_compile +from core.build.rules import project_rules_exec + +__all__ = [ + 'VxProject' +] + + +#--- +# Public object +#--- + +class VxProject(VxProjectMeta): + r"""Represent a Vhex project + + A Vhex project is a folder which contains a 'vxsdk.toml' file. This file + describe information about the project, like its name, version, description, + and many other information. + + This class will handle this particular file, and will abstract some complex + tasks like build abstraction, assets conversion, manage project dependencies + and more. + + All project information and file generation (file objects, build logs, + assets conversion, ...) will be stored in a '.vxsdk' folder at the root of + the project folder. + + The VxProject exposes: + + ================================== ========================================= + Property VxprojectMeta Description + ================================== ========================================= + name (str) * Holds name + type (str) * Holds type (lib, app or addin) + path (str) * Holds project path + dependencies (list) * Holds project dependencies + description (str) * Holds description + build_rules (dict) * Holds custom build information + board_support (list,str) * List of all supported board (None if all) + extra_conf (dict) Holds extra rules information + parent (str) Holds project parent path + version (str) Holds version + assets (list) Holds project assets paths + is_original (bool) Return True if its the original package + ================================== ========================================= + + ================================== ========================================= + Method Description + ================================== ========================================= + update() Try to update project's dependencies + install() Install the project + uninstall() Uninstall the project + build() Build the entiere project (dep, src, ...) + rebuild() Rebuild the entiere project + ================================== ========================================= + """ + def __init__(self, path=None, parent_path=None, extra_conf=None): + r""" Try to read the TOML project file of a project + + This constructor will simply try to read the vxsdk.toml file stored at + the root of the project directory . The project path can be provided by + `path` but if the path is not provided, the vxSDK will select the + current working directory. + + The `parent` path is used to know if the project is the original one or + a dependency. If it a dependency, its build genereted files will be done + in the `.vxsdk/*` directory of the original one. + + Extra information for each building steps can be provided in the form of + dictionary. Like this: + + ``` + { + 'configure' : '--static --enable-xana', + 'build': '--verbose' + } + ``` + + @args + > path (str) - the project path + > parent_path (str) - the project parent path + > extra (dict) - extra flags information about particular steps + + @raise + > Exception - if the project file cannot be loaded + """ + super().__init__( + os.path.abspath(path) if path else os.getcwd(), + parent_path, + extra_conf + ) + + #--- + # Public methods + #--- + + def update(self): + r"""Update project's dependencies. + + This method will try to bump all dependencies to the possible newest + version, always validating the dependencies versioning match + + TODO: + > display dependency name and tag and "old version -> new version" + > try to update and build the new dependencies before override + """ + log.error('dependencies update not implemented yet') + return -1 + + def install(self, target, verbose=False): + r""" Install the project. + + @args + > target (str) : targeted board for operation (fxcg50, SDL, ...) + > verbose (bool) : display project logs or just strict minimum + + @return + > True if no error, False otherwise + """ + if target and self.target_support and target not in self.target_support: + log.error(f"[{self.name}] target '{target}' not supported") + return -1 + return project_rules_exec( + self, + target, + ['install'], + verbose + ) + + def uninstall(self, target, verbose=False): + """ Uninstall the project + + @args + > target (str) : targted board for operation (fxcg50, SDL, ...) + > verbose (bool) : display project logs or just strict minimum + + @return + > True if no error, False otherwise + """ + if target and self.target_support and target not in self.target_support: + log.error(f"[{self.name}] target '{target}' not supported") + return -1 + return project_rules_exec( + self, + target, + ['uninstall'], + verbose + ) + + def build(self, target=None, verbose=False): + """Build the entire project + + Args: + > target (str) : target build operation + > verbose (bool) : display project logs or just strict minimum + + Return: + > True if success, False otherwise + """ + log.user(f"[{self.name}] start building package...") + + # check target availability + if target and self.target_support and target not in self.target_support: + log.error(f"[{self.name}] target '{target}' not supported") + return -1 + + # check if the project is compatible with the board + build_rules = self.get_build_rules(target) + if not 'config' in build_rules \ + and not 'build' in build_rules \ + and not 'install' in build_rules: + log.error(f"[{self.name}] project not compatible for '{target}'") + return -2 + + # clone dependencies + log.debug(f"target test = {target}") + dep_graph = project_dependency_clone(self, target) + if not dep_graph: + log.error(f"[{self.name}] unable to perform dependeincy relovation") + return -3 + + # compile the entire project + log.debug(f"dep_graph = {dep_graph}") + return project_compile(dep_graph, verbose) diff --git a/vxsdk/core/build/rules.py b/vxsdk/core/build/rules.py new file mode 100644 index 0000000..95812bc --- /dev/null +++ b/vxsdk/core/build/rules.py @@ -0,0 +1,160 @@ +""" +Vhex build shell command abstraction +""" +import os +import subprocess + +from core.logger import log +from core.config import config_get + +__all__ = [ + 'project_rules_exec' +] + +#--- +# Internals +#--- + +def __project_generate_env(env_extra, pkg_meta, target): + r""" Generate environment variables + + ================================ ======================================= + Type Description + ================================ ======================================= + VXSDK_PKG_NAME project name + VXSDK_PKG_VERSION project version + VXSDK_PKG_IS_ORIGINAL 'true' if the package is the original + VXSDK_PREFIX_BUILD project build prefix (for object files) + VXSDK_PREFIX_INSTALL project installation prefix + VXSDK_PREFIX_LIB prefix for all stored library + VXSDK_GCC_CFLAGS include flags for GCC + VXSDK_ASSETS_SRC assets sources file directory + VXSDK_ASSETS_BUILD assets build directory + VXSDK_PATH_SYSROOT_LIB sysroot library path + VXSDK_PATH_SYSROOT_INCLUDE sysroot include path + ================================ ======================================= + + @arg + > board_target (str) - targeted board + """ + prefix = f"{pkg_meta.parent_path}/.vxsdk" + + # generate VXSDK_PREFIX_BUILD information + pkg_prefix_build = f'{prefix}/build/{pkg_meta.name}' + if not os.path.exists(pkg_prefix_build): + os.makedirs(pkg_prefix_build) + + # generate VXSDK_PREFIX_INSTALL information + pkg_prefix_install = f'{prefix}/lib' + if pkg_meta.type == 'app': + pkg_prefix_install = os.path.expanduser(config_get('path.bin')) + if not os.path.exists(pkg_prefix_install): + os.makedirs(pkg_prefix_install) + + # generate VXSDK_PREFIX_LIB information + pkg_prefix_lib = f'{prefix}/lib' + if not os.path.exists(pkg_prefix_lib): + os.makedirs(pkg_prefix_lib) + + # generate VXSDK_GCC_CFLAGS information + #@todo : compiler specific ! + #pkg_gcc_cflags = f'-L. -Llib/ -L{prefix}/lib/' + #pkg_gcc_cflags += f' -I. -Iinclude/ -I{prefix}/lib/include/' + + # generate VXSDK_ASSETS_* information + pkg_assets_src = f'{prefix}/converter/{pkg_meta.name}/src' + pkg_assets_obj = f'{prefix}/converter/{pkg_meta.name}/obj' + + # generate VXSDK_PKG_* information + pkg_name = pkg_meta.name + pkg_is_original = str(pkg_meta.is_original) + pkg_version = pkg_meta.version + pkg_target = target + + # generate "dependence-specific" env + envp = { + 'VXSDK_PKG_NAME' : pkg_name, + 'VXSDK_PKG_TARGET' : pkg_target, + 'VXSDK_PKG_VERSION' : pkg_version, + 'VXSDK_PKG_IS_ORIGINAL' : pkg_is_original, + 'VXSDK_PREFIX_BUILD' : pkg_prefix_build, + 'VXSDK_PREFIX_INSTALL' : pkg_prefix_install, + 'VXSDK_PREFIX_LIB' : pkg_prefix_lib, + 'VXSDK_ASSETS_SRC' : pkg_assets_src, + 'VXSDK_ASSETS_BUILD' : pkg_assets_obj, + 'VXSDK_CURRENT_SOURCE_DIR' : pkg_meta.path + } + + # merge extra env configuration + for key in env_extra: + if key in envp: + log.warn(f"[{pkg_name}] extra env key '{key}' already exist") + envp[key] += f' {env_extra[key]}' + + # update env + log.debug(f"{envp}") + os.environ.update(envp) + +#--- +# Public +#--- + +def project_rules_exec(pkg_meta, target, rule_list, env_extra, verbose): + """ Walk through project rules and performs target operation if needed + + This method will build the project source using the custom build + information set by the author in the TOML description. + + Sometimes, we need to pass extra flags information in some steps. You + can pass this information using the 'rules_extra' property of this class + (see more information at 'VxProject.extra') + + @args + > board_target (str) : targeted board for operation (fxcg50, SDL, ...) + > rules (array,str) : list of wanted operations to performs + > extra (dict,str) : user extra information about operations + > verbose (bool) : capture (verbose=False) or not the operation print + + @return + > 0 if success, negative value otherwise + """ + # save env information + saved_pwd = os.getcwd() + saved_env = os.environ.copy() + + # move to the project path + os.chdir(pkg_meta.path) + + # fetch target specific rules + project_rules = pkg_meta.get_build_rules(target) + + # main loop. Fetch operation, check if available, generate env + # information, and perform said operation + ret = 0 + verbose = not verbose + for rule in rule_list: + if rule not in project_rules: + continue + cmd = project_rules[rule].strip() + if pkg_meta.extra_conf and rule in pkg_meta.extra_conf: + cmd += ' ' + pkg_meta.extra_conf[rule] + __project_generate_env(env_extra, pkg_meta, target) + log.debug(f"[{pkg_meta.name}] rule : {rule} -> cmd : ${cmd}$") + ret = subprocess.run( + cmd.split(), + capture_output=verbose, + check=False + ) + if ret.returncode != 0: + if ret.stdout: + log.error(ret.stdout.decode('utf8')) + if ret.stderr: + log.error(ret.stderr.decode('utf8')) + ret = ret.returncode + break + ret = 0 + + # restore env information + os.environ.update(saved_env) + os.chdir(saved_pwd) + return ret diff --git a/vxsdk/core/config.py b/vxsdk/core/config.py new file mode 100644 index 0000000..825c4dc --- /dev/null +++ b/vxsdk/core/config.py @@ -0,0 +1,168 @@ +""" +Configuration file wrapper +""" +import os +import toml + +from core.logger import log + +__all__ = [ + 'config_get', + 'config_set', + 'config_set_default' +] + +DEFAULT_CONFIG_KEYVAL = [ + ('path.config', '~/.config/vxsdk/config.toml'), + ('path.sysroot', '~/.local/share/vxsdk/sysroot'), + ('path.packages', '~/.local/share/vxsdk/packages'), + ('path.assets', '~/.local/share/vxsdk/assets'), + ('path.bin', '~/.local/bin'), + + ('build.default_target', 'superh'), + + ('pkg.backend.name', 'gitea'), + ('pkg.backend.url', 'https://gitea.planet-casio.com'), + ('pkg.local_storage', '{path.packages}') +] + +__CACHED_CONFIG_FILE = None +__CACHED_CONFIG_PATH = None + +#--- +# Internals +#--- + +def __setitem_dots(dictionary, key, value, path=""): + if "." not in key: + old = dictionary[key] if key in dictionary else None + dictionary[key] = value + return old + group, key = key.split(".", 1) + if group in dictionary and not isinstance(dictionary[group], dict): + raise ValueError(f"cannot assign {value} into value {path+group}") + if group not in dictionary: + dictionary[group] = {} + return __setitem_dots(dictionary[group], key, value, path + group + ".") + + +def __config_control(name, value=None): + """ Common configuration file manipulation + + If the `value` parameter is not set, only read operation will be performed, + otherwise the complete file will be updated. + + @args + > name (str) - dot-separated ker (ex : `default.board.name`) + > value (str) - value for the key + """ + global __CACHED_CONFIG_PATH + global __CACHED_CONFIG_FILE + + # check if config file information are cached + # @notes + # - create the configuration file folder if needed + # - load the TOML content + # - cache pathname to avoid path manipulation + if not __CACHED_CONFIG_PATH: + __CACHED_CONFIG_PATH = os.path.expanduser('~/.config/vxsdk/config.toml') + if not __CACHED_CONFIG_FILE: + cache_basename = os.path.basename(__CACHED_CONFIG_PATH) + if not os.path.exists(cache_basename): + os.makedirs(cache_basename) + with open(__CACHED_CONFIG_PATH, 'r+', encoding='utf-8') as file: + __CACHED_CONFIG_FILE = toml.loads(file.read()) + + # check "read-only" request (just fetch value) + if not value: + conf = __CACHED_CONFIG_FILE + targets = name.split('.') + while targets: + if not targets[0] in conf: + log.debug(f"[config] unable to find target '{name}'") + return None + conf = conf[targets[0]] + targets = targets[1:] + return conf + + # perform "write-only" request (update the configuration file) + old = __setitem_dots(__CACHED_CONFIG_FILE, name, value) + with open(__CACHED_CONFIG_PATH, "w", encoding='utf-8') as file: + file.write(toml.dumps(__CACHED_CONFIG_FILE)) + + # return the previous information of the update field (if available) + return old + + +def _generate_value(name, val): + if not val: + return None + while val.find('{') >= 0: + if val.find('}') < 0: + break + key = val[val.find('{') + 1: val.find('}')] + res = __config_control(key) + if not res: + log.warn(f"[config] {name} = {val} : unable to find '{key}'") + return None + val = val[:val.find('{')] + res + val[val.find('}') + 1:] + return val + +#--- +# Public functions +#--- + +def config_get(key: str, default_value: str = None) -> str: + """ Get configuration key/value + + This function will try to find the key value of `key`. If the key doest not + exists then None will be returned. You can specify a default value if the + key doest not exist. + + @args + > key: string - the key name + > default_value: string - the key default value if not found + + @return + > return the key value or None if not found nor default value set + """ + if ret := __config_control(key): + return _generate_value(key, ret) + default_value = _generate_value(key, default_value) + if default_value: + __config_control(key, default_value) + return default_value + +def config_set(key: str, value: str) -> str: + """ Set configuration key = value + + This function will try to update the user vxSDK configuration file to add + key / value information. Note that the `value` can have placeholder in its + content like `{path.sysroot}/superh` wich will fetch the 'path.sysroot' + configuration key. + + @args + > key: string - the key name + > name: string - the key value + + @return + > the old key value or None if new + """ + return __config_control(key, value) + +def config_set_default(list_of_keyval: list): + """ Set default key / value + + This function will setup all default key value if the key doest not exists. + This is usefull to centralise all default user configuration information in + this file instead of in all project files. + + @arg + > list_of_keyval: list of tuple - [(key, value), ...] + """ + for key, value in list_of_keyval: + if not __config_control(key): + __config_control(key, value) + +# workaround +config_set_default(DEFAULT_CONFIG_KEYVAL) diff --git a/vxsdk/core/conv/__init__.py b/vxsdk/core/conv/__init__.py new file mode 100644 index 0000000..cdc2281 --- /dev/null +++ b/vxsdk/core/conv/__init__.py @@ -0,0 +1,68 @@ +""" +Vhex converter module +""" + +from core.conv.asset import conv_assets_generate + +__all__ = [ + 'assets_generate' +] + +def assets_generate(prefix_assets, prefix_src): + r"""Generate Vhex assets. + + This function abstract the asset convertion for the Vhex Operating System. + It will walk througt the `` folder and will try to find some + files named 'vxconv.txt' wich discribe assets information of a potential + project. Then it will use them to convert assets into an appropriate source + file in C without using any Vhex-specific function (so, you can use this + converter for other project). + + The vxconv.txt file is structured like basic key/value file: + + ``` + : + type: (font, bitmap) - required + path: - required + ... + + : + ... + ``` + + Each asset file description should have at least type and name information, + and each type have potentially its own requierements. + + type = bitmap: + ================================== ========================================= + Keys name and value type Description + ================================== ========================================= + profile: Select the bitmap pixel profile + | rgb4 | RGB 4 (indexed) + | rgb4a | RGBA 4 (indexed) + | rgb8 | RGB 8 (indexed) + | rgb8a | RGBA 8 (indexed) + | rgb16 | RGB 16 (5:R, 6:G, 5:B) + | rgb16a | RGBA 16 (5:R, 5:G, 5:B, 1:A) + ================================== ========================================= + + type = font: + ================================== ========================================= + Keys name and value type Description + ================================== ========================================= + grid.size: 8x9 (widthxheight) caracter size in pixel + grid.padding: space between caracter + grig.border: space around grid + proportional: caracter are cropped + line_height: caracter line alignement + charset: charset specification + ================================== ========================================= + + @args: + > path (str) - the path to find assets + > source_prefix (str) - the path to generate image source file + + @return: + > a list of string which represents all assets sources files path + """ + return conv_assets_generate(prefix_assets, prefix_src) diff --git a/vxsdk/core/conv/addin.py b/vxsdk/core/conv/addin.py new file mode 100644 index 0000000..20f55d7 --- /dev/null +++ b/vxsdk/core/conv/addin.py @@ -0,0 +1,67 @@ +from core.conv.pixel import rgba8conv + +from PIL import Image +import os + +__all__ = [ + 'generate_addin' +] + + +def generate_addin(binary, icon=None, name=None, output=None, version=None): + r"""Generate an addin for the Vhex Operating System. + + The addin name (passed through the `name` argument) is optional here. In + this case, the name will use the internal name...which can be guessed using + the binary name (e.i '/path/to/the/super_addin.elf' -> internal name = + 'super_addin' -> output name = '/path/to/the/super_addin'). + + The output path for the generated file is, by defautl, the same path that + the binary but the suffix '.vxos' will be added. + + if the icon is not specified, a default blank icon will be used. + + Args: + > binary (str) - binary path + > icon (str) - addin icon path (optional) + > name (str) - addin name (displayed in the menu) (optional) + > output (str) - output path for the generated addin (optional) + + FIXME: + > generate default internal name + > change 8-bits icon into rgb565 + > add internal addin version in the header + """ + if not os.path.exists(binary): + logger(LOG_ERR, 'binary path is invalid') + sys.exit(84) + if icon and not os.path.exists(icon): + logger(LOG_WARN, f'{icon}: icon does not exists, ignored') + icon = None + if not name: + name = '' + if not output: + output = binary + '.vxos' + + if icon: + bitmap = Image.open(icon) + if bitmap.size != (92, 64): + logger( + LOG_ERR, + f'{icon}:icon size does not match {bitmap.size} != (92, 64)', + exit=84 + ) + + with open(binary, 'rb') as b: + with open(output, 'wb') as a: + a.write(b'VHEX') + a.write(name.encode('utf8')) + a.write(b'\x00') + if icon: + for pixel in bitmap.getdata(): + a.write(rgba8conv(pixel).to_bytes(1, 'big')) + else: + a.write(bytes(92*64)) + a.write(b.read()) + + return 0 diff --git a/vxsdk/core/conv/asset.py b/vxsdk/core/conv/asset.py new file mode 100644 index 0000000..19e0592 --- /dev/null +++ b/vxsdk/core/conv/asset.py @@ -0,0 +1,126 @@ +""" +Vhex assets converter +""" +import os +import toml + +from core.logger import log +from core.conv.type.font import conv_font_generate +from core.conv.type.image import conv_image_generate + + +__all__ = [ + 'conv_assets_generate' +] + + +#--- +# Private +#--- + +class _VxAsset(): + """Represent a asset object + + This is an internal class which represents assets information with some + methods to abstract conversion and file type manipulation (for asset type + font and asset type bitmap). + + Also note that this class is private because we use a tricky optimization to + parse the `vxconv.txt` file, this is why we have no "private" property with + setter and getter, and why this class is "hidden". + + Some important methods to note: + + ================================== ========================================= + Name Description + ================================== ========================================= + generate() Generate the source file (C) + ================================== ========================================= + + """ + def __init__(self, prefix, name, meta): + if 'path' not in meta: + raise Exception(f"[{name}] missing required path information") + if 'type' not in meta: + raise Exception(f"[{name}] missing required type information") + if meta['type'] not in ['font', 'image']: + raise Exception(f"asset type '{meta[type]}' is not known") + + self._name = name + self._meta = meta + self._type = meta['type'] + self._path = prefix + '/' + meta['path'] + if not os.path.exists(self.path): + raise Exception("asset path '{self._path}' cannot be openned") + + def __repr__(self): + return f'<_VxAssetObj, {self.name}>' + + def __str__(self): + content = f"[{self.name}]\n" + content += f" - type: {self.type}\n" + content += f" - path: {self.path}\n" + return content + + #--- + # Getter + #--- + + @property + def path(self): + """ path""" + return self._path + + @property + def name(self): + """ name""" + return self._name + + @property + def type(self): + """ type""" + return self._type + + @property + def meta(self): + """ meta""" + return self._meta + + #--- + # Public method + #--- + + def generate_source_file(self, prefix_output): + """generate source file """ + if self.type == 'font': + return conv_font_generate(self, prefix_output) + return conv_image_generate(self, prefix_output) + + +#--- +# Public +#--- + +def conv_assets_generate(prefix_assets, prefix_output): + """ Walk through the assets prefix and generate all source file + + @args + > prefix_asset (str) - prefix used for recursivly search for asset info + > prefix_output (str) - prefix used for the output of generated file + + @return + > a list of all generated sources pathname + """ + if not os.path.exists(prefix_output): + os.makedirs(prefix_output) + generated = [] + for root, _, files in os.walk(prefix_assets): + if not 'vxconv.toml' in files: + continue + with open(root + '/vxconv.toml', "r", encoding='utf-8') as inf: + content = toml.loads(inf.read()) + for asset_name in content: + log.user(f"converting {asset_name}...") + asset = _VxAsset(root, asset_name, content[asset_name]) + generated += asset.generate_source_file(prefix_output) + return generated diff --git a/vxsdk/core/conv/pixel.py b/vxsdk/core/conv/pixel.py new file mode 100644 index 0000000..aa153cc --- /dev/null +++ b/vxsdk/core/conv/pixel.py @@ -0,0 +1,45 @@ +""" Pixel converter utilities + +This file expose many 32 bits RGBA into various pixel format +""" + +__all__ = [ + 'rgb1conv', + 'rgb8conv', + 'rgba8conv', + 'rgb16conv', + 'rgba16conv' +] + +def rgb24to16(rgb): + _r = (rgb[0] & 0xff) >> 3 + _g = (rgb[1] & 0xff) >> 2 + _b = (rgb[2] & 0xff) >> 3 + return (_r << 11) | (_g << 5) | _b + + + +def rgb1conv(pixel): + return pixel == (0, 0, 0) + +def rgb8conv(pixel): + return int((pixel[0] * 7) / 255) << 5 \ + | int((pixel[1] * 4) / 255) << 3 \ + | int((pixel[2] * 7) / 255) << 0 + +def rgba8conv(pixel): + return int((pixel[0] * 4) / 256) << 6 \ + | int((pixel[1] * 8) / 256) << 3 \ + | int((pixel[2] * 4) / 256) << 1 \ + | (len(pixel) >= 4 and pixel[3] == 0) + +def rgb16conv(pixel): + return int((pixel[0] * 31) / 255) << 11 \ + | int((pixel[1] * 63) / 255) << 5 \ + | int((pixel[2] * 31) / 255) << 0 + +def rgba16conv(pixel): + return int((pixel[0] * 31) / 255) << 11 \ + | int((pixel[1] * 63) / 255) << 6 \ + | int((pixel[2] * 31) / 255) << 1 \ + | (pixel[3] != 0) diff --git a/vxsdk/core/conv/type/__init__.py b/vxsdk/core/conv/type/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/vxsdk/core/conv/type/font.py b/vxsdk/core/conv/type/font.py new file mode 100644 index 0000000..fc50413 --- /dev/null +++ b/vxsdk/core/conv/type/font.py @@ -0,0 +1,466 @@ +""" +Vhex font converter +""" + +from PIL import Image + +from core.logger import log + +__all__ = [ + 'conv_font_generate' +] + +#--- +# Internal vxconv.tmol handling routines +#--- + +def __font_fetch_info(asset): + """ Check and fetch font information + + @arg + > asset (VxAsset) - asset information + + @return + > dictionary with font information + """ + # generate font default information + font_info = { + # user can customise + 'charset' : 'normal', + 'grid_size_x' : 0, + 'grid_size_y' : 0, + 'grid_padding' : 1, + 'grid_border' : 1, + 'is_proportional' : False, + 'line_height' : 0, + 'char_spacing' : 1, + + # generated "on-the-fly" by the conversion step + # @notes + # This is mainly to provide cache for the Vhex operating system to + # speed-up render calculation by avoiding recurent caculation. + 'glyph_size' : 0, + 'glyph_height' : 0, + 'font_size' : 0, + 'data' : [] + } + + # handle user meta-indication + if 'charset' in asset.meta: + if asset.meta['charset'] not in ['default', 'unicode']: + log.error(f"Unknown charset '{asset.meta['charset']}', abord") + return None + font_info['charset'] = asset.meta['charset'] + if 'grid_size' not in asset.meta: + log.error("Missing critical grid size information, abord") + return None + grid_size = asset.meta['grid_size'].split('x') + font_info['grid_size_x'] = int(grid_size[0]) + font_info['grid_size_y'] = int(grid_size[1]) + if 'grid_padding' in asset.meta: + font_info['grid_padding'] = int(asset.meta['grid_padding']) + if 'grid_border' in asset.meta: + font_info['grid_border'] = int(asset.meta['grid_border']) + if 'proportional' in asset.meta: + font_info['is_proportional'] = asset.meta['proportional'] + font_info['line_height'] = font_info['grid_size_y'] + if 'line_height' in asset.meta: + font_info['line_height'] = asset.meta['line_height'] + if 'char_spacing' in asset.meta: + font_info['char_spacing'] = asset.meta['char_spacing'] + font_info['glyph_height'] = font_info['grid_size_y'] + + # return font information + return font_info + +#--- +# Internal glyph routines +#--- + +def __glyph_get_wgeometry(geometry_info, img_raw, img_size, pos, grid_size): + """ Generate glyph width geometry information + + @args + > geometry_info (dict) - geometry information + > img_raw (list) - list of all pixel of the image + > img_size (tuple) - image width and image height + > pos (tuple) - glyph position information (X and Y in pixel) + > grid_size (tuple) - glyph grid size information (width and height) + + @return + > Nothing + """ + + geometry_info['wstart'] = -1 + geometry_info['wend'] = -1 + + _px = pos[0] + _py = pos[1] + log.debug(f'[geometry] X:{pos[0]} Y:{int(pos[1]/img_size[0])}') + log.debug(f' - grid_size = {grid_size}') + for _ in range(0, grid_size[1]): + for offx in range(0, grid_size[0]): + if img_raw[_py + (_px + offx)][:3] == (255, 255, 255): + continue + if geometry_info['wstart'] < 0 or offx < geometry_info['wstart']: + geometry_info['wstart'] = offx + if geometry_info['wstart'] < 0 or offx > geometry_info['wend']: + geometry_info['wend'] = offx + _py += img_size[0] + geometry_info['wend'] += 1 + log.debug(f' - geometry = {geometry_info}') + +def __glyph_encode(data_info, img_info, geometry, posx, posy): + """ Encode glyph bitmap + + @args + > data_info (dict) - internal data information (list, index and shift) + > img_info (dict) - image-related information (object and raw content) + > geometry (dict) - geometry information + > posx (int) - X-axis position in pixel + > posy (int) - Y-axis position in pixel + + @return + > Nothing + """ + # fetch information + img = img_info['obj'] + img_raw = img_info['raw'] + data = data_info['table'] + data_idx = data_info['idx'] + data_shift = data_info['shift'] + wstart = geometry['wstart'] + wend = geometry['wend'] + + # encode the glyph + yoff = 0 + log.debug(f'[encode] X:{posx} Y:{int(posy/img.size[0])}') + for _h in range(geometry['hstart'], geometry['hend']): + for _w in range(wstart, wend): + if img_raw[(posy + yoff) + (posx + _w)][:3] == (0, 0, 0): + log.debug('#', end='') + data[data_idx] |= 0x80000000 >> data_shift + else: + log.debug('.', end='') + data[data_idx] &= ~(0x80000000 >> data_shift) + if (data_shift := data_shift + 1) >= 32: + data_shift = 0 + data_idx += 1 + log.debug('') + yoff += img.size[0] + + # commit modification + data_info['idx'] = data_idx + data_info['shift'] = data_shift + +#--- +# Intenal font conversion +#--- + +def __font_convert_proportional(packed_info): + """ Generate proportional font + + Proportional font means that each character have its own width size (but + have a common height). We need to performs more complexe handling than the + monospaced one. + + @args + > asset (VxAsset) - asset information + > font_information (dict) - font indication + + @return + > 0 if success, negative value otherwise + """ + # unpack information + font_info = packed_info[0] + img_info = packed_info[1] + glyph_info = packed_info[2] + data_info = packed_info[4] + geometry_info = packed_info[5] + + # isolate needed information + img = img_info['obj'] + img_raw = img_info['raw'] + nb_col = packed_info[3][0] + nb_row = packed_info[3][1] + gwidth = glyph_info[0] + gheight = glyph_info[1] + + # main loop, walk glyph per glyph + _py = (font_info['grid_border'] + font_info['grid_padding']) * img.size[0] + for _ in range(0, nb_row): + _px = font_info['grid_border'] + font_info['grid_padding'] + for _ in range(0, nb_col): + # generate width geometry information + __glyph_get_wgeometry( + geometry_info, + img_raw, + img.size, + (_px, _py), + (font_info['grid_size_x'], font_info['grid_size_y']) + ) + + # save critical glyph geometry information that will be encoded in + # the final C source file + font_info['glyph_props'].append(( + geometry_info['wend'] - geometry_info['wstart'], + data_info['idx'], + data_info['shift'] + )) + + # encode glyph information + __glyph_encode(data_info, img_info, geometry_info, _px, _py) + + # update loop information + font_info['glyph_count'] += 1 + _px += gwidth + _py += gheight * img.size[0] + return 0 + +def __font_convert_monospaced(packed_info): + """ Generate proportional font + + Proportional font means that each character have its own width size (but + have a common height). We need to performs more complexe handling than the + monospaced one. + + @args + > asset (VxAsset) - asset information + > font_information (dict) - font indication + + @return + > 0 if success, negative value otherwise + """ + # unpack information + font_info = packed_info[0] + img_info = packed_info[1] + glyph_info = packed_info[2] + grid_info = packed_info[3] + data_info = packed_info[4] + geometry_info = packed_info[5] + + # isolate needed information + img = img_info['obj'] + nb_row = grid_info[1] + nb_col = grid_info[0] + gwidth = glyph_info[0] + gheight = glyph_info[1] + + # main loop, walk glyph per glyph + _py = (font_info['grid_border'] + font_info['grid_padding']) * img.size[0] + for _ in range(0, nb_row): + _px = font_info['grid_border'] + font_info['grid_padding'] + for _ in range(0, nb_col): + __glyph_encode(data_info, img_info, geometry_info, _px, _py) + font_info['glyph_count'] += 1 + _px += gwidth + _py += gheight * img.size[0] + return 0 + +def __font_convert(asset, font_info): + """ Generate font information + + @args + > asset (VxAsset) - asset information + > font_info (dict) - font information + + @return + > 0 if success, negative value otherwise + """ + # generate image information + img = Image.open(asset.path) + img_raw = img.getdata() + img_info = { + 'obj' : img, + 'raw' : img_raw + } + + # pre-calculate the "real" glyph width and height using padding information + glyph_info = [0, 0] + glyph_info[0] = font_info['grid_size_x'] + font_info['grid_padding'] + glyph_info[1] = font_info['grid_size_y'] + font_info['grid_padding'] + gheight = glyph_info[1] + gwidth = glyph_info[0] + log.debug(f"gwidth = {gwidth} && gheight = {gheight}") + + # pre-calculate the number of row and column of the font + grid_info = [0, 0] + grid_info[0] = int((img.size[0] - (font_info['grid_border'] * 2)) / gwidth) + grid_info[1] = int((img.size[1] - (font_info['grid_border'] * 2)) / gheight) + nb_col = grid_info[0] + nb_row = grid_info[1] + log.debug(f"nb_row = {nb_row} && nb_col = {nb_col}") + + # pre-calculate and prepare per-glyph information + # @note + # The generated data is designed for 4-alignement padding. This to have + # speed-up on drawing function. + font_info['glyph_size'] = font_info['grid_size_x'] * font_info['grid_size_y'] + font_info['font_size'] = font_info['glyph_size'] * nb_row * nb_col + font_info['glyph_count'] = 0 + font_info['glyph_props'] = [] + font_info['data'] = [0] * int((font_info['font_size'] + 31) / 32) + log.debug(f"data original = {id(font_info['data'])}") + + # generate data information + data_info = { + 'table' : font_info['data'], + 'idx' : 0, + 'shift' : 0 + } + log.debug(f"data packed = {id(data_info['table'])}") + + # generate geometry information + geometry_info = { + 'hstart' : 0, + 'hend' : font_info['grid_size_y'], + 'wstart' : 0, + 'wend' : font_info['grid_size_x'], + } + + # select the converter + converter = __font_convert_monospaced + if font_info['is_proportional']: + converter = __font_convert_proportional + + # convert font + converter(( + font_info, + img_info, + glyph_info, + grid_info, + data_info, + geometry_info + )) + + log.debug(f"data packed end = {id(data_info['table'])}") + return 0 + +#--- +# Source file generation +#--- + +def __font_generate_unicode_source(_): + """Unicode special chaset directory""" + log.error("unicode conversion not implemented yet o(x_x)o") + return '' + +def __font_generate_normal_source(font_info): + """Print chaset is a image file + + """ + content = "\t.glyph = {\n" + content += f"\t\t.height = {font_info['glyph_height']},\n" + content += f"\t\t.line_height = {font_info['line_height']},\n" + + # encode font bitmap + line = 0 + log.debug(f"data = {font_info['data']}") + content += "\t\t.data = (uint32_t[]){\n" + for pixel in font_info['data']: + if line == 0: + content += '\t\t\t' + if line >= 1: + content += ' ' + content += f"{pixel:#010x}," + if (line := line + 1) == 4: + content += '\n' + line = 0 + if line != 0: + content += '\n' + content += '\t\t},\n' + + # indicate the number of glyph in the bitmap + content += f"\t\t.count = {font_info['glyph_count']},\n" + + # encode proportional information if needed + if font_info['is_proportional']: + content += '\t\t.prop = (struct __workaround[]){\n' + for prop in font_info['glyph_props']: + content += "\t\t\t{\n" + content += f"\t\t\t\t.width = {prop[0]},\n" + content += f"\t\t\t\t.index = {prop[1]},\n" + content += f"\t\t\t\t.shift = {prop[2]},\n" + content += "\t\t\t},\n" + else: + content += "\t\t.mono = {,\n" + content += f"\t\t\t.width = {font_info['glyph_width']},\n" + content += f"\t\t\t.size = {font_info['glyph_size']},\n" + content += "\t\t},\n" + content += "\t},\n" + + # skip unicode struct + content += "\t.unicode = {\n" + content += "\t\t.blocks = NULL,\n" + content += "\t\t.block_count = 0,\n" + content += "\t}\n" + return content + +def __font_generate_source_file(asset, font_info): + """Generate font source file content + + @args + > asset (VxAsset) - asset information + > info (dict) - hold font information + + @return + > file C content string + """ + # generate basic header + content = "#include \n" + content += "\n" + content += f"/* {asset.name} - Vhex asset\n" + content += " This object has been converted by using the vxSDK " + content += "converter */\n" + content += f"struct font const {asset.name} = " + "{\n" + content += f"\t.name = \"{asset.name}\",\n" + + # shape information + content += "\t.shape = {\n" + content += "\t\t.bold = 0,\n" + content += "\t\t.italic = 0,\n" + content += "\t\t.serif = 0,\n" + content += "\t\t.mono = 0,\n" + content += f"\t\t.prop = {int(font_info['is_proportional'])},\n" + content += "\t},\n" + + # manage display indication + content += f"\t.char_spacing = {font_info['char_spacing']},\n" + + # handle special charset behaviour + if font_info['charset'] == 'unicode': + content += __font_generate_unicode_source(font_info) + else: + content += __font_generate_normal_source(font_info) + + # closure and return + content += '};\n' + return content + +#--- +# Public +#--- + +def conv_font_generate(asset, prefix_output): + """ Convert an image asset to a C source file + + @args + > asset (_VxAsset) - minimal asset information + > prefix_output (str) - prefix for source file generation + + @return + > pathname of the generated file + """ + # generate font information + if not (font_info := __font_fetch_info(asset)): + return '' + if __font_convert(asset, font_info) != 0: + return '' + content = __font_generate_source_file(asset, font_info) + + # create the source file + asset_src = f'{prefix_output}/{asset.name}_vxfont.c' + with open(asset_src, "w", encoding='utf8') as file: + file.write(content) + log.debug(f"source file generated at {asset_src}") + return asset_src diff --git a/vxsdk/core/conv/type/image.py b/vxsdk/core/conv/type/image.py new file mode 100644 index 0000000..743602f --- /dev/null +++ b/vxsdk/core/conv/type/image.py @@ -0,0 +1,397 @@ +""" +Vhex image converter +""" + +from PIL import Image + +from core.logger import log +from core.conv.pixel import rgb24to16 + +__all__ = [ + 'conv_image_generate' +] + +#--- +# Private profile color management +#--- + +def __profile_gen(profile, name, palette=None, alpha=None): + r""" Internal image profile class + + ================================== ========================================= + Property Description + ================================== ========================================= + id (int) profile ID + names (array of str) list all profile names + format (str) profile format name (vhex API) + has_alpha (bool) indicate if the profil has alpha + alpha (int) alpha index in the palette (or mask) + is_indexed (bool) indicate if the profile should be indexed + palette_base (int) indicate base index for color inserting + palette_color_count (int) indicate the number of color (palette) + palette_trim (bool) indicate if the palette should be trimed + ================================== ========================================= + """ + profile = { + 'profile' : profile, + 'name' : name, + 'has_alpha' : (alpha is not None), + 'alpha' : alpha, + 'is_indexed': (palette is not None), + 'palette' : None + } + if palette is not None: + profile['palette_base'] = palette[0] + profile['palette_color_count'] = palette[1] + profile['palette_trim'] = palette[2] + return profile + +# all supported profile information +VX_PROFILES = [ + __profile_gen('IMAGE_RGB565', "p16"), + __profile_gen('IMAGE_RGB565A', "p16a", alpha=0x0001), + __profile_gen('IMAGE_P8_RGB565', "p8", palette=(0,256,True)), + __profile_gen('IMAGE_P8_RGB565A', "p8a", palette=(1,256,True), alpha=0), + __profile_gen('IMAGE_P4_RGB565', "p4", palette=(0,16,False)), + __profile_gen('IMAGE_P4_RGB565A', "p4a", palette=(1,16,False), alpha=0), +] + +def __profile_find(name): + """Find a profile by name.""" + for profile in VX_PROFILES: + if name == profile['name']: + return profile + return None + +#--- +# Private image manipulation +#--- + +def __image_isolate_alpha(info): + """ Isolate alpha corlor of the image + + Vhex use a particular handling for alpha color and this information should + use a strict encoding way. Things that Pillow don't do properly. So, lets + manually setup our alpha isolation and patch Pillow alpha palette handling. + + @args + > info (dict) - contains all needed information (image, data, ...) + + @return + > Nothing + """ + # fetch needed information + img = info['img'] + profile = info['profile'] + + # Save the alpha channel and make it 1-bit. We need to do this because + # the alpha value is handled specialy in Vhex and the image conversion + # to palette-oriented image is weird : the alpha colors is also converted + # in the palette + if profile['has_alpha']: + alpha_channel = img.getchannel("A").convert("1", dither=Image.NONE) + else: + alpha_channel = Image.new("1", img.size, 1) + + alpha_pixels = alpha_channel.load() + img = img.convert("RGB") + + # Transparent pixels have random values on the RGB channels, causing + # them to use up palette entries during quantization. To avoid that, set + # their RGB data to a color used somewhere else in the image. + pixels = img.load() + bg_color = next( + ( + pixels[x,y] + for x in range(img.width) + for y in range(img.height) + if alpha_pixels[x,y] > 0 + ), + (0,0,0) + ) + for _y in range(img.height): + for _x in range(img.width): + if alpha_pixels[_x, _y] == 0: + pixels[_x, _y] = bg_color + + # update external information + info['img'] = img + info['img_pixel_list_alpha'] = alpha_pixels + info['img_pixel_list_clean'] = pixels + +def __image_encode_palette(info): + """ Generate palette information + + This routine is involved only if the targeted profile is indexed. We need to + generate (and isolate) color palette. + + @args + > info (dict) - contains all needed information (image, data, ...) + + @return + > Nothing + """ + # fetch needed information + img = info['img'] + profile = info['profile'] + + # convert image into palette format + # note: we remove one color slot in the palette for the alpha one + color_count = profile['palette_color_count'] - int(profile['has_alpha']) + img = img.convert( + 'P', + dither=Image.NONE, + palette=Image.ADAPTIVE, + colors=color_count + ) + + # The palette format is a list of N triplets ([r, g, b, ...]). But, + # sometimes, colors after img.convert() are not numbered 0 to + # `color_count`, because the palette don't need to be that big. So, + # we calculate the "palette size" by walking throuth the bitmap and + # by saving the biggest index used. + pixels = img.load() + nb_triplet = 1 + max( + pixels[x,y] + for y in range(img.height) + for x in range(img.width) + ) + palette = img.getpalette()[:3 * nb_triplet] + palette = list(zip(palette[::3], palette[1::3], palette[2::3])) + + # For formats with transparency, add an "unused" palette slot which + # will used has pink/purple in case of a bad application try to use + # this value anyway + if profile['has_alpha']: + palette = [(255, 0, 255)] + palette + nb_triplet += 1 + + # Also keep track of how to remap indices from the values generated + # by img.convert() into the palette, which is shifted by 1 due to + # alpha and also starts at profile.palette_base. + # + # Note: profile.palette_base already starts 1 value later for + # formats with alpha. + palette_map = [ + (profile['palette_base'] + i) % profile['palette_color_count'] + for i in range(nb_triplet) + ] + + # Encode the palette + palette_color_count = nb_triplet + if not profile['palette_trim']: + palette_color_count = profile['palette_color_count'] + + palette_data = [0] * palette_color_count + for i, rgb24 in enumerate(palette): + palette_data[i] = rgb24to16(rgb24) + + # update internal information + info['palette_map'] = palette_map + info['palette_data'] = palette_data + info['palette_color_count'] = palette_color_count + info['nb_triplet'] = nb_triplet + info['img_pixel_list_clean'] = pixels + +def __image_encode_bitmap(info): + """ Encode the bitmap + + This routine will generate the main data list which will contains the bitmap + using Vhex-specific encoding. + + @args + > info (dict) - contains all needed information (image, data, ...) + + @return + > Nothing + """ + # fetch needed information + img = info['img'] + profile = info['profile'] + alpha_pixels = info['img_pixel_list_alpha'] + pixels = info['img_pixel_list_clean'] + palette_map = info['palette_map'] + + # generate profile-specific geometry information + if profile['name'] in ['p16', 'p16a']: + # Preserve alignment between rows by padding to 4 bytes + nb_stride = ((img.width + 1) // 2) * 4 + data_size = (nb_stride * img.height) * 2 + elif profile['name'] in ['p8', 'p8a']: + nb_stride = img.width + data_size = img.width * img.height + else: + # Pad whole bytes + nb_stride = (img.width + 1) // 2 + data_size = nb_stride * img.height + + # Generate the real data map + data = [0] * data_size + + # encode the bitmap + for _y in range(img.height): + for _x in range(img.width): + # get alpha information about this pixel + _a = alpha_pixels[_x, _y] + + if profile['name'] in ['p16', 'p16a']: + # If c lands on the alpha value, flip its lowest bit to avoid + # ambiguity with alpha + _c = profile['alpha'] if _a else rgb24to16(pixels[_x, _y]) & ~1 + data[(img.width * _y) + _x] = _c + + elif profile['name'] in ['p8', 'p8a']: + _c = palette_map[pixels[_x, _y]] if _a > 0 else profile['alpha'] + data[(img.width * _y) + _x] = _c + + else: + _c = palette_map[pixels[_x, _y]] if _a > 0 else profile['alpha'] + offset = (nb_stride * _y) + (_x // 2) + if _x % 2 == 0: + data[offset] |= (_c << 4) + else: + data[offset] |= _c + + # update external information + info['data'] = data + info['data_size'] = data_size + info['nb_stride'] = nb_stride + info['data_size'] = data_size + +def __image_convert(asset, profile_name): + """ Image asset convertion + + @args + > asset (_VxAsset) - asset information + > profile_name (str) - profile name information + + @return + > a dictionary with all image information + """ + # generate critical information and check posible error + img_info = { + 'img' : Image.open(asset.path), + 'profile' : __profile_find(profile_name) + } + if not img_info['img']: + log.error(f"unable to open the asset '{asset.path}', abord") + return None + if not img_info['profile']: + log.error(f"unable to find the color profile '{profile_name}', abord") + return None + + # convert the bitmap and generate critical information + __image_isolate_alpha(img_info) + if img_info['profile']['is_indexed']: + __image_encode_palette(img_info) + __image_encode_bitmap(img_info) + + # return generated information + return img_info + +#--- +# Internal source file content generation +#--- + +def __display_array(array, prefix='\t\t'): + """ Display array information (only for p16* profile) """ + line = 0 + content = '' + for pixels in array: + if line == 0: + content += prefix + if line >= 1: + content += ' ' + content += f'{pixels:#06x},' + if (line := line + 1) >= 8: + content += '\n' + line = 0 + if line != 0: + content += '\n' + return content + +def __image_generate_source_file(asset, info): + """Generate image source file + + @args + > asset (VxAsset) - asset information + > info (dict) - hold image information + + @return + > file C content string + """ + img = info['img'] + profile = info['profile'] + + # generate basic header + content = "#include \n" + content += "\n" + content += f"/* {asset.name} - Vhex asset\n" + content += " This object has been converted by using the vxSDK " + content += "converter */\n" + content += "const image_t " + f"{asset.name} = " + "{\n" + content += f"\t.format = {profile['profile']},\n" + content += "\t.flags = IMAGE_FLAGS_RO | IMAGE_FLAGS_OWN,\n" + content += f"\t.color_count = {profile['palette_color_count']},\n" + content += f"\t.width = {img.width},\n" + content += f"\t.height = {img.height},\n" + content += f"\t.stride = {info['nb_stride']},\n" + + # encode bitmap table + encode = 16 if profile['profile'] in ['p16', 'p16a'] else 8 + content += f"\t.data = (void*)(const uint{encode}_t [])" + "{\n" + for _y in range(img.height): + content += '\t\t' + for _x in range(info['nb_stride']): + pixel = info['data'][(_y * info['nb_stride']) + _x] + if profile['profile'] in ['p16', 'p16a']: + content += f'{pixel:#06x},' + elif profile['profile'] in ['p8', 'p8a']: + content += f'{pixel:#04x},' + else: + content += f'{pixel:3},' + content += '\n' + + content += '\t},\n' + + # add palette information + if 'palette_data' in info: + content += "\t.palette = (void*)(const uint16_t []){\n" + content += __display_array(info['palette_data']) + content += "\t},\n" + else: + content += "\t.palette = NULL,\n" + + # closure and return + content += '};' + return content + +#--- +# Public +#--- + +def conv_image_generate(asset, prefix_output): + """ Convert an image asset to a C source file + + @args + > asset (_VxAsset) - minimal asset information + > prefix_output (str) - prefix for source file generation + + @return + > pathname of the generated file + """ + # check critical requirement + if 'profile' not in asset.meta: + log.error(f"[{asset.name}] missing profile information!") + return '' + + #generate the source file content + if not (img_info := __image_convert(asset, asset.meta['profile'])): + return '' + content = __image_generate_source_file(asset, img_info) + + # generate the source file + asset_src = f'{prefix_output}/{asset.name}_vximage.c' + with open(asset_src, "w", encoding='utf8') as file: + file.write(content) + return asset_src diff --git a/vxsdk/core/logger.py b/vxsdk/core/logger.py new file mode 100644 index 0000000..3ee290c --- /dev/null +++ b/vxsdk/core/logger.py @@ -0,0 +1,115 @@ +""" +Log wrapper +""" +import sys + +__all__ = [ + 'log' +] + +LOG_DEBUG = 7 +LOG_INFO = 6 +LOG_NOTICE = 5 +LOG_USER = 4 +LOG_WARN = 3 +LOG_ERR = 2 +LOG_CRIT = 1 +LOG_EMERG = 0 + +#--- +# Internals +#--- + +class _VxLogger(): + def __init__(self, logfile=None): + self._logfile = logfile + self._level = LOG_USER + self._indent = 0 + + #--- + # Internals + #--- + + def _print(self, level, text, skip_indent, fileno): + if self._level < level: + return 0 + if not skip_indent and self._level == LOG_DEBUG and self._indent > 0: + text = ('>>> ' * self._indent) + text + print(text, file=fileno, end='', flush=True) + return len(text) + 1 + + #--- + # Public properties + #--- + + @property + def level(self): + """ handle print level """ + return self._level + + @level.setter + def level(self, level): + """ handle print level """ + if level < LOG_EMERG or level > LOG_DEBUG: + print(f"[log] level update to {level} is not possible, ignored") + return + self._level = level + + @property + def indent(self): + """ handle indentation level for LOG_DEBUG """ + return self._indent + + @indent.setter + def indent(self, indent): + """ handle indentation level for LOG_DEBUG """ + if indent < 0: + print(f"[log] indent update to {indent} is not possible, ignored") + return + self._indent = indent + + #--- + # Public methods + #--- + + def debug(self, text, end='\n', skip_indent=False): + """ print debug log """ + return self._print(LOG_DEBUG, text + end, skip_indent, sys.stdout) + + def info(self, text, end='\n', skip_indent=False): + """ print info log """ + return self._print(LOG_INFO, text + end, skip_indent, sys.stdout) + + def notice(self, text, end='\n', skip_indent=False): + """ print notice log """ + return self._print(LOG_NOTICE, text + end, skip_indent, sys.stdout) + + def user(self, text, end='\n', skip_indent=False): + """ print user log """ + return self._print(LOG_USER, text + end, skip_indent, sys.stdout) + + def warn(self, text, end='\n', skip_indent=False): + """ print warning log """ + return self._print(LOG_WARN, text + end, skip_indent, sys.stderr) + + def error(self, text, end='\n', skip_indent=False): + """ print error log """ + return self._print(LOG_ERR, text + end, skip_indent, sys.stderr) + + def critical(self, text, end='\n', skip_indent=False): + """ print critical log """ + return self._print(LOG_CRIT, text + end, skip_indent, sys.stderr) + + def emergency(self, text, end='\n', skip_indent=False): + """ print emergency log """ + return self._print(LOG_EMERG, text + end, skip_indent, sys.stderr) + + + + + +#--- +# Public functions +#--- + +log = _VxLogger() diff --git a/vxsdk/core/pkg/__init__.py b/vxsdk/core/pkg/__init__.py new file mode 100644 index 0000000..c928c6e --- /dev/null +++ b/vxsdk/core/pkg/__init__.py @@ -0,0 +1,59 @@ +""" +Provide package primitives (mainly for syntax sugar) +""" +from core.pkg.find import pkg_find +from core.pkg.clone import pkg_clone + +__all__ = [ + 'find', + 'clone', +] + +#--- +# Public +#--- + +def find(name, version=None, local=True, remote=True): + r"""Try to find a particular package. + + @args + > name (str) - exact valid package name + > version (str) - version query string + > local (bool) - enable local search + > remote (bool) - enable remote search + + @return + > a list of all matched package dictionnary : + [ + { + 'name' : , + 'version' : [ + + ... + ] + ] + > None if error + """ + return pkg_find(name, version, local, remote) + +def clone(name, version=None, prefix=None, confirm=False): + r""" Clone package with appropriate version + + This function will try to find the wanted package with the appropriate + version then clone it into <`prefix`> if provided or in the internal + configuration key otherwise. + + The `name` should be a valid package name. However, the `version` argument + can contains version operation like carret (^), tilde (~) and start (*) as + described in + + @args + > prefix (str) - clone path prefix + > name (str) - exact valid package name + > version (str) - version query string + > confirm (bool) - display user input to confirm the clone + + @return + > the package path if successfully cloned, None otherwise + """ + return pkg_clone(name, version, prefix, confirm) diff --git a/vxsdk/core/pkg/backend/__init__.py b/vxsdk/core/pkg/backend/__init__.py new file mode 100644 index 0000000..8958de9 --- /dev/null +++ b/vxsdk/core/pkg/backend/__init__.py @@ -0,0 +1,57 @@ +"""Remote backend constructor + +This package will exposes the major important object for the package core remote +part of the vxsdk. + + =========================== ============================================ + Object name Description + =========================== ============================================ + PKG_CORE_BACKEND_REMOTE Remote ackend object + PKG_CORE_BACKEND_LOCAL Local backend object + VxRemoteBackend Abstract class for backend implementation + =========================== ====================================== + +This part of the vxsdk can be manually configured using a TOML configuration +which need to be located at <~/.config/vxsdk/configure.toml>. A special +section named 'package.remote' can be added to specify backend information: + + [pkg] + backend.name = 'gitea' + backend.url = 'https://personal.gitea.instance.gaming' + +This is the only configuration that you can set here. +""" +import os +import sys + +from core.pkg.backend.local import VxBackendLocal +from core.logger import log +from core.config import config_get + +__all__ = [ + 'PKG_CORE_BACKEND_REMOTE', + 'PKG_CORE_BACKEND_LOCAL', +] + +backend_remote_name = config_get('pkg.backend.name') +backend_remote_url = config_get('pkg.backend.url') +backend_local_url = os.path.expanduser(config_get('pkg.local_storage')) + +PKG_CORE_BACKEND_REMOTE = None +PKG_CORE_BACKEND_LOCAL = None +try: + mod = __import__( + f'core.pkg.backend.{backend_remote_name}', + fromlist=['VxBackendRemote'] + ) + if not hasattr(mod, 'VxBackendRemote'): + raise Exception( + f"backend '{backend_remote_name}' doesn't expose " + "VxBackendRemote class" + ) + PKG_CORE_BACKEND_REMOTE = mod.VxBackendRemote(backend_remote_url) + PKG_CORE_BACKEND_LOCAL = VxBackendLocal(backend_local_url) +except ImportError as err: + log.emergency("[backend] unable to load remote backend, abord") + log.emergency(err) + sys.exit(84) diff --git a/vxsdk/core/pkg/backend/core.py b/vxsdk/core/pkg/backend/core.py new file mode 100644 index 0000000..a518b3d --- /dev/null +++ b/vxsdk/core/pkg/backend/core.py @@ -0,0 +1,134 @@ +""" +Vhex package core abstract class +""" +import abc + +from core.logger import log + +__all__ = [ + 'VxRemoteBackendCore' +] + +class VxRemoteBackendCore(abc.ABC): + r"""Represent a remote backend 'core' class. + + This class is a simple abstract class that should be used by all internal + 'backend' wich should expose some common methods and property: + + ================================== ========================================= + Property Description + ================================== ========================================= + name (str) Backend name (for debug) + url (str) Backend URL (for internal use) + ================================== ========================================= + + ================================== ========================================= + Method Description + ================================== ========================================= + find() find packages + clone() clone a package + ================================== ========================================= + """ + def __init__(self, url): + self._url = url + self._pkg_list = [] + + @property + @abc.abstractmethod + def name(self): + """ hold pacakge name""" + + @property + @abc.abstractmethod + def url(self): + """ hold pacakge URL (or path for local package)""" + + @property + @abc.abstractmethod + def package_list(self): + """ return the pacakge list (cached)""" + + @abc.abstractmethod + def package_fetch_versions(self, pkg): + """ pacakge version fetch (cache)""" + + @abc.abstractmethod + def package_clone(self, pkg, prefix=None): + """ package primtive """ + + def package_find(self, name, version=None): + r"""Try to find package + + This method will try to find a particular package with a specific + version if the `version` argument is provided. Otherwise, the version + argument can holds special version requirement if the package use a + versionning sementic (more information in ) + + If name is empty, all package will be returned + + @args + > names (str) - package name + > versions (str) - version pattern + + Return: + > A list of dictionary with (at least): + [ + { + 'name' (str) : package name + 'full_name' (str) : full named package (author+name) + 'description' (str) : package description + 'versions' (list) : [ + List of :obj:VxVersion that match the `version` argument + ] + }, + ... + ] + """ + # check if we need to return the complet list + if not name: + ret = [] + for pkg in self.package_list: + if not self.package_fetch_versions(pkg): + continue + pkg_cpy = pkg.copy() + pkg_cpy['version'] = None + ret.append(pkg_cpy) + return ret + + # handle package name and version verification + # @note + # - return on the first matched package + for pkg in self.package_list: + # check package name + if pkg['name'] != name: + continue + + # fetch version information + if not self.package_fetch_versions(pkg): + log.warn("[pkg] '{pkg['name']}' unable to fetch versions") + continue + + # if the version is None, we just need to fetch available + # versions then return + if not version: + pkg_cpy = pkg.copy() + pkg_cpy['version'] = None + return [pkg_cpy] + + # perform version checking and select the best fit + ver_found = None + for ver in pkg['versions']: + if not ver.validate(version): + continue + if not ver_found or ver.compare(ver_found.name) < 0: + ver_found = ver + if not ver_found: + return [] + + # isolate the best version + pkg_cpy = pkg.copy() + pkg_cpy['version'] = ver_found + return [pkg_cpy] + + # not requested package found, return + return None diff --git a/vxsdk/core/pkg/backend/gitea.py b/vxsdk/core/pkg/backend/gitea.py new file mode 100644 index 0000000..f2913ed --- /dev/null +++ b/vxsdk/core/pkg/backend/gitea.py @@ -0,0 +1,170 @@ +""" +Vhex core backend for Gitea instance +""" +import os +import subprocess +import requests + +from core.pkg.backend.core import VxRemoteBackendCore +from core.pkg.version import VxVersion +from core.logger import log +from core.config import config_get + + +__all__ = [ + 'VxBackendRemote' +] + +class VxBackendRemote(VxRemoteBackendCore): + """ + Vhex Gitea backend class + """ + + #--- + # Public properties + #--- + + @property + def name(self): + return 'gitea' + + @property + def url(self): + return self._url + + @property + def package_list(self): + """ Return the list of all repository find in the remote instance. + + This property is used to cache request's information and avoid big HTTP + quesring to the remote instance of Gitea. + """ + # if cached, avoid quering operation + if self._pkg_list: + return self._pkg_list + + # raw HTTP request + resp = requests.get( + f'{self.url}/api/v1/repos/search', + params={ + 'q' : 'vxsdk', + 'topic': 'True' + } + ) + if not resp.ok: + log.warn( + f'[backend] gitea: remote package requests failed\n' + f'>>> {resp.status_code}' + ) + self._pkg_list = [] + return [] + + # generate the pacakge list + # @note + # - hide private repositories + self._pkg_list = [] + for pkg in resp.json()['data']: + if pkg['private']: + continue + self._pkg_list.append({ + 'name' : pkg['name'], + 'full_name' : pkg['full_name'], + 'description' : pkg['description'], + 'url' : pkg['clone_url'], + 'created' : pkg['created_at'].split('T')[0], + 'updated' : pkg['updated_at'].split('T')[0], + 'author' : pkg['owner']['login'], + 'default_branch' : pkg['default_branch'], + 'versions' : [] + }) + return self._pkg_list + + #--- + # Private methods + #--- + + def package_fetch_versions(self, pkg): + """ Fetch package version information. + + Same as the `package_list` property, we using cache to avoid too many + HTTP request. + """ + # if cached information, return it + if pkg['versions']: + return pkg['versions'] + + # request branches information + pkg['versions'] = [] + url = f"{self.url}/api/v1/repos/{pkg['full_name']}" + resp = requests.get(f'{url}/branches') + if not resp.ok: + log.warn( + f'[pkg]: backend: gitea: branches requests error\n' + f'>>> url = {url}/branches\n' + f'>>> status = {resp.status_code}' + ) + else: + for ver in resp.json(): + pkg['versions'].append( + VxVersion(ver['name'], 'branch', 'remote') + ) + + # request tag information + resp = requests.get(f'{url}/tags') + if not resp.ok: + log.warn( + f'[pkg]: backend: gitea: tags requests error\n' + f'>>> url = {url}/tags\n' + f'>>> status = {resp.status_code}' + ) + else: + for ver in resp.json(): + pkg['versions'].append(VxVersion(ver['name'], 'tag', 'remote')) + return pkg['versions'] + + #--- + # Public methods + #--- + + def package_clone(self, pkg, _=None): + """ Clone the package in global storage + + @args + > pkg (dict) - package information returned by package_find() + + @return + > Complet path for the package (str), or None if error + """ + # fetch global storage prefix + # @notes + # - create it if its doesn't exists + prefix = os.path.expanduser(config_get('path.packages')) + if not os.path.exists(prefix): + os.makedirs(prefix) + + # generate clone information + # @note + # - create clone folder if not exists + pkg_ver = pkg['version'] + pkg_name = f"{pkg['author']}@{pkg['name']}@{pkg_ver.name}@{pkg_ver.type}" + pkg_path = f"{prefix}/{pkg_name}" + if os.path.exists(pkg_path): + log.warn(f"[clone]: {pkg_name} already exists, skipped") + return pkg_path + + # perform high-level git clone + cmd = [ + 'git', + '-c', 'advice.detachedHead=false', + 'clone', '--branch', pkg_ver.name, + pkg['url'], pkg_path, + '--depth=1' + ] + log.debug(f"[gitea] {cmd}") + status = subprocess.run(cmd, capture_output=True, check=False) + if status.returncode != 0: + log.error(f"[clone] : unable to clone {pkg_name}, abord") + return [] + + # return the package path + return pkg_path diff --git a/vxsdk/core/pkg/backend/local.py b/vxsdk/core/pkg/backend/local.py new file mode 100644 index 0000000..011f4ef --- /dev/null +++ b/vxsdk/core/pkg/backend/local.py @@ -0,0 +1,89 @@ +""" +Vhex core backend for local package +""" +import os + +from core.logger import log +from core.pkg.backend.core import VxRemoteBackendCore +from core.pkg.version import VxVersion + +__all__ = [ + 'VxBackendLocal' +] + +class VxBackendLocal(VxRemoteBackendCore): + """ + Vhex backend local package class + """ + + #--- + # Public properties + #--- + + @property + def name(self): + return 'local' + + @property + def url(self): + return self._url + + @property + def package_list(self): + if self._pkg_list: + return self._pkg_list + + if not os.path.exists(self._url): + os.makedirs(self._url) + return [] + + self._pkg_list = [] + for file in os.listdir(self._url): + if not os.path.isdir(f"{self._url}/{file}"): + continue + exists = False + for pkg in self._pkg_list: + if pkg['name'] == file.split('@')[1]: + exists = True + break + if exists: + continue + self._pkg_list.append({ + 'name' : file.split('@')[1], + 'full_name' : f"{self._url}/{file.split('@')[1]}", + 'description' : None, + 'url' : f"{self._url}/{file}", + 'created' : None, + 'updated' : None, + 'author' : file.split('@')[0], + 'default_branch' : None, + 'versions' : [ + VxVersion(file.split('@')[2], file.split('@')[3], 'local') + ] + }) + return self._pkg_list + + #--- + # Private methods + #--- + + def package_fetch_versions(self, pkg): + return pkg['versions'] + + #--- + # Public methods + #--- + + def package_clone(self, pkg, prefix=None): + if not prefix: + prefix = os.getcwd() + if not os.path.exists(prefix): + os.makedirs(prefix) + if not os.path.exists(f"{prefix}/{pkg['name']}"): + log.debug(f"[local] link '{pkg['url']}' > '{prefix}/{pkg['name']}'") + os.symlink( + pkg['url'], + f"{prefix}/{pkg['name']}", + target_is_directory=True + ) + return f"{prefix}/{pkg['name']}" diff --git a/vxsdk/core/pkg/clone.py b/vxsdk/core/pkg/clone.py new file mode 100644 index 0000000..40c5726 --- /dev/null +++ b/vxsdk/core/pkg/clone.py @@ -0,0 +1,89 @@ +""" +Package clone backend abstraction +""" + +from core.pkg.backend import PKG_CORE_BACKEND_REMOTE, PKG_CORE_BACKEND_LOCAL +from core.pkg.version import VxVersion +from core.pkg.find import pkg_find +from core.logger import log + +__all__ = [ + 'pkg_clone' +] + +#--- +# Internals +#--- + +def _pkg_clone_core(pkg, prefix): + """ + + @args + > pkg (dict) - package information + + @return + > the package path if successfully cloned, None otherwise + """ + version = pkg['version'] + + # be sure that the package target is in the global storage + log.user(f"cloning package {pkg['name']}...") + pkg_path = PKG_CORE_BACKEND_REMOTE.package_clone(pkg) + if not pkg_path: + log.error( + f"{pkg['name']}@{version.name}: unable to clone the package, abord", + ) + return None + + # "clone" the package (create a symbolic link) + pkg['url'] = pkg_path + return PKG_CORE_BACKEND_LOCAL.package_clone(pkg, prefix) + +#--- +# Public +#--- + +def pkg_clone(name, version, prefix, confirm=False): + r""" Clone the package + + This function will try to clone the package with the exact selected + version, with all of its dependencies. See for more + information about the process. + + @args + > name (str) - exact valid package name + > version (str) - version query string + > prefix (str) - clone path prefix + > confirm (bool) - display user input to confirm the clone + + @return + > the package path if successfully cloned, None otherwise + """ + # try to find the package anywhere that the vxSDK allow + pkg_list = pkg_find(name, version, local=True, remote=True) + if not pkg_list: + log.error("[pkg] pacakge find error") + return None + if len(pkg_list) != 1: + log.warn("[pkg] multiple package found, other will be ignored") + + # check version information + pkg_info = pkg_list[0] + if not pkg_info['version']: + if version: + log.error(f"{name}@{version}: unable to find the version") + return None + pkg_info['version'] = VxVersion(pkg_info['default_branch'], 'branch') + + # wait user interaction if needed + if confirm and not 'local' in pkg_info['version'].sources: + log.user( + f"Do you want to install '{pkg_info['full_name']}'? (Y/n) ", + end = '' + ) + valid = input() + if valid and not valid in ['Y', 'y', 'yes', 'Yes']: + return None + + # "real" clone the package + return _pkg_clone_core(pkg_info, prefix) diff --git a/vxsdk/core/pkg/find.py b/vxsdk/core/pkg/find.py new file mode 100644 index 0000000..36de85a --- /dev/null +++ b/vxsdk/core/pkg/find.py @@ -0,0 +1,126 @@ +""" +Package find backend abstraction +""" + +from core.logger import log +from core.pkg.backend import PKG_CORE_BACKEND_REMOTE, PKG_CORE_BACKEND_LOCAL + +__all__ = [ + 'pkg_find' +] + +#--- +# Internals +#--- + +def _pkg_find_merge_result(pkg_list_remote, pkg_list_local): + """ Merge two custom package information list + @arg + > pkg_list_remote (list,dict) - package list + > pkg_list_local (list,dict) - package list + + @return + > The first list modified + """ + for pkg_local in pkg_list_local: + pkg_found = False + for pkg_remote in pkg_list_remote: + if pkg_remote['name'] != pkg_local['name']: + continue + pkg_found = True + for pkg_ver_local in pkg_local['versions']: + version_found = False + for pkg_ver_remote in pkg_remote['versions']: + if pkg_ver_remote.name != pkg_ver_local.name: + continue + pkg_ver_remote.addSource('local') + version_found = True + break + if not version_found: + pkg_remote['versions'].append(pkg_ver_local) + if not pkg_found: + pkg_list_remote.append(pkg_local) + return pkg_list_remote + +def _pkg_find_select_best(pkg_remote, pkg_local): + """ Select the best version + + @arg + > pkg_remote (dict) - pacakge information + > pkg_local (dict) - pacakge information + + @return + > a list with the best pacakge information version + """ + if not 'version' in pkg_remote or not 'version' in pkg_local: + log.warn("[log] misformed package search result, silently ignored") + return [] + if pkg_remote['version'].compare(pkg_local['version'].name) > 0: + pkg_remote['version'] = pkg_local['version'] + return [pkg_remote] + +#--- +# Public +#--- + +def pkg_find(name, version=None, local=False, remote=True): + r""" Find the most appropriate package information + + This function will request to the remote backend the list of all version of + the pacakge, which is basically all tags and branches. Then, we will check + all of these until a version match with the `version` query. + + Note that the version query can contains special operation as described in + + + @args + > name (str) - exact valid package name + > version (str) - version query string + > local (bool) - enable local search + > remote (bool) - enable remote search + + @return + > a list of all matched package dictionnary : + [ + { + 'name' : , + 'path' : + 'versions' : [ + + ... + ] + ] + > None if error + """ + if not local and not remote: + return [] + + # perform explicit request for each backend + pkg_list_local = [] + pkg_list_remote = [] + if remote: + pkg_list_remote = PKG_CORE_BACKEND_REMOTE.package_find(name, version) + if local: + pkg_list_local = PKG_CORE_BACKEND_LOCAL.package_find(name, version) + + + # verify useless checking and merging operation + if not pkg_list_remote: + return pkg_list_local + if not pkg_list_local: + return pkg_list_remote + + + # if a version is provided, each backend will return only one dictionary + # with the best match in local and remote one. We will select the best + # between these two. + if version: + return _pkg_find_select_best( + pkg_list_remote[0], + pkg_list_local[0], + ) + + # if no version information as been specified each `pkg_list_remote` and + # `pkg_list_local` will contains a list of all matched package information. + # So, we need to merge these two list. + return _pkg_find_merge_result(pkg_list_remote, pkg_list_local) diff --git a/vxsdk/core/pkg/version.py b/vxsdk/core/pkg/version.py new file mode 100644 index 0000000..f980d41 --- /dev/null +++ b/vxsdk/core/pkg/version.py @@ -0,0 +1,216 @@ +import os +import re +import subprocess + +__all__ = [ + 'VxVersion', + 'version_is_valid', + 'version_get' +] + +class VxVersion(object): + r"""Represent a Version object. + + This version mecanism is a strong part of the package managing because If + the 'version' is detected to respect the correct semantic versioning, you + can perform version operations: caret (^), tilde (~) and wildcard (*) + + Caret requirements (^) + ^1.2.3 := >=1.2.3, <2.0.0 + ^1.2 := >=1.2.0, <2.0.0 + ^1 := >=1.0.0, <2.0.0 + ^0.2.3 := >=0.2.3, <0.3.0 + ^0.2 := >=0.2.0, <0.3.0 + ^0.0.3 := >=0.0.3, <0.0.4 + ^0.0 := >=0.0.0, <0.1.0 + ^0 := >=0.0.0, <1.0.0 + + Tilde requirements (~) + ~1.2.3 := >=1.2.3, <1.3.0 + ~1.2 := >=1.2.0, <1.3.0 + ~1 := >=1.0.0, <2.0.0 + + Wildcard requirements (*) + * := >=0.0.0 + 1.* := >=1.0.0, <2.0.0 + 1.2.* := >=1.2.0, <1.3.0 + + Note that it's possible that a package can have two same versions (one + for branch name and another for a tag), by default, the tag is always + selected, but here the display information will explicitly describe if + the version is a tag and / or a branch. + + + TODO: + > Use 'releases' section too ? + > Define explicitely version type (tag, branch, releases ?) + """ + def __init__(self, name, type=None, source=None): + self._type = type + self._name = name + self._source_list = [source] + + def __repr__(self): + return '<' + self._name + ',(' + self._type + ')>' + + @property + def name(self): + return self._name + @property + def type(self): + return self._type + @property + def sources(self): + return self._source_list + + + def validate(self, pattern): + r""" Try to check if the version validate the pattern operation. + + As explainned in the class docstring, we can have 3 type of operation, + caret (^), tilde (~) and wildcard (*) and the pattern should follow the + semantic requirement for each operator. + + Note that the pattern can be None, in this case, the validation step is + always success. Beside, the pattern can be a non-valid version semantic + request, in this case the version name should match the entiere pattern. + + Args: + > pattern (None, str) - pattern used to check if the version is valid + + Return: + > True if the version validate the pattern, False otherwise + """ + return version_is_valid(self.name, pattern) + + def compare(self, version): + r"""Compare two version + + This methods will compare versionning inforation and return an interger + indicating the result of the comparison, as follow: + + > 0 - the two version match + > a negative value - self is less than 'version' + > a positive value - self is greater than 'version' + """ + try: + s1 = tuple([int(d) for d in self.name.split(".") if d] + [0,0,0])[:3] + s2 = tuple([int(d) for d in version.name.split(".") if d] + [0,0,0])[:3] + + if s1[0] - s2[0] != 0: return s1[0] - s2[0] + if s1[1] - s2[1] != 0: return s1[1] - s2[1] + return s1[2] - s2[2] + except Exception as _: + return 0 + + def addSource(self, source): + self._source_list.append(source) + + +def version_is_valid(version, pattern): + r"""Check if the version validate a pattern + + If the 'version' is detected to respect the correct semantic versioning, you + can perform version operations: caret (^), tilde (~) and wildcard (*) + + Caret requirements (^) + ^1.2.3 := >=1.2.3, <2.0.0 + ^1.2 := >=1.2.0, <2.0.0 + ^1 := >=1.0.0, <2.0.0 + ^0.2.3 := >=0.2.3, <0.3.0 + ^0.2 := >=0.2.0, <0.3.0 + ^0.0.3 := >=0.0.3, <0.0.4 + ^0.0 := >=0.0.0, <0.1.0 + ^0 := >=0.0.0, <1.0.0 + + Tilde requirements (~) + ~1.2.3 := >=1.2.3, <1.3.0 + ~1.2 := >=1.2.0, <1.3.0 + ~1 := >=1.0.0, <2.0.0 + + Wildcard requirements (*) + * := >=0.0.0 + 1.* := >=1.0.0, <2.0.0 + 1.2.* := >=1.2.0, <1.3.0 + + Note that if the pattern argument is None, then the version is always + validate and the function will return True + + Args: + > version (str) - version to check + > pattern (str, None) - version pattern + + Return: + > True if the version validate the pattern, False otherwise + """ + if not pattern: + return True + + # Parse a version string of the form "M", "M.m", or "M.m.p" into a triplet + def _parse_version(version_string): + digits = [int(d) for d in version_string.split(".") if d] + return tuple(digits + [0,0,0])[:3] + + # Index of first nonzero component + def _first_nonzero(v, default): + return next((i for i, value in enumerate(v) if value), default) + + # Increment at specified position + def _increment_at(v, position): + padding = len(v) - position - 1 + return tuple(list(v[:position]) + [v[position]+1] + [0] * padding) + + # Parse a spec like ^1.2.3, ~1.2 or 1.2.* into a min/max pair + def _parse_spec(spec): + regex = re.compile(r'[~^]?(\d+\.){,2}\d+|(\d+\.){,2}\*') + if not regex.fullmatch(spec): + return None + + spec_length = spec.count(".") + 1 + v = _parse_version( + spec.replace("^","").replace("~","").replace("*","") + ) + M, m, p = v + + if spec[0] == '^': + return v, _increment_at(v, _first_nonzero(v, spec_length-1)) + elif spec[0] == '~': + return v, _increment_at(v, min(spec_length-1, 1)) + elif spec == "*": + return ((0, 0, 0), None) + elif "*" in spec: + return v, _increment_at(v, spec_length-2) + else: + return (M,m,p), (M,m,p+1) + + # Check if version is between requested bounds + def _version_is_suitable(version, min_version, max_version): + return (min_version is None or min_version <= version) \ + and (max_version is None or version < max_version) + + # function entry + try: + pair = _parse_spec(pattern) + if not pair: + return version == pattern + fmt_version = _parse_version(version) + return _version_is_suitable(fmt_version, pair[0], pair[1]) + except: + return False + +def version_get(path): + saved_pwd = os.getcwd() + os.chdir(path) + ret = subprocess.run( + ['git', 'describe', '--tags', '--exact-match'], + capture_output=True, check=False + ) + if ret.returncode != 0: + ret = subprocess.run( + ['git', 'branch', '--show-current'], + capture_output=True, check=False + ) + os.chdir(saved_pwd) + if ret.returncode != 0: + return 'unknown' + return ret.stdout.decode("utf8").strip() diff --git a/vxsdk/core/project.py b/vxsdk/core/project.py new file mode 100644 index 0000000..b61801f --- /dev/null +++ b/vxsdk/core/project.py @@ -0,0 +1,20 @@ +import os +import shutil + +from core.logger import log + +__all__ = [ + 'project_new' +] + +#TODO: change internal project name +def project_new(project_path): + if os.path.exists(project_path): + logger(LOG_WARN, f'The path {project_path} already exists !') + return True + origin_path = os.path.dirname(__file__) + shutil.copytree( + origin_path + '/../../assets/project/', + project_path + ) + logger(LOG_USER, f"project '{project_path}' successfully created !")