From 42f2b5c175d7b7aa098fba000051227d81038634 Mon Sep 17 00:00:00 2001 From: Lephenixnoir Date: Mon, 11 Jan 2021 19:17:26 +0100 Subject: [PATCH] fxconv: support metadata discovery in fxconv-metadata.txt This change adds a new way for fxconv to discover metadata for file conversions. This complements the existing mechanism of passing parameters on the command-line. The new mechanism activates when fxconv is called without a type argument. Type information and metadata are searched in an fxconv-metadata.txt file in the same folder as the resource. The metadata file lists parameters, with some additional flexibility enabled by the use of wildcards. This way of declaring will replace command-line argument passing, which currently read parameters from the unreadable and not-so-maitainable project.cfg file. Both the GNU make and CMake build systems should use it in the future. The current way is still supported only for older projects and one-shot conversions outside of projects. --- Makefile | 6 +- fxconv/fxconv-main.py | 191 +++++++++++++++++++++++------------------- fxconv/fxconv.py | 2 +- 3 files changed, 107 insertions(+), 92 deletions(-) diff --git a/Makefile b/Makefile index a36a483..3ea5e6b 100755 --- a/Makefile +++ b/Makefile @@ -41,14 +41,12 @@ all: $(bin) all-fxsdk: bin/fxsdk.sh all-fxg1a: bin/fxg1a -all-fxconv: bin/fxconv-main.py +all-fxconv: # Explicit targets bin/fxsdk.sh: fxsdk/fxsdk.sh | bin/ sed $(sed) $< > $@ -bin/fxconv-main.py: fxconv/fxconv-main.py | bin/ - sed $(sed) $< > $@ bin/fxg1a: $(obj-fxg1a) | bin/ gcc $^ -o $@ $(lflags) @@ -98,7 +96,7 @@ install: $(bin) install -d $(PREFIX)/share/fxsdk/assets install fxsdk/assets/* $(m644) $(PREFIX)/share/fxsdk/assets install bin/fxsdk.sh $(m755) $(PREFIX)/bin/fxsdk - install bin/fxconv-main.py $(m755) $(PREFIX)/bin/fxconv + install fxconv/fxconv-main.py $(m755) $(PREFIX)/bin/fxconv install fxconv/fxconv.py $(m644) $(PREFIX)/bin uninstall: diff --git a/fxconv/fxconv-main.py b/fxconv/fxconv-main.py index cba14eb..7baf3d4 100755 --- a/fxconv/fxconv-main.py +++ b/fxconv/fxconv-main.py @@ -3,59 +3,98 @@ import getopt import sys import os +import re +import fnmatch import fxconv -import subprocess - -# Note: this line is edited at compile time to insert the install folder -PREFIX="""\ -""".strip() help_string = f""" -usage: fxconv [-s] [files...] - fxconv -b -o [parameters...] - fxconv -i -o (--fx|--cg) [parameters...] - fxconv -f -o [parameters...] +usage: fxconv [] -o [--fx|--cg] [...] -fxconv converts data files such as images and fonts into gint formats -optimized for fast execution, or into object files. +fxconv converts resources such as images and fonts into binary formats for +fxSDK applications, using gint and custom conversion formats. -Operating modes: - -s, --script Expose the fxconv module and run this Python script - -b, --binary Turn data into an object file, no conversion - -i, --image Convert to gint's bopti image format +When no TYPE is specified (automated mode), fxconv looks for type and +parameters in an fxconv-metadata.txt file in the same folder as the input. This +is normally the default for add-ins. + +When TYPE is specified (one-shot conversion), it should be one of: + -b, --binary Turn data into an object file without conversion -f, --font Convert to gint's topti font format + --bopti-image Convert to gint's bopti image format --libimg-image Convert to the libimg image format + --custom Use converters.py; you might want to specify an explicit type + by adding a parameter type:your_custom_type (see below) -When using -s, additional arguments are stored in the [fxconv.args] variable of -the module. This is intended to be a restricted list of file names specified by -a Makefile, used to convert only a subset of the files in the script. - -The operating mode options are shortcuts to convert single files without a -script. They accept parameters with a "category.key:value" syntax, for example: +During one-shot conversions, parameters can be specified with a "NAME:VALUE" +syntax (names can contain dots). For example: fxconv -f myfont.png -o myfont.o charset:ascii grid.padding:1 height:7 -When converting images, use --fx (black-and-white calculators) or --cg (16-bit -color calculators) to specify the target machine. - -Install PREFIX is set to '{PREFIX}'. +Some formats differ between platforms so you should specify it when possible: + --fx, --fx9860G Casio fx-9860G family (black-and-white calculators) + --cg, --fxCG50 Casio fx-CG 50 family (16-bit color calculators) """.strip() # Simple error-warnings system +FxconvError = fxconv.FxconvError + def err(msg): print("\x1b[31;1merror:\x1b[0m", msg, file=sys.stderr) + return 1 def warn(msg): print("\x1b[33;1mwarning:\x1b[0m", msg, file=sys.stderr) -# "converters" module from the user project +# "converters" module from the user project... if it exists try: import converters except ImportError: converters = None +def parse_parameters(params): + """Parse parameters of the form "NAME:VALUE" into a dictionary.""" + d = dict() + + def insert(d, path, value): + if len(path) == 1: + d[path[0]] = value + else: + if not path[0] in d: + d[path[0]] = dict() + insert(d[path[0]], path[1:], value) + + for decl in params: + if ":" not in decl: + raise FxconvError(f"invalid parameter {decl}, ignoring") + else: + name, value = decl.split(":", 1) + insert(d, name.split("."), value.strip()) + + return d + +def parse_parameters_metadata(contents): + """Parse parameters from a metadata file contents.""" + + RE_COMMENT = re.compile(r'#.*$', re.MULTILINE) + contents = re.sub(RE_COMMENT, "", contents) + + RE_WILDCARD = re.compile(r'^(\S(?:[^:\s]|\\:|\\ )*)\s*:\s*$', re.MULTILINE) + lead, *elements = [ s.strip() for s in re.split(RE_WILDCARD, contents) ] + + if lead: + raise FxconvError(f"invalid metadata: {lead} appears before wildcard") + + # Group elements by pairs (left: wildcard, right: list of properties) + elements = list(zip(elements[::2], elements[1::2])) + + metadata = [] + for (wildcard, params) in elements: + params = [ s.strip() for s in params.split("\n") if s.strip() ] + metadata.append((wildcard, parse_parameters(params))) + + return metadata + def main(): - # Default execution mode is to run a Python script for conversion - modes = "script binary image font bopti-image libimg-image" - mode = "s" + types = "binary image font bopti-image libimg-image custom" + mode = "" output = None model = None target = { 'toolchain': None, 'arch': None, 'section': None } @@ -68,24 +107,25 @@ def main(): sys.exit(1) try: - longs = "help output= fx cg toolchain= arch= section= custom " + modes + models = "fx cg fx9860G fxCG50" + longs = f"help output= toolchain= arch= section= {models} {types}" opts, args = getopt.gnu_getopt(sys.argv[1:], "hsbifo:", longs.split()) except getopt.GetoptError as error: - err(error) - sys.exit(1) + return err(error) for name, value in opts: # Print usage if name == "--help": - err(help_string, file=sys.stderr) - sys.exit(0) - # TODO: fxconv: verbose mode - elif name == "--verbose": - pass + print(help_string, file=sys.stderr) + return 0 elif name in [ "-o", "--output" ]: output = value elif name in [ "--fx", "--cg" ]: model = name[2:] + elif name == "--fx9860G": + model = "fx" + elif name == "--fxCG50": + model = "cg" elif name == "--toolchain": target['toolchain'] = value elif name == "--arch": @@ -101,54 +141,33 @@ def main(): # Remaining arguments if args == []: - err(f"execution mode -{mode} expects an input file") + err(f"no input file") sys.exit(1) input = args.pop(0) - # In --script mode, run the Python script with an augmented PYTHONPATH + # In automatic mode, look for information in fxconv-metadata.txt + if mode == "": + metadata_file = os.path.dirname(input) + "/fxconv-metadata.txt" + basename = os.path.basename(input) - if mode == "s": - if output is not None: - warn("option --output is ignored in script mode") + if not os.path.exists(metadata_file): + return err(f"using auto mode but {metadata_file} does not exist") - if PREFIX == "": - err("unknown or invalid install path x_x") - sys.exit(1) + with open(metadata_file, "r") as fp: + metadata = parse_parameters_metadata(fp.read()) - env = os.environ.copy() - if "PYTHONPATH" in env: - env["PYTHONPATH"] += f":{PREFIX}/bin" - else: - env["PYTHONPATH"] = f"{PREFIX}/bin" - - p = subprocess.run([ sys.executable, input ], env=env) - if p.returncode != 0: - sys.exit(1) - - # In shortcut conversion modes, read parameters from the command-line + params = dict() + for (wildcard, p) in metadata: + if fnmatch.fnmatchcase(basename, wildcard): + params.update(**p) + # In manual conversion modes, read parameters from the command-line else: - def check(arg): - if ':' not in arg: - warn(f"argument {arg} is not a valid parameter (ignored)") - return ':' in arg - - def insert(params, path, value): - if len(path) == 1: - params[path[0]] = value - return - if not path[0] in params: - params[path[0]] = {} - insert(params[path[0]], path[1:], value) - - args = [ arg.split(':', 1) for arg in args if check(arg) ] - params = {} - for (name, value) in args: - insert(params, name.split("."), value) + params = parse_parameters(args) if "type" in params: pass - elif(len(mode) == 1): + elif len(mode) == 1: params["type"] = { "b": "binary", "i": "image", "f": "font" }[mode] else: params["type"] = mode @@ -158,19 +177,17 @@ def main(): warn("type 'image' is deprecated, use 'bopti-image' instead") params["type"] = "bopti-image" - # Use the custom module - custom = None - if use_custom: - if converters is None: - err("--custom specified but no [converters] module in wd") - sys.exit(1) - custom = converters.convert + # Use the custom module + custom = None + if use_custom: + if converters is None: + return err("--custom specified but no [converters] module") + custom = converters.convert - try: - fxconv.convert(input, params, target, output, model, custom) - except fxconv.FxconvError as e: - err(e) - sys.exit(1) + fxconv.convert(input, params, target, output, model, custom) if __name__ == "__main__": - main() + try: + sys.exit(main()) + except fxconv.FxconvError as e: + sys.exit(err(e)) diff --git a/fxconv/fxconv.py b/fxconv/fxconv.py index e46de74..9c60861 100644 --- a/fxconv/fxconv.py +++ b/fxconv/fxconv.py @@ -162,7 +162,7 @@ def ref(base, offset=None, padding=None): assert padding is None return Ref(b"", base, offset or 0) - raise FxconvException(f"invalid type {type(base)} for ref()") + raise FxconvError(f"invalid type {type(base)} for ref()") Ref = collections.namedtuple("Ref", ["data", "name", "offset"])