From 34442dc0ae0f374c4eb41a562313fcbfff376da2 Mon Sep 17 00:00:00 2001 From: "Thomas \"Cakeisalie5\" Touhey" Date: Wed, 23 Nov 2016 06:10:45 +0100 Subject: [PATCH] Modified main handle MCS management, started adding G3P support --- Makefile.vars | 2 +- README.md | 50 ++++++++++++++++ include/libg1m.h | 6 +- include/libg1m/format.h | 17 ++++-- include/libg1m/format/picture_cg.h | 86 +++++++++++++++++++------- src/parse/main.c | 3 +- src/parse/mcs.c | 52 +++++++++------- src/parse/picture.c | 96 +++++++++++++++++++++++------- src/user/free.c | 10 ++-- 9 files changed, 239 insertions(+), 83 deletions(-) diff --git a/Makefile.vars b/Makefile.vars index 202406b..ba52815 100755 --- a/Makefile.vars +++ b/Makefile.vars @@ -61,7 +61,7 @@ # Linker LD := gcc # - Linker flags - LDFLAGS := -shared -lusb-1.0 \ + LDFLAGS := -shared -lz \ -e __lib$(NAME)_version \ -Wl,-soname,lib$(NAME).so.$(MAJOR) \ -Wl,-z,relro -Wl,-z,combreloc -Wl,-z,defs diff --git a/README.md b/README.md index e69de29..a4fb409 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,50 @@ +# libg1m - Casio File Format manipulation +## Introduction + +All of the files created around CASIO calculators are declinations of one +format, which I named by the name of one of them : G1M. These formats go +from main memory saves to pictures and e-activities (documents). + +This library aims to be able to parse and write in all of the available +formats, and to centralize the documentation and work on them, in order +to provide a simple interface for C programs, that can be adapted to +provide other interfaces such as a Python one, and to hide the odd +subtilities of this Patchwork As A Format. + +## Prerequisites + +Side note : the library might work with older versions of these dependencies, +I took these as a reference because these are the ones I work with. + +### Making-only dependencies + +| Name | Version | +| -------------------------------------------------- | -------- | +| [make](https://www.gnu.org/software/make/) | >= 4.0 | +| [gcc](https://gcc.gnu.org/) | >= 4.9 | +| [binutils](https://www.gnu.org/software/binutils/) | >= 2.25 | +| [asciidoc](http://asciidoc.org/) | >= 8.6.9 | +| [gzip](https://www.gnu.org/software/gzip/) | >= 1.6 | + +### Runtime dependencies + +| Name | Version | +| -------------------------------------------------- | -------- | +| [zlib](http://zlib.net/) | >= 1.2.8 | + +## Building + +Just `./configure` then `make`. +To install, use `make install`. + +To build and install only the lib, use `all-lib` then `install-lib`. +To build and install only the docs, use `all-doc` and `install-doc`. + +Other useful targets: + +- `uninstall`, `uninstall-lib`, `uninstall-bin`, `uninstall-doc`: + will try to uninstall using the current configuration (experimental); +- `mostlyclean`, `clean`, `clean-doc`, `mostlyclean-lib`, `clean-lib`: + remove built files at different levels; +- `re`, `re-lib`, `re-doc`: regenerate built files at different levels + (clean and build) -- useful when configuration is changed. diff --git a/include/libg1m.h b/include/libg1m.h index 76e3098..e2da73d 100644 --- a/include/libg1m.h +++ b/include/libg1m.h @@ -137,9 +137,9 @@ typedef struct { time_t creation_date; /* MCS RELATED DATA */ - int part_count; - int _parts_size; /* used internally for reallocs */ - g1m_mcs_t **parts; + int mcs_count; + int _mcs_size; /* used internally for reallocs */ + g1m_mcs_t **mcs; /* TODO: pictures, e-activities */ } g1m_t; diff --git a/include/libg1m/format.h b/include/libg1m/format.h index 7d5a363..6f4ddad 100644 --- a/include/libg1m/format.h +++ b/include/libg1m/format.h @@ -32,11 +32,13 @@ enum g1m_type { /* e-activity (document) */ g1m_typ_eact = 0x49, - /* g1r equivalent - different to g1m because of OS 2? - * (quick way to check expected compatibility? NOT RELIABLE) */ + /* g1r equivalent */ g1m_typ_g2r = 0x62, - /* g3p (fx-CGxx picture) */ + /* g3m (fx-CG program) */ + g1m_typ_g3m = 0x75, + + /* g3p (fx-CG picture) */ g1m_typ_g3p = 0x7d, /* add-in (compiled program) */ @@ -62,7 +64,7 @@ struct standard_header { uint8_t type; /* magic numbers, not always the same: - * - for g1m_type_addin_cg: {0x00, 0x01, 0x00, 0x01, 0x00} + * - for fx-CG formats: {0x00, 0x01, 0x00, 0x01, 0x00} * - otherwise: {0x00, 0x10, 0x00, 0x10, 0x00} */ uint8_t magic[5]; @@ -78,8 +80,11 @@ struct standard_header { /* second control byte: filesize LSB + 0xb8 */ uint8_t control2; - /* 4-bytes alignment */ - uint8_t align[9]; + /* alignment */ + uint8_t align[8]; + + /* is obfuscated - useful for G3P */ + uint8_t obfuscated; /* number of objects contained (useful for MCS filetype) */ uint16_t number; diff --git a/include/libg1m/format/picture_cg.h b/include/libg1m/format/picture_cg.h index 0519bb1..228b6ae 100644 --- a/include/libg1m/format/picture_cg.h +++ b/include/libg1m/format/picture_cg.h @@ -12,6 +12,8 @@ # include /* These are pictures for fx-CG. They only have one part. + * This mysteries over this format have been unraveled by many, but I'm using + * Simon Lothar's and Kerm Martian's documentations. * * Some color depth things: */ @@ -20,7 +22,7 @@ enum g3p_colorsize { g3p_color_16bit = 0x10 }; -/* The subheader is: */ +/* So, after the header, here is the G3P subheader: */ struct g3p_subheader { /* some magic sequence: "CP0100Ly755" @@ -28,42 +30,82 @@ struct g3p_subheader { uint8_t magic[11]; /* unused */ - uint8_t unused[11]; + uint8_t unused[5]; - /* plenty of unused space, or undocumented, for some reason */ - uint8_t undocumented[130]; + /* size of the file after standard header */ + uint32_t g3p_size; - /* magic sequence? "0100" */ - uint8_t magic2[4]; + /* undocumented */ + uint8_t unknown[4]; - /* control sequence? */ - uint32_t control; + /* size of the image data + 0x18 bytes of header */ + uint32_t image_size; - /* undocumented, again */ - uint16_t undocumented2; + /* undocumented gap */ + uint8_t undocumented_gap[124]; +}; - /* dimensions */ +/* What is after the G3P subheader is the image (data). + * + * The image has a header, some data and a footer. Here is the header: */ + +struct g3p_imageheader { + /* magic sequence? is: 0x00, 0x01, 0x00, 0x00 */ + uint8_t magic[4]; + + /* size of the image data + footer */ + uint32_t df_size; + + /* w0t */ + uint16_t unknown2; + + /* width */ uint16_t width; + + /* height */ uint16_t height; /* color depth - see `g3p_colorsize` */ uint16_t color_depth; - /* undocumented, the third coming */ - uint16_t undocumented3; + /* undocumented, again */ + uint32_t undocumented; - /* length of the deflated image */ - uint32_t dimage_length; + /* length of image + footer */ + uint32_t data_size; }; -/* And after the image: */ +/* Then there is the image data and the footer. + * + * At the beginning of the image data, there is a 2-byte ID. + * 0x3c1b is the ID for "Casio Provided" images with no footers (?). + * + * The image is deflated using the DEFLATE algorithm. + * + * For some images, CASIO added some obfuscation. To check if the image is + * encrypted, according to syscalls, you have to check standard header data. + * Image is encrypted if: + * + * [0x1C] != [0x08] + [0x12] + [0x13] + 0x7B + * + * Which, simplified and with our structures, is: + * + * std.obfuscated + 8 != (std.filesize & 0xff00) >> 8 + * + * then the deflated image is encrypted. + * + * If this is the case, before passing the data to the INFLATE algorithm, + * you have to apply this on each byte of the deflated image: + * + * 0b76543210 -> 0b21076543, + * which, in C, is: (byte >> 3) | ((byte & 0x7) << 5) + * + * The footer contains the Adler32 checksum of the raw and unobfuscated data. + * Here it is: */ -struct g3p_footer { - /* magic sequence: "0100" */ - uint8_t magic[4]; - - /* some unused bytes? */ - uint8_t unused[0x88]; +struct g3p_imagefooter { + /* checksum */ + uint32_t checksum; }; # pragma pack() diff --git a/src/parse/main.c b/src/parse/main.c index 4a69cbd..ee468d8 100644 --- a/src/parse/main.c +++ b/src/parse/main.c @@ -24,8 +24,7 @@ int g1m_parse(g1m_t *handle, FILE *stream) { /* initialize the handle */ - handle->type = 0x00; - handle->parts = NULL; + bzero(handle, sizeof(g1m_t)); /* get the standard header */ DREAD(hd, standard_header) diff --git a/src/parse/mcs.c b/src/parse/mcs.c index a93562b..c3e5211 100644 --- a/src/parse/mcs.c +++ b/src/parse/mcs.c @@ -526,18 +526,15 @@ int g1m_parse_mcs(g1m_t *handle, FILE *stream, uint_fast16_t num) * was the standard header. */ /* allocate memory for the subparts */ - handle->part_count = 0; - handle->_parts_size = num; - handle->parts = malloc(sizeof(g1m_mcs_t*) * num); - if (!handle->parts) return (g1m_error_alloc); - bzero(handle->parts, sizeof(g1m_mcs_t*) * num); - - /* proxy */ - g1m_mcs_t **pmcs = handle->parts; + handle->mcs_count = 0; + handle->_mcs_size = num; + handle->mcs = malloc(sizeof(g1m_mcs_t*) * num); + if (!handle->mcs) return (g1m_error_alloc); + bzero(handle->mcs, sizeof(g1m_mcs_t*) * num); /* read all of the parts */ - log_info("%ld parts to browse", num); - for (uint_fast16_t i = 0; i < num; i++) { + log_info("%ld total mcs files to browse", num); + for (int i = 0; num; i++) { /* get the subheader */ GDREAD(hd, mcs_subheader) @@ -545,17 +542,28 @@ int g1m_parse_mcs(g1m_t *handle, FILE *stream, uint_fast16_t num) hd.subcount = be32toh(hd.subcount); /* log info about part */ - log_info("[%ld] Internal name is '%.16s'", i, hd.intname); + log_info("[%d] Internal name is '%.16s'", i, hd.intname); + + /* check mcs tab length - in case of 0-files mcs */ + if (i >= handle->_mcs_size) { + size_t old_mcs_size = handle->_mcs_size; + handle->_mcs_size += num; /* should contain this element & others */ + g1m_mcs_t **nmcs = malloc(sizeof(g1m_mcs_t*) * handle->_mcs_size); + if (!nmcs) goto fail; + + memcpy((void*)nmcs, (void*)handle->mcs, old_mcs_size); + handle->mcs = nmcs; + } /* allocate tab and proxy */ err = g1m_error_alloc; - pmcs[i] = malloc(sizeof(g1m_mcs_t)); - if (!pmcs[i]) goto fail; - g1m_mcs_t *mcs = pmcs[i]; + handle->mcs[i] = malloc(sizeof(g1m_mcs_t)); + if (!handle->mcs[i]) goto fail; + g1m_mcs_t *mcs = handle->mcs[i]; /* increment parts count, and remove to num the "extra" subparts */ - handle->part_count++; - num -= hd.subcount - 1; + handle->mcs_count++; + num -= hd.subcount; /* prepare it */ mcs->file_count = hd.subcount; @@ -566,8 +574,8 @@ int g1m_parse_mcs(g1m_t *handle, FILE *stream, uint_fast16_t num) bzero(mcs->files, sizeof(g1m_mcsfile_t*) * hd.subcount); /* foreach subpart */ - log_info("[%ld] %d mcs files to browse", i, hd.subcount); - for (uint_fast32_t j = 0; j < hd.subcount; j++) { + log_info("[%d] %d mcs files to browse", i, hd.subcount); + for (int j = 0; j < (int)hd.subcount; j++) { /* get the part header */ GDREAD(fhd, mcs_fileheader) @@ -575,11 +583,11 @@ int g1m_parse_mcs(g1m_t *handle, FILE *stream, uint_fast16_t num) fhd.datalength = be32toh(fhd.datalength); /* log info about the subpart */ - log_info("[%ld,%ld] directory name is '%.8s'", i, j, fhd.dirname); - log_info("[%ld,%ld] filename is '%.8s'", i, j, fhd.filename); - log_info("[%ld,%ld] file type is '%s' (0x%02d)", + log_info("[%d,%d] directory name is '%.8s'", i, j, fhd.dirname); + log_info("[%d,%d] filename is '%.8s'", i, j, fhd.filename); + log_info("[%d,%d] file type is '%s' (0x%02d)", i, j, g1m_get_mcs_ftype_string(fhd.filetype), fhd.filetype); - log_info("[%ld,%ld] data length is %u", i, j, fhd.datalength); + log_info("[%d,%d] data length is %u", i, j, fhd.datalength); /* decode */ int err = g1m_parse_mcsfile_content(&mcs->files[j], stream, diff --git a/src/parse/picture.c b/src/parse/picture.c index 2911936..4b17b58 100644 --- a/src/parse/picture.c +++ b/src/parse/picture.c @@ -8,6 +8,23 @@ /* */ /* ************************************************************************** */ #include +#include + +/** + * g3p_deobfuscate: + * De-obfuscate the image data. + * + * @arg buf the buffer. + * @arg n the buffer size. + */ + +static void g3p_deobfuscate(uint8_t *buf, size_t n) +{ + while (n--) { + int byte = *buf; + *buf++ = (byte >> 3) | ((byte & 0x7) << 5); + } +} /** * g1m_parse_g3p: @@ -22,39 +39,74 @@ int g1m_parse_g3p(g1m_t *handle, FILE *stream, struct standard_header *std) { - /* get the header and footer */ + /* get the G3P global header */ DREAD(hd, g3p_subheader) - /* correct endianess */ - hd.control = be32toh(hd.control); - hd.width = be16toh(hd.width); - hd.height = be16toh(hd.height); - hd.color_depth = be16toh(hd.color_depth); - hd.dimage_length = be32toh(hd.dimage_length); - - /* check control */ + /* check magic */ if (memcmp(hd.magic, "CP", 2)) { - log_info("could not read CP."); - return (g1m_error_magic); - } else if (memcmp(hd.magic, "0100", 4)) { - log_info("could not read header 0100"); + log_fatal("could not read 'CP'."); return (g1m_error_magic); } - /* TODO: what about "length of packed data + 4" control? */ - /* TODO: get footer, check control 0100 */ + /* read the image header */ + DREAD(ihd, g3p_imageheader) - /* check if image is encrypted */ - int is_encrypted = (((std->filesize & 0xff) + - ((std->filesize & 0xff00) >> 8) + g1m_typ_g3p) & 0xff) ^ 0x1c; + /* correct endianness */ + ihd.df_size = be32toh(ihd.df_size); + ihd.width = be16toh(ihd.width); + ihd.height = be16toh(ihd.height); + ihd.color_depth = be16toh(ihd.color_depth); + ihd.data_size = be32toh(ihd.data_size); + + /* check some little things */ + int is_obfuscated = ((uint8_t)(std->obfuscated + 8) + != (uint8_t)((std->filesize & 0xff00) >> 8)); + int has_footer = 1; /* might change? */ + size_t deflated_image_size = ihd.data_size - has_footer * 0x4; /* log info */ - log_info("Width: %dpx, height: %dpx", hd.width, hd.height); + log_info("Width: %dpx, height: %dpx", ihd.width, ihd.height); log_info("Pixel depth: %s", - (hd.color_depth == g3p_color_4bit) ? "4-bit" : "16-bit"); - log_info("Deflated image length: %uo", hd.dimage_length); - log_info("Is encrypted: %s", is_encrypted ? "yes" : "no"); + (ihd.color_depth == g3p_color_4bit) ? "4-bit" : "16-bit"); + log_info("Deflated image length: %uo", ihd.data_size - 0x04); + log_info("Is obfuscated: %s", is_obfuscated ? "yes" : "no"); + /* read image */ + uint8_t deflated_image[deflated_image_size]; + READ(deflated_image, deflated_image_size) + + /* read footer and check sum */ + if (has_footer) { + DREAD(ft, g3p_imagefooter); + ft.checksum = be32toh(ft.checksum); + if (adler32(0, deflated_image, deflated_image_size) != ft.checksum) { + log_fatal("Incorrect Adler32 checksum!"); + return (g1m_error_magic); + } + } + + /* unobfuscate if required */ + if (is_obfuscated) + g3p_deobfuscate(deflated_image, deflated_image_size); + + /* find out length of inflated data */ + uLongf inflated_size = 0L; int z_err; + if ((z_err = uncompress(Z_NULL, &inflated_size, deflated_image, + deflated_image_size)) != Z_MEM_ERROR) { + log_fatal("Zlib error: error #%d", z_err); + return (g1m_error_magic); + } + log_info("Inflated image size is %zu", inflated_size); + + /* allocate space for it, and uncompress */ + uint8_t inflated_image[inflated_size]; + if ((z_err = uncompress(inflated_image, &inflated_size, deflated_image, + deflated_image_size))) { + log_fatal("Zlib error: error #%d", z_err); + return (g1m_error_magic); + } + + /* TODO: store pixels */ /* no error */ return (0); } diff --git a/src/user/free.c b/src/user/free.c index 8bf58a6..c5f980e 100644 --- a/src/user/free.c +++ b/src/user/free.c @@ -48,12 +48,12 @@ void g1m_free_mcsfile(g1m_mcsfile_t *handle) void g1m_free_mcs(g1m_t *handle) { /* check if mcs */ - if (!handle->parts) + if (!handle->mcs) return ; /* foreach mcs */ - g1m_mcs_t **mcs = handle->parts; - int mcs_count = handle->part_count; + g1m_mcs_t **mcs = handle->mcs; + int mcs_count = handle->mcs_count; for (int i = 0; i < mcs_count; i++) { /* check if used */ if (!mcs[i]) @@ -69,7 +69,7 @@ void g1m_free_mcs(g1m_t *handle) free(mcs[i]); } free(mcs); - handle->parts = NULL; + handle->mcs = NULL; } /** @@ -84,7 +84,7 @@ void g1m_free(g1m_t *handle) if (!handle) return ; /* mcs time! */ - if (handle->type & g1m_type_mcs && handle->parts) + if (handle->type & g1m_type_mcs) g1m_free_mcs(handle); /* free the handle */