define a library abstraction and logging helpers

-> The Library class handles the loading and parsing of data files. This
   is because any fxos application will use this since everyone will
   have only one library.
-> Add a logging function that automatically format()s everything in
   sight with basic logging levels and a verbose mode. Standard logs are
   prefixed with __func__ for debugging purposes.
-> Allow format() to take std::string arguments for %s by statically
   extracting c_str()s.
-> Add a simple timing utility to understand which file load or
   disassembler pass takes up the time.
This commit is contained in:
Lephenixnoir 2020-01-19 18:17:01 +01:00
parent b20731c829
commit c499ca1f90
11 changed files with 415 additions and 132 deletions

View File

@ -1,9 +1,11 @@
#include "fxos-cli.h"
#include <fxos/disassembly.h>
#include <fxos/library.h>
#include <fxos/errors.h>
#include <fxos/target.h>
#include <fxos/load.h>
#include <fxos/log.h>
#include <fxos/os.h>
#include <fxos/disasm-passes/cfg.h>
@ -19,9 +21,11 @@
namespace fs = std::filesystem;
using namespace FxOS;
using namespace FxOS::Log;
static char const *help_string = R"(
usage: fxos info <target>
usage: fxos library [-t] [-a]
fxos info <target>
fxos disasm <target> <region or function> [options...]
fxos analyze [-f] [-s] [-a] [-r] <number> <os file> [options...]
@ -32,11 +36,21 @@ editable database of platform, syscall, and OS knowledge.
General options:
-3, --sh3 Assume SH3 OS and platform (default: SH4)
-4, --sh4 Assume SH4 OS and platform (default: SH4)
-v, --verbose Print logs about what's happening
A <target> is either:
<targetname> A target in library (eg "fx@3.10")
-f <file> An arbitrary file which is loaded as ROM
LIBRARY COMMAND
Prints out the contents of the library. If an option is set, the results are
printed in a simple easily-parsable form without header.
Selectors:
-t Print all targets
-a Print all assembler instruction sets
INFO COMMAND
Identify an OS image: version, platform, date, checksums...
@ -85,85 +99,20 @@ Analysis options:
--occurrences <num> Show at most <num> occurrences (integer or "all")
)"+1;
#define usage(exitcode) { \
std::cerr << help_string; \
return exitcode; \
}
//---
// Configuration
//---
std::map<std::string, TargetDescription> targets;
std::vector<std::string> library { FXOS_INSTALL_PREFIX "/share/fxos" };
/* Load any fxos data file */
void load(std::string path)
void loadconfig(Library &lib)
{
Buffer file(path);
size_t offset;
int line;
/* First add the fxos install folder as PATH */
lib.add_path(FXOS_INSTALL_PREFIX "/share/fxos");
// std::cerr << "[fxos] loading resource file '" << path << "'...\n";
Header h = load_header(file, offset, line);
if(h.find("type") == h.end())
{
std::cerr << "error: no type in header of '" << path << "'\n";
return;
}
std::string type = h["type"];
if(type == "assembly")
{
try {
load_asm(file, offset, line);
}
catch(FxOS::SyntaxError &e) {
std::cerr << e.file() << ":" << e.line() << ": " <<
e.what() << "\n" << std::flush;
}
return;
}
else if(type == "target")
{
if(!h.count("name"))
{
std::cerr << "error: no name specified in '" << path
<< "'\n";
return;
}
try {
targets[h["name"]] = load_target(file, offset, line);
}
catch(FxOS::SyntaxError &e) {
std::cerr << e.file() << ":" << e.line() << ": " <<
e.what() << "\n" << std::flush;
}
return;
}
std::cerr << "unknown file type '" << type << "' in '" << path <<"'\n";
}
/* Load a whole folder into the database */
void loadfolder(std::string path)
{
try
{
fs::recursive_directory_iterator it(path);
for(auto &file: it) load(file.path());
}
catch(fs::filesystem_error &e)
{
if(e.code().value() == ENOENT)
{
std::cerr << "warning: directory '" << path << "' does"
" not exist\n";
}
else throw e;
}
}
void loadconfig(void)
{
std::string home = getenv("HOME");
fs::path configpath = home + "/.config/fxos/config";
@ -181,11 +130,11 @@ void loadconfig(void)
if(std::sscanf(line.c_str(), "library: %256s", path) == 1)
{
library.push_back(path);
lib.add_path(path);
}
else if(std::sscanf(line.c_str(), "load: %256s", path) == 1)
{
loadfolder(path);
lib.explore(path);
}
}
}
@ -196,35 +145,43 @@ void loadconfig(void)
int main_info(int argc, char **argv)
{
int error=0, option=0, mpu='4';
int error=0, option=0;
std::string path;
struct option const longs[] = {
{ "help", no_argument, NULL, 'h' },
{ "sh3", no_argument, NULL, '3' },
{ "sh4", no_argument, NULL, '4' },
{ "help", no_argument, NULL, 'h' },
{ "sh3", no_argument, NULL, '3' },
{ "sh4", no_argument, NULL, '4' },
{ "verbose", no_argument, NULL, 'v' },
};
while(option >= 0 && option != '?')
switch((option = getopt_long(argc, argv, "h34f:p:", longs, NULL)))
switch((option = getopt_long(argc, argv, "h34f:p:v", longs, NULL)))
{
case 'h':
std::cerr << help_string;
break;
usage(0);
case '3':
case '4':
/* TODO: Use sh3/sh4 information in [fxos info]? */
mpu = option;
break;
case 'f':
path = optarg;
break;
case 'v':
log_setminlevel(LEVEL_LOG);
break;
case '?':
error = 1;
}
if(error) return 1;
//~
/* Load the configuration and library */
Library lib;
loadconfig(lib);
/* Load from path if one is specified */
if(path.size())
{
@ -237,28 +194,23 @@ int main_info(int argc, char **argv)
os_info(t);
}
catch(std::exception &e) {
std::cerr << "error: " << e.what() << "\n";
log(ERR "%s", e.what());
return 1;
}
}
/* Load from target otherwise */
else
{
if(!argv[optind + 1])
if(!argv[optind + 1]) usage(1);
std::string tname = argv[optind + 1];
if(!lib.targets().count(tname))
{
std::cerr << help_string;
log(ERR "no target '%s' in library", tname);
return 1;
}
std::string targetname = argv[optind + 1];
if(!targets.count(targetname))
{
std::cerr << "error: no target '" << targetname
<< "' in library\n";
return 1;
}
Target t(targets[targetname], library);
Target t(lib.targets().at(tname), lib.paths());
os_info(t);
}
@ -274,17 +226,17 @@ int main_disassembly(int argc, char **argv)
std::string file;
struct option const longs[] = {
{ "help", no_argument, NULL, 'h' },
{ "sh3", no_argument, NULL, '3' },
{ "sh4", no_argument, NULL, '4' },
{ "help", no_argument, NULL, 'h' },
{ "sh3", no_argument, NULL, '3' },
{ "sh4", no_argument, NULL, '4' },
{ "verbose", no_argument, NULL, 'v' },
};
while(option >= 0 && option != '?')
switch((option = getopt_long(argc, argv, "h34p:f:", longs, NULL)))
switch((option = getopt_long(argc, argv, "h34p:f:v", longs, NULL)))
{
case 'h':
std::cerr << help_string;
break;
usage(0);
case '3':
case '4':
mpu = option;
@ -305,6 +257,9 @@ int main_disassembly(int argc, char **argv)
case 'f':
file = optarg;
break;
case 'v':
log_setminlevel(LEVEL_LOG);
break;
case '?':
error = 1;
}
@ -313,28 +268,33 @@ int main_disassembly(int argc, char **argv)
if(argc < optind + remaining_args + 1)
{
std::cerr << "error: missing file or address\n";
log(ERR "missing file or address");
error = 1;
}
else if(argc > optind + remaining_args + 1)
{
std::cerr << "error: excess arguments\n";
log(ERR "excess argument");
error = 1;
}
if(error) return 1;
//~
/* Load the configuration and library */
Library lib;
loadconfig(lib);
if(!file.size())
{
std::string targetname = argv[optind + 1];
std::string tname = argv[optind + 1];
if(!targets.count(targetname))
if(!lib.targets().count(tname))
{
std::cerr << "error: no target '" << targetname
<< "' in library\n";
log(ERR "no target '%s' in library", tname);
return 1;
}
Target target(targets[targetname], library);
Target target(lib.targets().at(tname), lib.paths());
char const *refstr = argv[optind + 2];
uint32_t ref;
@ -343,11 +303,12 @@ int main_disassembly(int argc, char **argv)
Disassembly disasm(target);
OS *os = nullptr;
std::cout << "disassembling target:" << targetname << " ref:" << refstr << "\n";
log(LOG "Disassembling target %s at %s", tname, refstr);
for(auto pass: passes)
{
std::cout << "running pass: " << pass << "\n";
log(LOG "Running pass %s", pass);
if(pass == "cfg")
{
CfgPass p(disasm);
@ -359,13 +320,14 @@ int main_disassembly(int argc, char **argv)
{
char const *ref = argv[optind + 1];
std::cout << "disassembling file:" << file << " ref:" << ref << "\n";
log(LOG "Disassembling file %s at %s", file, ref);
}
return 0;
}
int main_analyze(__attribute__((unused)) int argc, __attribute__((unused)) char **argv)
#define UN __attribute__((unused))
int main_analyze(UN int argc, UN char **argv)
{
std::cerr << "doing main_analyze, which is incomplete x_x\n";
return 0;
@ -373,18 +335,11 @@ int main_analyze(__attribute__((unused)) int argc, __attribute__((unused)) char
int main(int argc, char **argv)
{
if(argc < 2)
{
std::cerr << help_string;
return 1;
}
if(argc < 2) usage(1);
std::string cmd = argv[1];
argv[1] = (char *)"";
/* Load the configuration file if it exists */
loadconfig();
if(cmd == "info")
return main_info(argc, argv);
else if(cmd == "disasm")
@ -393,10 +348,7 @@ int main(int argc, char **argv)
return main_analyze(argc, argv);
else if(cmd == "-?" || cmd == "-h" || cmd == "--help")
{
std::cerr << help_string;
return 0;
}
usage(0);
std::cerr << "invalid operation '" << cmd << "'\n";
std::cerr << "Try '" << argv[0] << " --help'.\n";

View File

@ -5,6 +5,7 @@
#ifndef LIBFXOS_ERRORS_H
#define LIBFXOS_ERRORS_H
#include <fxos/util.h>
#include <exception>
#include <string>
@ -18,7 +19,7 @@ public:
SyntaxError(char const *file, int line, char const *what):
m_file(file), m_line(line), m_what(what) {}
/* Provides access to these free objects */
/* Provides access to these objects */
char const *file() const noexcept {
return m_file;
}
@ -29,12 +30,36 @@ public:
return m_what;
}
/* Additional friendly formatter */
std::string str() const noexcept {
return format("%s:%d: %s", m_file, m_line, m_what);
}
private:
char const *m_file;
int m_line;
char const *m_what;
};
/* Language errors for the disassembler */
class LangError: public std::exception
{
public:
LangError(uint32_t address, char const *what):
m_addr(address), m_what(what) {}
uint32_t addr() const noexcept {
return m_addr;
}
char const *what() const noexcept override {
return m_what;
}
private:
uint32_t m_addr;
char const *m_what;
};
} /* namespace FxOS */
#endif /* LIBFXOS_ERRORS_H */

51
include/fxos/library.h Normal file
View File

@ -0,0 +1,51 @@
//---
// fxos.library: Management of resource files into libraries
//---
#ifndef FXOS_LIBRARY_H
#define FXOS_LIBRARY_H
#include <fxos/target.h>
#include <string>
#include <vector>
#include <map>
namespace FxOS {
struct Library
{
Library() = default;
/* Add a new search path in the library. */
void add_path(std::string path);
/* Recursively explore the fxos data file in a folder. */
void explore(std::string path);
/* Directly load an fxos data file into the library. Normally called by
explore() only, but might be useful here. */
void load(std::string path);
/* List of search directories */
const std::vector<std::string> &paths() {
return m_paths;
}
/* Targets loaded by exploration */
const std::map<std::string, TargetDescription> &targets() {
return m_targets;
}
/* Simple list of assembly tables */
const std::vector<std::string> &asm_tables() {
return m_asmtables;
}
private:
std::vector<std::string> m_paths;
std::map<std::string, TargetDescription> m_targets;
std::vector<std::string> m_asmtables;
};
} /* namespace FxOS */
#endif /* FXOS_LIBRARY_H */

38
include/fxos/log.h Normal file
View File

@ -0,0 +1,38 @@
//---
// fxos.log: Logging functions
//---
#ifndef FXOS_LOG_H
#define FXOS_LOG_H
#include <fxos/util.h>
namespace FxOS::Log {
/* Message levels, used for masking and statistics */
#define LEVEL_LOG 0
#define LEVEL_WRN 1
#define LEVEL_ERR 4
/* Prefixes to set in the call to log() for brevity. The comma is included.
Typical usage would be log(ERR "logic is inconsistent"). */
#define LOG LEVEL_LOG,
#define WRN LEVEL_WRN,
#define ERR LEVEL_ERR,
/* Select the log level */
void log_setminlevel(int level);
/* General message logger */
void logmsg(int level, char const *function, std::string message);
/* Automatically apply format strings. Also force the first argument to be
expanded first, since this causes a comma to appear. */
#define log(level, ...) \
loghelper(level, __VA_ARGS__)
#define loghelper(level, fmtstr, ...) \
logmsg(level, __func__, format(fmtstr __VA_OPT__(,) __VA_ARGS__))
} /* namespace FxOS::Log */
#endif /* FXOS_LOG_H */

View File

@ -10,11 +10,12 @@
#include <string>
#include <memory>
#include <cstdio>
#include <ctime>
#include <vector>
/* Format a string with printf() syntax */
template<typename ... Args>
std::string format(std::string const &format, Args ... args)
std::string format_do(std::string const &format, Args && ... args)
{
/* Reserve space for snprintf() to put its NUL */
size_t size = snprintf(nullptr, 0, format.c_str(), args ...) + 1;
@ -26,6 +27,28 @@ std::string format(std::string const &format, Args ... args)
return std::string(buf.get(), buf.get() + size - 1);
}
/* Convert std::string to char * when printing (@Zitrax on Github) */
template<typename T>
auto format_convert(T&& t)
{
if constexpr(std::is_same<std::remove_cv_t<std::remove_reference_t<T>>,
std::string>::value)
{
return std::forward<T>(t).c_str();
}
else
{
return std::forward<T>(t);
}
}
/* String formatting with std::string support in %s */
template<typename ... Args>
std::string format(std::string const &format, Args && ... args)
{
return format_do(format, format_convert(std::forward<Args>(args))...);
}
/* An object extracted from a target, which has a virtual address */
template<typename T>
struct Addressable
@ -73,7 +96,8 @@ public:
Buffer(std::string filepath, std::vector<std::string> &folders,
ssize_t size=-1, int fill=0x00);
/* Create a buffer by copying and resizing another buffer */
/* Create a buffer by copying and resizing another buffer (we want no
copy constructor so the size must be explicit) */
Buffer(Buffer const &other, size_t new_size, int fill=0x00);
/* Free allocated data, obviously */
@ -87,4 +111,11 @@ public:
std::string path;
};
/* Generic timer which returns times in ns using CLOCK_REALTIME */
struct timespec timer_start(void);
long long timer_end(struct timespec start);
/* Format ns durations to us, ms or s depending on the value */
std::string timer_format(long long duration);
#endif /* LIBFXOS_UTIL_H */

View File

@ -163,6 +163,8 @@ void DisassemblyPass::run(uint32_t entry_pc)
ConcreteInstruction &ci = m_disasm.readins(pc);
analyze(pc, ci);
m_seen.insert(pc);
}
}

97
lib/library.cpp Normal file
View File

@ -0,0 +1,97 @@
#include <fxos/library.h>
#include <fxos/load.h>
#include <fxos/errors.h>
#include <fxos/util.h>
#include <fxos/log.h>
#include <filesystem>
#include <ctime>
namespace fs = std::filesystem;
using namespace FxOS::Log;
namespace FxOS {
/* Add a new search path in the library. */
void Library::add_path(std::string path)
{
m_paths.push_back(path);
}
/* Recursively explore the fxos data file in a folder. */
void Library::explore(std::string path)
{
try
{
fs::recursive_directory_iterator it(path);
for(auto &file: it) load(file.path());
}
catch(fs::filesystem_error &e)
{
if(e.code().value() == ENOENT)
{
log(WRN "directory %s does not exist", path);
}
else throw e;
}
}
/* Directly load an fxos data file into the library. */
void Library::load(std::string path)
{
Buffer file(path);
size_t offset;
int line;
log(LOG "Loading resource file '%s'...\\", path);
/* Time the loading */
auto start = timer_start();
Header h = load_header(file, offset, line);
if(h.find("type") == h.end())
{
log(ERR "no type set in the header of '%s'", path);
return;
}
std::string type = h["type"];
if(type == "assembly")
{
try {
load_asm(file, offset, line);
m_asmtables.push_back(path);
}
catch(FxOS::SyntaxError &e) {
log(ERR "%s", e.str());
}
}
else if(type == "target")
{
if(!h.count("name"))
{
log(ERR "no target name set in '%s'", path);
return;
}
try {
m_targets[h["name"]] = load_target(file, offset, line);
}
catch(FxOS::SyntaxError &e) {
log(ERR "%s", e.str());
}
}
else
{
log(ERR "unknown file type '%s' in '%s'", type, path);
return;
}
long long ns = timer_end(start);
log(LOG "done (%s)", timer_format(ns));
}
} /* namespace FxOS */

View File

@ -57,7 +57,6 @@ space [ \t]+
":" { return COLON; }
. { err("lex error near '%s'", yytext); }
<<EOF>> { err("EOF reached before header ends"); }
%%

56
lib/log.cpp Normal file
View File

@ -0,0 +1,56 @@
#include <fxos/log.h>
#include <iostream>
#include <map>
namespace FxOS::Log {
/* Currently configured log level */
static int loglevel = LEVEL_WRN;
/* Level of the last message */
static int lastlevel = -1;
/* Prefixes for each level */
static std::map<int,std::string> prefixes = {
{ LEVEL_WRN, "warning: " },
{ LEVEL_ERR, "error: " },
};
/* Select the log level */
void log_setminlevel(int level)
{
loglevel = level;
}
/* General message logger */
void logmsg(int level, char const *function, std::string message)
{
if(level < loglevel) return;
bool endline = true;
bool prefix = true;
/* Add a newline if last line was unfinished, but level changed */
if(lastlevel >= 0 && lastlevel != level) std::cerr << '\n';
else if(lastlevel >= 0) prefix = false;
if(message.size() && message.back() == '\\')
{
endline = false;
message.pop_back();
}
if(prefix)
{
if(level == LEVEL_LOG) std::cerr << "[" << function << "] ";
std::cerr << prefixes[level];
}
else std::cerr << " ";
std::cerr << message;
lastlevel = -1;
if(endline) std::cerr << '\n';
else lastlevel = level;
}
} /* namespace FxOS::Log */

View File

@ -2,7 +2,9 @@
// fxos.passes.cfg: CFG construction, as used by other passes
//---
#include <fxos/disasm-passes/cfg.h>
#include <fxos/disassembly.h>
#include <fxos/errors.h>
#include <cassert>
namespace FxOS {
@ -25,8 +27,8 @@ void CfgPass::analyze(uint32_t pc, ConcreteInstruction &ci)
if(ci.inst.mnemonic != mnemonic) continue;
auto &args = ci.inst.args;
assert((args.size() < 1 || args[0].kind != Argument::PcJump)
&& "invalid use of a jump instruction\n");
if(args.size() != 1 || args[0].kind != Argument::PcJump)
throw LangError(pc, "invalid jump instruction");
ci.jmptarget = pc + args[0].disp;
}

View File

@ -1,7 +1,8 @@
#include <fxos/util.h>
#include <filesystem>
#include <cstdlib>
#include <cstring>
#include <filesystem>
#include <ctime>
#include <unistd.h>
#include <fcntl.h>
@ -119,3 +120,32 @@ Buffer::~Buffer()
{
free(data);
}
/* Generic timer which returns times in ns using CLOCK_REALTIME */
struct timespec timer_start(void)
{
struct timespec start;
clock_gettime(CLOCK_REALTIME, &start);
return start;
}
long long timer_end(struct timespec start)
{
struct timespec end;
clock_gettime(CLOCK_REALTIME, &end);
long long ns = 1000000000 * (end.tv_sec - start.tv_sec);
ns += end.tv_nsec - start.tv_nsec;
return ns;
}
/* Format ns durations to us, ms or s depending on the value */
std::string timer_format(long long duration)
{
if(duration < 2000) return format("%lld ns", duration);
duration /= 1000;
if(duration < 2000) return format("%lld us", duration);
duration /= 1000;
if(duration < 2000) return format("%lld ms", duration);
duration /= 1000;
return format("%lld s", duration);
}