py: Add support for matmul operator @ as per PEP 465.

To make progress towards MicroPython supporting Python 3.5, adding the
matmul operator is important because it's a really "low level" part of the
language, being a new token and modifications to the grammar.

It doesn't make sense to make it configurable because 1) it would make the
grammar and lexer complicated/messy; 2) no other operators are
configurable; 3) it's not a feature that can be "dynamically plugged in"
via an import.

And matmul can be useful as a general purpose user-defined operator, it
doesn't have to be just for numpy use.

Based on work done by Jim Mussared.
This commit is contained in:
Damien George 2019-07-25 13:15:54 +10:00
parent 14e203282a
commit 2069c563f9
7 changed files with 34 additions and 25 deletions

View File

@ -55,7 +55,7 @@ DEF_RULE_NC(eval_input_2, and(1), tok(NEWLINE))
// varargslist: vfpdef ['=' test] (',' vfpdef ['=' test])* [',' ['*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef]] | '*' [vfpdef] (',' vfpdef ['=' test])* [',' '**' vfpdef] | '**' vfpdef
// vfpdef: NAME
DEF_RULE_NC(decorator, and(4), tok(DEL_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE))
DEF_RULE_NC(decorator, and(4), tok(OP_AT), rule(dotted_name), opt_rule(trailer_paren), tok(NEWLINE))
DEF_RULE_NC(decorators, one_or_more, rule(decorator))
DEF_RULE(decorated, c(decorated), and_ident(2), rule(decorators), rule(decorated_body))
#if MICROPY_PY_ASYNC_AWAIT
@ -96,7 +96,7 @@ DEF_RULE(simple_stmt_2, c(generic_all_nodes), list_with_end, rule(small_stmt), t
// small_stmt: expr_stmt | del_stmt | pass_stmt | flow_stmt | import_stmt | global_stmt | nonlocal_stmt | assert_stmt
// expr_stmt: testlist_star_expr (augassign (yield_expr|testlist) | ('=' (yield_expr|testlist_star_expr))*)
// testlist_star_expr: (test|star_expr) (',' (test|star_expr))* [',']
// augassign: '+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//='
// augassign: '+=' | '-=' | '*=' | '@=' | '/=' | '%=' | '&=' | '|=' | '^=' | '<<=' | '>>=' | '**=' | '//='
// # For normal assignments, additional restrictions enforced by the interpreter
DEF_RULE_NC(small_stmt, or(8), rule(del_stmt), rule(pass_stmt), rule(flow_stmt), rule(import_stmt), rule(global_stmt), rule(nonlocal_stmt), rule(assert_stmt), rule(expr_stmt))
@ -108,7 +108,7 @@ DEF_RULE_NC(expr_stmt_assign, and_ident(2), tok(DEL_EQUAL), rule(expr_stmt_6))
DEF_RULE_NC(expr_stmt_6, or(2), rule(yield_expr), rule(testlist_star_expr))
DEF_RULE(testlist_star_expr, c(generic_tuple), list_with_end, rule(testlist_star_expr_2), tok(DEL_COMMA))
DEF_RULE_NC(testlist_star_expr_2, or(2), rule(star_expr), rule(test))
DEF_RULE_NC(augassign, or(12), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL))
DEF_RULE_NC(augassign, or(13), tok(DEL_PLUS_EQUAL), tok(DEL_MINUS_EQUAL), tok(DEL_STAR_EQUAL), tok(DEL_AT_EQUAL), tok(DEL_SLASH_EQUAL), tok(DEL_PERCENT_EQUAL), tok(DEL_AMPERSAND_EQUAL), tok(DEL_PIPE_EQUAL), tok(DEL_CARET_EQUAL), tok(DEL_DBL_LESS_EQUAL), tok(DEL_DBL_MORE_EQUAL), tok(DEL_DBL_STAR_EQUAL), tok(DEL_DBL_SLASH_EQUAL))
// del_stmt: 'del' exprlist
// pass_stmt: 'pass'
@ -226,7 +226,7 @@ DEF_RULE(lambdef_nocond, c(lambdef), and_blank(4), tok(KW_LAMBDA), opt_rule(vara
// and_expr: shift_expr ('&' shift_expr)*
// shift_expr: arith_expr (('<<'|'>>') arith_expr)*
// arith_expr: term (('+'|'-') term)*
// term: factor (('*'|'/'|'%'|'//') factor)*
// term: factor (('*'|'@'|'/'|'%'|'//') factor)*
// factor: ('+'|'-'|'~') factor | power
// power: atom_expr ['**' factor]
// atom_expr: 'await' atom trailer* | atom trailer*
@ -249,7 +249,7 @@ DEF_RULE_NC(shift_op, or(2), tok(OP_DBL_LESS), tok(OP_DBL_MORE))
DEF_RULE(arith_expr, c(term), list, rule(term), rule(arith_op))
DEF_RULE_NC(arith_op, or(2), tok(OP_PLUS), tok(OP_MINUS))
DEF_RULE(term, c(term), list, rule(factor), rule(term_op))
DEF_RULE_NC(term_op, or(4), tok(OP_STAR), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH))
DEF_RULE_NC(term_op, or(5), tok(OP_STAR), tok(OP_AT), tok(OP_SLASH), tok(OP_PERCENT), tok(OP_DBL_SLASH))
DEF_RULE_NC(factor, or(2), rule(factor_2), rule(power))
DEF_RULE(factor_2, c(factor_2), and_ident(2), rule(factor_op), rule(factor))
DEF_RULE_NC(factor_op, or(3), tok(OP_PLUS), tok(OP_MINUS), tok(OP_TILDE))

View File

@ -174,7 +174,7 @@ STATIC void indent_pop(mp_lexer_t *lex) {
// this means if the start of two ops are the same then they are equal til the last char
STATIC const char *const tok_enc =
"()[]{},:;@~" // singles
"()[]{},:;~" // singles
"<e=c<e=" // < <= << <<=
">e=c>e=" // > >= >> >>=
"*e=c*e=" // * *= ** **=
@ -185,6 +185,7 @@ STATIC const char *const tok_enc =
"/e=c/e=" // / /= // //=
"%e=" // % %=
"^e=" // ^ ^=
"@e=" // @ @=
"=e=" // = ==
"!."; // start of special cases: != . ...
@ -193,7 +194,7 @@ STATIC const uint8_t tok_enc_kind[] = {
MP_TOKEN_DEL_PAREN_OPEN, MP_TOKEN_DEL_PAREN_CLOSE,
MP_TOKEN_DEL_BRACKET_OPEN, MP_TOKEN_DEL_BRACKET_CLOSE,
MP_TOKEN_DEL_BRACE_OPEN, MP_TOKEN_DEL_BRACE_CLOSE,
MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_DEL_AT, MP_TOKEN_OP_TILDE,
MP_TOKEN_DEL_COMMA, MP_TOKEN_DEL_COLON, MP_TOKEN_DEL_SEMICOLON, MP_TOKEN_OP_TILDE,
MP_TOKEN_OP_LESS, MP_TOKEN_OP_LESS_EQUAL, MP_TOKEN_OP_DBL_LESS, MP_TOKEN_DEL_DBL_LESS_EQUAL,
MP_TOKEN_OP_MORE, MP_TOKEN_OP_MORE_EQUAL, MP_TOKEN_OP_DBL_MORE, MP_TOKEN_DEL_DBL_MORE_EQUAL,
@ -205,6 +206,7 @@ STATIC const uint8_t tok_enc_kind[] = {
MP_TOKEN_OP_SLASH, MP_TOKEN_DEL_SLASH_EQUAL, MP_TOKEN_OP_DBL_SLASH, MP_TOKEN_DEL_DBL_SLASH_EQUAL,
MP_TOKEN_OP_PERCENT, MP_TOKEN_DEL_PERCENT_EQUAL,
MP_TOKEN_OP_CARET, MP_TOKEN_DEL_CARET_EQUAL,
MP_TOKEN_OP_AT, MP_TOKEN_DEL_AT_EQUAL,
MP_TOKEN_DEL_EQUAL, MP_TOKEN_OP_DBL_EQUAL,
};

View File

@ -106,7 +106,7 @@ typedef enum _mp_token_kind_t {
MP_TOKEN_OP_MORE_EQUAL,
MP_TOKEN_OP_NOT_EQUAL,
// Order of these 12 matches corresponding mp_binary_op_t operator
// Order of these 13 matches corresponding mp_binary_op_t operator
MP_TOKEN_OP_PIPE,
MP_TOKEN_OP_CARET,
MP_TOKEN_OP_AMPERSAND,
@ -115,12 +115,13 @@ typedef enum _mp_token_kind_t {
MP_TOKEN_OP_PLUS,
MP_TOKEN_OP_MINUS,
MP_TOKEN_OP_STAR,
MP_TOKEN_OP_AT,
MP_TOKEN_OP_DBL_SLASH,
MP_TOKEN_OP_SLASH,
MP_TOKEN_OP_PERCENT,
MP_TOKEN_OP_DBL_STAR,
// Order of these 12 matches corresponding mp_binary_op_t operator
// Order of these 13 matches corresponding mp_binary_op_t operator
MP_TOKEN_DEL_PIPE_EQUAL,
MP_TOKEN_DEL_CARET_EQUAL,
MP_TOKEN_DEL_AMPERSAND_EQUAL,
@ -129,6 +130,7 @@ typedef enum _mp_token_kind_t {
MP_TOKEN_DEL_PLUS_EQUAL,
MP_TOKEN_DEL_MINUS_EQUAL,
MP_TOKEN_DEL_STAR_EQUAL,
MP_TOKEN_DEL_AT_EQUAL,
MP_TOKEN_DEL_DBL_SLASH_EQUAL,
MP_TOKEN_DEL_SLASH_EQUAL,
MP_TOKEN_DEL_PERCENT_EQUAL,
@ -144,7 +146,6 @@ typedef enum _mp_token_kind_t {
MP_TOKEN_DEL_COLON,
MP_TOKEN_DEL_PERIOD,
MP_TOKEN_DEL_SEMICOLON,
MP_TOKEN_DEL_AT,
MP_TOKEN_DEL_EQUAL,
MP_TOKEN_DEL_MINUS_MORE,
} mp_token_kind_t;

View File

@ -478,6 +478,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = {
[MP_BINARY_OP_INPLACE_SUBTRACT] = MP_QSTR___isub__,
#if MICROPY_PY_ALL_INPLACE_SPECIAL_METHODS
[MP_BINARY_OP_INPLACE_MULTIPLY] = MP_QSTR___imul__,
[MP_BINARY_OP_INPLACE_MAT_MULTIPLY] = MP_QSTR___imatmul__,
[MP_BINARY_OP_INPLACE_FLOOR_DIVIDE] = MP_QSTR___ifloordiv__,
[MP_BINARY_OP_INPLACE_TRUE_DIVIDE] = MP_QSTR___itruediv__,
[MP_BINARY_OP_INPLACE_MODULO] = MP_QSTR___imod__,
@ -493,6 +494,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = {
[MP_BINARY_OP_SUBTRACT] = MP_QSTR___sub__,
#if MICROPY_PY_ALL_SPECIAL_METHODS
[MP_BINARY_OP_MULTIPLY] = MP_QSTR___mul__,
[MP_BINARY_OP_MAT_MULTIPLY] = MP_QSTR___matmul__,
[MP_BINARY_OP_FLOOR_DIVIDE] = MP_QSTR___floordiv__,
[MP_BINARY_OP_TRUE_DIVIDE] = MP_QSTR___truediv__,
[MP_BINARY_OP_MODULO] = MP_QSTR___mod__,
@ -510,6 +512,7 @@ const byte mp_binary_op_method_name[MP_BINARY_OP_NUM_RUNTIME] = {
[MP_BINARY_OP_REVERSE_SUBTRACT] = MP_QSTR___rsub__,
#if MICROPY_PY_ALL_SPECIAL_METHODS
[MP_BINARY_OP_REVERSE_MULTIPLY] = MP_QSTR___rmul__,
[MP_BINARY_OP_REVERSE_MAT_MULTIPLY] = MP_QSTR___rmatmul__,
[MP_BINARY_OP_REVERSE_FLOOR_DIVIDE] = MP_QSTR___rfloordiv__,
[MP_BINARY_OP_REVERSE_TRUE_DIVIDE] = MP_QSTR___rtruediv__,
[MP_BINARY_OP_REVERSE_MODULO] = MP_QSTR___rmod__,

View File

@ -135,8 +135,8 @@ STATIC const uint16_t rule_arg_combined_table[] = {
#define RULE_EXPAND(x) x
#define RULE_PADDING(rule, ...) RULE_PADDING2(rule, __VA_ARGS__, RULE_PADDING_IDS(rule))
#define RULE_PADDING2(rule, ...) RULE_EXPAND(RULE_PADDING3(rule, __VA_ARGS__))
#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, ...) __VA_ARGS__
#define RULE_PADDING_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r,
#define RULE_PADDING3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) __VA_ARGS__
#define RULE_PADDING_IDS(r) PAD13_##r, PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r,
// Use an enum to create constants specifying how much room a rule takes in rule_arg_combined_table
enum {
@ -155,8 +155,8 @@ enum {
// Macro to compute the start of a rule in rule_arg_combined_table
#define RULE_ARG_OFFSET(rule, ...) RULE_ARG_OFFSET2(rule, __VA_ARGS__, RULE_ARG_OFFSET_IDS(rule))
#define RULE_ARG_OFFSET2(rule, ...) RULE_EXPAND(RULE_ARG_OFFSET3(rule, __VA_ARGS__))
#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, ...) _13
#define RULE_ARG_OFFSET_IDS(r) PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r,
#define RULE_ARG_OFFSET3(rule, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, ...) _14
#define RULE_ARG_OFFSET_IDS(r) PAD13_##r, PAD12_##r, PAD11_##r, PAD10_##r, PAD9_##r, PAD8_##r, PAD7_##r, PAD6_##r, PAD5_##r, PAD4_##r, PAD3_##r, PAD2_##r, PAD1_##r, PAD0_##r,
// Use the above enum values to create a table of offsets for each rule's arg
// data, which indexes rule_arg_combined_table. The offsets require 9 bits of
@ -632,7 +632,7 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) {
} else if (rule_id == RULE_shift_expr
|| rule_id == RULE_arith_expr
|| rule_id == RULE_term) {
// folding for binary ops: << >> + - * / % //
// folding for binary ops: << >> + - * @ / % //
mp_parse_node_t pn = peek_result(parser, num_args - 1);
if (!mp_parse_node_get_int_maybe(pn, &arg0)) {
return false;
@ -644,8 +644,8 @@ STATIC bool fold_constants(parser_t *parser, uint8_t rule_id, size_t num_args) {
return false;
}
mp_token_kind_t tok = MP_PARSE_NODE_LEAF_ARG(peek_result(parser, i));
if (tok == MP_TOKEN_OP_SLASH || tok == MP_TOKEN_OP_DBL_STAR) {
// Can't fold / or **
if (tok == MP_TOKEN_OP_AT || tok == MP_TOKEN_OP_SLASH || tok == MP_TOKEN_OP_DBL_STAR) {
// Can't fold @ or / or **
return false;
}
mp_binary_op_t op = MP_BINARY_OP_LSHIFT + (tok - MP_TOKEN_OP_DBL_LESS);

View File

@ -81,7 +81,7 @@ typedef enum {
MP_BINARY_OP_IS,
MP_BINARY_OP_EXCEPTION_MATCH,
// 12 inplace arithmetic operations; order matches corresponding mp_token_kind_t
// 13 inplace arithmetic operations; order matches corresponding mp_token_kind_t
MP_BINARY_OP_INPLACE_OR,
MP_BINARY_OP_INPLACE_XOR,
MP_BINARY_OP_INPLACE_AND,
@ -90,12 +90,13 @@ typedef enum {
MP_BINARY_OP_INPLACE_ADD,
MP_BINARY_OP_INPLACE_SUBTRACT,
MP_BINARY_OP_INPLACE_MULTIPLY,
MP_BINARY_OP_INPLACE_MAT_MULTIPLY,
MP_BINARY_OP_INPLACE_FLOOR_DIVIDE,
MP_BINARY_OP_INPLACE_TRUE_DIVIDE,
MP_BINARY_OP_INPLACE_MODULO,
MP_BINARY_OP_INPLACE_POWER,
// 12 normal arithmetic operations; order matches corresponding mp_token_kind_t
// 13 normal arithmetic operations; order matches corresponding mp_token_kind_t
MP_BINARY_OP_OR,
MP_BINARY_OP_XOR,
MP_BINARY_OP_AND,
@ -104,6 +105,7 @@ typedef enum {
MP_BINARY_OP_ADD,
MP_BINARY_OP_SUBTRACT,
MP_BINARY_OP_MULTIPLY,
MP_BINARY_OP_MAT_MULTIPLY,
MP_BINARY_OP_FLOOR_DIVIDE,
MP_BINARY_OP_TRUE_DIVIDE,
MP_BINARY_OP_MODULO,
@ -123,6 +125,7 @@ typedef enum {
MP_BINARY_OP_REVERSE_ADD,
MP_BINARY_OP_REVERSE_SUBTRACT,
MP_BINARY_OP_REVERSE_MULTIPLY,
MP_BINARY_OP_REVERSE_MAT_MULTIPLY,
MP_BINARY_OP_REVERSE_FLOOR_DIVIDE,
MP_BINARY_OP_REVERSE_TRUE_DIVIDE,
MP_BINARY_OP_REVERSE_MODULO,

View File

@ -43,9 +43,9 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
bc=\\d\+ line=126
00 LOAD_CONST_NONE
01 LOAD_CONST_FALSE
02 BINARY_OP 26 __add__
02 BINARY_OP 27 __add__
03 LOAD_CONST_TRUE
04 BINARY_OP 26 __add__
04 BINARY_OP 27 __add__
05 STORE_FAST 0
06 LOAD_CONST_SMALL_INT 0
07 STORE_FAST 0
@ -84,7 +84,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
\\d\+ STORE_FAST 7
\\d\+ LOAD_FAST 0
\\d\+ LOAD_DEREF 14
\\d\+ BINARY_OP 26 __add__
58 BINARY_OP 27 __add__
\\d\+ STORE_FAST 8
\\d\+ LOAD_FAST 0
\\d\+ UNARY_OP 1
@ -363,7 +363,7 @@ Raw bytecode (code_info_size=\\d\+, bytecode_size=\\d\+):
42 STORE_FAST_N 19
44 LOAD_FAST 9
45 LOAD_FAST_N 19
47 BINARY_OP 26 __add__
47 BINARY_OP 27 __add__
48 POP_TOP
49 LOAD_CONST_NONE
50 RETURN_VALUE
@ -520,7 +520,7 @@ arg names: *
bc=\\d\+ line=113
00 LOAD_DEREF 0
02 LOAD_CONST_SMALL_INT 1
03 BINARY_OP 26 __add__
03 BINARY_OP 27 __add__
04 STORE_FAST 1
05 LOAD_CONST_SMALL_INT 1
06 STORE_DEREF 0
@ -539,7 +539,7 @@ arg names: * b
bc=\\d\+ line=139
00 LOAD_FAST 1
01 LOAD_DEREF 0
03 BINARY_OP 26 __add__
03 BINARY_OP 27 __add__
04 RETURN_VALUE
mem: total=\\d\+, current=\\d\+, peak=\\d\+
stack: \\d\+ out of \\d\+