From 4cf741062b20fa34062d236d56b2168515e990b4 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Tue, 31 Oct 2023 15:14:05 +1100 Subject: [PATCH] extmod/vfs_reader: Add file ioctl to set read buffer size. Can be used to speed up importing a file from a vfs based filesystem. Signed-off-by: Andrew Leech --- extmod/vfs_reader.c | 50 ++++++++++++++++++++++++---------- py/stream.h | 1 + tests/extmod/vfs_userfs.py | 18 +++++++++++- tests/extmod/vfs_userfs.py.exp | 16 +++++++++++ 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/extmod/vfs_reader.c b/extmod/vfs_reader.c index 3c6ff52a9..13fc31bee 100644 --- a/extmod/vfs_reader.c +++ b/extmod/vfs_reader.c @@ -26,6 +26,7 @@ #include #include +#include #include "py/runtime.h" #include "py/stream.h" @@ -34,33 +35,39 @@ #if MICROPY_READER_VFS +#ifndef MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE +#define MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE (2 * MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf)) +#endif +#define MICROPY_READER_VFS_MIN_BUFFER_SIZE (MICROPY_BYTES_PER_GC_BLOCK - offsetof(mp_reader_vfs_t, buf)) +#define MICROPY_READER_VFS_MAX_BUFFER_SIZE (255) + typedef struct _mp_reader_vfs_t { mp_obj_t file; - uint16_t len; - uint16_t pos; - byte buf[24]; + uint8_t bufpos; + uint8_t buflen; + uint8_t bufsize; + byte buf[]; } mp_reader_vfs_t; STATIC mp_uint_t mp_reader_vfs_readbyte(void *data) { mp_reader_vfs_t *reader = (mp_reader_vfs_t *)data; - if (reader->pos >= reader->len) { - if (reader->len < sizeof(reader->buf)) { + if (reader->bufpos >= reader->buflen) { + if (reader->buflen < reader->bufsize) { return MP_READER_EOF; } else { int errcode; - reader->len = mp_stream_rw(reader->file, reader->buf, sizeof(reader->buf), - &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); + reader->buflen = mp_stream_rw(reader->file, reader->buf, reader->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); if (errcode != 0) { // TODO handle errors properly return MP_READER_EOF; } - if (reader->len == 0) { + if (reader->buflen == 0) { return MP_READER_EOF; } - reader->pos = 0; + reader->bufpos = 0; } } - return reader->buf[reader->pos++]; + return reader->buf[reader->bufpos++]; } STATIC void mp_reader_vfs_close(void *data) { @@ -70,18 +77,31 @@ STATIC void mp_reader_vfs_close(void *data) { } void mp_reader_new_file(mp_reader_t *reader, qstr filename) { - mp_reader_vfs_t *rf = m_new_obj(mp_reader_vfs_t); mp_obj_t args[2] = { MP_OBJ_NEW_QSTR(filename), MP_OBJ_NEW_QSTR(MP_QSTR_rb), }; - rf->file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map); - int errcode; - rf->len = mp_stream_rw(rf->file, rf->buf, sizeof(rf->buf), &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); + mp_obj_t file = mp_vfs_open(MP_ARRAY_SIZE(args), &args[0], (mp_map_t *)&mp_const_empty_map); + + const mp_stream_p_t *stream_p = mp_get_stream(file); + int errcode = 0; + mp_uint_t bufsize = stream_p->ioctl(file, MP_STREAM_GET_BUFFER_SIZE, 0, &errcode); + if (bufsize == MP_STREAM_ERROR || bufsize == 0) { + // bufsize == 0 is included here to support mpremote v1.21 and older where mount file ioctl + // returned 0 by default. + bufsize = MICROPY_READER_VFS_DEFAULT_BUFFER_SIZE; + } else { + bufsize = MIN(MICROPY_READER_VFS_MAX_BUFFER_SIZE, MAX(MICROPY_READER_VFS_MIN_BUFFER_SIZE, bufsize)); + } + + mp_reader_vfs_t *rf = m_new_obj_var(mp_reader_vfs_t, buf, byte, bufsize); + rf->file = file; + rf->bufsize = bufsize; + rf->buflen = mp_stream_rw(rf->file, rf->buf, rf->bufsize, &errcode, MP_STREAM_RW_READ | MP_STREAM_RW_ONCE); if (errcode != 0) { mp_raise_OSError(errcode); } - rf->pos = 0; + rf->bufpos = 0; reader->data = rf; reader->readbyte = mp_reader_vfs_readbyte; reader->close = mp_reader_vfs_close; diff --git a/py/stream.h b/py/stream.h index 6b5c85ab1..e6e6f283d 100644 --- a/py/stream.h +++ b/py/stream.h @@ -43,6 +43,7 @@ #define MP_STREAM_GET_DATA_OPTS (8) // Get data/message options #define MP_STREAM_SET_DATA_OPTS (9) // Set data/message options #define MP_STREAM_GET_FILENO (10) // Get fileno of underlying file +#define MP_STREAM_GET_BUFFER_SIZE (11) // Get preferred buffer size for file // These poll ioctl values are compatible with Linux #define MP_STREAM_POLL_RD (0x0001) diff --git a/tests/extmod/vfs_userfs.py b/tests/extmod/vfs_userfs.py index 36f108887..5c487315e 100644 --- a/tests/extmod/vfs_userfs.py +++ b/tests/extmod/vfs_userfs.py @@ -16,6 +16,8 @@ except (ImportError, AttributeError): class UserFile(io.IOBase): + buffer_size = 16 + def __init__(self, mode, data): assert isinstance(data, bytes) self.is_text = mode.find("b") == -1 @@ -39,7 +41,11 @@ class UserFile(io.IOBase): def ioctl(self, req, arg): print("ioctl", req, arg) - return 0 + if req == 4: # MP_STREAM_CLOSE + return 0 + if req == 11: # MP_STREAM_GET_BUFFER_SIZE + return UserFile.buffer_size + return -1 class UserFS: @@ -70,6 +76,8 @@ user_files = { "/usermod2.py": b"print('in usermod2')", "/usermod3.py": b"syntax error", "/usermod4.mpy": b"syntax error", + "/usermod5.py": b"print('in usermod5')", + "/usermod6.py": b"print('in usermod6')", } os.mount(UserFS(user_files), "/userfs") @@ -93,6 +101,14 @@ try: except ValueError: print("ValueError in usermod4") +# Test an import with largest buffer size +UserFile.buffer_size = 255 +import usermod5 + +# Test an import with over-size buffer size (should be safely limited internally) +UserFile.buffer_size = 1024 +import usermod6 + # unmount and undo path addition os.umount("/userfs") sys.path.pop() diff --git a/tests/extmod/vfs_userfs.py.exp b/tests/extmod/vfs_userfs.py.exp index 05cff0452..be8011d56 100644 --- a/tests/extmod/vfs_userfs.py.exp +++ b/tests/extmod/vfs_userfs.py.exp @@ -3,21 +3,37 @@ some data in a text file stat /usermod1 stat /usermod1.py open /usermod1.py rb +ioctl 11 0 ioctl 4 0 in usermod1 stat /usermod2 stat /usermod2.py open /usermod2.py rb +ioctl 11 0 ioctl 4 0 in usermod2 stat /usermod3 stat /usermod3.py open /usermod3.py rb +ioctl 11 0 ioctl 4 0 SyntaxError in usermod3 stat /usermod4 stat /usermod4.py stat /usermod4.mpy open /usermod4.mpy rb +ioctl 11 0 ioctl 4 0 ValueError in usermod4 +stat /usermod5 +stat /usermod5.py +open /usermod5.py rb +ioctl 11 0 +ioctl 4 0 +in usermod5 +stat /usermod6 +stat /usermod6.py +open /usermod6.py rb +ioctl 11 0 +ioctl 4 0 +in usermod6