Skip to content

compression: Replace global variables with per-fd context #6913

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

compression: Replace global variables with per-fd context #6913

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion os/binfmt/libelf/libelf_uninit.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ int elf_uninit(struct elf_loadinfo_s *loadinfo)

/* Free buffers used for decompression */
#ifdef CONFIG_COMPRESSED_BINARY
compress_uninit();
compress_uninit(loadinfo->filfd);
#endif
#if defined(CONFIG_ELF_CACHE_READ)
elf_cache_uninit();
Expand Down
170 changes: 110 additions & 60 deletions os/compression/compress_read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
#include <string.h>
#include <debug.h>
#include <errno.h>
#include <queue.h>

#include <tinyara/fs/fs.h>
#include <tinyara/binfmt/compression/compress_read.h>
Expand All @@ -41,14 +42,27 @@
* Private Declarations
****************************************************************************/

static struct s_header *compression_header;
static struct s_buffer buffers;
static int active_filefd = -1;
static bool g_ctx_list_init;
static dq_queue_t comp_ctx_list;

/****************************************************************************
* Private Functions
****************************************************************************/


static struct comp_ctx *find_comp_ctx(int filfd)
{
struct comp_ctx *ctx = (struct comp_ctx *)dq_peek(&comp_ctx_list);

while(ctx) {
if (ctx->fd == filfd) {
return ctx;
}
ctx = dq_next(ctx);
}

return NULL;
}

/****************************************************************************
* Name: compress_blocks_to_read
*
Expand All @@ -59,11 +73,11 @@ static int active_filefd = -1;
* Returned Value:
* None
****************************************************************************/
static void compress_blocks_to_read(int *first_block, int *last_block, int *no_blocks, int offset, int readsize)
static void compress_blocks_to_read(struct comp_ctx *ctx ,int *first_block, int *last_block, int *no_blocks, int offset, int readsize)
{
int blocksize;

blocksize = compression_header->blocksize;
blocksize = ctx->compression_header->blocksize;

*first_block = offset / blocksize;
*last_block = (offset + readsize - 1) / blocksize;
Expand All @@ -81,7 +95,7 @@ static void compress_blocks_to_read(int *first_block, int *last_block, int *no_b
* OK (0) is Success
* Negative value on Failure
****************************************************************************/
static int compress_parse_header(int filfd, uint16_t offset)
static int compress_parse_header(struct comp_ctx *ctx, int filfd, uint16_t offset)
{
off_t rpos; /* Position returned by lseek */
int nbytes; /* Number of bytes read */
Expand All @@ -103,23 +117,23 @@ static int compress_parse_header(int filfd, uint16_t offset)
}

/* Allocate memory for compression header now that we know it's size */
compression_header = (struct s_header *)kmm_malloc(compheader_size);
if (!compression_header) {
ctx->compression_header = (struct s_header *)kmm_malloc(compheader_size);
if (!ctx->compression_header) {
bcmpdbg("Failed kmm_malloc for compression_header\n");
return -ENOMEM;
}

/* Assign compresssion_header->size_header */
compression_header->size_header = compheader_size;
ctx->compression_header->size_header = compheader_size;

/* Read remaining compression header, including section offsets */
nbytes = read(filfd, ((uint8_t *)compression_header + sizeof(compression_header->size_header)), compheader_size - sizeof(compression_header->size_header));
if (nbytes != (compheader_size - sizeof(compression_header->size_header))) {
nbytes = read(filfd, ((uint8_t *)ctx->compression_header + sizeof(ctx->compression_header->size_header)), compheader_size - sizeof(ctx->compression_header->size_header));
if (nbytes != (compheader_size - sizeof(ctx->compression_header->size_header))) {
bcmpdbg("Read for compression header from offset %lu failed\n", offset);
return ERROR;
}

bcmpvdbg("Compressed Binary Header info: size (%d), compression format (%d), blocksize (%d), No. sections (%d), Uncompressed binary size = %d\n", compression_header->size_header, compression_header->compression_format, compression_header->blocksize, compression_header->sections, compression_header->binary_size);
bcmpvdbg("Compressed Binary Header info: size (%d), compression format (%d), blocksize (%d), No. sections (%d), Uncompressed binary size = %d\n", ctx->compression_header->size_header, ctx->compression_header->compression_format, ctx->compression_header->blocksize, ctx->compression_header->sections, ctx->compression_header->binary_size);

return OK;
}
Expand All @@ -134,12 +148,12 @@ static int compress_parse_header(int filfd, uint16_t offset)
* 'block_offset' value (positive) on Success
* Negative value on Failure
****************************************************************************/
static off_t compress_offset_block(int filfd, uint16_t binary_header_size, int block_number)
static off_t compress_offset_block(struct comp_ctx *ctx, int filfd, uint16_t binary_header_size, int block_number)
{
off_t position;

/* Return position for 'block_number' block */
position = binary_header_size + compression_header->size_header + compression_header->secoff[block_number];
position = binary_header_size + ctx->compression_header->size_header + ctx->compression_header->secoff[block_number];

return position;
}
Expand All @@ -154,13 +168,13 @@ static off_t compress_offset_block(int filfd, uint16_t binary_header_size, int b
* Starting offset (positive) of 'block_number' block in binary on Success
* Negative value on Failure
****************************************************************************/
static off_t compress_lseek_block(int filfd, uint16_t binary_header_size, int block_number)
static off_t compress_lseek_block(struct comp_ctx *ctx, int filfd, uint16_t binary_header_size, int block_number)
{
off_t rpos;
off_t block_offset;

/* Calculate block offset from start of file for start of 'block_number' block */
block_offset = compress_offset_block(filfd, binary_header_size, block_number);
block_offset = compress_offset_block(ctx, filfd, binary_header_size, block_number);
if (block_offset < 0) {
bcmpdbg("Incorrect offset for block number %d\n", block_number);
return block_offset;
Expand All @@ -187,7 +201,7 @@ static off_t compress_lseek_block(int filfd, uint16_t binary_header_size, int bl
* Number of bytes read into read_buffer on Success
* Negative value on Failure
****************************************************************************/
static off_t compress_read_block(int filfd, uint16_t binary_header_size, FAR uint8_t *buf, int block_number)
static off_t compress_read_block(struct comp_ctx *ctx, int filfd, uint16_t binary_header_size, FAR uint8_t *buf, int block_number)
{
off_t rpos;
off_t readsize;
Expand All @@ -196,13 +210,13 @@ static off_t compress_read_block(int filfd, uint16_t binary_header_size, FAR uin
off_t next_block_offset;

/* Find out size of 'block_number' block in compressed file. Assign to readsize */
next_block_offset = compress_offset_block(filfd, binary_header_size, block_number + 1);
next_block_offset = compress_offset_block(ctx, filfd, binary_header_size, block_number + 1);
if (next_block_offset < 0) {
bcmpdbg("Incorrect offset for block number %d\n", block_number + 1);
return ERROR;
}

current_block_offset = compress_offset_block(filfd, binary_header_size, block_number);
current_block_offset = compress_offset_block(ctx, filfd, binary_header_size, block_number);
if (current_block_offset < 0) {
bcmpdbg("Incorrect offset for block number %d\n", block_number);
return ERROR;
Expand All @@ -215,7 +229,7 @@ static off_t compress_read_block(int filfd, uint16_t binary_header_size, FAR uin
}

/* Seek to location of 'block_number' block in compressed file */
rpos = compress_lseek_block(filfd, binary_header_size, block_number);
rpos = compress_lseek_block(ctx, filfd, binary_header_size, block_number);
if (rpos < 0) {
bcmpdbg("Failed to seek to offset of block number %d\n", block_number);
return rpos;
Expand Down Expand Up @@ -256,6 +270,7 @@ int compress_read(int filfd, uint16_t binary_header_size, FAR uint8_t *buffer, s
int buffer_index;
int blocksize;
int block_readsize;
struct comp_ctx *ctx;
#if CONFIG_COMPRESSION_TYPE == LZMA
unsigned int writesize;
unsigned int size;
Expand All @@ -264,9 +279,16 @@ int compress_read(int filfd, uint16_t binary_header_size, FAR uint8_t *buffer, s
long unsigned int size;
#endif

ctx = find_comp_ctx(filfd);
if (ctx == NULL) {
bcmpdbg("Find compression context failed.\n");
buffer_index = ERROR;
goto error_compress_read;
}

/* Setting first block, end block and number of blocks to read and decompressed */
blocksize = compression_header->blocksize;
compress_blocks_to_read(&first_block, &last_block, &no_blocks, offset, readsize);
blocksize = ctx->compression_header->blocksize;
compress_blocks_to_read(ctx, &first_block, &last_block, &no_blocks, offset, readsize);
if (first_block < 0 || no_blocks < 0) {
bcmpdbg("Incorrect first_block, no_blocks info\n");
buffer_index = ERROR;
Expand All @@ -281,7 +303,7 @@ int compress_read(int filfd, uint16_t binary_header_size, FAR uint8_t *buffer, s
/* Reading and decompressing blocks from first_block to last_block. Then writing to buffer. */
for (; index < first_block + no_blocks; index++) {
/* Read compressed 'index' block into read_buffer */
block_readsize = compress_read_block(filfd, binary_header_size, buffers.read_buffer, index);
block_readsize = compress_read_block(ctx, filfd, binary_header_size, ctx->buffers.read_buffer, index);
if (block_readsize < 0) {
bcmpdbg("Read for compressed block %d failed\n", index);
buffer_index = block_readsize;
Expand All @@ -293,9 +315,9 @@ int compress_read(int filfd, uint16_t binary_header_size, FAR uint8_t *buffer, s
#elif CONFIG_COMPRESSION_TYPE == MINIZ
size = (long unsigned int)block_readsize;
#endif
writesize = compression_header->blocksize;
writesize = ctx->compression_header->blocksize;
/* Decompress block in read_buffer to out_buffer */
ret = decompress_block(buffers.out_buffer, &writesize, buffers.read_buffer, &size);
ret = decompress_block(ctx->buffers.out_buffer, &writesize, ctx->buffers.read_buffer, &size);
if (ret == ERROR) {
bcmpdbg("Failed to decompress %d block of this binary\n", index);
buffer_index = ret;
Expand All @@ -309,7 +331,7 @@ int compress_read(int filfd, uint16_t binary_header_size, FAR uint8_t *buffer, s
* Otherwise, write from start_offset to end_offset into buffer.
*/
block_size_to_write = ((index + 1) * blocksize - 1 > actual_offset + readsize - 1 ? readsize : (index + 1) * blocksize - actual_offset);
memcpy(&buffer[buffer_index], &buffers.out_buffer[actual_offset - (index * blocksize)], block_size_to_write);
memcpy(&buffer[buffer_index], &ctx->buffers.out_buffer[actual_offset - (index * blocksize)], block_size_to_write);
buffer_index += block_size_to_write;
} else if (index == last_block) {
/*
Expand All @@ -318,15 +340,15 @@ int compress_read(int filfd, uint16_t binary_header_size, FAR uint8_t *buffer, s
* Write from start_offset to end_offset from this block into buffer.
*/
block_size_to_write = actual_offset + readsize - (index * blocksize);
memcpy(&buffer[buffer_index], &buffers.out_buffer[0], block_size_to_write);
memcpy(&buffer[buffer_index], &ctx->buffers.out_buffer[0], block_size_to_write);
buffer_index += block_size_to_write;
} else {
/*
* This is not the first or last block. This is intermediary block.
* So, write entire block into buffer.
*/
block_size_to_write = blocksize;
memcpy(&buffer[buffer_index], &buffers.out_buffer[0], block_size_to_write);
memcpy(&buffer[buffer_index], &ctx->buffers.out_buffer[0], block_size_to_write);
buffer_index += block_size_to_write;
}
}
Expand All @@ -348,45 +370,58 @@ int compress_read(int filfd, uint16_t binary_header_size, FAR uint8_t *buffer, s
int compress_init(int filfd, uint16_t offset, off_t *filelen)
{
int ret;
struct comp_ctx *ctx;

if (active_filefd != -1 && active_filefd != filfd) {
bcmpdbg("Another file decompression is in process\n");
return -EBUSY;
if (!g_ctx_list_init) {
dq_init(&comp_ctx_list);
g_ctx_list_init = true;
}
active_filefd = filfd;

ctx = find_comp_ctx(filfd);
if (ctx == NULL) {
ctx = (struct comp_ctx *)kmm_malloc(sizeof(struct comp_ctx));
if (!ctx) {
bcmpdbg("Memory allocation fail for compress context\n");
return -ENOMEM;
}
ctx->fd = filfd;
ctx->from_binfmt = true;
}
dq_addlast((dq_entry_t *)ctx, &comp_ctx_list);

/* Parsing compression header for compressed file */
ret = compress_parse_header(filfd, offset);
ret = compress_parse_header(ctx, filfd, offset);
if (ret != OK) {
bcmpdbg("Failed to parse compression header from file\n");
goto error_compress_init;
}

/* Assign file length as that of uncompressed file */
*filelen = compression_header->binary_size;
*filelen = ctx->compression_header->binary_size;

#if CONFIG_COMPRESSION_TYPE == LZMA
/* Allocating memory for read and out buffer to be used for LZMA decompression */
if (compression_header->compression_format == COMPRESSION_TYPE_LZMA) {
buffers.read_buffer = (unsigned char *)kmm_malloc(compression_header->blocksize + LZMA_PROPS_SIZE);
if (buffers.read_buffer == NULL) {
if (ctx->compression_header->compression_format == COMPRESSION_TYPE_LZMA) {
ctx->buffers.read_buffer = (unsigned char *)kmm_malloc(ctx->compression_header->blocksize + LZMA_PROPS_SIZE);
if (ctx->buffers.read_buffer == NULL) {
return -ENOMEM;
}
buffers.out_buffer = (unsigned char *)kmm_malloc(compression_header->blocksize);
if (buffers.out_buffer == NULL) {
kmm_free(buffers.read_buffer);
ctx->buffers.out_buffer = (unsigned char *)kmm_malloc(ctx->compression_header->blocksize);
if (ctx->buffers.out_buffer == NULL) {
kmm_free(ctx->buffers.read_buffer);
return -ENOMEM;
}
}
#elif CONFIG_COMPRESSION_TYPE == MINIZ
/* Allocating memory for read and out buffer to be used for Miniz decompression */
if (compression_header->compression_format == COMPRESSION_TYPE_MINIZ) {
buffers.read_buffer = (unsigned char *)kmm_malloc(compressBound(compression_header->blocksize));
if (buffers.read_buffer == NULL) {
if (ctx->compression_header->compression_format == COMPRESSION_TYPE_MINIZ) {
ctx->buffers.read_buffer = (unsigned char *)kmm_malloc(compressBound(ctx->compression_header->blocksize));
if (ctx->buffers.read_buffer == NULL) {
return -ENOMEM;
}
buffers.out_buffer = (unsigned char *)kmm_malloc(compression_header->blocksize);
if (buffers.out_buffer == NULL) {
kmm_free(buffers.read_buffer);
ctx->buffers.out_buffer = (unsigned char *)kmm_malloc(ctx->compression_header->blocksize);
if (ctx->buffers.out_buffer == NULL) {
kmm_free(ctx->buffers.read_buffer);
return -ENOMEM;
}
}
Expand All @@ -405,29 +440,44 @@ int compress_init(int filfd, uint16_t offset, off_t *filelen)
* Returned Value:
* None
****************************************************************************/
void compress_uninit(void)
void compress_uninit(int filfd)
{
struct comp_ctx *ctx = find_comp_ctx(filfd);

if (!ctx) {
return;
}
#if CONFIG_COMPRESSION_TYPE == LZMA || CONFIG_COMPRESSION_TYPE == MINIZ
/* Freeing memory allocated to read_buffer and out_buffer for file decompression */
if (compression_header->compression_format == COMPRESSION_TYPE_LZMA || compression_header->compression_format == COMPRESSION_TYPE_MINIZ) {
if (buffers.read_buffer) {
kmm_free(buffers.read_buffer);
buffers.read_buffer = NULL;
if (ctx->compression_header->compression_format == COMPRESSION_TYPE_LZMA || ctx->compression_header->compression_format == COMPRESSION_TYPE_MINIZ) {
if (ctx->buffers.read_buffer) {
kmm_free(ctx->buffers.read_buffer);
ctx->buffers.read_buffer = NULL;
}
if (buffers.out_buffer) {
kmm_free(buffers.out_buffer);
buffers.out_buffer = NULL;
if (ctx->buffers.out_buffer) {
kmm_free(ctx->buffers.out_buffer);
ctx->buffers.out_buffer = NULL;
}
}
#endif

kmm_free(compression_header);
compression_header = NULL;
kmm_free(ctx->compression_header);
ctx->compression_header = NULL;
dq_rem((dq_entry_t *)ctx, &comp_ctx_list);

active_filefd = -1;
if (ctx->from_binfmt) {
ctx->fd = -1;
kmm_free(ctx);
}
}

struct s_header *get_compression_header(void)
struct s_header *get_compression_header(int filfd)
{
return compression_header;
struct comp_ctx *ctx = find_comp_ctx(filfd);

if (!ctx) {
return NULL;
}

return ctx->compression_header;
}
Loading