decompressing a file from zlib library

[Zahme.2]

Hi all,

I have a small test project setup in STM32CubeIDE where an sd-card is interfaced with stm32h743 MCU. The program uses zlib library where it compresses the file from 3.2MB to 26KB. That's good but I cannot find a way to decompress this file on the PC (7zip or zip) nor with the inflate() function so I can write the decompress file on the sd-card. I have used the demo example which the zlib provides (zpipe.c).

The sd-card interfacing is all FATFS based so I can't use local file operation functions. I have replaced them with FATFS calls. Any help would be appreciated.

/* header files */
#include "main.h"
#include "fatfs.h"
#include "sdmmc.h"
#include "gpio.h"
#include "zlib.h"
#include <stdint.h>
#include <string.h>
 
#define CHUNK 16384
 
FATFS myfatfs;
FIL srcFil;
FIL destFIL;
DIR myDir;
FILINFO myFileInfo;
 
const char DirPath[] = "/ADC File Dir";
char filepath[_MAX_LFN];
char destFilePath[_MAX_LFN];
char decompfile[_MAX_LFN];
 
int def(char *source, char *dest, int level);
int inf(char *source, char *dest);
 
int main(void)
{
  /* USER CODE BEGIN 1 */
 
  /* USER CODE END 1 */
 
  /* MCU Configuration--------------------------------------------------------*/
 
  /* Reset of all peripherals, Initializes the Flash interface and the Systick. */
  HAL_Init();
 
  /* USER CODE BEGIN Init */
FRESULT fresMain;
  /* USER CODE END Init */
 
  /* Configure the system clock */
  SystemClock_Config();
 
  /* USER CODE BEGIN SysInit */
 
  /* USER CODE END SysInit */
 
  /* Initialize all configured peripherals */
  MX_GPIO_Init();
  MX_SDMMC1_SD_Init();
  MX_FATFS_Init();
  /* USER CODE BEGIN 2 */
 
  if(f_mount(&myfatfs, (TCHAR const *) SDPath, 0) == FR_OK) {
	  // open directory
	  if(f_opendir(&myDir, DirPath) == FR_OK) {
		  // read first file name
		  if(f_readdir(&myDir, &myFileInfo) == FR_OK) {
			  // get file name
			  memset(filepath, 0, sizeof(filepath));
			  memset(destFilePath, 0, sizeof(destFilePath));
			  snprintf(filepath, sizeof(filepath), "%s/%s", DirPath, (char *) myFileInfo.fname);
			  snprintf(destFilePath, sizeof(destFilePath), "%s/compress.z", DirPath);
			  // execute the compression method
			  def(filepath, destFilePath, Z_DEFAULT_COMPRESSION);
 
			  // close both files
			  fresMain = f_close(&srcFil);
			  fresMain = f_close(&destFIL);
 
			  snprintf(decompfile, sizeof(decompfile), "%s/decomp", DirPath);
			  inf(destFilePath, decompfile);
			  if(f_closedir(&myDir) == FR_OK) {
				  // dismount device
				  f_mount(0, "", 0);
			  }
 
		  }
	  }
  }
  /* USER CODE END 2 */
 
  /* Infinite loop */
  /* USER CODE BEGIN WHILE */
  while (1)
  {
    /* USER CODE END WHILE */
 
    /* USER CODE BEGIN 3 */
  }
  /* USER CODE END 3 */
}
 
/* Compress from file source to file dest until EOF on source.
   def() returns Z_OK on success, Z_MEM_ERROR if memory could not be
   allocated for processing, Z_STREAM_ERROR if an invalid compression
   level is supplied, Z_VERSION_ERROR if the version of zlib.h and the
   version of the library linked do not match, or Z_ERRNO if there is
   an error reading or writing the files. */
int def(char *source, char *dest, int level)
{
    int ret, flush;
    unsigned have;
    z_stream strm;
    unsigned char in[CHUNK];
    unsigned char out[CHUNK];
    FRESULT fres = FR_OK;
    uint32_t bw = 0U;
    uint32_t srcPtr = 0U;
    uint32_t destPtr = 0U;
 
    /* allocate deflate state */
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    ret = deflateInit(&strm, level);
    if (ret != Z_OK)
        return ret;
 
 
    /* compress until end of file */
    do {
 
    	fres = f_open(&srcFil, source, FA_OPEN_EXISTING | FA_READ);
    	fres = f_read(&srcFil, in, CHUNK, &strm.avail_in);
//        strm.avail_in = fread(in, 1, CHUNK, source);
//        if (ferror(source)) {
//            (void)deflateEnd(&strm);
//            return Z_ERRNO;
//        }
    	if(fres != FR_OK) {
    		 (void)deflateEnd(&strm);
    		  return Z_ERRNO;
    	} else if(fres == FR_OK) {
    		// move src pointer
    		srcPtr += CHUNK;
    		if(f_lseek(&srcFil, srcPtr) == FR_OK) {
 
    		}
    	}
    	fres = f_close(&srcFil);
        //flush = feof(source) ? Z_FINISH : Z_NO_FLUSH;
        flush = f_eof(&srcFil) ? Z_FINISH : Z_NO_FLUSH;
    	strm.next_in = in;
 
        /* run deflate() on input until output buffer not full, finish
           compression if all of source has been read in */
        do {
            strm.avail_out = CHUNK;
            strm.next_out = out;
            ret = deflate(&strm, flush);    /* no bad return value */
            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
            have = CHUNK - strm.avail_out;
 
            fres = f_open(&destFIL, dest, FA_OPEN_ALWAYS | FA_WRITE);
            fres = f_write(&destFIL, out, have, (UINT *)&bw);
            if((bw != have) || (fres != FR_OK)) {
                (void)deflateEnd(&strm);
                return Z_ERRNO;
            } else if(fres == FR_OK) {
            	destPtr += have;
            	if(f_lseek(&destFIL, destPtr) == FR_OK) {
 
            	}
            }
            fres = f_close(&destFIL);
//            if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
//                (void)deflateEnd(&strm);
//                return Z_ERRNO;
//            }
        } while (strm.avail_out == 0);
        assert(strm.avail_in == 0);     /* all input will be used */
 
        /* done when last data in file processed */
    } while (flush != Z_FINISH);
    assert(ret == Z_STREAM_END);        /* stream will be complete */
 
    /* clean up and return */
    (void)deflateEnd(&strm);
    return Z_OK;
}
 
/* Decompress from file source to file dest until stream ends or EOF.
   inf() returns Z_OK on success, Z_MEM_ERROR if memory could not be
   allocated for processing, Z_DATA_ERROR if the deflate data is
   invalid or incomplete, Z_VERSION_ERROR if the version of zlib.h and
   the version of the library linked do not match, or Z_ERRNO if there
   is an error reading or writing the files. */
int inf(char *source, char *dest)
{
    int ret;
    unsigned have;
    z_stream strm;
    unsigned char in[CHUNK];
    unsigned char out[CHUNK];
    FRESULT fres = FR_OK;
    uint32_t bw = 0U;
    uint32_t srcPtr = 0U;
    uint32_t destPtr = 0U;
 
    /* allocate inflate state */
    strm.zalloc = Z_NULL;
    strm.zfree = Z_NULL;
    strm.opaque = Z_NULL;
    strm.avail_in = 0;
    strm.next_in = Z_NULL;
    ret = inflateInit(&strm);
    if (ret != Z_OK)
        return ret;
 
    /* decompress until deflate stream ends or end of file */
    do {
    	fres = f_open(&srcFil, source, FA_OPEN_EXISTING | FA_READ);
    	fres = f_read(&srcFil, in, CHUNK, &strm.avail_in);
    	if(fres != FR_OK) {
    		 (void)inflateEnd(&strm);
    		  return Z_ERRNO;
    	} else if(fres == FR_OK) {
    		// move src pointer
    		srcPtr += CHUNK;
    		if(f_lseek(&srcFil, srcPtr) == FR_OK) {
 
    		}
    	}
    	fres = f_close(&srcFil);
//        strm.avail_in = fread(in, 1, CHUNK, source);
//        if (ferror(source)) {
//            (void)inflateEnd(&strm);
//            return Z_ERRNO;
//        }
        if (strm.avail_in == 0)
            break;
        strm.next_in = in;
 
        /* run inflate() on input until output buffer not full */
        do {
            strm.avail_out = CHUNK;
            strm.next_out = out;
            ret = inflate(&strm, Z_NO_FLUSH);
            assert(ret != Z_STREAM_ERROR);  /* state not clobbered */
            switch (ret) {
            case Z_NEED_DICT:
                ret = Z_DATA_ERROR;     /* and fall through */
            case Z_DATA_ERROR:
            case Z_MEM_ERROR:
                (void)inflateEnd(&strm);
                return ret;
            }
            have = CHUNK - strm.avail_out;
 
            fres = f_open(&destFIL, dest, FA_OPEN_ALWAYS | FA_WRITE);
            fres = f_write(&destFIL, out, have, (UINT *)&bw);
            if((bw != have) || (fres != FR_OK)) {
                (void)inflateEnd(&strm);
                return Z_ERRNO;
            } else if(fres == FR_OK) {
            	destPtr += have;
            	if(f_lseek(&destFIL, destPtr) == FR_OK) {
 
            	}
            }
            fres = f_close(&destFIL);
 
//            if (fwrite(out, 1, have, dest) != have || ferror(dest)) {
//                (void)inflateEnd(&strm);
//                return Z_ERRNO;
//            }
        } while (strm.avail_out == 0);
 
        /* done when inflate() says it's done */
    } while (ret != Z_STREAM_END);
 
    /* clean up and return */
    (void)inflateEnd(&strm);
    return ret == Z_STREAM_END ? Z_OK : Z_DATA_ERROR;
}

I wasn't sure of what would be the extension of the compressed file so I've added ".z" at the end which the windows do recognise as a compressed file but 7zip cannot decompress. When using the inflate() function it returns Z_DATA_ERROR (-3)

[Tesla DeLorean]

Could it be that it's not even packed properly?

Typically when building/validating this stuff you'd build PC side pack/unpack programs using ZLIB, prove you got all the logic right, perhaps test on the unpacked equivalent to your 3MB to 26KB case, and make sure it's not outlandish.

Are you sure your stack is adequate?

Perhaps use static allocations instead of auto/local's ?

[Zahme.2]

I haven't tried running this program on PC as it was an example guide from zlib website. I think I need to try it on the PC first and see if the logic is right or not. Stack is set to default "0x400"

[Haithem Rahmani]

@Zahme.2​ 

the cleanest way to compress/decompress files usiong zlib is to use the API below:

ZEXTERN int ZEXPORT gzwrite OF((gzFile file,voidpc buf, unsigned len));

ZEXTERN int ZEXPORT gzread OF((gzFile file, voidp buf, unsigned len));

but you'll need to update the zlib APIs to use the FatFs objects and API (FIL* , f_read, f_write)

regards

Haithem.