From 150850b80012f852521c9935145cf966946334d5 Mon Sep 17 00:00:00 2001 From: Adam Ierymenko Date: Thu, 4 Jul 2013 16:56:19 -0400 Subject: New git repository for release - version 0.2.0 tagged --- ext/lz4/bench.c | 477 +++++++++++++++++++ ext/lz4/bench.h | 41 ++ ext/lz4/fuzzer.c | 227 ++++++++++ ext/lz4/lz4.c | 906 +++++++++++++++++++++++++++++++++++++ ext/lz4/lz4.h | 128 ++++++ ext/lz4/lz4_format_description.txt | 121 +++++ ext/lz4/lz4demo.c | 402 ++++++++++++++++ ext/lz4/lz4hc.c | 730 ++++++++++++++++++++++++++++++ ext/lz4/lz4hc.h | 60 +++ 9 files changed, 3092 insertions(+) create mode 100644 ext/lz4/bench.c create mode 100644 ext/lz4/bench.h create mode 100644 ext/lz4/fuzzer.c create mode 100644 ext/lz4/lz4.c create mode 100644 ext/lz4/lz4.h create mode 100644 ext/lz4/lz4_format_description.txt create mode 100644 ext/lz4/lz4demo.c create mode 100644 ext/lz4/lz4hc.c create mode 100644 ext/lz4/lz4hc.h (limited to 'ext/lz4') diff --git a/ext/lz4/bench.c b/ext/lz4/bench.c new file mode 100644 index 00000000..d1858116 --- /dev/null +++ b/ext/lz4/bench.c @@ -0,0 +1,477 @@ +/* + bench.c - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_DEPRECATE // VS2005 + +// Unix Large Files support (>4GB) +#if (defined(__sun__) && (!defined(__LP64__))) // Sun Solaris 32-bits requires specific definitions +# define _LARGEFILE_SOURCE +# define FILE_OFFSET_BITS=64 +#elif ! defined(__LP64__) // No point defining Large file for 64 bit +# define _LARGEFILE64_SOURCE +#endif + +// S_ISREG & gettimeofday() are not supported by MSVC +#if defined(_MSC_VER) +# define S_ISREG(x) (((x) & S_IFMT) == S_IFREG) +# define BMK_LEGACY_TIMER 1 +#endif + +// GCC does not support _rotl outside of Windows +#if !defined(_WIN32) +# define _rotl(x,r) ((x << r) | (x >> (32 - r))) +#endif + + +//************************************** +// Includes +//************************************** +#include // malloc +#include // fprintf, fopen, ftello64 +#include // stat64 +#include // stat64 + +// Use ftime() if gettimeofday() is not available on your target +#if defined(BMK_LEGACY_TIMER) +# include // timeb, ftime +#else +# include // gettimeofday +#endif + +#include "lz4.h" +#define COMPRESSOR0 LZ4_compress +#include "lz4hc.h" +#define COMPRESSOR1 LZ4_compressHC +#define DEFAULTCOMPRESSOR LZ4_compress + + + +//************************************** +// Basic Types +//************************************** +#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively +#define BYTE unsigned __int8 +#define U16 unsigned __int16 +#define U32 unsigned __int32 +#define S32 __int32 +#define U64 unsigned __int64 +#else +#include +#define BYTE uint8_t +#define U16 uint16_t +#define U32 uint32_t +#define S32 int32_t +#define U64 uint64_t +#endif + + +//************************************** +// Constants +//************************************** +#define NBLOOPS 3 +#define TIMELOOP 2000 + +#define KNUTH 2654435761U +#define MAX_MEM (1984<<20) +#define DEFAULT_CHUNKSIZE (8<<20) + + +//************************************** +// Local structures +//************************************** +struct chunkParameters +{ + U32 id; + char* inputBuffer; + char* outputBuffer; + int inputSize; + int outputSize; +}; + +struct compressionParameters +{ + int (*compressionFunction)(const char*, char*, int); + int (*decompressionFunction)(const char*, char*, int); +}; + + +//************************************** +// MACRO +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + + + +//************************************** +// Benchmark Parameters +//************************************** +static int chunkSize = DEFAULT_CHUNKSIZE; +static int nbIterations = NBLOOPS; +static int BMK_pause = 0; + +void BMK_SetBlocksize(int bsize) +{ + chunkSize = bsize; + DISPLAY("-Using Block Size of %i KB-", chunkSize>>10); +} + +void BMK_SetNbIterations(int nbLoops) +{ + nbIterations = nbLoops; + DISPLAY("- %i iterations-", nbIterations); +} + +void BMK_SetPause() +{ + BMK_pause = 1; +} + +//********************************************************* +// Private functions +//********************************************************* + +#if defined(BMK_LEGACY_TIMER) + +static int BMK_GetMilliStart() +{ + // Based on Legacy ftime() + // Rolls over every ~ 12.1 days (0x100000/24/60/60) + // Use GetMilliSpan to correct for rollover + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + +#else + +static int BMK_GetMilliStart() +{ + // Based on newer gettimeofday() + // Use GetMilliSpan to correct for rollover + struct timeval tv; + int nCount; + gettimeofday(&tv, NULL); + nCount = (int) (tv.tv_usec/1000 + (tv.tv_sec & 0xfffff) * 1000); + return nCount; +} + +#endif + + +static int BMK_GetMilliSpan( int nTimeStart ) +{ + int nSpan = BMK_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +static U32 BMK_checksum_MMH3A (char* buff, U32 length) +{ + const BYTE* data = (const BYTE*)buff; + const int nblocks = length >> 2; + + U32 h1 = KNUTH; + U32 c1 = 0xcc9e2d51; + U32 c2 = 0x1b873593; + + const U32* blocks = (const U32*)(data + nblocks*4); + int i; + + for(i = -nblocks; i; i++) + { + U32 k1 = blocks[i]; + + k1 *= c1; + k1 = _rotl(k1,15); + k1 *= c2; + + h1 ^= k1; + h1 = _rotl(h1,13); + h1 = h1*5+0xe6546b64; + } + + { + const BYTE* tail = (const BYTE*)(data + nblocks*4); + U32 k1 = 0; + + switch(length & 3) + { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; + k1 *= c1; k1 = _rotl(k1,15); k1 *= c2; h1 ^= k1; + }; + } + + h1 ^= length; + h1 ^= h1 >> 16; + h1 *= 0x85ebca6b; + h1 ^= h1 >> 13; + h1 *= 0xc2b2ae35; + h1 ^= h1 >> 16; + + return h1; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t step = (64U<<20); // 64 MB + BYTE* testmem=NULL; + + requiredMem = (((requiredMem >> 25) + 1) << 26); + if (requiredMem > MAX_MEM) requiredMem = MAX_MEM; + + requiredMem += 2*step; + while (!testmem) + { + requiredMem -= step; + testmem = malloc ((size_t)requiredMem); + } + + free (testmem); + return (size_t) (requiredMem - step); +} + + +static U64 BMK_GetFileSize(char* infilename) +{ + int r; +#if defined(_MSC_VER) + struct _stat64 statbuf; + r = _stat64(infilename, &statbuf); +#else + struct stat statbuf; + r = stat(infilename, &statbuf); +#endif + if (r || !S_ISREG(statbuf.st_mode)) return 0; // No good... + return (U64)statbuf.st_size; +} + + +//********************************************************* +// Public function +//********************************************************* + +int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel) +{ + int fileIdx=0; + FILE* fileIn; + char* infilename; + U64 largefilesize; + size_t benchedsize; + int nbChunks; + int maxCChunkSize; + size_t readSize; + char* in_buff; + char* out_buff; int out_buff_size; + struct chunkParameters* chunkP; + U32 crcc, crcd=0; + struct compressionParameters compP; + + U64 totals = 0; + U64 totalz = 0; + double totalc = 0.; + double totald = 0.; + + + // Init + switch (cLevel) + { +#ifdef COMPRESSOR0 + case 0 : compP.compressionFunction = COMPRESSOR0; break; +#endif +#ifdef COMPRESSOR1 + case 1 : compP.compressionFunction = COMPRESSOR1; break; +#endif + default : compP.compressionFunction = DEFAULTCOMPRESSOR; + } + compP.decompressionFunction = LZ4_uncompress; + + // Loop for each file + while (fileIdx largefilesize) benchedsize = (size_t)largefilesize; + if (benchedsize < largefilesize) + { + DISPLAY("Not enough memory for '%s' full size; testing %i MB only...\n", infilename, (int)(benchedsize>>20)); + } + + // Alloc + chunkP = (struct chunkParameters*) malloc(((benchedsize / chunkSize)+1) * sizeof(struct chunkParameters)); + in_buff = malloc((size_t )benchedsize); + nbChunks = (int) (benchedsize / chunkSize) + 1; + maxCChunkSize = LZ4_compressBound(chunkSize); + out_buff_size = nbChunks * maxCChunkSize; + out_buff = malloc((size_t )out_buff_size); + + + if(!in_buff || !out_buff) + { + DISPLAY("\nError: not enough memory!\n"); + free(in_buff); + free(out_buff); + fclose(fileIn); + return 12; + } + + // Init chunks data + { + int i; + size_t remaining = benchedsize; + char* in = in_buff; + char* out = out_buff; + for (i=0; i chunkSize) { chunkP[i].inputSize = chunkSize; remaining -= chunkSize; } else { chunkP[i].inputSize = (int)remaining; remaining = 0; } + chunkP[i].outputBuffer = out; out += maxCChunkSize; + chunkP[i].outputSize = 0; + } + } + + // Fill input buffer + DISPLAY("Loading %s... \r", infilename); + readSize = fread(in_buff, 1, benchedsize, fileIn); + fclose(fileIn); + + if(readSize != benchedsize) + { + DISPLAY("\nError: problem reading file '%s' !! \n", infilename); + free(in_buff); + free(out_buff); + return 13; + } + + // Calculating input Checksum + crcc = BMK_checksum_MMH3A(in_buff, (unsigned int)benchedsize); + + + // Bench + { + int loopNb, nb_loops, chunkNb; + size_t cSize=0; + int milliTime; + double fastestC = 100000000., fastestD = 100000000.; + double ratio=0.; + + DISPLAY("\r%79s\r", ""); + for (loopNb = 1; loopNb <= nbIterations; loopNb++) + { + // Compression + DISPLAY("%1i-%-14.14s : %9i ->\r", loopNb, infilename, (int)benchedsize); + { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s\r", loopNb, infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000.); + + // Decompression + { size_t i; for (i=0; i %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\r", loopNb, infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.); + + // CRC Checking + crcd = BMK_checksum_MMH3A(in_buff, (unsigned int)benchedsize); + if (crcc!=crcd) { DISPLAY("\n!!! WARNING !!! %14s : Invalid Checksum : %x != %x\n", infilename, (unsigned)crcc, (unsigned)crcd); break; } + } + + if (crcc==crcd) + { + if (ratio<100.) + DISPLAY("%-16.16s : %9i -> %9i (%5.2f%%),%7.1f MB/s ,%7.1f MB/s\n", infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.); + else + DISPLAY("%-16.16s : %9i -> %9i (%5.1f%%),%7.1f MB/s ,%7.1f MB/s \n", infilename, (int)benchedsize, (int)cSize, ratio, (double)benchedsize / fastestC / 1000., (double)benchedsize / fastestD / 1000.); + } + totals += benchedsize; + totalz += cSize; + totalc += fastestC; + totald += fastestD; + } + + free(in_buff); + free(out_buff); + free(chunkP); + } + + if (nbFiles > 1) + printf("%-16.16s :%10llu ->%10llu (%5.2f%%), %6.1f MB/s , %6.1f MB/s\n", " TOTAL", (long long unsigned int)totals, (long long unsigned int)totalz, (double)totalz/(double)totals*100., (double)totals/totalc/1000., (double)totals/totald/1000.); + + if (BMK_pause) { printf("press enter...\n"); getchar(); } + + return 0; +} + + + diff --git a/ext/lz4/bench.h b/ext/lz4/bench.h new file mode 100644 index 00000000..a55c3ce2 --- /dev/null +++ b/ext/lz4/bench.h @@ -0,0 +1,41 @@ +/* + bench.h - Demo program to benchmark open-source compression algorithm + Copyright (C) Yann Collet 2012 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ +#pragma once + +#if defined (__cplusplus) +extern "C" { +#endif + + +int BMK_benchFile(char** fileNamesTable, int nbFiles, int cLevel); + +// Parameters +void BMK_SetBlocksize(int bsize); +void BMK_SetNbIterations(int nbLoops); +void BMK_SetPause(); + + + +#if defined (__cplusplus) +} +#endif diff --git a/ext/lz4/fuzzer.c b/ext/lz4/fuzzer.c new file mode 100644 index 00000000..11697fd0 --- /dev/null +++ b/ext/lz4/fuzzer.c @@ -0,0 +1,227 @@ +/* + fuzzer.c - Fuzzer test tool for LZ4 + Copyright (C) Andrew Mahone - Yann Collet 2012 + Original code by Andrew Mahone / Modified by Yann Collet + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Remove Visual warning messages +//************************************** +#define _CRT_SECURE_NO_WARNINGS // fgets + + +//************************************** +// Includes +//************************************** +#include +#include // fgets, sscanf +#include // timeb +#include "lz4.h" + + +//************************************** +// Constants +//************************************** +#define NB_ATTEMPTS (1<<18) +#define LEN ((1<<15)) +#define SEQ_POW 2 +#define NUM_SEQ (1 << SEQ_POW) +#define SEQ_MSK ((NUM_SEQ) - 1) +#define MOD_SEQ(x) ((((x) >> 8) & 255) == 0) +#define NEW_SEQ(x) ((((x) >> 10) %10) == 0) +#define PAGE_SIZE 4096 +#define ROUND_PAGE(x) (((x) + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1)) +#define PRIME1 2654435761U +#define PRIME2 2246822519U +#define PRIME3 3266489917U + + + +//********************************************************* +// Functions +//********************************************************* +static int FUZ_GetMilliStart() +{ + struct timeb tb; + int nCount; + ftime( &tb ); + nCount = (int) (tb.millitm + (tb.time & 0xfffff) * 1000); + return nCount; +} + +static int FUZ_GetMilliSpan( int nTimeStart ) +{ + int nSpan = FUZ_GetMilliStart() - nTimeStart; + if ( nSpan < 0 ) + nSpan += 0x100000 * 1000; + return nSpan; +} + + +unsigned int FUZ_rand(unsigned int* src) +{ + *src = ((*src) * PRIME1) + PRIME2; + return *src; +} + + +int test_canary(unsigned char *buf) { + int i; + for (i = 0; i < 2048; i++) + if (buf[i] != buf[i + 2048]) + return 0; + return 1; +} + +int FUZ_SecurityTest() +{ + char* output; + char* input; + int i, r; + + printf("Starting security tests..."); + input = (char*) malloc (20<<20); + output = (char*) malloc (20<<20); + input[0] = 0x0F; + input[1] = 0x00; + input[2] = 0x00; + for(i = 3; i < 16840000; i++) + input[i] = 0xff; + r = LZ4_uncompress(input, output, 20<<20); + + free(input); + free(output); + printf(" Completed (r=%i)\n",r); + return 0; +} + + +//int main(int argc, char *argv[]) { +int main() { + unsigned long long bytes = 0; + unsigned long long cbytes = 0; + unsigned char buf[LEN]; + unsigned char testOut[LEN+1]; +# define FUZ_max LZ4_COMPRESSBOUND(LEN) +# define FUZ_avail ROUND_PAGE(FUZ_max) + const int off_full = FUZ_avail - FUZ_max; + unsigned char cbuf[FUZ_avail + PAGE_SIZE]; + unsigned int seed, cur_seq=PRIME3, seeds[NUM_SEQ], timestamp=FUZ_GetMilliStart(); + int i, j, k, ret, len; + char userInput[30] = {0}; + + printf("starting LZ4 fuzzer\n"); + printf("Select an Initialisation number (default : random) : "); + fflush(stdout); + if ( fgets(userInput, sizeof userInput, stdin) ) + { + if ( sscanf(userInput, "%d", &seed) == 1 ) {} + else seed = FUZ_GetMilliSpan(timestamp); + } + printf("Seed = %u\n", seed); + + FUZ_SecurityTest(); + + for (i = 0; i < 2048; i++) + cbuf[FUZ_avail + i] = cbuf[FUZ_avail + 2048 + i] = FUZ_rand(&seed) >> 16; + + for (i = 0; i < NB_ATTEMPTS; i++) + { + printf("\r%7i /%7i\r", i, NB_ATTEMPTS); + + FUZ_rand(&seed); + for (j = 0; j < NUM_SEQ; j++) { + seeds[j] = FUZ_rand(&seed) << 8; + seeds[j] ^= (FUZ_rand(&seed) >> 8) & 65535; + } + for (j = 0; j < LEN; j++) { + k = FUZ_rand(&seed); + if (j == 0 || NEW_SEQ(k)) + cur_seq = seeds[(FUZ_rand(&seed) >> 16) & SEQ_MSK]; + if (MOD_SEQ(k)) { + k = (FUZ_rand(&seed) >> 16) & SEQ_MSK; + seeds[k] = FUZ_rand(&seed) << 8; + seeds[k] ^= (FUZ_rand(&seed) >> 8) & 65535; + } + buf[j] = FUZ_rand(&cur_seq) >> 16; + } + + // Test compression + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[off_full], LEN, FUZ_max); + if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + len = ret; + + // Test decoding with output size being exactly what's necessary => must work + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN); + if (ret<0) { printf("decompression failed despite correct space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with one byte missing => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with one byte too much => must fail + ret = LZ4_uncompress((char*)&cbuf[off_full], (char*)testOut, LEN+1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with enough output size => must work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN+1); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with output size being exactly what's necessary => must work + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN); + if (ret<0) { printf("decompression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with output size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len, LEN-1); + if (ret>=0) { printf("decompression should have failed, due to Output Size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with input size being one byte too short => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len-1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too small : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test decoding with input size being one byte too large => must fail + ret = LZ4_uncompress_unknownOutputSize((char*)&cbuf[off_full], (char*)testOut, len+1, LEN); + if (ret>=0) { printf("decompression should have failed, due to input size being too large : seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test compression with output size being exactly what's necessary (should work) + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-len], LEN, len); + if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d\n", seed, LEN, len); goto _output_error; } + if (ret == 0) { printf("compression failed despite sufficient space: seed %u, len %d\n", seed, LEN); goto _output_error; } + + // Test compression with just one missing byte into output buffer => must fail + ret = LZ4_compress_limitedOutput((const char*)buf, (char*)&cbuf[FUZ_avail-(len-1)], LEN, len-1); + if (ret) { printf("compression overran output buffer: seed %u, len %d, olen %d => ret %d", seed, LEN, len-1, ret); goto _output_error; } + if (!test_canary(&cbuf[FUZ_avail])) { printf("compression overran output buffer: seed %u, len %d, olen %d", seed, LEN, len-1); goto _output_error; } + + bytes += LEN; + cbytes += len; + } + + printf("all tests completed successfully \n"); + printf("compression ratio: %0.3f%%\n", (double)cbytes/bytes*100); + getchar(); + return 0; + +_output_error: + getchar(); + return 1; +} diff --git a/ext/lz4/lz4.c b/ext/lz4/lz4.c new file mode 100644 index 00000000..1f2eafde --- /dev/null +++ b/ext/lz4/lz4.c @@ -0,0 +1,906 @@ +/* + LZ4 - Fast LZ compression algorithm + Copyright (C) 2011-2012, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + +//************************************** +// Tuning parameters +//************************************** +// MEMORY_USAGE : +// Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +// Increasing memory usage improves compression ratio +// Reduced memory usage can improve speed, due to cache effect +// Default value is 14, for 16KB, which nicely fits into Intel x86 L1 cache +#define MEMORY_USAGE 14 + +// BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE : +// This will provide a small boost to performance for big endian cpu, but the resulting compressed stream will be incompatible with little-endian CPU. +// You can set this option to 1 in situations where data will remain within closed environment +// This option is useless on Little_Endian CPU (such as x86) +//#define BIG_ENDIAN_NATIVE_BUT_INCOMPATIBLE 1 + + + +//************************************** +// CPU Feature Detection +//************************************** +// 32 or 64 bits ? +#if (defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64) || defined(__ppc64__) || defined(_WIN64) || defined(__LP64__) || defined(_LP64) ) // Detects 64 bits mode +# define LZ4_ARCH64 1 +#else +# define LZ4_ARCH64 0 +#endif + +// Little Endian or Big Endian ? +// Overwrite the #define below if you know your architecture endianess +#if defined (__GLIBC__) +# include +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define LZ4_BIG_ENDIAN 1 +# endif +#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) +# define LZ4_BIG_ENDIAN 1 +#elif defined(__sparc) || defined(__sparc__) \ + || defined(__ppc__) || defined(_POWER) || defined(__powerpc__) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__PPC) || defined(PPC) || defined(__powerpc__) || defined(__powerpc) || defined(powerpc) \ + || defined(__hpux) || defined(__hppa) \ + || defined(_MIPSEB) || defined(__s390__) +# define LZ4_BIG_ENDIAN 1 +#else +// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. +#endif + +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +#if defined(__ARM_FEATURE_UNALIGNED) +# define LZ4_FORCE_UNALIGNED_ACCESS 1 +#endif + +// Define this parameter if your target system or compiler does not support hardware bit count +#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +//************************************** +// Compiler Options +//************************************** +#if __STDC_VERSION__ >= 199901L // C99 +/* "restrict" is a known keyword */ +#else +# define restrict // Disable restrict +#endif + +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#ifdef _MSC_VER // Visual Studio +# include // For Visual 2005 +# if LZ4_ARCH64 // 64-bit +# pragma intrinsic(_BitScanForward64) // For Visual 2005 +# pragma intrinsic(_BitScanReverse64) // For Visual 2005 +# else +# pragma intrinsic(_BitScanForward) // For Visual 2005 +# pragma intrinsic(_BitScanReverse) // For Visual 2005 +# endif +#endif + +#ifdef _MSC_VER +# define lz4_bswap16(x) _byteswap_ushort(x) +#else +# define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) +#endif + +#if (GCC_VERSION >= 302) || (__INTEL_COMPILER >= 800) || defined(__clang__) +# define expect(expr,value) (__builtin_expect ((expr),(value)) ) +#else +# define expect(expr,value) (expr) +#endif + +#define likely(expr) expect((expr) != 0, 1) +#define unlikely(expr) expect((expr) != 0, 0) + + +//************************************** +// Includes +//************************************** +#include // for malloc +#include // for memset +#include "lz4.h" + + +//************************************** +// Basic Types +//************************************** +#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively +# define BYTE unsigned __int8 +# define U16 unsigned __int16 +# define U32 unsigned __int32 +# define S32 __int32 +# define U64 unsigned __int64 +#else +# include +# define BYTE uint8_t +# define U16 uint16_t +# define U32 uint32_t +# define S32 int32_t +# define U64 uint64_t +#endif + +#ifndef LZ4_FORCE_UNALIGNED_ACCESS +# pragma pack(push, 1) +#endif + +typedef struct _U16_S { U16 v; } U16_S; +typedef struct _U32_S { U32 v; } U32_S; +typedef struct _U64_S { U64 v; } U64_S; + +#ifndef LZ4_FORCE_UNALIGNED_ACCESS +# pragma pack(pop) +#endif + +#define A64(x) (((U64_S *)(x))->v) +#define A32(x) (((U32_S *)(x))->v) +#define A16(x) (((U16_S *)(x))->v) + + +//************************************** +// Constants +//************************************** +#define MINMATCH 4 + +#define HASH_LOG (MEMORY_USAGE-2) +#define HASHTABLESIZE (1 << HASH_LOG) +#define HASH_MASK (HASHTABLESIZE - 1) + +// NOTCOMPRESSIBLE_DETECTIONLEVEL : +// Decreasing this value will make the algorithm skip faster data segments considered "incompressible" +// This may decrease compression ratio dramatically, but will be faster on incompressible data +// Increasing this value will make the algorithm search more before declaring a segment "incompressible" +// This could improve compression a bit, but will be slower on incompressible data +// The default value (6) is recommended +#define NOTCOMPRESSIBLE_DETECTIONLEVEL 6 +#define SKIPSTRENGTH (NOTCOMPRESSIBLE_DETECTIONLEVEL>2?NOTCOMPRESSIBLE_DETECTIONLEVEL:2) +#define STACKLIMIT 13 +#define HEAPMODE (HASH_LOG>STACKLIMIT) // Defines if memory is allocated into the stack (local variable), or into the heap (malloc()). +#define COPYLENGTH 8 +#define LASTLITERALS 5 +#define MFLIMIT (COPYLENGTH+MINMATCH) +#define MINLENGTH (MFLIMIT+1) + +#define MAXD_LOG 16 +#define MAX_DISTANCE ((1 << MAXD_LOG) - 1) + +#define ML_BITS 4 +#define ML_MASK ((1U<> ((MINMATCH*8)-HASH_LOG)) +#define LZ4_HASH_VALUE(p) LZ4_HASH_FUNCTION(A32(p)) +#define LZ4_WILDCOPY(s,d,e) do { LZ4_COPYPACKET(s,d) } while (d>3); + #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll(val) >> 3); + #else + int r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; + #endif +#else + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll(val) >> 3); + #else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; + #endif +#endif +} + +#else + +static inline int LZ4_NbCommonBytes (register U32 val) +{ +#if defined(LZ4_BIG_ENDIAN) + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz(val) >> 3); + #else + int r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; + #endif +#else + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && (GCC_VERSION >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz(val) >> 3); + #else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; + #endif +#endif +} + +#endif + + + +//****************************** +// Compression functions +//****************************** + +// LZ4_compressCtx : +// ----------------- +// Compress 'isize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'. +// If it cannot achieve it, compression will stop, and result of the function will be zero. +// return : the number of bytes written in buffer 'dest', or 0 if the compression fails + +static inline int LZ4_compressCtx(void** ctx, + const char* source, + char* dest, + int isize, + int maxOutputSize) +{ +#if HEAPMODE + struct refTables *srt = (struct refTables *) (*ctx); + HTYPE* HashTable; +#else + HTYPE HashTable[HASHTABLESIZE] = {0}; +#endif + + const BYTE* ip = (BYTE*) source; + INITBASE(base); + const BYTE* anchor = ip; + const BYTE* const iend = ip + isize; + const BYTE* const mflimit = iend - MFLIMIT; +#define matchlimit (iend - LASTLITERALS) + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + int length; + const int skipStrength = SKIPSTRENGTH; + U32 forwardH; + + + // Init + if (isizehashTable); + memset((void*)HashTable, 0, sizeof(srt->hashTable)); +#else + (void) ctx; +#endif + + + // First Byte + HashTable[LZ4_HASH_VALUE(ip)] = ip - base; + ip++; forwardH = LZ4_HASH_VALUE(ip); + + // Main Loop + for ( ; ; ) + { + int findMatchAttempts = (1U << skipStrength) + 3; + const BYTE* forwardIp = ip; + const BYTE* ref; + BYTE* token; + + // Find a match + do { + U32 h = forwardH; + int step = findMatchAttempts++ >> skipStrength; + ip = forwardIp; + forwardIp = ip + step; + + if unlikely(forwardIp > mflimit) { goto _last_literals; } + + forwardH = LZ4_HASH_VALUE(forwardIp); + ref = base + HashTable[h]; + HashTable[h] = ip - base; + + } while ((ref < ip - MAX_DISTANCE) || (A32(ref) != A32(ip))); + + // Catch up + while ((ip>anchor) && (ref>(BYTE*)source) && unlikely(ip[-1]==ref[-1])) { ip--; ref--; } + + // Encode Literal length + length = (int)(ip - anchor); + token = op++; + if unlikely(op + length + (2 + 1 + LASTLITERALS) + (length>>8) > oend) return 0; // Check output limit +#ifdef _MSC_VER + if (length>=(int)RUN_MASK) + { + int len = length-RUN_MASK; + *token=(RUN_MASK<254) + { + do { *op++ = 255; len -= 255; } while (len>254); + *op++ = (BYTE)len; + memcpy(op, anchor, length); + op += length; + goto _next_match; + } + else + *op++ = (BYTE)len; + } + else *token = (length<=(int)RUN_MASK) + { + int len; + *token=(RUN_MASK< 254 ; len-=255) *op++ = 255; + *op++ = (BYTE)len; + } + else *token = (length<>8) > oend) return 0; // Check output limit + if (length>=(int)ML_MASK) + { + *token += ML_MASK; + length -= ML_MASK; + for (; length > 509 ; length-=510) { *op++ = 255; *op++ = 255; } + if (length > 254) { length-=255; *op++ = 255; } + *op++ = (BYTE)length; + } + else *token += length; + + // Test end of chunk + if (ip > mflimit) { anchor = ip; break; } + + // Fill table + HashTable[LZ4_HASH_VALUE(ip-2)] = ip - 2 - base; + + // Test next position + ref = base + HashTable[LZ4_HASH_VALUE(ip)]; + HashTable[LZ4_HASH_VALUE(ip)] = ip - base; + if ((ref > ip - (MAX_DISTANCE + 1)) && (A32(ref) == A32(ip))) { token = op++; *token=0; goto _next_match; } + + // Prepare next loop + anchor = ip++; + forwardH = LZ4_HASH_VALUE(ip); + } + +_last_literals: + // Encode Last Literals + { + int lastRun = (int)(iend - anchor); + if (((char*)op - dest) + lastRun + 1 + ((lastRun+255-RUN_MASK)/255) > (U32)maxOutputSize) return 0; + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (lastRun<> ((MINMATCH*8)-HASHLOG64K)) +#define LZ4_HASH64K_VALUE(p) LZ4_HASH64K_FUNCTION(A32(p)) +static inline int LZ4_compress64kCtx(void** ctx, + const char* source, + char* dest, + int isize, + int maxOutputSize) +{ +#if HEAPMODE + struct refTables *srt = (struct refTables *) (*ctx); + U16* HashTable; +#else + U16 HashTable[HASH64KTABLESIZE] = {0}; +#endif + + const BYTE* ip = (BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const base = ip; + const BYTE* const iend = ip + isize; + const BYTE* const mflimit = iend - MFLIMIT; +#define matchlimit (iend - LASTLITERALS) + + BYTE* op = (BYTE*) dest; + BYTE* const oend = op + maxOutputSize; + + int len, length; + const int skipStrength = SKIPSTRENGTH; + U32 forwardH; + + + // Init + if (isizehashTable); + memset((void*)HashTable, 0, sizeof(srt->hashTable)); +#else + (void) ctx; +#endif + + + // First Byte + ip++; forwardH = LZ4_HASH64K_VALUE(ip); + + // Main Loop + for ( ; ; ) + { + int findMatchAttempts = (1U << skipStrength) + 3; + const BYTE* forwardIp = ip; + const BYTE* ref; + BYTE* token; + + // Find a match + do { + U32 h = forwardH; + int step = findMatchAttempts++ >> skipStrength; + ip = forwardIp; + forwardIp = ip + step; + + if (forwardIp > mflimit) { goto _last_literals; } + + forwardH = LZ4_HASH64K_VALUE(forwardIp); + ref = base + HashTable[h]; + HashTable[h] = (U16)(ip - base); + + } while (A32(ref) != A32(ip)); + + // Catch up + while ((ip>anchor) && (ref>(BYTE*)source) && (ip[-1]==ref[-1])) { ip--; ref--; } + + // Encode Literal length + length = (int)(ip - anchor); + token = op++; + if unlikely(op + length + (2 + 1 + LASTLITERALS) + (length>>8) > oend) return 0; // Check output limit +#ifdef _MSC_VER + if (length>=(int)RUN_MASK) + { + int len = length-RUN_MASK; + *token=(RUN_MASK<254) + { + do { *op++ = 255; len -= 255; } while (len>254); + *op++ = (BYTE)len; + memcpy(op, anchor, length); + op += length; + goto _next_match; + } + else + *op++ = (BYTE)len; + } + else *token = (length<=(int)RUN_MASK) { *token=(RUN_MASK< 254 ; len-=255) *op++ = 255; *op++ = (BYTE)len; } + else *token = (length<>8) > oend) return 0; // Check output limit + if (len>=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *op++ = 255; *op++ = 255; } if (len > 254) { len-=255; *op++ = 255; } *op++ = (BYTE)len; } + else *token += len; + + // Test end of chunk + if (ip > mflimit) { anchor = ip; break; } + + // Fill table + HashTable[LZ4_HASH64K_VALUE(ip-2)] = (U16)(ip - 2 - base); + + // Test next position + ref = base + HashTable[LZ4_HASH64K_VALUE(ip)]; + HashTable[LZ4_HASH64K_VALUE(ip)] = (U16)(ip - base); + if (A32(ref) == A32(ip)) { token = op++; *token=0; goto _next_match; } + + // Prepare next loop + anchor = ip++; + forwardH = LZ4_HASH64K_VALUE(ip); + } + +_last_literals: + // Encode Last Literals + { + int lastRun = (int)(iend - anchor); + if (op + lastRun + 1 + (lastRun-RUN_MASK+255)/255 > oend) return 0; + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (lastRun<>ML_BITS)) == RUN_MASK) { size_t len; for (;(len=*ip++)==255;length+=255){} length += len; } + + // copy literals + cpy = op+length; + if (cpy>oend-COPYLENGTH) + { + if (cpy != oend) goto _output_error; // Error : not enough place for another match (min 4) + 5 literals + memcpy(op, ip, length); + ip += length; + break; // EOF + } + LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy; + + // get offset + LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; + if unlikely(ref < (BYTE* const)dest) goto _output_error; // Error : offset outside destination buffer + + // get matchlength + if ((length=(token&ML_MASK)) == ML_MASK) { for (;*ip==255;length+=255) {ip++;} length += *ip++; } + + // copy repeated sequence + if unlikely((op-ref)oend-(COPYLENGTH)-(STEPSIZE-4)) + { + if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals + LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH)); + while(op>ML_BITS)) == RUN_MASK) + { + int s=255; + while (likely(ipoend-MFLIMIT) || (ip+length>iend-(2+1+LASTLITERALS))) + { + if (cpy > oend) goto _output_error; // Error : writes beyond output buffer + if (ip+length != iend) goto _output_error; // Error : LZ4 format requires to consume all input at this stage (no match within the last 11 bytes, and at least 8 remaining input bytes for another match+literals) + memcpy(op, ip, length); + op += length; + break; // Necessarily EOF, due to parsing restrictions + } + LZ4_WILDCOPY(ip, op, cpy); ip -= (op-cpy); op = cpy; + + // get offset + LZ4_READ_LITTLEENDIAN_16(ref,cpy,ip); ip+=2; + if unlikely(ref < (BYTE* const)dest) goto _output_error; // Error : offset outside of destination buffer + + // get matchlength + if ((length=(token&ML_MASK)) == ML_MASK) + { + while likely(ipoend-(COPYLENGTH+(STEPSIZE-4))) + { + if (cpy > oend-LASTLITERALS) goto _output_error; // Error : last 5 bytes must be literals + LZ4_SECURECOPY(ref, op, (oend-COPYLENGTH)); + while(op= 255 +- 10 : (=280 - 15 - 255) ) remaining length to reach 280 + +Example 3 : A length of 15 will be represented as : +- 15 : value for the 4-bits High field +- 0 : (=15-15) yes, the zero must be output + +Following the token and optional length bytes, are the literals themselves. +They are exactly as numerous as previously decoded (length of literals). +It's possible that there are zero literal. + + +Following the literals is the match copy operation. + +It starts by the offset. +This is a 2 bytes value, in little endian format : +the lower byte is the first one in the stream. + +The offset represents the position of the match to be copied from. +1 means "current position - 1 byte". +The maximum offset value is 65535, 65536 cannot be coded. +Note that 0 is an invalid value, not used. + +Then we need to extract the match length. +For this, we use the second token field, the low 4-bits. +Value, obviously, ranges from 0 to 15. +However here, 0 means that the copy operation will be minimal. +The minimum length of a match, called minmatch, is 4. +As a consequence, a 0 value means 4 bytes, and a value of 15 means 19+ bytes. +Similar to literal length, on reaching the highest possible value (15), +we output additional bytes, one at a time, with values ranging from 0 to 255. +They are added to total to provide the final match length. +A 255 value means there is another byte to read and add. +There is no limit to the number of optional bytes that can be output this way. +(This points towards a maximum achievable compression ratio of ~250). + +With the offset and the matchlength, +the decoder can now proceed to copy the data from the already decoded buffer. +On decoding the matchlength, we reach the end of the compressed sequence, +and therefore start another one. + + +-- Parsing restrictions -- + +There are specific parsing rules to respect in order to remain compatible +with assumptions made by the decoder : +1) The last 5 bytes are always literals +2) The last match must start at least 12 bytes before end of stream +Consequently, a file with less than 13 bytes cannot be compressed. +These rules are in place to ensure that the decoder +will never read beyond the input buffer, nor write beyond the output buffer. + +Note that the last sequence is also incomplete, +and stops right after literals. + + +-- Additional notes -- + +There is no assumption nor limits to the way the compressor +searches and selects matches within the source stream. +It could be a fast scan, a multi-probe, a full search using BST, +standard hash chains or MMC, well whatever. + +Advanced parsing strategies can also be implemented, such as lazy match, +or full optimal parsing. + +All these trade-off offer distinctive speed/memory/compression advantages. +Whatever the method used by the compressor, its result will be decodable +by any LZ4 decoder if it follows the format specification described above. + diff --git a/ext/lz4/lz4demo.c b/ext/lz4/lz4demo.c new file mode 100644 index 00000000..85edcf00 --- /dev/null +++ b/ext/lz4/lz4demo.c @@ -0,0 +1,402 @@ +/* + LZ4Demo - Demo CLI program using LZ4 compression + Copyright (C) Yann Collet 2011-2012 + GPL v2 License + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ +/* + Note : this is *only* a demo program, an example to show how LZ4 can be used. + It is not considered part of LZ4 compression library. + The license of LZ4 is BSD. + The license of the demo program is GPL. +*/ + +//************************************** +// Compiler Options +//************************************** +// Disable some Visual warning messages +#define _CRT_SECURE_NO_WARNINGS +#define _CRT_SECURE_NO_DEPRECATE // VS2005 + + +//**************************** +// Includes +//**************************** +#include // fprintf, fopen, fread, _fileno(?) +#include // malloc +#include // strcmp +#include // clock +#ifdef _WIN32 +#include // _setmode +#include // _O_BINARY +#endif +#include "lz4.h" +#include "lz4hc.h" +#include "bench.h" + + +//************************************** +// Compiler-specific functions +//************************************** +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +#if defined(_MSC_VER) // Visual Studio +#define swap32 _byteswap_ulong +#elif GCC_VERSION >= 403 +#define swap32 __builtin_bswap32 +#else +static inline unsigned int swap32(unsigned int x) { + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); + } +#endif + + +//**************************** +// Constants +//**************************** +#define COMPRESSOR_NAME "Compression CLI using LZ4 algorithm" +#define COMPRESSOR_VERSION "" +#define COMPILED __DATE__ +#define AUTHOR "Yann Collet" +#define EXTENSION ".lz4" +#define WELCOME_MESSAGE "*** %s %s, by %s (%s) ***\n", COMPRESSOR_NAME, COMPRESSOR_VERSION, AUTHOR, COMPILED + +#define CHUNKSIZE (8<<20) // 8 MB +#define CACHELINE 64 +#define ARCHIVE_MAGICNUMBER 0x184C2102 +#define ARCHIVE_MAGICNUMBER_SIZE 4 + + +//************************************** +// Architecture Macros +//************************************** +static const int one = 1; +#define CPU_LITTLE_ENDIAN (*(char*)(&one)) +#define CPU_BIG_ENDIAN (!CPU_LITTLE_ENDIAN) +#define LITTLE_ENDIAN32(i) if (CPU_BIG_ENDIAN) { i = swap32(i); } + + +//************************************** +// Macros +//************************************** +#define DISPLAY(...) fprintf(stderr, __VA_ARGS__) + + +//**************************** +// Functions +//**************************** +int usage(char* exename) +{ + DISPLAY( "Usage :\n"); + DISPLAY( " %s [arg] input output\n", exename); + DISPLAY( "Arguments :\n"); + DISPLAY( " -c0: Fast compression (default) \n"); + DISPLAY( " -c1: High compression \n"); + DISPLAY( " -d : decompression \n"); + DISPLAY( " -b#: Benchmark files, using # compression level\n"); + DISPLAY( " -t : check compressed file \n"); + DISPLAY( " -h : help (this text)\n"); + DISPLAY( "input : can be 'stdin' (pipe) or a filename\n"); + DISPLAY( "output : can be 'stdout'(pipe) or a filename or 'null'\n"); + return 0; +} + + +int badusage(char* exename) +{ + DISPLAY("Wrong parameters\n"); + usage(exename); + return 0; +} + + + +int get_fileHandle(char* input_filename, char* output_filename, FILE** pfinput, FILE** pfoutput) +{ + char stdinmark[] = "stdin"; + char stdoutmark[] = "stdout"; + + if (!strcmp (input_filename, stdinmark)) { + DISPLAY( "Using stdin for input\n"); + *pfinput = stdin; +#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows + _setmode( _fileno( stdin ), _O_BINARY ); +#endif + } else { + *pfinput = fopen( input_filename, "rb" ); + } + + if (!strcmp (output_filename, stdoutmark)) { + DISPLAY( "Using stdout for output\n"); + *pfoutput = stdout; +#ifdef _WIN32 // Need to set stdin/stdout to binary mode specifically for windows + _setmode( _fileno( stdout ), _O_BINARY ); +#endif + } else { + *pfoutput = fopen( output_filename, "wb" ); + } + + if ( *pfinput==0 ) { DISPLAY( "Pb opening %s\n", input_filename); return 2; } + if ( *pfoutput==0) { DISPLAY( "Pb opening %s\n", output_filename); return 3; } + + return 0; +} + + + +int compress_file(char* input_filename, char* output_filename, int compressionlevel) +{ + int (*compressionFunction)(const char*, char*, int); + unsigned long long filesize = 0; + unsigned long long compressedfilesize = ARCHIVE_MAGICNUMBER_SIZE; + unsigned int u32var; + char* in_buff; + char* out_buff; + FILE* finput; + FILE* foutput; + int r; + int displayLevel = (compressionlevel>0); + clock_t start, end; + size_t sizeCheck; + + + // Init + switch (compressionlevel) + { + case 0 : compressionFunction = LZ4_compress; break; + case 1 : compressionFunction = LZ4_compressHC; break; + default : compressionFunction = LZ4_compress; + } + start = clock(); + r = get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (r) return r; + + // Allocate Memory + in_buff = (char*)malloc(CHUNKSIZE); + out_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); + if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 8; } + + // Write Archive Header + u32var = ARCHIVE_MAGICNUMBER; + LITTLE_ENDIAN32(u32var); + *(unsigned int*)out_buff = u32var; + sizeCheck = fwrite(out_buff, 1, ARCHIVE_MAGICNUMBER_SIZE, foutput); + if (sizeCheck!=ARCHIVE_MAGICNUMBER_SIZE) { DISPLAY("write error\n"); return 10; } + + // Main Loop + while (1) + { + int outSize; + // Read Block + int inSize = (int) fread(in_buff, (size_t)1, (size_t)CHUNKSIZE, finput); + if( inSize<=0 ) break; + filesize += inSize; + if (displayLevel) DISPLAY("Read : %i MB \r", (int)(filesize>>20)); + + // Compress Block + outSize = compressionFunction(in_buff, out_buff+4, inSize); + compressedfilesize += outSize+4; + if (displayLevel) DISPLAY("Read : %i MB ==> %.2f%%\r", (int)(filesize>>20), (double)compressedfilesize/filesize*100); + + // Write Block + LITTLE_ENDIAN32(outSize); + * (unsigned int*) out_buff = outSize; + LITTLE_ENDIAN32(outSize); + sizeCheck = fwrite(out_buff, 1, outSize+4, foutput); + if (sizeCheck!=(size_t)(outSize+4)) { DISPLAY("write error\n"); return 11; } + } + + // Status + end = clock(); + DISPLAY( "Compressed %llu bytes into %llu bytes ==> %.2f%%\n", + (unsigned long long) filesize, (unsigned long long) compressedfilesize, (double)compressedfilesize/filesize*100); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +int decode_file(char* input_filename, char* output_filename) +{ + unsigned long long filesize = 0; + char* in_buff; + char* out_buff; + size_t uselessRet; + int sinkint; + unsigned int chunkSize; + FILE* finput; + FILE* foutput; + clock_t start, end; + int r; + size_t sizeCheck; + + + // Init + start = clock(); + r = get_fileHandle(input_filename, output_filename, &finput, &foutput); + if (r) return r; + + // Allocate Memory + in_buff = (char*)malloc(LZ4_compressBound(CHUNKSIZE)); + out_buff = (char*)malloc(CHUNKSIZE); + if (!in_buff || !out_buff) { DISPLAY("Allocation error : not enough memory\n"); return 7; } + + // Check Archive Header + chunkSize = 0; + uselessRet = fread(&chunkSize, 1, ARCHIVE_MAGICNUMBER_SIZE, finput); + LITTLE_ENDIAN32(chunkSize); + if (chunkSize != ARCHIVE_MAGICNUMBER) { DISPLAY("Unrecognized header : file cannot be decoded\n"); return 6; } + + // Main Loop + while (1) + { + // Block Size + uselessRet = fread(&chunkSize, 1, 4, finput); + if( uselessRet==0 ) break; // Nothing to read : file read is completed + LITTLE_ENDIAN32(chunkSize); + if (chunkSize == ARCHIVE_MAGICNUMBER) + continue; // appended compressed stream + + // Read Block + uselessRet = fread(in_buff, 1, chunkSize, finput); + + // Decode Block + sinkint = LZ4_uncompress_unknownOutputSize(in_buff, out_buff, chunkSize, CHUNKSIZE); + if (sinkint < 0) { DISPLAY("Decoding Failed ! Corrupted input !\n"); return 9; } + filesize += sinkint; + + // Write Block + sizeCheck = fwrite(out_buff, 1, sinkint, foutput); + if (sizeCheck != (size_t)sinkint) { DISPLAY("write error\n"); return 12; } + } + + // Status + end = clock(); + DISPLAY( "Successfully decoded %llu bytes \n", (unsigned long long)filesize); + { + double seconds = (double)(end - start)/CLOCKS_PER_SEC; + DISPLAY( "Done in %.2f s ==> %.2f MB/s\n", seconds, (double)filesize / seconds / 1024 / 1024); + } + + // Close & Free + free(in_buff); + free(out_buff); + fclose(finput); + fclose(foutput); + + return 0; +} + + +int main(int argc, char** argv) +{ + int i, + cLevel=0, + decode=0, + bench=0, + filenamesStart=2; + char* exename=argv[0]; + char* input_filename=0; + char* output_filename=0; +#ifdef _WIN32 + char nulmark[] = "nul"; +#else + char nulmark[] = "/dev/null"; +#endif + char nullinput[] = "null"; + + // Welcome message + DISPLAY( WELCOME_MESSAGE); + + if (argc<2) { badusage(exename); return 1; } + + for(i=1; i='0') cLevel=argument[1] - '0'; continue; } + + // Decoding + if ( argument[0] =='d' ) { decode=1; continue; } + + // Bench + if ( argument[0] =='b' ) { bench=1; if (argument[1] >= '0') cLevel=argument[1] - '0'; continue; } + + // Modify Block Size (benchmark only) + if ( argument[0] =='B' ) { int B = argument[1] - '0'; int S = 1 << (10 + 2*B); BMK_SetBlocksize(S); continue; } + + // Modify Nb Iterations (benchmark only) + if ( argument[0] =='i' ) { int iters = argument[1] - '0'; BMK_SetNbIterations(iters); continue; } + + // Pause at the end (benchmark only) + if ( argument[0] =='p' ) { BMK_SetPause(); continue; } + + // Test + if ( argument[0] =='t' ) { decode=1; output_filename=nulmark; continue; } + } + + // first provided filename is input + if (!input_filename) { input_filename=argument; filenamesStart=i; continue; } + + // second provided filename is output + if (!output_filename) + { + output_filename=argument; + if (!strcmp (output_filename, nullinput)) output_filename = nulmark; + continue; + } + } + + // No input filename ==> Error + if(!input_filename) { badusage(exename); return 1; } + + if (bench) return BMK_benchFile(argv+filenamesStart, argc-filenamesStart, cLevel); + + // No output filename ==> Error + if (!output_filename) { badusage(exename); return 1; } + + if (decode) return decode_file(input_filename, output_filename); + + return compress_file(input_filename, output_filename, cLevel); // Compression is 'default' action + +} diff --git a/ext/lz4/lz4hc.c b/ext/lz4/lz4hc.c new file mode 100644 index 00000000..7324492e --- /dev/null +++ b/ext/lz4/lz4hc.c @@ -0,0 +1,730 @@ +/* + LZ4 HC - High Compression Mode of LZ4 + Copyright (C) 2011-2012, Yann Collet. + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html + - LZ4 source repository : http://code.google.com/p/lz4/ +*/ + + +//************************************** +// CPU Feature Detection +//************************************** +// 32 or 64 bits ? +#if (defined(__x86_64__) || defined(__x86_64) || defined(__amd64__) || defined(__amd64) || defined(__ppc64__) || defined(_WIN64) || defined(__LP64__) || defined(_LP64) ) // Detects 64 bits mode +# define LZ4_ARCH64 1 +#else +# define LZ4_ARCH64 0 +#endif + +// Little Endian or Big Endian ? +// Overwrite the #define below if you know your architecture endianess +#if defined (__GLIBC__) +# include +# if (__BYTE_ORDER == __BIG_ENDIAN) +# define LZ4_BIG_ENDIAN 1 +# endif +#elif (defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined(_BIG_ENDIAN)) && !(defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined(_LITTLE_ENDIAN)) +# define LZ4_BIG_ENDIAN 1 +#elif defined(__sparc) || defined(__sparc__) \ + || defined(__ppc__) || defined(_POWER) || defined(__powerpc__) || defined(_ARCH_PPC) || defined(__PPC__) || defined(__PPC) || defined(PPC) || defined(__powerpc__) || defined(__powerpc) || defined(powerpc) \ + || defined(__hpux) || defined(__hppa) \ + || defined(_MIPSEB) || defined(__s390__) +# define LZ4_BIG_ENDIAN 1 +#else +// Little Endian assumed. PDP Endian and other very rare endian format are unsupported. +#endif + +// Unaligned memory access is automatically enabled for "common" CPU, such as x86. +// For others CPU, the compiler will be more cautious, and insert extra code to ensure aligned access is respected +// If you know your target CPU supports unaligned memory access, you want to force this option manually to improve performance +#if defined(__ARM_FEATURE_UNALIGNED) +# define LZ4_FORCE_UNALIGNED_ACCESS 1 +#endif + +// Define this parameter if your target system or compiler does not support hardware bit count +#if defined(_MSC_VER) && defined(_WIN32_WCE) // Visual Studio for Windows CE does not support Hardware bit count +# define LZ4_FORCE_SW_BITCOUNT +#endif + + +//************************************** +// Compiler Options +//************************************** +#if __STDC_VERSION__ >= 199901L // C99 + /* "restrict" is a known keyword */ +#else +# define restrict // Disable restrict +#endif + +#ifdef _MSC_VER +# define inline __inline // Visual is not C99, but supports some kind of inline +# define forceinline __forceinline +# include // For Visual 2005 +# if LZ4_ARCH64 // 64-bit +# pragma intrinsic(_BitScanForward64) // For Visual 2005 +# pragma intrinsic(_BitScanReverse64) // For Visual 2005 +# else +# pragma intrinsic(_BitScanForward) // For Visual 2005 +# pragma intrinsic(_BitScanReverse) // For Visual 2005 +# endif +#else +# ifdef __GNUC__ +# define forceinline inline __attribute__((always_inline)) +# else +# define forceinline inline +# endif +#endif + +#ifdef _MSC_VER // Visual Studio +#define lz4_bswap16(x) _byteswap_ushort(x) +#else +#define lz4_bswap16(x) ((unsigned short int) ((((x) >> 8) & 0xffu) | (((x) & 0xffu) << 8))) +#endif + + +//************************************** +// Includes +//************************************** +#include // calloc, free +#include // memset, memcpy +#include "lz4hc.h" + +#define ALLOCATOR(s) calloc(1,s) +#define FREEMEM free +#define MEM_INIT memset + + +//************************************** +// Basic Types +//************************************** +#if defined(_MSC_VER) // Visual Studio does not support 'stdint' natively +#define BYTE unsigned __int8 +#define U16 unsigned __int16 +#define U32 unsigned __int32 +#define S32 __int32 +#define U64 unsigned __int64 +#else +#include +#define BYTE uint8_t +#define U16 uint16_t +#define U32 uint32_t +#define S32 int32_t +#define U64 uint64_t +#endif + +#ifndef LZ4_FORCE_UNALIGNED_ACCESS +#pragma pack(push, 1) +#endif + +typedef struct _U16_S { U16 v; } U16_S; +typedef struct _U32_S { U32 v; } U32_S; +typedef struct _U64_S { U64 v; } U64_S; + +#ifndef LZ4_FORCE_UNALIGNED_ACCESS +#pragma pack(pop) +#endif + +#define A64(x) (((U64_S *)(x))->v) +#define A32(x) (((U32_S *)(x))->v) +#define A16(x) (((U16_S *)(x))->v) + + +//************************************** +// Constants +//************************************** +#define MINMATCH 4 + +#define DICTIONARY_LOGSIZE 16 +#define MAXD (1<> ((MINMATCH*8)-HASH_LOG)) +#define HASH_VALUE(p) HASH_FUNCTION(A32(p)) +#define HASH_POINTER(p) (HashTable[HASH_VALUE(p)] + base) +#define DELTANEXT(p) chainTable[(size_t)(p) & MAXD_MASK] +#define GETNEXT(p) ((p) - (size_t)DELTANEXT(p)) + + +//************************************** +// Private functions +//************************************** +#if LZ4_ARCH64 + +inline static int LZ4_NbCommonBytes (register U64 val) +{ +#if defined(LZ4_BIG_ENDIAN) + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clzll(val) >> 3); + #else + int r; + if (!(val>>32)) { r=4; } else { r=0; val>>=32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; + #endif +#else + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r = 0; + _BitScanForward64( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctzll(val) >> 3); + #else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -val) * 0x0218A392CDABBD3F)) >> 58]; + #endif +#endif +} + +#else + +inline static int LZ4_NbCommonBytes (register U32 val) +{ +#if defined(LZ4_BIG_ENDIAN) + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanReverse( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_clz(val) >> 3); + #else + int r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; + #endif +#else + #if defined(_MSC_VER) && !defined(LZ4_FORCE_SW_BITCOUNT) + unsigned long r; + _BitScanForward( &r, val ); + return (int)(r>>3); + #elif defined(__GNUC__) && ((__GNUC__ * 100 + __GNUC_MINOR__) >= 304) && !defined(LZ4_FORCE_SW_BITCOUNT) + return (__builtin_ctz(val) >> 3); + #else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; + #endif +#endif +} + +#endif + + +inline static int LZ4HC_Init (LZ4HC_Data_Structure* hc4, const BYTE* base) +{ + MEM_INIT((void*)hc4->hashTable, 0, sizeof(hc4->hashTable)); + MEM_INIT(hc4->chainTable, 0xFF, sizeof(hc4->chainTable)); + hc4->nextToUpdate = base + LZ4_ARCH64; + hc4->base = base; + return 1; +} + + +inline static void* LZ4HC_Create (const BYTE* base) +{ + void* hc4 = ALLOCATOR(sizeof(LZ4HC_Data_Structure)); + + LZ4HC_Init ((LZ4HC_Data_Structure*)hc4, base); + return hc4; +} + + +inline static int LZ4HC_Free (void** LZ4HC_Data) +{ + FREEMEM(*LZ4HC_Data); + *LZ4HC_Data = NULL; + return (1); +} + + +// Update chains up to ip (excluded) +forceinline static void LZ4HC_Insert (LZ4HC_Data_Structure* hc4, const BYTE* ip) +{ + U16* chainTable = hc4->chainTable; + HTYPE* HashTable = hc4->hashTable; + INITBASE(base,hc4->base); + + while(hc4->nextToUpdate < ip) + { + const BYTE* p = hc4->nextToUpdate; + size_t delta = (p) - HASH_POINTER(p); + if (delta>MAX_DISTANCE) delta = MAX_DISTANCE; + DELTANEXT(p) = (U16)delta; + HashTable[HASH_VALUE(p)] = (p) - base; + hc4->nextToUpdate++; + } +} + + +forceinline static size_t LZ4HC_CommonLength (const BYTE* p1, const BYTE* p2, const BYTE* const matchlimit) +{ + const BYTE* p1t = p1; + + while (p1tchainTable; + HTYPE* const HashTable = hc4->hashTable; + const BYTE* ref; + INITBASE(base,hc4->base); + int nbAttempts=MAX_NB_ATTEMPTS; + size_t repl=0, ml=0; + U16 delta; + + // HC4 match finder + LZ4HC_Insert(hc4, ip); + ref = HASH_POINTER(ip); + +#define REPEAT_OPTIMIZATION +#ifdef REPEAT_OPTIMIZATION + // Detect repetitive sequences of length <= 4 + if (ref >= ip-4) // potential repetition + { + if (A32(ref) == A32(ip)) // confirmed + { + delta = (U16)(ip-ref); + repl = ml = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; + *matchpos = ref; + } + ref = GETNEXT(ref); + } +#endif + + while ((ref >= ip-MAX_DISTANCE) && (nbAttempts)) + { + nbAttempts--; + if (*(ref+ml) == *(ip+ml)) + if (A32(ref) == A32(ip)) + { + size_t mlt = LZ4HC_CommonLength(ip+MINMATCH, ref+MINMATCH, matchlimit) + MINMATCH; + if (mlt > ml) { ml = mlt; *matchpos = ref; } + } + ref = GETNEXT(ref); + } + +#ifdef REPEAT_OPTIMIZATION + // Complete table + if (repl) + { + const BYTE* ptr = ip; + const BYTE* end; + + end = ip + repl - (MINMATCH-1); + while(ptr < end-delta) + { + DELTANEXT(ptr) = delta; // Pre-Load + ptr++; + } + do + { + DELTANEXT(ptr) = delta; + HashTable[HASH_VALUE(ptr)] = (ptr) - base; // Head of chain + ptr++; + } while(ptr < end); + hc4->nextToUpdate = end; + } +#endif + + return (int)ml; +} + + +forceinline static int LZ4HC_InsertAndGetWiderMatch (LZ4HC_Data_Structure* hc4, const BYTE* ip, const BYTE* startLimit, const BYTE* matchlimit, int longest, const BYTE** matchpos, const BYTE** startpos) +{ + U16* const chainTable = hc4->chainTable; + HTYPE* const HashTable = hc4->hashTable; + INITBASE(base,hc4->base); + const BYTE* ref; + int nbAttempts = MAX_NB_ATTEMPTS; + int delta = (int)(ip-startLimit); + + // First Match + LZ4HC_Insert(hc4, ip); + ref = HASH_POINTER(ip); + + while ((ref >= ip-MAX_DISTANCE) && (nbAttempts)) + { + nbAttempts--; + if (*(startLimit + longest) == *(ref - delta + longest)) + if (A32(ref) == A32(ip)) + { +#if 1 + const BYTE* reft = ref+MINMATCH; + const BYTE* ipt = ip+MINMATCH; + const BYTE* startt = ip; + + while (iptstartLimit) && (reft > hc4->base) && (startt[-1] == reft[-1])) {startt--; reft--;} + + if ((ipt-startt) > longest) + { + longest = (int)(ipt-startt); + *matchpos = reft; + *startpos = startt; + } + } + ref = GETNEXT(ref); + } + + return longest; +} + + +forceinline static int LZ4_encodeSequence(const BYTE** ip, BYTE** op, const BYTE** anchor, int ml, const BYTE* ref) +{ + int length, len; + BYTE* token; + + // Encode Literal length + length = (int)(*ip - *anchor); + token = (*op)++; + if (length>=(int)RUN_MASK) { *token=(RUN_MASK< 254 ; len-=255) *(*op)++ = 255; *(*op)++ = (BYTE)len; } + else *token = (length<=(int)ML_MASK) { *token+=ML_MASK; len-=ML_MASK; for(; len > 509 ; len-=510) { *(*op)++ = 255; *(*op)++ = 255; } if (len > 254) { len-=255; *(*op)++ = 255; } *(*op)++ = (BYTE)len; } + else *token += len; + + // Prepare next loop + *ip += ml; + *anchor = *ip; + + return 0; +} + + +//**************************** +// Compression CODE +//**************************** + +int LZ4_compressHCCtx(LZ4HC_Data_Structure* ctx, + const char* source, + char* dest, + int isize) +{ + const BYTE* ip = (const BYTE*) source; + const BYTE* anchor = ip; + const BYTE* const iend = ip + isize; + const BYTE* const mflimit = iend - MFLIMIT; + const BYTE* const matchlimit = (iend - LASTLITERALS); + + BYTE* op = (BYTE*) dest; + + int ml, ml2, ml3, ml0; + const BYTE* ref=NULL; + const BYTE* start2=NULL; + const BYTE* ref2=NULL; + const BYTE* start3=NULL; + const BYTE* ref3=NULL; + const BYTE* start0; + const BYTE* ref0; + + ip++; + + // Main Loop + while (ip < mflimit) + { + ml = LZ4HC_InsertAndFindBestMatch (ctx, ip, matchlimit, (&ref)); + if (!ml) { ip++; continue; } + + // saved, in case we would skip too much + start0 = ip; + ref0 = ref; + ml0 = ml; + +_Search2: + if (ip+ml < mflimit) + ml2 = LZ4HC_InsertAndGetWiderMatch(ctx, ip + ml - 2, ip + 1, matchlimit, ml, &ref2, &start2); + else ml2 = ml; + + if (ml2 == ml) // No better match + { + LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); + continue; + } + + if (start0 < ip) + { + if (start2 < ip + ml0) // empirical + { + ip = start0; + ref = ref0; + ml = ml0; + } + } + + // Here, start0==ip + if ((start2 - ip) < 3) // First Match too small : removed + { + ml = ml2; + ip = start2; + ref =ref2; + goto _Search2; + } + +_Search3: + // Currently we have : + // ml2 > ml1, and + // ip1+3 <= ip2 (usually < ip1+ml1) + if ((start2 - ip) < OPTIMAL_ML) + { + int correction; + int new_ml = ml; + if (new_ml > OPTIMAL_ML) new_ml = OPTIMAL_ML; + if (ip+new_ml > start2 + ml2 - MINMATCH) new_ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = new_ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + // Now, we have start2 = ip+new_ml, with new_ml = min(ml, OPTIMAL_ML=18) + + if (start2 + ml2 < mflimit) + ml3 = LZ4HC_InsertAndGetWiderMatch(ctx, start2 + ml2 - 3, start2, matchlimit, ml2, &ref3, &start3); + else ml3 = ml2; + + if (ml3 == ml2) // No better match : 2 sequences to encode + { + // ip & ref are known; Now for ml + if (start2 < ip+ml) ml = (int)(start2 - ip); + // Now, encode 2 sequences + LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); + ip = start2; + LZ4_encodeSequence(&ip, &op, &anchor, ml2, ref2); + continue; + } + + if (start3 < ip+ml+3) // Not enough space for match 2 : remove it + { + if (start3 >= (ip+ml)) // can write Seq1 immediately ==> Seq2 is removed, so Seq3 becomes Seq1 + { + if (start2 < ip+ml) + { + int correction = (int)(ip+ml - start2); + start2 += correction; + ref2 += correction; + ml2 -= correction; + if (ml2 < MINMATCH) + { + start2 = start3; + ref2 = ref3; + ml2 = ml3; + } + } + + LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); + ip = start3; + ref = ref3; + ml = ml3; + + start0 = start2; + ref0 = ref2; + ml0 = ml2; + goto _Search2; + } + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + goto _Search3; + } + + // OK, now we have 3 ascending matches; let's write at least the first one + // ip & ref are known; Now for ml + if (start2 < ip+ml) + { + if ((start2 - ip) < (int)ML_MASK) + { + int correction; + if (ml > OPTIMAL_ML) ml = OPTIMAL_ML; + if (ip + ml > start2 + ml2 - MINMATCH) ml = (int)(start2 - ip) + ml2 - MINMATCH; + correction = ml - (int)(start2 - ip); + if (correction > 0) + { + start2 += correction; + ref2 += correction; + ml2 -= correction; + } + } + else + { + ml = (int)(start2 - ip); + } + } + LZ4_encodeSequence(&ip, &op, &anchor, ml, ref); + + ip = start2; + ref = ref2; + ml = ml2; + + start2 = start3; + ref2 = ref3; + ml2 = ml3; + + goto _Search3; + + } + + // Encode Last Literals + { + int lastRun = (int)(iend - anchor); + if (lastRun>=(int)RUN_MASK) { *op++=(RUN_MASK< 254 ; lastRun-=255) *op++ = 255; *op++ = (BYTE) lastRun; } + else *op++ = (lastRun<