]>
Commit | Line | Data |
---|---|---|
26c7fdad | 1 | // SPDX-License-Identifier: GPL 2.0+ OR BSD-2-Clause |
027b728d | 2 | /* |
26c7fdad HJ |
3 | * LZ4 - Fast LZ compression algorithm |
4 | * Copyright (C) 2011 - 2016, Yann Collet. | |
5 | * BSD 2 - Clause License (http://www.opensource.org/licenses/bsd - license.php) | |
6 | * Redistribution and use in source and binary forms, with or without | |
7 | * modification, are permitted provided that the following conditions are | |
8 | * met: | |
9 | * * Redistributions of source code must retain the above copyright | |
10 | * notice, this list of conditions and the following disclaimer. | |
11 | * * Redistributions in binary form must reproduce the above | |
12 | * copyright notice, this list of conditions and the following disclaimer | |
13 | * in the documentation and/or other materials provided with the | |
14 | * distribution. | |
15 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
16 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
17 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
18 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
19 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
21 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
22 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
23 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
24 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
25 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
26 | * You can contact the author at : | |
27 | * - LZ4 homepage : http://www.lz4.org | |
28 | * - LZ4 source repository : https://github.com/lz4/lz4 | |
29 | */ | |
26c7fdad HJ |
30 | #include <compiler.h> |
31 | #include <linux/kernel.h> | |
32 | #include <linux/types.h> | |
33 | #include <linux/bug.h> | |
34 | #include <asm/unaligned.h> | |
35 | #include <u-boot/lz4.h> | |
36 | ||
37 | #define FORCE_INLINE inline __attribute__((always_inline)) | |
38 | ||
39 | static FORCE_INLINE u16 LZ4_readLE16(const void *src) | |
40 | { | |
41 | return get_unaligned_le16(src); | |
42 | } | |
027b728d | 43 | |
26c7fdad HJ |
44 | static FORCE_INLINE void LZ4_copy8(void *dst, const void *src) |
45 | { | |
46 | put_unaligned(get_unaligned((const u64 *)src), (u64 *)dst); | |
47 | } | |
48 | ||
49 | typedef uint8_t BYTE; | |
50 | typedef uint16_t U16; | |
51 | typedef uint32_t U32; | |
52 | typedef int32_t S32; | |
53 | typedef uint64_t U64; | |
54 | typedef uintptr_t uptrval; | |
027b728d | 55 | |
26c7fdad HJ |
56 | static FORCE_INLINE void LZ4_write32(void *memPtr, U32 value) |
57 | { | |
58 | put_unaligned(value, (U32 *)memPtr); | |
59 | } | |
027b728d JW |
60 | |
61 | /************************************** | |
62 | * Reading and writing into memory | |
63 | **************************************/ | |
64 | ||
65 | /* customized version of memcpy, which may overwrite up to 7 bytes beyond dstEnd */ | |
66 | static void LZ4_wildCopy(void* dstPtr, const void* srcPtr, void* dstEnd) | |
67 | { | |
68 | BYTE* d = (BYTE*)dstPtr; | |
69 | const BYTE* s = (const BYTE*)srcPtr; | |
70 | BYTE* e = (BYTE*)dstEnd; | |
71 | do { LZ4_copy8(d,s); d+=8; s+=8; } while (d<e); | |
72 | } | |
73 | ||
027b728d JW |
74 | /************************************** |
75 | * Common Constants | |
76 | **************************************/ | |
77 | #define MINMATCH 4 | |
78 | ||
26c7fdad | 79 | #define WILDCOPYLENGTH 8 |
027b728d | 80 | #define LASTLITERALS 5 |
26c7fdad | 81 | #define MFLIMIT (WILDCOPYLENGTH + MINMATCH) |
027b728d | 82 | |
26c7fdad HJ |
83 | /* |
84 | * ensure it's possible to write 2 x wildcopyLength | |
85 | * without overflowing output buffer | |
86 | */ | |
87 | #define MATCH_SAFEGUARD_DISTANCE ((2 * WILDCOPYLENGTH) - MINMATCH) | |
88 | ||
89 | #define KB (1 <<10) | |
027b728d JW |
90 | |
91 | #define MAXD_LOG 16 | |
92 | #define MAX_DISTANCE ((1 << MAXD_LOG) - 1) | |
93 | ||
94 | #define ML_BITS 4 | |
95 | #define ML_MASK ((1U<<ML_BITS)-1) | |
96 | #define RUN_BITS (8-ML_BITS) | |
97 | #define RUN_MASK ((1U<<RUN_BITS)-1) | |
98 | ||
26c7fdad | 99 | #define LZ4_STATIC_ASSERT(c) BUILD_BUG_ON(!(c)) |
027b728d JW |
100 | |
101 | /************************************** | |
102 | * Local Structures and types | |
103 | **************************************/ | |
104 | typedef enum { noDict = 0, withPrefix64k, usingExtDict } dict_directive; | |
105 | typedef enum { endOnOutputSize = 0, endOnInputSize = 1 } endCondition_directive; | |
26c7fdad | 106 | typedef enum { decode_full_block = 0, partial_decode = 1 } earlyEnd_directive; |
027b728d | 107 | |
26c7fdad | 108 | #define DEBUGLOG(l, ...) {} /* disabled */ |
027b728d | 109 | |
26c7fdad HJ |
110 | #ifndef assert |
111 | #define assert(condition) ((void)0) | |
112 | #endif | |
027b728d | 113 | |
027b728d | 114 | /* |
26c7fdad HJ |
115 | * LZ4_decompress_generic() : |
116 | * This generic decompression function covers all use cases. | |
117 | * It shall be instantiated several times, using different sets of directives. | |
118 | * Note that it is important for performance that this function really get inlined, | |
027b728d JW |
119 | * in order to remove useless branches during compilation optimization. |
120 | */ | |
26c7fdad HJ |
121 | static FORCE_INLINE int LZ4_decompress_generic( |
122 | const char * const src, | |
123 | char * const dst, | |
124 | int srcSize, | |
125 | /* | |
126 | * If endOnInput == endOnInputSize, | |
127 | * this value is `dstCapacity` | |
128 | */ | |
129 | int outputSize, | |
130 | /* endOnOutputSize, endOnInputSize */ | |
131 | endCondition_directive endOnInput, | |
132 | /* full, partial */ | |
133 | earlyEnd_directive partialDecoding, | |
134 | /* noDict, withPrefix64k, usingExtDict */ | |
135 | dict_directive dict, | |
136 | /* always <= dst, == dst when no prefix */ | |
137 | const BYTE * const lowPrefix, | |
138 | /* only if dict == usingExtDict */ | |
139 | const BYTE * const dictStart, | |
140 | /* note : = 0 if noDict */ | |
141 | const size_t dictSize | |
142 | ) | |
027b728d | 143 | { |
26c7fdad HJ |
144 | const BYTE *ip = (const BYTE *) src; |
145 | const BYTE * const iend = ip + srcSize; | |
146 | ||
147 | BYTE *op = (BYTE *) dst; | |
148 | BYTE * const oend = op + outputSize; | |
149 | BYTE *cpy; | |
150 | ||
151 | const BYTE * const dictEnd = (const BYTE *)dictStart + dictSize; | |
152 | static const unsigned int inc32table[8] = {0, 1, 2, 1, 0, 4, 4, 4}; | |
153 | static const int dec64table[8] = {0, 0, 0, -1, -4, 1, 2, 3}; | |
154 | ||
155 | const int safeDecode = (endOnInput == endOnInputSize); | |
156 | const int checkOffset = ((safeDecode) && (dictSize < (int)(64 * KB))); | |
157 | ||
158 | /* Set up the "end" pointers for the shortcut. */ | |
159 | const BYTE *const shortiend = iend - | |
160 | (endOnInput ? 14 : 8) /*maxLL*/ - 2 /*offset*/; | |
161 | const BYTE *const shortoend = oend - | |
162 | (endOnInput ? 14 : 8) /*maxLL*/ - 18 /*maxML*/; | |
163 | ||
164 | DEBUGLOG(5, "%s (srcSize:%i, dstSize:%i)", __func__, | |
165 | srcSize, outputSize); | |
166 | ||
167 | /* Special cases */ | |
168 | assert(lowPrefix <= op); | |
169 | assert(src != NULL); | |
170 | ||
171 | /* Empty output buffer */ | |
172 | if ((endOnInput) && (unlikely(outputSize == 0))) | |
173 | return ((srcSize == 1) && (*ip == 0)) ? 0 : -1; | |
174 | ||
175 | if ((!endOnInput) && (unlikely(outputSize == 0))) | |
176 | return (*ip == 0 ? 1 : -1); | |
177 | ||
178 | if ((endOnInput) && unlikely(srcSize == 0)) | |
179 | return -1; | |
180 | ||
181 | /* Main Loop : decode sequences */ | |
182 | while (1) { | |
183 | size_t length; | |
184 | const BYTE *match; | |
185 | size_t offset; | |
186 | ||
187 | /* get literal length */ | |
188 | unsigned int const token = *ip++; | |
189 | length = token>>ML_BITS; | |
190 | ||
191 | /* ip < iend before the increment */ | |
192 | assert(!endOnInput || ip <= iend); | |
193 | ||
194 | /* | |
195 | * A two-stage shortcut for the most common case: | |
196 | * 1) If the literal length is 0..14, and there is enough | |
197 | * space, enter the shortcut and copy 16 bytes on behalf | |
198 | * of the literals (in the fast mode, only 8 bytes can be | |
199 | * safely copied this way). | |
200 | * 2) Further if the match length is 4..18, copy 18 bytes | |
201 | * in a similar manner; but we ensure that there's enough | |
202 | * space in the output for those 18 bytes earlier, upon | |
203 | * entering the shortcut (in other words, there is a | |
204 | * combined check for both stages). | |
205 | * | |
206 | * The & in the likely() below is intentionally not && so that | |
207 | * some compilers can produce better parallelized runtime code | |
208 | */ | |
209 | if ((endOnInput ? length != RUN_MASK : length <= 8) | |
210 | /* | |
211 | * strictly "less than" on input, to re-enter | |
212 | * the loop with at least one byte | |
213 | */ | |
214 | && likely((endOnInput ? ip < shortiend : 1) & | |
215 | (op <= shortoend))) { | |
216 | /* Copy the literals */ | |
217 | memcpy(op, ip, endOnInput ? 16 : 8); | |
218 | op += length; ip += length; | |
219 | ||
220 | /* | |
221 | * The second stage: | |
222 | * prepare for match copying, decode full info. | |
223 | * If it doesn't work out, the info won't be wasted. | |
224 | */ | |
225 | length = token & ML_MASK; /* match length */ | |
226 | offset = LZ4_readLE16(ip); | |
227 | ip += 2; | |
228 | match = op - offset; | |
229 | assert(match <= op); /* check overflow */ | |
230 | ||
231 | /* Do not deal with overlapping matches. */ | |
232 | if ((length != ML_MASK) && | |
233 | (offset >= 8) && | |
234 | (dict == withPrefix64k || match >= lowPrefix)) { | |
235 | /* Copy the match. */ | |
236 | memcpy(op + 0, match + 0, 8); | |
237 | memcpy(op + 8, match + 8, 8); | |
238 | memcpy(op + 16, match + 16, 2); | |
239 | op += length + MINMATCH; | |
240 | /* Both stages worked, load the next token. */ | |
241 | continue; | |
242 | } | |
243 | ||
244 | /* | |
245 | * The second stage didn't work out, but the info | |
246 | * is ready. Propel it right to the point of match | |
247 | * copying. | |
248 | */ | |
249 | goto _copy_match; | |
250 | } | |
251 | ||
252 | /* decode literal length */ | |
253 | if (length == RUN_MASK) { | |
254 | unsigned int s; | |
255 | ||
256 | if (unlikely(endOnInput ? ip >= iend - RUN_MASK : 0)) { | |
257 | /* overflow detection */ | |
258 | goto _output_error; | |
259 | } | |
260 | do { | |
261 | s = *ip++; | |
262 | length += s; | |
263 | } while (likely(endOnInput | |
264 | ? ip < iend - RUN_MASK | |
265 | : 1) & (s == 255)); | |
266 | ||
267 | if ((safeDecode) | |
268 | && unlikely((uptrval)(op) + | |
269 | length < (uptrval)(op))) { | |
270 | /* overflow detection */ | |
271 | goto _output_error; | |
272 | } | |
273 | if ((safeDecode) | |
274 | && unlikely((uptrval)(ip) + | |
275 | length < (uptrval)(ip))) { | |
276 | /* overflow detection */ | |
277 | goto _output_error; | |
278 | } | |
279 | } | |
280 | ||
281 | /* copy literals */ | |
282 | cpy = op + length; | |
283 | LZ4_STATIC_ASSERT(MFLIMIT >= WILDCOPYLENGTH); | |
284 | ||
285 | if (((endOnInput) && ((cpy > oend - MFLIMIT) | |
286 | || (ip + length > iend - (2 + 1 + LASTLITERALS)))) | |
287 | || ((!endOnInput) && (cpy > oend - WILDCOPYLENGTH))) { | |
288 | if (partialDecoding) { | |
289 | if (cpy > oend) { | |
290 | /* | |
291 | * Partial decoding : | |
292 | * stop in the middle of literal segment | |
293 | */ | |
294 | cpy = oend; | |
295 | length = oend - op; | |
296 | } | |
297 | if ((endOnInput) | |
298 | && (ip + length > iend)) { | |
299 | /* | |
300 | * Error : | |
301 | * read attempt beyond | |
302 | * end of input buffer | |
303 | */ | |
304 | goto _output_error; | |
305 | } | |
306 | } else { | |
307 | if ((!endOnInput) | |
308 | && (cpy != oend)) { | |
309 | /* | |
310 | * Error : | |
311 | * block decoding must | |
312 | * stop exactly there | |
313 | */ | |
314 | goto _output_error; | |
315 | } | |
316 | if ((endOnInput) | |
317 | && ((ip + length != iend) | |
318 | || (cpy > oend))) { | |
319 | /* | |
320 | * Error : | |
321 | * input must be consumed | |
322 | */ | |
323 | goto _output_error; | |
324 | } | |
325 | } | |
326 | ||
327 | /* | |
328 | * supports overlapping memory regions; only matters | |
329 | * for in-place decompression scenarios | |
330 | */ | |
331 | memmove(op, ip, length); | |
332 | ip += length; | |
333 | op += length; | |
334 | ||
335 | /* Necessarily EOF, due to parsing restrictions */ | |
336 | if (!partialDecoding || (cpy == oend)) | |
337 | break; | |
338 | } else { | |
339 | /* may overwrite up to WILDCOPYLENGTH beyond cpy */ | |
340 | LZ4_wildCopy(op, ip, cpy); | |
341 | ip += length; | |
342 | op = cpy; | |
343 | } | |
344 | ||
345 | /* get offset */ | |
346 | offset = LZ4_readLE16(ip); | |
347 | ip += 2; | |
348 | match = op - offset; | |
349 | ||
350 | /* get matchlength */ | |
351 | length = token & ML_MASK; | |
352 | ||
353 | _copy_match: | |
354 | if ((checkOffset) && (unlikely(match + dictSize < lowPrefix))) { | |
355 | /* Error : offset outside buffers */ | |
356 | goto _output_error; | |
357 | } | |
358 | ||
359 | /* costs ~1%; silence an msan warning when offset == 0 */ | |
360 | /* | |
361 | * note : when partialDecoding, there is no guarantee that | |
362 | * at least 4 bytes remain available in output buffer | |
363 | */ | |
364 | if (!partialDecoding) { | |
365 | assert(oend > op); | |
366 | assert(oend - op >= 4); | |
367 | ||
368 | LZ4_write32(op, (U32)offset); | |
369 | } | |
370 | ||
371 | if (length == ML_MASK) { | |
372 | unsigned int s; | |
373 | ||
374 | do { | |
375 | s = *ip++; | |
376 | ||
377 | if ((endOnInput) && (ip > iend - LASTLITERALS)) | |
378 | goto _output_error; | |
379 | ||
380 | length += s; | |
381 | } while (s == 255); | |
382 | ||
383 | if ((safeDecode) | |
384 | && unlikely( | |
385 | (uptrval)(op) + length < (uptrval)op)) { | |
386 | /* overflow detection */ | |
387 | goto _output_error; | |
388 | } | |
389 | } | |
390 | ||
391 | length += MINMATCH; | |
392 | ||
393 | /* match starting within external dictionary */ | |
394 | if ((dict == usingExtDict) && (match < lowPrefix)) { | |
395 | if (unlikely(op + length > oend - LASTLITERALS)) { | |
396 | /* doesn't respect parsing restriction */ | |
397 | if (!partialDecoding) | |
398 | goto _output_error; | |
399 | length = min(length, (size_t)(oend - op)); | |
400 | } | |
401 | ||
402 | if (length <= (size_t)(lowPrefix - match)) { | |
403 | /* | |
404 | * match fits entirely within external | |
405 | * dictionary : just copy | |
406 | */ | |
407 | memmove(op, dictEnd - (lowPrefix - match), | |
408 | length); | |
409 | op += length; | |
410 | } else { | |
411 | /* | |
412 | * match stretches into both external | |
413 | * dictionary and current block | |
414 | */ | |
415 | size_t const copySize = (size_t)(lowPrefix - match); | |
416 | size_t const restSize = length - copySize; | |
417 | ||
418 | memcpy(op, dictEnd - copySize, copySize); | |
419 | op += copySize; | |
420 | if (restSize > (size_t)(op - lowPrefix)) { | |
421 | /* overlap copy */ | |
422 | BYTE * const endOfMatch = op + restSize; | |
423 | const BYTE *copyFrom = lowPrefix; | |
424 | ||
425 | while (op < endOfMatch) | |
426 | *op++ = *copyFrom++; | |
427 | } else { | |
428 | memcpy(op, lowPrefix, restSize); | |
429 | op += restSize; | |
430 | } | |
431 | } | |
432 | continue; | |
433 | } | |
434 | ||
435 | /* copy match within block */ | |
436 | cpy = op + length; | |
437 | ||
438 | /* | |
439 | * partialDecoding : | |
440 | * may not respect endBlock parsing restrictions | |
441 | */ | |
442 | assert(op <= oend); | |
443 | if (partialDecoding && | |
444 | (cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { | |
445 | size_t const mlen = min(length, (size_t)(oend - op)); | |
446 | const BYTE * const matchEnd = match + mlen; | |
447 | BYTE * const copyEnd = op + mlen; | |
448 | ||
449 | if (matchEnd > op) { | |
450 | /* overlap copy */ | |
451 | while (op < copyEnd) | |
452 | *op++ = *match++; | |
453 | } else { | |
454 | memcpy(op, match, mlen); | |
455 | } | |
456 | op = copyEnd; | |
457 | if (op == oend) | |
458 | break; | |
459 | continue; | |
460 | } | |
461 | ||
462 | if (unlikely(offset < 8)) { | |
463 | op[0] = match[0]; | |
464 | op[1] = match[1]; | |
465 | op[2] = match[2]; | |
466 | op[3] = match[3]; | |
467 | match += inc32table[offset]; | |
468 | memcpy(op + 4, match, 4); | |
469 | match -= dec64table[offset]; | |
470 | } else { | |
471 | LZ4_copy8(op, match); | |
472 | match += 8; | |
473 | } | |
474 | ||
475 | op += 8; | |
476 | ||
477 | if (unlikely(cpy > oend - MATCH_SAFEGUARD_DISTANCE)) { | |
478 | BYTE * const oCopyLimit = oend - (WILDCOPYLENGTH - 1); | |
479 | ||
480 | if (cpy > oend - LASTLITERALS) { | |
481 | /* | |
482 | * Error : last LASTLITERALS bytes | |
483 | * must be literals (uncompressed) | |
484 | */ | |
485 | goto _output_error; | |
486 | } | |
487 | ||
488 | if (op < oCopyLimit) { | |
489 | LZ4_wildCopy(op, match, oCopyLimit); | |
490 | match += oCopyLimit - op; | |
491 | op = oCopyLimit; | |
492 | } | |
493 | while (op < cpy) | |
494 | *op++ = *match++; | |
495 | } else { | |
496 | LZ4_copy8(op, match); | |
497 | if (length > 16) | |
498 | LZ4_wildCopy(op + 8, match + 8, cpy); | |
499 | } | |
500 | op = cpy; /* wildcopy correction */ | |
501 | } | |
502 | ||
503 | /* end of decoding */ | |
504 | if (endOnInput) { | |
505 | /* Nb of output bytes decoded */ | |
506 | return (int) (((char *)op) - dst); | |
507 | } else { | |
508 | /* Nb of input bytes read */ | |
509 | return (int) (((const char *)ip) - src); | |
510 | } | |
511 | ||
512 | /* Overflow error detected */ | |
027b728d | 513 | _output_error: |
26c7fdad HJ |
514 | return (int) (-(((const char *)ip) - src)) - 1; |
515 | } | |
516 | ||
517 | int LZ4_decompress_safe(const char *source, char *dest, | |
518 | int compressedSize, int maxDecompressedSize) | |
519 | { | |
520 | return LZ4_decompress_generic(source, dest, | |
521 | compressedSize, maxDecompressedSize, | |
522 | endOnInputSize, decode_full_block, | |
523 | noDict, (BYTE *)dest, NULL, 0); | |
524 | } | |
525 | ||
526 | int LZ4_decompress_safe_partial(const char *src, char *dst, | |
527 | int compressedSize, int targetOutputSize, int dstCapacity) | |
528 | { | |
529 | dstCapacity = min(targetOutputSize, dstCapacity); | |
530 | return LZ4_decompress_generic(src, dst, compressedSize, dstCapacity, | |
531 | endOnInputSize, partial_decode, | |
532 | noDict, (BYTE *)dst, NULL, 0); | |
027b728d | 533 | } |