xz-utils (5.4.5)
1 /**
2 * \file lzma/lzma12.h
3 * \brief LZMA1 and LZMA2 filters
4 * \note Never include this file directly. Use <lzma.h> instead.
5 */
6
7 /*
8 * Author: Lasse Collin
9 *
10 * This file has been put into the public domain.
11 * You can do whatever you want with this file.
12 */
13
14 #ifndef LZMA_H_INTERNAL
15 # error Never include this file directly. Use <lzma.h> instead.
16 #endif
17
18
19 /**
20 * \brief LZMA1 Filter ID (for raw encoder/decoder only, not in .xz)
21 *
22 * LZMA1 is the very same thing as what was called just LZMA in LZMA Utils,
23 * 7-Zip, and LZMA SDK. It's called LZMA1 here to prevent developers from
24 * accidentally using LZMA when they actually want LZMA2.
25 */
26 #define LZMA_FILTER_LZMA1 LZMA_VLI_C(0x4000000000000001)
27
28 /**
29 * \brief LZMA1 Filter ID with extended options (for raw encoder/decoder)
30 *
31 * This is like LZMA_FILTER_LZMA1 but with this ID a few extra options
32 * are supported in the lzma_options_lzma structure:
33 *
34 * - A flag to tell the encoder if the end of payload marker (EOPM) alias
35 * end of stream (EOS) marker must be written at the end of the stream.
36 * In contrast, LZMA_FILTER_LZMA1 always writes the end marker.
37 *
38 * - Decoder needs to be told the uncompressed size of the stream
39 * or that it is unknown (using the special value UINT64_MAX).
40 * If the size is known, a flag can be set to allow the presence of
41 * the end marker anyway. In contrast, LZMA_FILTER_LZMA1 always
42 * behaves as if the uncompressed size was unknown.
43 *
44 * This allows handling file formats where LZMA1 streams are used but where
45 * the end marker isn't allowed or where it might not (always) be present.
46 * This extended LZMA1 functionality is provided as a Filter ID for raw
47 * encoder and decoder instead of adding new encoder and decoder initialization
48 * functions because this way it is possible to also use extra filters,
49 * for example, LZMA_FILTER_X86 in a filter chain with LZMA_FILTER_LZMA1EXT,
50 * which might be needed to handle some file formats.
51 */
52 #define LZMA_FILTER_LZMA1EXT LZMA_VLI_C(0x4000000000000002)
53
54 /**
55 * \brief LZMA2 Filter ID
56 *
57 * Usually you want this instead of LZMA1. Compared to LZMA1, LZMA2 adds
58 * support for LZMA_SYNC_FLUSH, uncompressed chunks (smaller expansion
59 * when trying to compress incompressible data), possibility to change
60 * lc/lp/pb in the middle of encoding, and some other internal improvements.
61 */
62 #define LZMA_FILTER_LZMA2 LZMA_VLI_C(0x21)
63
64
65 /**
66 * \brief Match finders
67 *
68 * Match finder has major effect on both speed and compression ratio.
69 * Usually hash chains are faster than binary trees.
70 *
71 * If you will use LZMA_SYNC_FLUSH often, the hash chains may be a better
72 * choice, because binary trees get much higher compression ratio penalty
73 * with LZMA_SYNC_FLUSH.
74 *
75 * The memory usage formulas are only rough estimates, which are closest to
76 * reality when dict_size is a power of two. The formulas are more complex
77 * in reality, and can also change a little between liblzma versions. Use
78 * lzma_raw_encoder_memusage() to get more accurate estimate of memory usage.
79 */
80 typedef enum {
81 LZMA_MF_HC3 = 0x03,
82 /**<
83 * \brief Hash Chain with 2- and 3-byte hashing
84 *
85 * Minimum nice_len: 3
86 *
87 * Memory usage:
88 * - dict_size <= 16 MiB: dict_size * 7.5
89 * - dict_size > 16 MiB: dict_size * 5.5 + 64 MiB
90 */
91
92 LZMA_MF_HC4 = 0x04,
93 /**<
94 * \brief Hash Chain with 2-, 3-, and 4-byte hashing
95 *
96 * Minimum nice_len: 4
97 *
98 * Memory usage:
99 * - dict_size <= 32 MiB: dict_size * 7.5
100 * - dict_size > 32 MiB: dict_size * 6.5
101 */
102
103 LZMA_MF_BT2 = 0x12,
104 /**<
105 * \brief Binary Tree with 2-byte hashing
106 *
107 * Minimum nice_len: 2
108 *
109 * Memory usage: dict_size * 9.5
110 */
111
112 LZMA_MF_BT3 = 0x13,
113 /**<
114 * \brief Binary Tree with 2- and 3-byte hashing
115 *
116 * Minimum nice_len: 3
117 *
118 * Memory usage:
119 * - dict_size <= 16 MiB: dict_size * 11.5
120 * - dict_size > 16 MiB: dict_size * 9.5 + 64 MiB
121 */
122
123 LZMA_MF_BT4 = 0x14
124 /**<
125 * \brief Binary Tree with 2-, 3-, and 4-byte hashing
126 *
127 * Minimum nice_len: 4
128 *
129 * Memory usage:
130 * - dict_size <= 32 MiB: dict_size * 11.5
131 * - dict_size > 32 MiB: dict_size * 10.5
132 */
133 } lzma_match_finder;
134
135
136 /**
137 * \brief Test if given match finder is supported
138 *
139 * It is safe to call this with a value that isn't listed in
140 * lzma_match_finder enumeration; the return value will be false.
141 *
142 * There is no way to list which match finders are available in this
143 * particular liblzma version and build. It would be useless, because
144 * a new match finder, which the application developer wasn't aware,
145 * could require giving additional options to the encoder that the older
146 * match finders don't need.
147 *
148 * \param match_finder Match finder ID
149 *
150 * \return lzma_bool:
151 * - true if the match finder is supported by this liblzma build.
152 * - false otherwise.
153 */
154 extern LZMA_API(lzma_bool) lzma_mf_is_supported(lzma_match_finder match_finder)
155 lzma_nothrow lzma_attr_const;
156
157
158 /**
159 * \brief Compression modes
160 *
161 * This selects the function used to analyze the data produced by the match
162 * finder.
163 */
164 typedef enum {
165 LZMA_MODE_FAST = 1,
166 /**<
167 * \brief Fast compression
168 *
169 * Fast mode is usually at its best when combined with
170 * a hash chain match finder.
171 */
172
173 LZMA_MODE_NORMAL = 2
174 /**<
175 * \brief Normal compression
176 *
177 * This is usually notably slower than fast mode. Use this
178 * together with binary tree match finders to expose the
179 * full potential of the LZMA1 or LZMA2 encoder.
180 */
181 } lzma_mode;
182
183
184 /**
185 * \brief Test if given compression mode is supported
186 *
187 * It is safe to call this with a value that isn't listed in lzma_mode
188 * enumeration; the return value will be false.
189 *
190 * There is no way to list which modes are available in this particular
191 * liblzma version and build. It would be useless, because a new compression
192 * mode, which the application developer wasn't aware, could require giving
193 * additional options to the encoder that the older modes don't need.
194 *
195 * \param mode Mode ID.
196 *
197 * \return lzma_bool:
198 * - true if the compression mode is supported by this liblzma
199 * build.
200 * - false otherwise.
201 */
202 extern LZMA_API(lzma_bool) lzma_mode_is_supported(lzma_mode mode)
203 lzma_nothrow lzma_attr_const;
204
205
206 /**
207 * \brief Options specific to the LZMA1 and LZMA2 filters
208 *
209 * Since LZMA1 and LZMA2 share most of the code, it's simplest to share
210 * the options structure too. For encoding, all but the reserved variables
211 * need to be initialized unless specifically mentioned otherwise.
212 * lzma_lzma_preset() can be used to get a good starting point.
213 *
214 * For raw decoding, both LZMA1 and LZMA2 need dict_size, preset_dict, and
215 * preset_dict_size (if preset_dict != NULL). LZMA1 needs also lc, lp, and pb.
216 */
217 typedef struct {
218 /**
219 * \brief Dictionary size in bytes
220 *
221 * Dictionary size indicates how many bytes of the recently processed
222 * uncompressed data is kept in memory. One method to reduce size of
223 * the uncompressed data is to store distance-length pairs, which
224 * indicate what data to repeat from the dictionary buffer. Thus,
225 * the bigger the dictionary, the better the compression ratio
226 * usually is.
227 *
228 * Maximum size of the dictionary depends on multiple things:
229 * - Memory usage limit
230 * - Available address space (not a problem on 64-bit systems)
231 * - Selected match finder (encoder only)
232 *
233 * Currently the maximum dictionary size for encoding is 1.5 GiB
234 * (i.e. (UINT32_C(1) << 30) + (UINT32_C(1) << 29)) even on 64-bit
235 * systems for certain match finder implementation reasons. In the
236 * future, there may be match finders that support bigger
237 * dictionaries.
238 *
239 * Decoder already supports dictionaries up to 4 GiB - 1 B (i.e.
240 * UINT32_MAX), so increasing the maximum dictionary size of the
241 * encoder won't cause problems for old decoders.
242 *
243 * Because extremely small dictionaries sizes would have unneeded
244 * overhead in the decoder, the minimum dictionary size is 4096 bytes.
245 *
246 * \note When decoding, too big dictionary does no other harm
247 * than wasting memory.
248 */
249 uint32_t dict_size;
250 # define LZMA_DICT_SIZE_MIN UINT32_C(4096)
251 # define LZMA_DICT_SIZE_DEFAULT (UINT32_C(1) << 23)
252
253 /**
254 * \brief Pointer to an initial dictionary
255 *
256 * It is possible to initialize the LZ77 history window using
257 * a preset dictionary. It is useful when compressing many
258 * similar, relatively small chunks of data independently from
259 * each other. The preset dictionary should contain typical
260 * strings that occur in the files being compressed. The most
261 * probable strings should be near the end of the preset dictionary.
262 *
263 * This feature should be used only in special situations. For
264 * now, it works correctly only with raw encoding and decoding.
265 * Currently none of the container formats supported by
266 * liblzma allow preset dictionary when decoding, thus if
267 * you create a .xz or .lzma file with preset dictionary, it
268 * cannot be decoded with the regular decoder functions. In the
269 * future, the .xz format will likely get support for preset
270 * dictionary though.
271 */
272 const uint8_t *preset_dict;
273
274 /**
275 * \brief Size of the preset dictionary
276 *
277 * Specifies the size of the preset dictionary. If the size is
278 * bigger than dict_size, only the last dict_size bytes are
279 * processed.
280 *
281 * This variable is read only when preset_dict is not NULL.
282 * If preset_dict is not NULL but preset_dict_size is zero,
283 * no preset dictionary is used (identical to only setting
284 * preset_dict to NULL).
285 */
286 uint32_t preset_dict_size;
287
288 /**
289 * \brief Number of literal context bits
290 *
291 * How many of the highest bits of the previous uncompressed
292 * eight-bit byte (also known as `literal') are taken into
293 * account when predicting the bits of the next literal.
294 *
295 * E.g. in typical English text, an upper-case letter is
296 * often followed by a lower-case letter, and a lower-case
297 * letter is usually followed by another lower-case letter.
298 * In the US-ASCII character set, the highest three bits are 010
299 * for upper-case letters and 011 for lower-case letters.
300 * When lc is at least 3, the literal coding can take advantage of
301 * this property in the uncompressed data.
302 *
303 * There is a limit that applies to literal context bits and literal
304 * position bits together: lc + lp <= 4. Without this limit the
305 * decoding could become very slow, which could have security related
306 * results in some cases like email servers doing virus scanning.
307 * This limit also simplifies the internal implementation in liblzma.
308 *
309 * There may be LZMA1 streams that have lc + lp > 4 (maximum possible
310 * lc would be 8). It is not possible to decode such streams with
311 * liblzma.
312 */
313 uint32_t lc;
314 # define LZMA_LCLP_MIN 0
315 # define LZMA_LCLP_MAX 4
316 # define LZMA_LC_DEFAULT 3
317
318 /**
319 * \brief Number of literal position bits
320 *
321 * lp affects what kind of alignment in the uncompressed data is
322 * assumed when encoding literals. A literal is a single 8-bit byte.
323 * See pb below for more information about alignment.
324 */
325 uint32_t lp;
326 # define LZMA_LP_DEFAULT 0
327
328 /**
329 * \brief Number of position bits
330 *
331 * pb affects what kind of alignment in the uncompressed data is
332 * assumed in general. The default means four-byte alignment
333 * (2^ pb =2^2=4), which is often a good choice when there's
334 * no better guess.
335 *
336 * When the alignment is known, setting pb accordingly may reduce
337 * the file size a little. E.g. with text files having one-byte
338 * alignment (US-ASCII, ISO-8859-*, UTF-8), setting pb=0 can
339 * improve compression slightly. For UTF-16 text, pb=1 is a good
340 * choice. If the alignment is an odd number like 3 bytes, pb=0
341 * might be the best choice.
342 *
343 * Even though the assumed alignment can be adjusted with pb and
344 * lp, LZMA1 and LZMA2 still slightly favor 16-byte alignment.
345 * It might be worth taking into account when designing file formats
346 * that are likely to be often compressed with LZMA1 or LZMA2.
347 */
348 uint32_t pb;
349 # define LZMA_PB_MIN 0
350 # define LZMA_PB_MAX 4
351 # define LZMA_PB_DEFAULT 2
352
353 /** Compression mode */
354 lzma_mode mode;
355
356 /**
357 * \brief Nice length of a match
358 *
359 * This determines how many bytes the encoder compares from the match
360 * candidates when looking for the best match. Once a match of at
361 * least nice_len bytes long is found, the encoder stops looking for
362 * better candidates and encodes the match. (Naturally, if the found
363 * match is actually longer than nice_len, the actual length is
364 * encoded; it's not truncated to nice_len.)
365 *
366 * Bigger values usually increase the compression ratio and
367 * compression time. For most files, 32 to 128 is a good value,
368 * which gives very good compression ratio at good speed.
369 *
370 * The exact minimum value depends on the match finder. The maximum
371 * is 273, which is the maximum length of a match that LZMA1 and
372 * LZMA2 can encode.
373 */
374 uint32_t nice_len;
375
376 /** Match finder ID */
377 lzma_match_finder mf;
378
379 /**
380 * \brief Maximum search depth in the match finder
381 *
382 * For every input byte, match finder searches through the hash chain
383 * or binary tree in a loop, each iteration going one step deeper in
384 * the chain or tree. The searching stops if
385 * - a match of at least nice_len bytes long is found;
386 * - all match candidates from the hash chain or binary tree have
387 * been checked; or
388 * - maximum search depth is reached.
389 *
390 * Maximum search depth is needed to prevent the match finder from
391 * wasting too much time in case there are lots of short match
392 * candidates. On the other hand, stopping the search before all
393 * candidates have been checked can reduce compression ratio.
394 *
395 * Setting depth to zero tells liblzma to use an automatic default
396 * value, that depends on the selected match finder and nice_len.
397 * The default is in the range [4, 200] or so (it may vary between
398 * liblzma versions).
399 *
400 * Using a bigger depth value than the default can increase
401 * compression ratio in some cases. There is no strict maximum value,
402 * but high values (thousands or millions) should be used with care:
403 * the encoder could remain fast enough with typical input, but
404 * malicious input could cause the match finder to slow down
405 * dramatically, possibly creating a denial of service attack.
406 */
407 uint32_t depth;
408
409 /**
410 * \brief For LZMA_FILTER_LZMA1EXT: Extended flags
411 *
412 * This is used only with LZMA_FILTER_LZMA1EXT.
413 *
414 * Currently only one flag is supported, LZMA_LZMA1EXT_ALLOW_EOPM:
415 *
416 * - Encoder: If the flag is set, then end marker is written just
417 * like it is with LZMA_FILTER_LZMA1. Without this flag the
418 * end marker isn't written and the application has to store
419 * the uncompressed size somewhere outside the compressed stream.
420 * To decompress streams without the end marker, the application
421 * has to set the correct uncompressed size in ext_size_low and
422 * ext_size_high.
423 *
424 * - Decoder: If the uncompressed size in ext_size_low and
425 * ext_size_high is set to the special value UINT64_MAX
426 * (indicating unknown uncompressed size) then this flag is
427 * ignored and the end marker must always be present, that is,
428 * the behavior is identical to LZMA_FILTER_LZMA1.
429 *
430 * Otherwise, if this flag isn't set, then the input stream
431 * must not have the end marker; if the end marker is detected
432 * then it will result in LZMA_DATA_ERROR. This is useful when
433 * it is known that the stream must not have the end marker and
434 * strict validation is wanted.
435 *
436 * If this flag is set, then it is autodetected if the end marker
437 * is present after the specified number of uncompressed bytes
438 * has been decompressed (ext_size_low and ext_size_high). The
439 * end marker isn't allowed in any other position. This behavior
440 * is useful when uncompressed size is known but the end marker
441 * may or may not be present. This is the case, for example,
442 * in .7z files (valid .7z files that have the end marker in
443 * LZMA1 streams are rare but they do exist).
444 */
445 uint32_t ext_flags;
446 # define LZMA_LZMA1EXT_ALLOW_EOPM UINT32_C(0x01)
447
448 /**
449 * \brief For LZMA_FILTER_LZMA1EXT: Uncompressed size (low bits)
450 *
451 * The 64-bit uncompressed size is needed for decompression with
452 * LZMA_FILTER_LZMA1EXT. The size is ignored by the encoder.
453 *
454 * The special value UINT64_MAX indicates that the uncompressed size
455 * is unknown and that the end of payload marker (also known as
456 * end of stream marker) must be present to indicate the end of
457 * the LZMA1 stream. Any other value indicates the expected
458 * uncompressed size of the LZMA1 stream. (If LZMA1 was used together
459 * with filters that change the size of the data then the uncompressed
460 * size of the LZMA1 stream could be different than the final
461 * uncompressed size of the filtered stream.)
462 *
463 * ext_size_low holds the least significant 32 bits of the
464 * uncompressed size. The most significant 32 bits must be set
465 * in ext_size_high. The macro lzma_ext_size_set(opt_lzma, u64size)
466 * can be used to set these members.
467 *
468 * The 64-bit uncompressed size is split into two uint32_t variables
469 * because there were no reserved uint64_t members and using the
470 * same options structure for LZMA_FILTER_LZMA1, LZMA_FILTER_LZMA1EXT,
471 * and LZMA_FILTER_LZMA2 was otherwise more convenient than having
472 * a new options structure for LZMA_FILTER_LZMA1EXT. (Replacing two
473 * uint32_t members with one uint64_t changes the ABI on some systems
474 * as the alignment of this struct can increase from 4 bytes to 8.)
475 */
476 uint32_t ext_size_low;
477
478 /**
479 * \brief For LZMA_FILTER_LZMA1EXT: Uncompressed size (high bits)
480 *
481 * This holds the most significant 32 bits of the uncompressed size.
482 */
483 uint32_t ext_size_high;
484
485 /*
486 * Reserved space to allow possible future extensions without
487 * breaking the ABI. You should not touch these, because the names
488 * of these variables may change. These are and will never be used
489 * with the currently supported options, so it is safe to leave these
490 * uninitialized.
491 */
492
493 /** \private Reserved member. */
494 uint32_t reserved_int4;
495
496 /** \private Reserved member. */
497 uint32_t reserved_int5;
498
499 /** \private Reserved member. */
500 uint32_t reserved_int6;
501
502 /** \private Reserved member. */
503 uint32_t reserved_int7;
504
505 /** \private Reserved member. */
506 uint32_t reserved_int8;
507
508 /** \private Reserved member. */
509 lzma_reserved_enum reserved_enum1;
510
511 /** \private Reserved member. */
512 lzma_reserved_enum reserved_enum2;
513
514 /** \private Reserved member. */
515 lzma_reserved_enum reserved_enum3;
516
517 /** \private Reserved member. */
518 lzma_reserved_enum reserved_enum4;
519
520 /** \private Reserved member. */
521 void *reserved_ptr1;
522
523 /** \private Reserved member. */
524 void *reserved_ptr2;
525
526 } lzma_options_lzma;
527
528
529 /**
530 * \brief Macro to set the 64-bit uncompressed size in ext_size_*
531 *
532 * This might be convenient when decoding using LZMA_FILTER_LZMA1EXT.
533 * This isn't used with LZMA_FILTER_LZMA1 or LZMA_FILTER_LZMA2.
534 */
535 #define lzma_set_ext_size(opt_lzma2, u64size) \
536 do { \
537 (opt_lzma2).ext_size_low = (uint32_t)(u64size); \
538 (opt_lzma2).ext_size_high = (uint32_t)((uint64_t)(u64size) >> 32); \
539 } while (0)
540
541
542 /**
543 * \brief Set a compression preset to lzma_options_lzma structure
544 *
545 * 0 is the fastest and 9 is the slowest. These match the switches -0 .. -9
546 * of the xz command line tool. In addition, it is possible to bitwise-or
547 * flags to the preset. Currently only LZMA_PRESET_EXTREME is supported.
548 * The flags are defined in container.h, because the flags are used also
549 * with lzma_easy_encoder().
550 *
551 * The preset levels are subject to changes between liblzma versions.
552 *
553 * This function is available only if LZMA1 or LZMA2 encoder has been enabled
554 * when building liblzma.
555 *
556 * If features (like certain match finders) have been disabled at build time,
557 * then the function may return success (false) even though the resulting
558 * LZMA1/LZMA2 options may not be usable for encoder initialization
559 * (LZMA_OPTIONS_ERROR).
560 *
561 * \param[out] options Pointer to LZMA1 or LZMA2 options to be filled
562 * \param preset Preset level bitwse-ORed with preset flags
563 *
564 * \return lzma_bool:
565 * - true if the preset is not supported (failure).
566 * - false otherwise (success).
567 */
568 extern LZMA_API(lzma_bool) lzma_lzma_preset(
569 lzma_options_lzma *options, uint32_t preset) lzma_nothrow;
Browse
Usage
Versions