src/third-party/lzma/XzEnc.c (view raw)
1/* XzEnc.c -- Xz Encode
22014-12-30 : Igor Pavlov : Public domain */
3
4#include "Precomp.h"
5
6#include <stdlib.h>
7#include <string.h>
8
9#include "7zCrc.h"
10#include "Alloc.h"
11#include "Bra.h"
12#include "CpuArch.h"
13#ifdef USE_SUBBLOCK
14#include "Bcj3Enc.c"
15#include "SbFind.c"
16#include "SbEnc.c"
17#endif
18
19#include "XzEnc.h"
20
21static void *SzBigAlloc(void *p, size_t size) { p = p; return BigAlloc(size); }
22static void SzBigFree(void *p, void *address) { p = p; BigFree(address); }
23static ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree };
24
25static void *SzAlloc(void *p, size_t size) { p = p; return MyAlloc(size); }
26static void SzFree(void *p, void *address) { p = p; MyFree(address); }
27static ISzAlloc g_Alloc = { SzAlloc, SzFree };
28
29#define XzBlock_ClearFlags(p) (p)->flags = 0;
30#define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1);
31#define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE;
32#define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE;
33
34static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size)
35{
36 return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE;
37}
38
39static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc)
40{
41 *crc = CrcUpdate(*crc, buf, size);
42 return WriteBytes(s, buf, size);
43}
44
45SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s)
46{
47 UInt32 crc;
48 Byte header[XZ_STREAM_HEADER_SIZE];
49 memcpy(header, XZ_SIG, XZ_SIG_SIZE);
50 header[XZ_SIG_SIZE] = (Byte)(f >> 8);
51 header[XZ_SIG_SIZE + 1] = (Byte)(f & 0xFF);
52 crc = CrcCalc(header + XZ_SIG_SIZE, XZ_STREAM_FLAGS_SIZE);
53 SetUi32(header + XZ_SIG_SIZE + XZ_STREAM_FLAGS_SIZE, crc);
54 return WriteBytes(s, header, XZ_STREAM_HEADER_SIZE);
55}
56
57SRes XzBlock_WriteHeader(const CXzBlock *p, ISeqOutStream *s)
58{
59 Byte header[XZ_BLOCK_HEADER_SIZE_MAX];
60
61 unsigned pos = 1;
62 int numFilters, i;
63 header[pos++] = p->flags;
64
65 if (XzBlock_HasPackSize(p)) pos += Xz_WriteVarInt(header + pos, p->packSize);
66 if (XzBlock_HasUnpackSize(p)) pos += Xz_WriteVarInt(header + pos, p->unpackSize);
67 numFilters = XzBlock_GetNumFilters(p);
68 for (i = 0; i < numFilters; i++)
69 {
70 const CXzFilter *f = &p->filters[i];
71 pos += Xz_WriteVarInt(header + pos, f->id);
72 pos += Xz_WriteVarInt(header + pos, f->propsSize);
73 memcpy(header + pos, f->props, f->propsSize);
74 pos += f->propsSize;
75 }
76 while((pos & 3) != 0)
77 header[pos++] = 0;
78 header[0] = (Byte)(pos >> 2);
79 SetUi32(header + pos, CrcCalc(header, pos));
80 return WriteBytes(s, header, pos + 4);
81}
82
83SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s)
84{
85 Byte buf[32];
86 UInt64 globalPos;
87 {
88 UInt32 crc = CRC_INIT_VAL;
89 unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks);
90 size_t i;
91
92 globalPos = pos;
93 buf[0] = 0;
94 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
95 for (i = 0; i < p->numBlocks; i++)
96 {
97 const CXzBlockSizes *block = &p->blocks[i];
98 pos = Xz_WriteVarInt(buf, block->totalSize);
99 pos += Xz_WriteVarInt(buf + pos, block->unpackSize);
100 globalPos += pos;
101 RINOK(WriteBytesAndCrc(s, buf, pos, &crc));
102 }
103 pos = ((unsigned)globalPos & 3);
104 if (pos != 0)
105 {
106 buf[0] = buf[1] = buf[2] = 0;
107 RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc));
108 globalPos += 4 - pos;
109 }
110 {
111 SetUi32(buf, CRC_GET_DIGEST(crc));
112 RINOK(WriteBytes(s, buf, 4));
113 globalPos += 4;
114 }
115 }
116
117 {
118 UInt32 indexSize = (UInt32)((globalPos >> 2) - 1);
119 SetUi32(buf + 4, indexSize);
120 buf[8] = (Byte)(p->flags >> 8);
121 buf[9] = (Byte)(p->flags & 0xFF);
122 SetUi32(buf, CrcCalc(buf + 4, 6));
123 memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE);
124 return WriteBytes(s, buf, 12);
125 }
126}
127
128SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc)
129{
130 if (p->blocks == 0 || p->numBlocksAllocated == p->numBlocks)
131 {
132 size_t num = (p->numBlocks + 1) * 2;
133 size_t newSize = sizeof(CXzBlockSizes) * num;
134 CXzBlockSizes *blocks;
135 if (newSize / sizeof(CXzBlockSizes) != num)
136 return SZ_ERROR_MEM;
137 blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize);
138 if (blocks == 0)
139 return SZ_ERROR_MEM;
140 if (p->numBlocks != 0)
141 {
142 memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes));
143 Xz_Free(p, alloc);
144 }
145 p->blocks = blocks;
146 p->numBlocksAllocated = num;
147 }
148 {
149 CXzBlockSizes *block = &p->blocks[p->numBlocks++];
150 block->totalSize = totalSize;
151 block->unpackSize = unpackSize;
152 }
153 return SZ_OK;
154}
155
156/* ---------- CSeqCheckInStream ---------- */
157
158typedef struct
159{
160 ISeqInStream p;
161 ISeqInStream *realStream;
162 UInt64 processed;
163 CXzCheck check;
164} CSeqCheckInStream;
165
166void SeqCheckInStream_Init(CSeqCheckInStream *p, int mode)
167{
168 p->processed = 0;
169 XzCheck_Init(&p->check, mode);
170}
171
172void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)
173{
174 XzCheck_Final(&p->check, digest);
175}
176
177static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size)
178{
179 CSeqCheckInStream *p = (CSeqCheckInStream *)pp;
180 SRes res = p->realStream->Read(p->realStream, data, size);
181 XzCheck_Update(&p->check, data, *size);
182 p->processed += *size;
183 return res;
184}
185
186/* ---------- CSeqSizeOutStream ---------- */
187
188typedef struct
189{
190 ISeqOutStream p;
191 ISeqOutStream *realStream;
192 UInt64 processed;
193} CSeqSizeOutStream;
194
195static size_t MyWrite(void *pp, const void *data, size_t size)
196{
197 CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp;
198 size = p->realStream->Write(p->realStream, data, size);
199 p->processed += size;
200 return size;
201}
202
203/* ---------- CSeqInFilter ---------- */
204
205#define FILTER_BUF_SIZE (1 << 20)
206
207typedef struct
208{
209 ISeqInStream p;
210 ISeqInStream *realStream;
211 IStateCoder StateCoder;
212 Byte *buf;
213 size_t curPos;
214 size_t endPos;
215 int srcWasFinished;
216} CSeqInFilter;
217
218static SRes SeqInFilter_Read(void *pp, void *data, size_t *size)
219{
220 CSeqInFilter *p = (CSeqInFilter *)pp;
221 size_t sizeOriginal = *size;
222 if (sizeOriginal == 0)
223 return SZ_OK;
224 *size = 0;
225 for (;;)
226 {
227 if (!p->srcWasFinished && p->curPos == p->endPos)
228 {
229 p->curPos = 0;
230 p->endPos = FILTER_BUF_SIZE;
231 RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos));
232 if (p->endPos == 0)
233 p->srcWasFinished = 1;
234 }
235 {
236 SizeT srcLen = p->endPos - p->curPos;
237 int wasFinished;
238 SRes res;
239 *size = sizeOriginal;
240 res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen,
241 p->srcWasFinished, CODER_FINISH_ANY, &wasFinished);
242 p->curPos += srcLen;
243 if (*size != 0 || srcLen == 0 || res != 0)
244 return res;
245 }
246 }
247}
248
249static void SeqInFilter_Construct(CSeqInFilter *p)
250{
251 p->buf = NULL;
252 p->p.Read = SeqInFilter_Read;
253}
254
255static void SeqInFilter_Free(CSeqInFilter *p)
256{
257 if (p->buf)
258 {
259 g_Alloc.Free(&g_Alloc, p->buf);
260 p->buf = NULL;
261 }
262}
263
264SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc);
265
266static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props)
267{
268 if (!p->buf)
269 {
270 p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE);
271 if (!p->buf)
272 return SZ_ERROR_MEM;
273 }
274 p->curPos = p->endPos = 0;
275 p->srcWasFinished = 0;
276 RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc));
277 RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc));
278 p->StateCoder.Init(p->StateCoder.p);
279 return SZ_OK;
280}
281
282/* ---------- CSbEncInStream ---------- */
283
284#ifdef USE_SUBBLOCK
285
286typedef struct
287{
288 ISeqInStream p;
289 ISeqInStream *inStream;
290 CSbEnc enc;
291} CSbEncInStream;
292
293static SRes SbEncInStream_Read(void *pp, void *data, size_t *size)
294{
295 CSbEncInStream *p = (CSbEncInStream *)pp;
296 size_t sizeOriginal = *size;
297 if (sizeOriginal == 0)
298 return S_OK;
299 for (;;)
300 {
301 if (p->enc.needRead && !p->enc.readWasFinished)
302 {
303 size_t processed = p->enc.needReadSizeMax;
304 RINOK(p->inStream->Read(p->inStream, p->enc.buf + p->enc.readPos, &processed));
305 p->enc.readPos += processed;
306 if (processed == 0)
307 {
308 p->enc.readWasFinished = True;
309 p->enc.isFinalFinished = True;
310 }
311 p->enc.needRead = False;
312 }
313 *size = sizeOriginal;
314 RINOK(SbEnc_Read(&p->enc, data, size));
315 if (*size != 0 || !p->enc.needRead)
316 return S_OK;
317 }
318}
319
320void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc)
321{
322 SbEnc_Construct(&p->enc, alloc);
323 p->p.Read = SbEncInStream_Read;
324}
325
326SRes SbEncInStream_Init(CSbEncInStream *p)
327{
328 return SbEnc_Init(&p->enc);
329}
330
331void SbEncInStream_Free(CSbEncInStream *p)
332{
333 SbEnc_Free(&p->enc);
334}
335
336#endif
337
338
339typedef struct
340{
341 CLzma2EncHandle lzma2;
342 #ifdef USE_SUBBLOCK
343 CSbEncInStream sb;
344 #endif
345 CSeqInFilter filter;
346 ISzAlloc *alloc;
347 ISzAlloc *bigAlloc;
348} CLzma2WithFilters;
349
350
351static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc)
352{
353 p->alloc = alloc;
354 p->bigAlloc = bigAlloc;
355 p->lzma2 = NULL;
356 #ifdef USE_SUBBLOCK
357 SbEncInStream_Construct(&p->sb, alloc);
358 #endif
359 SeqInFilter_Construct(&p->filter);
360}
361
362static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p)
363{
364 p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc);
365 if (p->lzma2 == 0)
366 return SZ_ERROR_MEM;
367 return SZ_OK;
368}
369
370static void Lzma2WithFilters_Free(CLzma2WithFilters *p)
371{
372 SeqInFilter_Free(&p->filter);
373 #ifdef USE_SUBBLOCK
374 SbEncInStream_Free(&p->sb);
375 #endif
376 if (p->lzma2)
377 {
378 Lzma2Enc_Destroy(p->lzma2);
379 p->lzma2 = NULL;
380 }
381}
382
383void XzProps_Init(CXzProps *p)
384{
385 p->lzma2Props = 0;
386 p->filterProps = 0;
387 p->checkId = XZ_CHECK_CRC32;
388}
389
390void XzFilterProps_Init(CXzFilterProps *p)
391{
392 p->id = 0;
393 p->delta = 0;
394 p->ip= 0;
395 p->ipDefined = False;
396}
397
398static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf,
399 ISeqOutStream *outStream, ISeqInStream *inStream,
400 const CXzProps *props, ICompressProgress *progress)
401{
402 xz->flags = (Byte)props->checkId;
403
404 RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props));
405 RINOK(Xz_WriteHeader(xz->flags, outStream));
406
407 {
408 CSeqCheckInStream checkInStream;
409 CSeqSizeOutStream seqSizeOutStream;
410 CXzBlock block;
411 int filterIndex = 0;
412 CXzFilter *filter = NULL;
413 const CXzFilterProps *fp = props->filterProps;
414
415 XzBlock_ClearFlags(&block);
416 XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0));
417
418 if (fp)
419 {
420 filter = &block.filters[filterIndex++];
421 filter->id = fp->id;
422 filter->propsSize = 0;
423 if (fp->id == XZ_ID_Delta)
424 {
425 filter->props[0] = (Byte)(fp->delta - 1);
426 filter->propsSize = 1;
427 }
428 else if (fp->ipDefined)
429 {
430 SetUi32(filter->props, fp->ip);
431 filter->propsSize = 4;
432 }
433 }
434
435 {
436 CXzFilter *f = &block.filters[filterIndex++];
437 f->id = XZ_ID_LZMA2;
438 f->propsSize = 1;
439 f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2);
440 }
441
442 seqSizeOutStream.p.Write = MyWrite;
443 seqSizeOutStream.realStream = outStream;
444 seqSizeOutStream.processed = 0;
445
446 RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p));
447
448 checkInStream.p.Read = SeqCheckInStream_Read;
449 checkInStream.realStream = inStream;
450 SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags));
451
452 if (fp)
453 {
454 #ifdef USE_SUBBLOCK
455 if (fp->id == XZ_ID_Subblock)
456 {
457 lzmaf->sb.inStream = &checkInStream.p;
458 RINOK(SbEncInStream_Init(&lzmaf->sb));
459 }
460 else
461 #endif
462 {
463 lzmaf->filter.realStream = &checkInStream.p;
464 RINOK(SeqInFilter_Init(&lzmaf->filter, filter));
465 }
466 }
467
468 {
469 UInt64 packPos = seqSizeOutStream.processed;
470 SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p,
471 fp ?
472 #ifdef USE_SUBBLOCK
473 (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p:
474 #endif
475 &lzmaf->filter.p:
476 &checkInStream.p,
477 progress);
478 RINOK(res);
479 block.unpackSize = checkInStream.processed;
480 block.packSize = seqSizeOutStream.processed - packPos;
481 }
482
483 {
484 unsigned padSize = 0;
485 Byte buf[128];
486 while((((unsigned)block.packSize + padSize) & 3) != 0)
487 buf[padSize++] = 0;
488 SeqCheckInStream_GetDigest(&checkInStream, buf + padSize);
489 RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags)));
490 RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc));
491 }
492 }
493 return Xz_WriteFooter(xz, outStream);
494}
495
496SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream,
497 const CXzProps *props, ICompressProgress *progress)
498{
499 SRes res;
500 CXzStream xz;
501 CLzma2WithFilters lzmaf;
502 Xz_Construct(&xz);
503 Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc);
504 res = Lzma2WithFilters_Create(&lzmaf);
505 if (res == SZ_OK)
506 res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress);
507 Lzma2WithFilters_Free(&lzmaf);
508 Xz_Free(&xz, &g_Alloc);
509 return res;
510}
511
512SRes Xz_EncodeEmpty(ISeqOutStream *outStream)
513{
514 SRes res;
515 CXzStream xz;
516 Xz_Construct(&xz);
517 res = Xz_WriteHeader(xz.flags, outStream);
518 if (res == SZ_OK)
519 res = Xz_WriteFooter(&xz, outStream);
520 Xz_Free(&xz, &g_Alloc);
521 return res;
522}