Merge branch 'master' (early part) into medusa
Vicki Pfau vi@endrift.com
Mon, 17 Feb 2020 16:47:24 -0800
111 files changed,
13050 insertions(+),
3823 deletions(-)
jump to
M
CHANGES
→
CHANGES
@@ -112,13 +112,16 @@ - ARM: Partially fix LDM/STM writeback with empty register list
- ARM: Fix stepping when events are pending - GBA DMA: Fix case where DMAs could get misaligned (fixes mgba.io/i/1092) - GBA Memory: Fix open bus from IWRAM (fixes mgba.io/i/1575) + - GBA Timers: Improve timer startup emulation - GBA Video: Fix OpenGL renderer 512x512 backgrounds (fixes mgba.io/i/1572) - GBA Video: Fix BLDY for semitransparent sprite on non-target-2 backgrounds - GBA Video: Fix effects blending improperly in some non-last windows + - GBA Video: Copy higher priority target 1 flag Other fixes: - 3DS: Fix screen darkening (fixes mgba.io/i/1562) - Core: Fix uninitialized memory issues with graphics caches - Core: Return null for out of bounds cached tile VRAM querying + - Debugger: Fix tracing skipping instructions (fixes mgba.io/i/1614) - OpenGL: Only invalidate texture if dimensions change (fixes mgba.io/i/1612) - Qt: Fix fast forward mute being reset (fixes mgba.io/i/1574) - Qt: Fix scrollbar arrows in memory view (fixes mgba.io/i/1558)@@ -128,16 +131,22 @@ - Qt: Fix "restart needed" dialog after first config (fixes mgba.io/i/1601)
- Qt: Fix undesired screen filtering when paused (fixes mgba.io/i/1602) - Qt: Fix sprite view using wrong base address (fixes mgba.io/i/1603) - Qt: Fix inability to clear default keybindings + - Qt: Release held actions if they get rebound - Vita: Fix analog controls (fixes mgba.io/i/1554) + - Wii: Fix game fast-forwarding after slowing down + - Wii: Improve audio buffering (fixes mgba.io/i/1617) Misc: - GB Memory: Support manual SRAM editing (fixes mgba.io/i/1580) + - GBA: Trim non-movie ROMs to 32 MiB if applicable - GBA Audio: Redo channel 4 batching for GBA only - GBA I/O: Stop logging several harmless invalid register reads - Debugger: Separate aliases from main commands - Debugger: Print break-/watchpoint ID when breaking in CLI + - Debugger: Minor interface cleanup - SDL: Use controller GUID instead of name - SM83: Rename LR35902 to SM83 - Tools: Allow using threaded renderer in perf.py + - Tools: Clean up unused argument in mgba-perf 0.8 beta 1: (2019-10-20) - Initial beta for 0.8
M
include/mgba-util/platform/switch/threading.h
→
include/mgba-util/platform/switch/threading.h
@@ -64,7 +64,7 @@ static inline int ThreadCreate(Thread* thread, ThreadEntry entry, void* context) {
if (!entry || !thread) { return 1; } - int res = threadCreate(thread, entry, context, 0x8000, 0x3B, 1); + int res = threadCreate(thread, entry, context, NULL, 0x8000, 0x3B, 1); if(R_FAILED(res)) { return res; }
M
include/mgba/debugger/debugger.h
→
include/mgba/debugger/debugger.h
@@ -20,17 +20,16 @@ extern const uint32_t DEBUGGER_ID;
enum mDebuggerType { DEBUGGER_NONE = 0, + DEBUGGER_CUSTOM, DEBUGGER_CLI, -#ifdef USE_GDB_STUB DEBUGGER_GDB, -#endif DEBUGGER_MAX }; enum mDebuggerState { DEBUGGER_PAUSED, DEBUGGER_RUNNING, - DEBUGGER_CUSTOM, + DEBUGGER_CALLBACK, DEBUGGER_SHUTDOWN };
M
src/arm/debugger/cli-debugger.c
→
src/arm/debugger/cli-debugger.c
@@ -86,7 +86,7 @@ if (!dv || dv->type != CLIDV_INT_TYPE) {
size = 1; } else { size = dv->intValue; - dv = dv->next; // TODO: Check for excess args + // TODO: Check for excess args } int i;
M
src/debugger/cli-debugger.c
→
src/debugger/cli-debugger.c
@@ -155,7 +155,7 @@ #endif
static void _continue(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { UNUSED(dv); - debugger->d.state = debugger->traceRemaining != 0 ? DEBUGGER_CUSTOM : DEBUGGER_RUNNING; + debugger->d.state = debugger->traceRemaining != 0 ? DEBUGGER_CALLBACK : DEBUGGER_RUNNING; } static void _next(struct CLIDebugger* debugger, struct CLIDebugVector* dv) {@@ -700,7 +700,7 @@ if (dv->next && dv->next->charValue) {
debugger->traceVf = VFileOpen(dv->next->charValue, O_CREAT | O_WRONLY | O_APPEND); } if (_doTrace(debugger)) { - debugger->d.state = DEBUGGER_CUSTOM; + debugger->d.state = DEBUGGER_CALLBACK; } else { debugger->system->printStatus(debugger->system); }@@ -709,7 +709,6 @@
static bool _doTrace(struct CLIDebugger* debugger) { char trace[1024]; trace[sizeof(trace) - 1] = '\0'; - debugger->d.core->step(debugger->d.core); size_t traceSize = sizeof(trace) - 2; debugger->d.platform->trace(debugger->d.platform, trace, &traceSize); if (traceSize + 1 <= sizeof(trace)) {@@ -884,6 +883,7 @@ }
if (dvNext->type == CLIDV_ERROR_TYPE) { debugger->backend->printf(debugger->backend, "Parse error\n"); _DVFree(dv); + _DVFree(dvNext); return 0; }@@ -1044,7 +1044,7 @@ }
if (cliDebugger->system) { retain = cliDebugger->system->custom(cliDebugger->system) && retain; } - if (!retain && debugger->state == DEBUGGER_CUSTOM) { + if (!retain && debugger->state == DEBUGGER_CALLBACK) { debugger->state = next; } }
M
src/debugger/debugger.c
→
src/debugger/debugger.c
@@ -50,13 +50,14 @@ CLIDebuggerCreate(&debugger->cli);
struct CLIDebuggerSystem* sys = core->cliDebuggerSystem(core); CLIDebuggerAttachSystem(&debugger->cli, sys); break; -#ifdef USE_GDB_STUB case DEBUGGER_GDB: +#ifdef USE_GDB_STUB GDBStubCreate(&debugger->gdb); GDBStubListen(&debugger->gdb, 2345, 0); break; #endif case DEBUGGER_NONE: + case DEBUGGER_CUSTOM: case DEBUGGER_MAX: free(debugger); return 0;@@ -89,7 +90,7 @@ debugger->core->step(debugger->core);
debugger->platform->checkBreakpoints(debugger->platform); } break; - case DEBUGGER_CUSTOM: + case DEBUGGER_CALLBACK: debugger->core->step(debugger->core); debugger->platform->checkBreakpoints(debugger->platform); debugger->custom(debugger);
M
src/debugger/gdb-stub.c
→
src/debugger/gdb-stub.c
@@ -205,7 +205,7 @@ _sendMessage(stub);
} static void _continue(struct GDBStub* stub, const char* message) { - stub->d.state = DEBUGGER_CUSTOM; + stub->d.state = DEBUGGER_CALLBACK; stub->untilPoll = GDB_STUB_INTERVAL; // TODO: parse message UNUSED(message);
M
src/gb/audio.c
→
src/gb/audio.c
@@ -936,7 +936,7 @@ cycles *= 8 * audio->timingFactor;
uint32_t last = 0; uint32_t now = cycles; - uint32_t next = cycles - cyclesLate; + int32_t next = cycles - cyclesLate; if (audio->style == GB_AUDIO_GBA) { last = ch->lastEvent;
M
src/gb/core.c
→
src/gb/core.c
@@ -520,7 +520,7 @@ bios->close(bios);
bios = NULL; } } - if (bios) { + if (found && bios) { GBLoadBIOS(gb, bios); } }@@ -757,6 +757,7 @@ #ifdef USE_DEBUGGERS
static bool _GBCoreSupportsDebuggerType(struct mCore* core, enum mDebuggerType type) { UNUSED(core); switch (type) { + case DEBUGGER_CUSTOM: case DEBUGGER_CLI: return true; default:
M
src/gb/debugger/cli.c
→
src/gb/debugger/cli.c
@@ -66,7 +66,7 @@ }
static void _frame(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { UNUSED(dv); - debugger->d.state = DEBUGGER_CUSTOM; + debugger->d.state = DEBUGGER_CALLBACK; struct GBCLIDebugger* gbDebugger = (struct GBCLIDebugger*) debugger->system; gbDebugger->frameAdvance = true;
M
src/gb/extra/proxy.c
→
src/gb/extra/proxy.c
@@ -64,7 +64,7 @@ proxyRenderer->backend->cache = NULL;
} } -static void _reset(struct GBVideoProxyRenderer* proxyRenderer, enum GBModel model) { +static void _reset(struct GBVideoProxyRenderer* proxyRenderer) { memcpy(proxyRenderer->logger->oam, &proxyRenderer->d.oam->raw, GB_SIZE_OAM); memcpy(proxyRenderer->logger->vram, proxyRenderer->d.vram, GB_SIZE_VRAM);@@ -83,7 +83,7 @@ renderer->d.cache = renderer->backend->cache;
renderer->d.vram = video->vram; renderer->d.oam = &video->oam; _init(renderer); - _reset(renderer, video->p->model); + _reset(renderer); } void GBVideoProxyRendererUnshim(struct GBVideo* video, struct GBVideoProxyRenderer* renderer) {
M
src/gb/video.c
→
src/gb/video.c
@@ -187,7 +187,7 @@ renderer->vram = video->vram;
video->renderer->init(video->renderer, video->p->model, video->sgbBorders); } -static bool _statIRQAsserted(struct GBVideo* video, GBRegisterSTAT stat) { +static bool _statIRQAsserted(GBRegisterSTAT stat) { // TODO: variable for the IRQ line value? if (GBRegisterSTATIsLYCIRQ(stat) && GBRegisterSTATIsLYC(stat)) { return true;@@ -236,20 +236,20 @@
mTimingDeschedule(&video->p->timing, &video->frameEvent); mTimingSchedule(&video->p->timing, &video->frameEvent, -cyclesLate); - if (!_statIRQAsserted(video, oldStat) && GBRegisterSTATIsOAMIRQ(video->stat)) { + if (!_statIRQAsserted(oldStat) && GBRegisterSTATIsOAMIRQ(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); } video->p->memory.io[REG_IF] |= (1 << GB_IRQ_VBLANK); } video->stat = GBRegisterSTATSetMode(video->stat, video->mode); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); } // LYC stat is delayed 1 T-cycle oldStat = video->stat; video->stat = GBRegisterSTATSetLYC(video->stat, lyc == video->ly); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); }@@ -287,7 +287,7 @@
GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, video->mode); video->stat = GBRegisterSTATSetLYC(video->stat, lyc == video->p->memory.io[REG_LY]); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); }@@ -305,7 +305,7 @@ video->mode = 3;
video->modeEvent.callback = _endMode3; GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, video->mode); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); }@@ -326,7 +326,7 @@ video->mode = 0;
video->modeEvent.callback = _endMode0; GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, video->mode); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); }@@ -416,7 +416,7 @@ video->p->memory.io[REG_LY] = 0;
GBRegisterSTAT oldStat = video->stat; video->stat = GBRegisterSTATSetMode(video->stat, 0); video->stat = GBRegisterSTATSetLYC(video->stat, video->ly == video->p->memory.io[REG_LYC]); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); }@@ -447,7 +447,7 @@ video->stat = (video->stat & 0x7) | (value & 0x78);
if (!GBRegisterLCDCIsEnable(video->p->memory.io[REG_LCDC]) || video->p->model >= GB_MODEL_CGB) { return; } - if (!_statIRQAsserted(video, oldStat) && video->mode < 3) { + if (!_statIRQAsserted(oldStat) && video->mode < 3) { // TODO: variable for the IRQ line value? video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p);@@ -458,7 +458,7 @@ void GBVideoWriteLYC(struct GBVideo* video, uint8_t value) {
GBRegisterSTAT oldStat = video->stat; if (GBRegisterLCDCIsEnable(video->p->memory.io[REG_LCDC])) { video->stat = GBRegisterSTATSetLYC(video->stat, value == video->ly); - if (!_statIRQAsserted(video, oldStat) && _statIRQAsserted(video, video->stat)) { + if (!_statIRQAsserted(oldStat) && _statIRQAsserted(video->stat)) { video->p->memory.io[REG_IF] |= (1 << GB_IRQ_LCDSTAT); GBUpdateIRQs(video->p); }
M
src/gba/core.c
→
src/gba/core.c
@@ -362,6 +362,7 @@ #if defined(BUILD_GLES2) || defined(BUILD_GLES3)
struct GBACore* gbacore = (struct GBACore*) core; int scale = gbacore->glRenderer.scale; #else + UNUSED(core); int scale = 1; #endif@@ -518,7 +519,7 @@ #if defined(BUILD_GLES2) || defined(BUILD_GLES3)
|| gbacore->glRenderer.outputTex != (unsigned) -1 #endif ) { - struct GBAVideoRenderer* renderer; + struct GBAVideoRenderer* renderer = NULL; if (gbacore->renderer.outputBuffer) { renderer = &gbacore->renderer.d; }@@ -537,13 +538,15 @@ }
} #endif #ifndef MINIMAL_CORE - if (core->videoLogger) { + if (renderer && core->videoLogger) { gbacore->proxyRenderer.logger = core->videoLogger; GBAVideoProxyRendererCreate(&gbacore->proxyRenderer, renderer); renderer = &gbacore->proxyRenderer.d; } #endif - GBAVideoAssociateRenderer(&gba->video, renderer); + if (renderer) { + GBAVideoAssociateRenderer(&gba->video, renderer); + } } #ifndef MINIMAL_CORE@@ -595,7 +598,7 @@ bios->close(bios);
bios = NULL; } } - if (bios) { + if (found && bios) { GBALoadBIOS(gba, bios); } }@@ -841,12 +844,10 @@ #ifdef USE_DEBUGGERS
static bool _GBACoreSupportsDebuggerType(struct mCore* core, enum mDebuggerType type) { UNUSED(core); switch (type) { + case DEBUGGER_CUSTOM: case DEBUGGER_CLI: - return true; -#ifdef USE_GDB_STUB case DEBUGGER_GDB: return true; -#endif default: return false; }
M
src/gba/debugger/cli.c
→
src/gba/debugger/cli.c
@@ -65,7 +65,7 @@ }
static void _frame(struct CLIDebugger* debugger, struct CLIDebugVector* dv) { UNUSED(dv); - debugger->d.state = DEBUGGER_CUSTOM; + debugger->d.state = DEBUGGER_CALLBACK; struct GBACLIDebugger* gbaDebugger = (struct GBACLIDebugger*) debugger->system; gbaDebugger->frameAdvance = true;
M
src/gba/gba.c
→
src/gba/gba.c
@@ -238,8 +238,16 @@ gba->idleDetectionFailures = 0;
gba->debug = false; memset(gba->debugString, 0, sizeof(gba->debugString)); - if (gba->pristineRomSize > SIZE_CART0) { - GBAMatrixReset(gba); + + + if (gba->romVf && gba->pristineRomSize > SIZE_CART0) { + char ident; + gba->romVf->seek(gba->romVf, 0xAC, SEEK_SET); + gba->romVf->read(gba->romVf, &ident, 1); + gba->romVf->seek(gba->romVf, 0, SEEK_SET); + if (ident == 'M') { + GBAMatrixReset(gba); + } } }@@ -376,12 +384,20 @@ gba->pristineRomSize = vf->size(vf);
vf->seek(vf, 0, SEEK_SET); if (gba->pristineRomSize > SIZE_CART0) { gba->isPristine = false; - gba->memory.romSize = 0x01000000; + char ident; + vf->seek(vf, 0xAC, SEEK_SET); + vf->read(vf, &ident, 1); + if (ident == 'M') { + gba->memory.romSize = 0x01000000; #ifdef FIXED_ROM_BUFFER - gba->memory.rom = romBuffer; + gba->memory.rom = romBuffer; #else - gba->memory.rom = anonymousMemoryMap(SIZE_CART0); + gba->memory.rom = anonymousMemoryMap(SIZE_CART0); #endif + } else { + gba->memory.rom = vf->map(vf, SIZE_CART0, MAP_READ); + gba->memory.romSize = SIZE_CART0; + } } else { gba->isPristine = true; gba->memory.rom = vf->map(vf, gba->pristineRomSize, MAP_READ);
M
src/gba/io.c
→
src/gba/io.c
@@ -712,16 +712,16 @@
switch (address) { // Reading this takes two cycles (1N+1I), so let's remove them preemptively case REG_TM0CNT_LO: - GBATimerUpdateRegister(gba, 0, 4); + GBATimerUpdateRegister(gba, 0, 2); break; case REG_TM1CNT_LO: - GBATimerUpdateRegister(gba, 1, 4); + GBATimerUpdateRegister(gba, 1, 2); break; case REG_TM2CNT_LO: - GBATimerUpdateRegister(gba, 2, 4); + GBATimerUpdateRegister(gba, 2, 2); break; case REG_TM3CNT_LO: - GBATimerUpdateRegister(gba, 3, 4); + GBATimerUpdateRegister(gba, 3, 2); break; case REG_KEYINPUT:
M
src/gba/renderers/gl.c
→
src/gba/renderers/gl.c
@@ -585,9 +585,6 @@ "uniform isampler2D backdropFlags;\n"
"out vec4 color;\n" "void composite(vec4 pixel, ivec4 flags, inout vec4 topPixel, inout ivec4 topFlags, inout vec4 bottomPixel, inout ivec4 bottomFlags) {\n" - " if (pixel.a == 0.) {\n" - " return;\n" - " }\n" " if (flags.x >= topFlags.x) {\n" " if (flags.x >= bottomFlags.x) {\n" " return;\n"@@ -607,32 +604,43 @@ " vec4 topPixel = texelFetch(backdrop, ivec2(0, texCoord.y), 0);\n"
" vec4 bottomPixel = topPixel;\n" " ivec4 topFlags = ivec4(texelFetch(backdropFlags, ivec2(0, texCoord.y), 0));\n" " ivec4 bottomFlags = topFlags;\n" - " ivec4 windowFlags = texelFetch(window, ivec2(texCoord * float(scale)), 0);\n" + " ivec2 coord = ivec2(texCoord * float(scale));\n" + " ivec4 windowFlags = texelFetch(window, coord, 0);\n" " int layerWindow = windowFlags.x;\n" " if ((layerWindow & 16) != 0) {\n" - " vec4 pix = texelFetch(layers[4], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[4], ivec2(texCoord * float(scale)), 0));\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[4], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[4], coord, 0));\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 1) != 0) {\n" - " vec4 pix = texelFetch(layers[0], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[0], ivec2(texCoord * float(scale)), 0).xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[0], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[0], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 2) != 0) {\n" - " vec4 pix = texelFetch(layers[1], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[1], ivec2(texCoord * float(scale)), 0).xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[1], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[1], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 4) != 0) {\n" - " vec4 pix = texelFetch(layers[2], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[2], ivec2(texCoord * float(scale)), 0).xyz.xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[2], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[2], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 8) != 0) {\n" - " vec4 pix = texelFetch(layers[3], ivec2(texCoord * float(scale)), 0);\n" - " ivec4 inflags = ivec4(texelFetch(flags[3], ivec2(texCoord * float(scale)), 0).xyz, 0);\n" - " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " vec4 pix = texelFetch(layers[3], coord, 0);\n" + " if (pix.a != 0.) {\n" + " ivec4 inflags = ivec4(texelFetch(flags[3], coord, 0).xyz, 0);\n" + " composite(pix, inflags, topPixel, topFlags, bottomPixel, bottomFlags);\n" + " }\n" " }\n" " if ((layerWindow & 32) == 0) {\n" " topFlags.y &= ~1;\n"@@ -1153,7 +1161,6 @@ GBAVideoGLRendererWriteBGCNT(&glRenderer->bg[3], value);
break; case REG_BLDCNT: GBAVideoGLRendererWriteBLDCNT(glRenderer, value); - value &= 0x3FFF; break; case REG_BLDALPHA: glRenderer->blda = value & 0x1F;@@ -1164,7 +1171,6 @@ glRenderer->bldb = (value >> 8) & 0x1F;
if (glRenderer->bldb > 0x10) { glRenderer->bldb = 0x10; } - value &= 0x1F1F; break; case REG_BLDY: glRenderer->bldy = value;@@ -1503,7 +1509,10 @@ *pixels = glRenderer->temporaryBuffer;
} void GBAVideoGLRendererPutPixels(struct GBAVideoRenderer* renderer, size_t stride, const void* pixels) { - + // TODO + UNUSED(renderer); + UNUSED(stride); + UNUSED(pixels); } static void _enableBg(struct GBAVideoGLRenderer* renderer, int bg, bool active) {
M
src/gba/renderers/software-obj.c
→
src/gba/renderers/software-obj.c
@@ -89,7 +89,7 @@ if ((current & FLAG_ORDER_MASK) > flags) { \
if (tileData) { \ renderer->spriteLayer[outX] = palette[tileData] | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ }@@ -107,7 +107,7 @@ if (tileData) { \
unsigned color = (renderer->row[outX] & FLAG_OBJWIN) ? objwinPalette[tileData] : palette[tileData]; \ renderer->spriteLayer[outX] = color | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ }@@ -139,7 +139,7 @@ if ((current & FLAG_ORDER_MASK) > flags) { \
if (tileData) { \ renderer->spriteLayer[outX] = palette[tileData] | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ }@@ -157,7 +157,7 @@ if (tileData) { \
unsigned color = (renderer->row[outX] & FLAG_OBJWIN) ? objwinPalette[tileData] : palette[tileData]; \ renderer->spriteLayer[outX] = color | flags; \ } else if (current != FLAG_UNWRITTEN) { \ - renderer->spriteLayer[outX] = (current & ~FLAG_ORDER_MASK) | GBAObjAttributesCGetPriority(sprite->c) << OFFSET_PRIORITY; \ + renderer->spriteLayer[outX] = (current & ~(FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)) | (flags & (FLAG_ORDER_MASK | FLAG_REBLEND | FLAG_TARGET_1)); \ } \ }
M
src/gba/timer.c
→
src/gba/timer.c
@@ -8,9 +8,6 @@
#include <mgba/internal/gba/gba.h> #include <mgba/internal/gba/io.h> -#define TIMER_RELOAD_DELAY 0 -#define TIMER_STARTUP_DELAY 2 - #define REG_TMCNT_LO(X) (REG_TM0CNT_LO + ((X) << 2)) static void GBATimerIrq(struct GBA* gba, int timerId, uint32_t cyclesLate) {@@ -24,7 +21,7 @@ void GBATimerUpdate(struct mTiming* timing, struct GBATimer* timer, uint16_t* io, uint32_t cyclesLate) {
if (GBATimerFlagsIsCountUp(timer->flags)) { *io = timer->reload; } else { - GBATimerUpdateRegisterInternal(timer, timing, io, TIMER_RELOAD_DELAY + cyclesLate); + GBATimerUpdateRegisterInternal(timer, timing, io, cyclesLate); } }@@ -182,14 +179,14 @@ if (!wasEnabled && GBATimerFlagsIsEnable(timer->flags)) {
mTimingDeschedule(timing, &timer->event); *io = timer->reload; int32_t tickMask = (1 << prescaleBits) - 1; - timer->lastEvent = (mTimingCurrentTime(timing) - TIMER_STARTUP_DELAY) & ~tickMask; - GBATimerUpdateRegisterInternal(timer, timing, io, TIMER_STARTUP_DELAY); + timer->lastEvent = mTimingCurrentTime(timing) & ~tickMask; + GBATimerUpdateRegisterInternal(timer, timing, io, 0); } else if (wasEnabled && !GBATimerFlagsIsEnable(timer->flags)) { mTimingDeschedule(timing, &timer->event); } else if (GBATimerFlagsIsEnable(timer->flags) && GBATimerFlagsGetPrescaleBits(timer->flags) != oldPrescale && !GBATimerFlagsIsCountUp(timer->flags)) { mTimingDeschedule(timing, &timer->event); int32_t tickMask = (1 << prescaleBits) - 1; - timer->lastEvent = (mTimingCurrentTime(timing) - TIMER_STARTUP_DELAY) & ~tickMask; - GBATimerUpdateRegisterInternal(timer, timing, io, TIMER_STARTUP_DELAY); + timer->lastEvent = mTimingCurrentTime(timing) & ~tickMask; + GBATimerUpdateRegisterInternal(timer, timing, io, 0); } }
M
src/platform/libretro/libretro.c
→
src/platform/libretro/libretro.c
@@ -26,6 +26,8 @@ #endif
#include <mgba-util/memory.h> #include <mgba-util/vfs.h> +#include "libretro_core_options.h" + #define SAMPLES 1024 #define RUMBLE_PWM 35@@ -49,7 +51,7 @@ static void _stopImage(struct mImageSource*);
static void _requestImage(struct mImageSource*, const void** buffer, size_t* stride, enum mColorFormat* colorFormat); static struct mCore* core; -static void* outputBuffer; +static color_t* outputBuffer = NULL; static void* data; static size_t dataSize; static void* savedata;@@ -122,6 +124,13 @@ mCoreConfigSetDefaultIntValue(&core->config, "sgb.borders", false);
} } + var.key = "mgba_frameskip"; + var.value = 0; + if (environCallback(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value) { + opts.frameskip = strtol(var.value, NULL, 10); + + } + var.key = "mgba_idle_optimization"; var.value = 0; if (environCallback(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value) {@@ -132,13 +141,6 @@ mCoreConfigSetDefaultValue(&core->config, "idleOptimization", "remove");
} else if (strcmp(var.value, "Detect and Remove") == 0) { mCoreConfigSetDefaultValue(&core->config, "idleOptimization", "detect"); } - } - - var.key = "mgba_frameskip"; - var.value = 0; - if (environCallback(RETRO_ENVIRONMENT_GET_VARIABLE, &var) && var.value) { - opts.frameskip = strtol(var.value, NULL, 10); - } mCoreConfigLoadDefaults(&core->config, &opts);@@ -152,19 +154,7 @@
void retro_set_environment(retro_environment_t env) { environCallback = env; - struct retro_variable vars[] = { - { "mgba_solar_sensor_level", "Solar sensor level; 0|1|2|3|4|5|6|7|8|9|10" }, - { "mgba_allow_opposing_directions", "Allow opposing directional input; OFF|ON" }, - { "mgba_gb_model", "Game Boy model (requires restart); Autodetect|Game Boy|Super Game Boy|Game Boy Color|Game Boy Advance" }, - { "mgba_use_bios", "Use BIOS file if found (requires restart); ON|OFF" }, - { "mgba_skip_bios", "Skip BIOS intro (requires restart); OFF|ON" }, - { "mgba_sgb_borders", "Use Super Game Boy borders (requires restart); ON|OFF" }, - { "mgba_idle_optimization", "Idle loop removal; Remove Known|Detect and Remove|Don't Remove" }, - { "mgba_frameskip", "Frameskip; 0|1|2|3|4|5|6|7|8|9|10" }, - { 0, 0 } - }; - - environCallback(RETRO_ENVIRONMENT_SET_VARIABLES, vars); + libretro_set_core_options(environCallback); } void retro_set_video_refresh(retro_video_refresh_t video) {@@ -825,6 +815,7 @@ static void _updateCamera(const uint32_t* buffer, unsigned width, unsigned height, size_t pitch) {
if (!camData || width > camWidth || height > camHeight) { if (camData) { free(camData); + camData = NULL; } unsigned bufPitch = pitch / sizeof(*buffer); unsigned bufHeight = height;
M
src/platform/libretro/libretro.h
→
src/platform/libretro/libretro.h
@@ -1,4 +1,4 @@
-/* Copyright (C) 2010-2016 The RetroArch team +/* Copyright (C) 2010-2018 The RetroArch team * * --------------------------------------------------------------------------------------- * The following license statement only applies to this libretro API header (libretro.h).@@ -32,7 +32,7 @@ extern "C" {
#endif #ifndef __cplusplus -#if defined(_MSC_VER) && !defined(SN_TARGET_PS3) +#if defined(_MSC_VER) && _MSC_VER < 1800 && !defined(SN_TARGET_PS3) /* Hack applied for MSVC when compiling in C89 mode * as it isn't C99-compliant. */ #define bool unsigned char@@ -77,7 +77,7 @@ # endif
# endif #endif -/* Used for checking API/ABI mismatches that can break libretro +/* Used for checking API/ABI mismatches that can break libretro * implementations. * It is not incremented for compatible changes to the API. */@@ -87,13 +87,13 @@ /*
* Libretro's fundamental device abstractions. * * Libretro's input system consists of some standardized device types, - * such as a joypad (with/without analog), mouse, keyboard, lightgun + * such as a joypad (with/without analog), mouse, keyboard, lightgun * and a pointer. * - * The functionality of these devices are fixed, and individual cores + * The functionality of these devices are fixed, and individual cores * map their own concept of a controller to libretro's abstractions. - * This makes it possible for frontends to map the abstract types to a - * real input device, and not having to worry about binding input + * This makes it possible for frontends to map the abstract types to a + * real input device, and not having to worry about binding input * correctly to arbitrary controller layouts. */@@ -104,43 +104,52 @@
/* Input disabled. */ #define RETRO_DEVICE_NONE 0 -/* The JOYPAD is called RetroPad. It is essentially a Super Nintendo - * controller, but with additional L2/R2/L3/R3 buttons, similar to a +/* The JOYPAD is called RetroPad. It is essentially a Super Nintendo + * controller, but with additional L2/R2/L3/R3 buttons, similar to a * PS1 DualShock. */ #define RETRO_DEVICE_JOYPAD 1 /* The mouse is a simple mouse, similar to Super Nintendo's mouse. * X and Y coordinates are reported relatively to last poll (poll callback). - * It is up to the libretro implementation to keep track of where the mouse + * It is up to the libretro implementation to keep track of where the mouse * pointer is supposed to be on the screen. - * The frontend must make sure not to interfere with its own hardware + * The frontend must make sure not to interfere with its own hardware * mouse pointer. */ #define RETRO_DEVICE_MOUSE 2 /* KEYBOARD device lets one poll for raw key pressed. - * It is poll based, so input callback will return with the current + * It is poll based, so input callback will return with the current * pressed state. * For event/text based keyboard input, see * RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK. */ #define RETRO_DEVICE_KEYBOARD 3 -/* Lightgun X/Y coordinates are reported relatively to last poll, - * similar to mouse. */ +/* LIGHTGUN device is similar to Guncon-2 for PlayStation 2. + * It reports X/Y coordinates in screen space (similar to the pointer) + * in the range [-0x8000, 0x7fff] in both axes, with zero being center and + * -0x8000 being out of bounds. + * As well as reporting on/off screen state. It features a trigger, + * start/select buttons, auxiliary action buttons and a + * directional pad. A forced off-screen shot can be requested for + * auto-reloading function in some games. + */ #define RETRO_DEVICE_LIGHTGUN 4 /* The ANALOG device is an extension to JOYPAD (RetroPad). - * Similar to DualShock it adds two analog sticks. - * This is treated as a separate device type as it returns values in the - * full analog range of [-0x8000, 0x7fff]. Positive X axis is right. - * Positive Y axis is down. - * Only use ANALOG type when polling for analog values of the axes. + * Similar to DualShock2 it adds two analog sticks and all buttons can + * be analog. This is treated as a separate device type as it returns + * axis values in the full analog range of [-0x7fff, 0x7fff], + * although some devices may return -0x8000. + * Positive X axis is right. Positive Y axis is down. + * Buttons are returned in the range [0, 0x7fff]. + * Only use ANALOG type when polling for analog values. */ #define RETRO_DEVICE_ANALOG 5 /* Abstracts the concept of a pointing mechanism, e.g. touch. - * This allows libretro to query in absolute coordinates where on the + * This allows libretro to query in absolute coordinates where on the * screen a mouse (or something similar) is being placed. * For a touch centric device, coordinates reported are the coordinates * of the press.@@ -148,33 +157,34 @@ *
* Coordinates in X and Y are reported as: * [-0x7fff, 0x7fff]: -0x7fff corresponds to the far left/top of the screen, * and 0x7fff corresponds to the far right/bottom of the screen. - * The "screen" is here defined as area that is passed to the frontend and + * The "screen" is here defined as area that is passed to the frontend and * later displayed on the monitor. * * The frontend is free to scale/resize this screen as it sees fit, however, - * (X, Y) = (-0x7fff, -0x7fff) will correspond to the top-left pixel of the + * (X, Y) = (-0x7fff, -0x7fff) will correspond to the top-left pixel of the * game image, etc. * - * To check if the pointer coordinates are valid (e.g. a touch display + * To check if the pointer coordinates are valid (e.g. a touch display * actually being touched), PRESSED returns 1 or 0. * - * If using a mouse on a desktop, PRESSED will usually correspond to the + * If using a mouse on a desktop, PRESSED will usually correspond to the * left mouse button, but this is a frontend decision. * PRESSED will only return 1 if the pointer is inside the game screen. * - * For multi-touch, the index variable can be used to successively query + * For multi-touch, the index variable can be used to successively query * more presses. * If index = 0 returns true for _PRESSED, coordinates can be extracted - * with _X, _Y for index = 0. One can then query _PRESSED, _X, _Y with + * with _X, _Y for index = 0. One can then query _PRESSED, _X, _Y with * index = 1, and so on. - * Eventually _PRESSED will return false for an index. No further presses + * Eventually _PRESSED will return false for an index. No further presses * are registered at this point. */ #define RETRO_DEVICE_POINTER 6 /* Buttons for the RetroPad (JOYPAD). - * The placement of these is equivalent to placements on the + * The placement of these is equivalent to placements on the * Super Nintendo controller. - * L2/R2/L3/R3 buttons correspond to the PS1 DualShock. */ + * L2/R2/L3/R3 buttons correspond to the PS1 DualShock. + * Also used as id values for RETRO_DEVICE_INDEX_ANALOG_BUTTON */ #define RETRO_DEVICE_ID_JOYPAD_B 0 #define RETRO_DEVICE_ID_JOYPAD_Y 1 #define RETRO_DEVICE_ID_JOYPAD_SELECT 2@@ -192,11 +202,14 @@ #define RETRO_DEVICE_ID_JOYPAD_R2 13
#define RETRO_DEVICE_ID_JOYPAD_L3 14 #define RETRO_DEVICE_ID_JOYPAD_R3 15 +#define RETRO_DEVICE_ID_JOYPAD_MASK 256 + /* Index / Id values for ANALOG device. */ -#define RETRO_DEVICE_INDEX_ANALOG_LEFT 0 -#define RETRO_DEVICE_INDEX_ANALOG_RIGHT 1 -#define RETRO_DEVICE_ID_ANALOG_X 0 -#define RETRO_DEVICE_ID_ANALOG_Y 1 +#define RETRO_DEVICE_INDEX_ANALOG_LEFT 0 +#define RETRO_DEVICE_INDEX_ANALOG_RIGHT 1 +#define RETRO_DEVICE_INDEX_ANALOG_BUTTON 2 +#define RETRO_DEVICE_ID_ANALOG_X 0 +#define RETRO_DEVICE_ID_ANALOG_Y 1 /* Id values for MOUSE. */ #define RETRO_DEVICE_ID_MOUSE_X 0@@ -208,20 +221,36 @@ #define RETRO_DEVICE_ID_MOUSE_WHEELDOWN 5
#define RETRO_DEVICE_ID_MOUSE_MIDDLE 6 #define RETRO_DEVICE_ID_MOUSE_HORIZ_WHEELUP 7 #define RETRO_DEVICE_ID_MOUSE_HORIZ_WHEELDOWN 8 +#define RETRO_DEVICE_ID_MOUSE_BUTTON_4 9 +#define RETRO_DEVICE_ID_MOUSE_BUTTON_5 10 -/* Id values for LIGHTGUN types. */ -#define RETRO_DEVICE_ID_LIGHTGUN_X 0 -#define RETRO_DEVICE_ID_LIGHTGUN_Y 1 -#define RETRO_DEVICE_ID_LIGHTGUN_TRIGGER 2 -#define RETRO_DEVICE_ID_LIGHTGUN_CURSOR 3 -#define RETRO_DEVICE_ID_LIGHTGUN_TURBO 4 -#define RETRO_DEVICE_ID_LIGHTGUN_PAUSE 5 -#define RETRO_DEVICE_ID_LIGHTGUN_START 6 +/* Id values for LIGHTGUN. */ +#define RETRO_DEVICE_ID_LIGHTGUN_SCREEN_X 13 /*Absolute Position*/ +#define RETRO_DEVICE_ID_LIGHTGUN_SCREEN_Y 14 /*Absolute*/ +#define RETRO_DEVICE_ID_LIGHTGUN_IS_OFFSCREEN 15 /*Status Check*/ +#define RETRO_DEVICE_ID_LIGHTGUN_TRIGGER 2 +#define RETRO_DEVICE_ID_LIGHTGUN_RELOAD 16 /*Forced off-screen shot*/ +#define RETRO_DEVICE_ID_LIGHTGUN_AUX_A 3 +#define RETRO_DEVICE_ID_LIGHTGUN_AUX_B 4 +#define RETRO_DEVICE_ID_LIGHTGUN_START 6 +#define RETRO_DEVICE_ID_LIGHTGUN_SELECT 7 +#define RETRO_DEVICE_ID_LIGHTGUN_AUX_C 8 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_UP 9 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_DOWN 10 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_LEFT 11 +#define RETRO_DEVICE_ID_LIGHTGUN_DPAD_RIGHT 12 +/* deprecated */ +#define RETRO_DEVICE_ID_LIGHTGUN_X 0 /*Relative Position*/ +#define RETRO_DEVICE_ID_LIGHTGUN_Y 1 /*Relative*/ +#define RETRO_DEVICE_ID_LIGHTGUN_CURSOR 3 /*Use Aux:A*/ +#define RETRO_DEVICE_ID_LIGHTGUN_TURBO 4 /*Use Aux:B*/ +#define RETRO_DEVICE_ID_LIGHTGUN_PAUSE 5 /*Use Start*/ /* Id values for POINTER. */ #define RETRO_DEVICE_ID_POINTER_X 0 #define RETRO_DEVICE_ID_POINTER_Y 1 #define RETRO_DEVICE_ID_POINTER_PRESSED 2 +#define RETRO_DEVICE_ID_POINTER_COUNT 3 /* Returned from retro_get_region(). */ #define RETRO_REGION_NTSC 0@@ -230,28 +259,33 @@
/* Id values for LANGUAGE */ enum retro_language { - RETRO_LANGUAGE_ENGLISH = 0, - RETRO_LANGUAGE_JAPANESE = 1, - RETRO_LANGUAGE_FRENCH = 2, - RETRO_LANGUAGE_SPANISH = 3, - RETRO_LANGUAGE_GERMAN = 4, - RETRO_LANGUAGE_ITALIAN = 5, - RETRO_LANGUAGE_DUTCH = 6, - RETRO_LANGUAGE_PORTUGUESE = 7, - RETRO_LANGUAGE_RUSSIAN = 8, - RETRO_LANGUAGE_KOREAN = 9, - RETRO_LANGUAGE_CHINESE_TRADITIONAL = 10, - RETRO_LANGUAGE_CHINESE_SIMPLIFIED = 11, - RETRO_LANGUAGE_ESPERANTO = 12, - RETRO_LANGUAGE_POLISH = 13, + RETRO_LANGUAGE_ENGLISH = 0, + RETRO_LANGUAGE_JAPANESE = 1, + RETRO_LANGUAGE_FRENCH = 2, + RETRO_LANGUAGE_SPANISH = 3, + RETRO_LANGUAGE_GERMAN = 4, + RETRO_LANGUAGE_ITALIAN = 5, + RETRO_LANGUAGE_DUTCH = 6, + RETRO_LANGUAGE_PORTUGUESE_BRAZIL = 7, + RETRO_LANGUAGE_PORTUGUESE_PORTUGAL = 8, + RETRO_LANGUAGE_RUSSIAN = 9, + RETRO_LANGUAGE_KOREAN = 10, + RETRO_LANGUAGE_CHINESE_TRADITIONAL = 11, + RETRO_LANGUAGE_CHINESE_SIMPLIFIED = 12, + RETRO_LANGUAGE_ESPERANTO = 13, + RETRO_LANGUAGE_POLISH = 14, + RETRO_LANGUAGE_VIETNAMESE = 15, + RETRO_LANGUAGE_ARABIC = 16, + RETRO_LANGUAGE_GREEK = 17, + RETRO_LANGUAGE_TURKISH = 18, RETRO_LANGUAGE_LAST, /* Ensure sizeof(enum) == sizeof(int) */ - RETRO_LANGUAGE_DUMMY = INT_MAX + RETRO_LANGUAGE_DUMMY = INT_MAX }; /* Passed to retro_get_memory_data/size(). - * If the memory type doesn't apply to the + * If the memory type doesn't apply to the * implementation NULL/0 can be returned. */ #define RETRO_MEMORY_MASK 0xff@@ -349,6 +383,10 @@ RETROK_w = 119,
RETROK_x = 120, RETROK_y = 121, RETROK_z = 122, + RETROK_LEFTBRACE = 123, + RETROK_BAR = 124, + RETROK_RIGHTBRACE = 125, + RETROK_TILDE = 126, RETROK_DELETE = 127, RETROK_KP0 = 256,@@ -419,6 +457,7 @@ RETROK_MENU = 319,
RETROK_POWER = 320, RETROK_EURO = 321, RETROK_UNDO = 322, + RETROK_OEM_102 = 323, RETROK_LAST,@@ -441,7 +480,7 @@
RETROKMOD_DUMMY = INT_MAX /* Ensure sizeof(enum) == sizeof(int) */ }; -/* If set, this call is not part of the public libretro API yet. It can +/* If set, this call is not part of the public libretro API yet. It can * change or be removed at any time. */ #define RETRO_ENVIRONMENT_EXPERIMENTAL 0x10000 /* Environment callback to be used internally in frontend. */@@ -450,12 +489,14 @@
/* Environment commands. */ #define RETRO_ENVIRONMENT_SET_ROTATION 1 /* const unsigned * -- * Sets screen rotation of graphics. - * Is only implemented if rotation can be accelerated by hardware. - * Valid values are 0, 1, 2, 3, which rotates screen by 0, 90, 180, + * Valid values are 0, 1, 2, 3, which rotates screen by 0, 90, 180, * 270 degrees counter-clockwise respectively. */ #define RETRO_ENVIRONMENT_GET_OVERSCAN 2 /* bool * -- - * Boolean value whether or not the implementation should use overscan, + * NOTE: As of 2019 this callback is considered deprecated in favor of + * using core options to manage overscan in a more nuanced, core-specific way. + * + * Boolean value whether or not the implementation should use overscan, * or crop away overscan. */ #define RETRO_ENVIRONMENT_GET_CAN_DUPE 3 /* bool * --@@ -463,15 +504,15 @@ * Boolean value whether or not frontend supports frame duping,
* passing NULL to video frame callback. */ - /* Environ 4, 5 are no longer supported (GET_VARIABLE / SET_VARIABLES), + /* Environ 4, 5 are no longer supported (GET_VARIABLE / SET_VARIABLES), * and reserved to avoid possible ABI clash. */ #define RETRO_ENVIRONMENT_SET_MESSAGE 6 /* const struct retro_message * -- - * Sets a message to be displayed in implementation-specific manner + * Sets a message to be displayed in implementation-specific manner * for a certain amount of 'frames'. - * Should not be used for trivial messages, which should simply be - * logged via RETRO_ENVIRONMENT_GET_LOG_INTERFACE (or as a + * Should not be used for trivial messages, which should simply be + * logged via RETRO_ENVIRONMENT_GET_LOG_INTERFACE (or as a * fallback, stderr). */ #define RETRO_ENVIRONMENT_SHUTDOWN 7 /* N/A (NULL) --@@ -499,15 +540,15 @@ */
#define RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY 9 /* const char ** -- * Returns the "system" directory of the frontend. - * This directory can be used to store system specific + * This directory can be used to store system specific * content such as BIOSes, configuration data, etc. * The returned value can be NULL. * If so, no such directory is defined, * and it's up to the implementation to find a suitable directory. * - * NOTE: Some cores used this folder also for "save" data such as + * NOTE: Some cores used this folder also for "save" data such as * memory cards, etc, for lack of a better place to put it. - * This is now discouraged, and if possible, cores should try to + * This is now discouraged, and if possible, cores should try to * use the new GET_SAVE_DIRECTORY. */ #define RETRO_ENVIRONMENT_SET_PIXEL_FORMAT 10@@ -515,19 +556,19 @@ /* const enum retro_pixel_format * --
* Sets the internal pixel format used by the implementation. * The default pixel format is RETRO_PIXEL_FORMAT_0RGB1555. * This pixel format however, is deprecated (see enum retro_pixel_format). - * If the call returns false, the frontend does not support this pixel + * If the call returns false, the frontend does not support this pixel * format. * - * This function should be called inside retro_load_game() or + * This function should be called inside retro_load_game() or * retro_get_system_av_info(). */ #define RETRO_ENVIRONMENT_SET_INPUT_DESCRIPTORS 11 /* const struct retro_input_descriptor * -- * Sets an array of retro_input_descriptors. * It is up to the frontend to present this in a usable way. - * The array is terminated by retro_input_descriptor::description + * The array is terminated by retro_input_descriptor::description * being set to NULL. - * This function can be called at any time, but it is recommended + * This function can be called at any time, but it is recommended * to call it as early as possible. */ #define RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK 12@@ -536,52 +577,55 @@ * Sets a callback function used to notify core about keyboard events.
*/ #define RETRO_ENVIRONMENT_SET_DISK_CONTROL_INTERFACE 13 /* const struct retro_disk_control_callback * -- - * Sets an interface which frontend can use to eject and insert + * Sets an interface which frontend can use to eject and insert * disk images. - * This is used for games which consist of multiple images and + * This is used for games which consist of multiple images and * must be manually swapped out by the user (e.g. PSX). */ #define RETRO_ENVIRONMENT_SET_HW_RENDER 14 /* struct retro_hw_render_callback * -- - * Sets an interface to let a libretro core render with + * Sets an interface to let a libretro core render with * hardware acceleration. * Should be called in retro_load_game(). - * If successful, libretro cores will be able to render to a + * If successful, libretro cores will be able to render to a * frontend-provided framebuffer. - * The size of this framebuffer will be at least as large as + * The size of this framebuffer will be at least as large as * max_width/max_height provided in get_av_info(). - * If HW rendering is used, pass only RETRO_HW_FRAME_BUFFER_VALID or + * If HW rendering is used, pass only RETRO_HW_FRAME_BUFFER_VALID or * NULL to retro_video_refresh_t. */ #define RETRO_ENVIRONMENT_GET_VARIABLE 15 /* struct retro_variable * -- * Interface to acquire user-defined information from environment * that cannot feasibly be supported in a multi-system way. - * 'key' should be set to a key which has already been set by + * 'key' should be set to a key which has already been set by * SET_VARIABLES. * 'data' will be set to a value or NULL. */ #define RETRO_ENVIRONMENT_SET_VARIABLES 16 /* const struct retro_variable * -- * Allows an implementation to signal the environment - * which variables it might want to check for later using + * which variables it might want to check for later using * GET_VARIABLE. - * This allows the frontend to present these variables to + * This allows the frontend to present these variables to * a user dynamically. - * This should be called as early as possible (ideally in - * retro_set_environment). + * This should be called the first time as early as + * possible (ideally in retro_set_environment). + * Afterward it may be called again for the core to communicate + * updated options to the frontend, but the number of core + * options must not change from the number in the initial call. * - * 'data' points to an array of retro_variable structs + * 'data' points to an array of retro_variable structs * terminated by a { NULL, NULL } element. - * retro_variable::key should be namespaced to not collide - * with other implementations' keys. E.g. A core called + * retro_variable::key should be namespaced to not collide + * with other implementations' keys. E.g. A core called * 'foo' should use keys named as 'foo_option'. - * retro_variable::value should contain a human readable - * description of the key as well as a '|' delimited list + * retro_variable::value should contain a human readable + * description of the key as well as a '|' delimited list * of expected values. * - * The number of possible options should be very limited, - * i.e. it should be feasible to cycle through options + * The number of possible options should be very limited, + * i.e. it should be feasible to cycle through options * without a keyboard. * * First entry should be treated as a default.@@ -589,11 +633,11 @@ *
* Example entry: * { "foo_option", "Speed hack coprocessor X; false|true" } * - * Text before first ';' is description. This ';' must be - * followed by a space, and followed by a list of possible + * Text before first ';' is description. This ';' must be + * followed by a space, and followed by a list of possible * values split up with '|'. * - * Only strings are operated on. The possible values will + * Only strings are operated on. The possible values will * generally be displayed and stored as-is by the frontend. */ #define RETRO_ENVIRONMENT_GET_VARIABLE_UPDATE 17@@ -604,72 +648,72 @@ * Variables should be queried with GET_VARIABLE.
*/ #define RETRO_ENVIRONMENT_SET_SUPPORT_NO_GAME 18 /* const bool * -- - * If true, the libretro implementation supports calls to + * If true, the libretro implementation supports calls to * retro_load_game() with NULL as argument. * Used by cores which can run without particular game data. * This should be called within retro_set_environment() only. */ #define RETRO_ENVIRONMENT_GET_LIBRETRO_PATH 19 /* const char ** -- - * Retrieves the absolute path from where this libretro + * Retrieves the absolute path from where this libretro * implementation was loaded. - * NULL is returned if the libretro was loaded statically - * (i.e. linked statically to frontend), or if the path cannot be + * NULL is returned if the libretro was loaded statically + * (i.e. linked statically to frontend), or if the path cannot be * determined. - * Mostly useful in cooperation with SET_SUPPORT_NO_GAME as assets can + * Mostly useful in cooperation with SET_SUPPORT_NO_GAME as assets can * be loaded without ugly hacks. */ - - /* Environment 20 was an obsolete version of SET_AUDIO_CALLBACK. + + /* Environment 20 was an obsolete version of SET_AUDIO_CALLBACK. * It was not used by any known core at the time, * and was removed from the API. */ +#define RETRO_ENVIRONMENT_SET_FRAME_TIME_CALLBACK 21 + /* const struct retro_frame_time_callback * -- + * Lets the core know how much time has passed since last + * invocation of retro_run(). + * The frontend can tamper with the timing to fake fast-forward, + * slow-motion, frame stepping, etc. + * In this case the delta time will use the reference value + * in frame_time_callback.. + */ #define RETRO_ENVIRONMENT_SET_AUDIO_CALLBACK 22 /* const struct retro_audio_callback * -- - * Sets an interface which is used to notify a libretro core about audio + * Sets an interface which is used to notify a libretro core about audio * being available for writing. - * The callback can be called from any thread, so a core using this must + * The callback can be called from any thread, so a core using this must * have a thread safe audio implementation. - * It is intended for games where audio and video are completely + * It is intended for games where audio and video are completely * asynchronous and audio can be generated on the fly. - * This interface is not recommended for use with emulators which have + * This interface is not recommended for use with emulators which have * highly synchronous audio. * - * The callback only notifies about writability; the libretro core still + * The callback only notifies about writability; the libretro core still * has to call the normal audio callbacks - * to write audio. The audio callbacks must be called from within the + * to write audio. The audio callbacks must be called from within the * notification callback. * The amount of audio data to write is up to the implementation. * Generally, the audio callback will be called continously in a loop. * - * Due to thread safety guarantees and lack of sync between audio and - * video, a frontend can selectively disallow this interface based on - * internal configuration. A core using this interface must also + * Due to thread safety guarantees and lack of sync between audio and + * video, a frontend can selectively disallow this interface based on + * internal configuration. A core using this interface must also * implement the "normal" audio interface. * - * A libretro core using SET_AUDIO_CALLBACK should also make use of + * A libretro core using SET_AUDIO_CALLBACK should also make use of * SET_FRAME_TIME_CALLBACK. */ -#define RETRO_ENVIRONMENT_SET_FRAME_TIME_CALLBACK 21 - /* const struct retro_frame_time_callback * -- - * Lets the core know how much time has passed since last - * invocation of retro_run(). - * The frontend can tamper with the timing to fake fast-forward, - * slow-motion, frame stepping, etc. - * In this case the delta time will use the reference value - * in frame_time_callback.. - */ #define RETRO_ENVIRONMENT_GET_RUMBLE_INTERFACE 23 /* struct retro_rumble_interface * -- - * Gets an interface which is used by a libretro core to set + * Gets an interface which is used by a libretro core to set * state of rumble motors in controllers. - * A strong and weak motor is supported, and they can be + * A strong and weak motor is supported, and they can be * controlled indepedently. */ #define RETRO_ENVIRONMENT_GET_INPUT_DEVICE_CAPABILITIES 24 /* uint64_t * -- - * Gets a bitmask telling which device type are expected to be + * Gets a bitmask telling which device type are expected to be * handled properly in a call to retro_input_state_t. - * Devices which are not handled or recognized always return + * Devices which are not handled or recognized always return * 0 in retro_input_state_t. * Example bitmask: caps = (1 << RETRO_DEVICE_JOYPAD) | (1 << RETRO_DEVICE_ANALOG). * Should only be called in retro_run().@@ -678,56 +722,56 @@ #define RETRO_ENVIRONMENT_GET_SENSOR_INTERFACE (25 | RETRO_ENVIRONMENT_EXPERIMENTAL)
/* struct retro_sensor_interface * -- * Gets access to the sensor interface. * The purpose of this interface is to allow - * setting state related to sensors such as polling rate, + * setting state related to sensors such as polling rate, * enabling/disable it entirely, etc. - * Reading sensor state is done via the normal + * Reading sensor state is done via the normal * input_state_callback API. */ #define RETRO_ENVIRONMENT_GET_CAMERA_INTERFACE (26 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* struct retro_camera_callback * -- * Gets an interface to a video camera driver. - * A libretro core can use this interface to get access to a + * A libretro core can use this interface to get access to a * video camera. - * New video frames are delivered in a callback in same + * New video frames are delivered in a callback in same * thread as retro_run(). * * GET_CAMERA_INTERFACE should be called in retro_load_game(). * - * Depending on the camera implementation used, camera frames + * Depending on the camera implementation used, camera frames * will be delivered as a raw framebuffer, * or as an OpenGL texture directly. * - * The core has to tell the frontend here which types of + * The core has to tell the frontend here which types of * buffers can be handled properly. - * An OpenGL texture can only be handled when using a + * An OpenGL texture can only be handled when using a * libretro GL core (SET_HW_RENDER). - * It is recommended to use a libretro GL core when + * It is recommended to use a libretro GL core when * using camera interface. * - * The camera is not started automatically. The retrieved start/stop + * The camera is not started automatically. The retrieved start/stop * functions must be used to explicitly * start and stop the camera driver. */ #define RETRO_ENVIRONMENT_GET_LOG_INTERFACE 27 /* struct retro_log_callback * -- - * Gets an interface for logging. This is useful for + * Gets an interface for logging. This is useful for * logging in a cross-platform way - * as certain platforms cannot use use stderr for logging. + * as certain platforms cannot use stderr for logging. * It also allows the frontend to * show logging information in a more suitable way. - * If this interface is not used, libretro cores should + * If this interface is not used, libretro cores should * log to stderr as desired. */ #define RETRO_ENVIRONMENT_GET_PERF_INTERFACE 28 /* struct retro_perf_callback * -- - * Gets an interface for performance counters. This is useful - * for performance logging in a cross-platform way and for detecting + * Gets an interface for performance counters. This is useful + * for performance logging in a cross-platform way and for detecting * architecture-specific features, such as SIMD support. */ #define RETRO_ENVIRONMENT_GET_LOCATION_INTERFACE 29 /* struct retro_location_callback * -- * Gets access to the location interface. - * The purpose of this interface is to be able to retrieve + * The purpose of this interface is to be able to retrieve * location-based information from the host device, * such as current latitude / longitude. */@@ -735,7 +779,7 @@ #define RETRO_ENVIRONMENT_GET_CONTENT_DIRECTORY 30 /* Old name, kept for compatibility. */
#define RETRO_ENVIRONMENT_GET_CORE_ASSETS_DIRECTORY 30 /* const char ** -- * Returns the "core assets" directory of the frontend. - * This directory can be used to store specific assets that the + * This directory can be used to store specific assets that the * core relies upon, such as art assets, * input data, etc etc. * The returned value can be NULL.@@ -744,76 +788,77 @@ * and it's up to the implementation to find a suitable directory.
*/ #define RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY 31 /* const char ** -- - * Returns the "save" directory of the frontend. - * This directory can be used to store SRAM, memory cards, - * high scores, etc, if the libretro core + * Returns the "save" directory of the frontend, unless there is no + * save directory available. The save directory should be used to + * store SRAM, memory cards, high scores, etc, if the libretro core * cannot use the regular memory interface (retro_get_memory_data()). * - * NOTE: libretro cores used to check GET_SYSTEM_DIRECTORY for - * similar things before. - * They should still check GET_SYSTEM_DIRECTORY if they want to - * be backwards compatible. - * The path here can be NULL. It should only be non-NULL if the - * frontend user has set a specific save path. + * If the frontend cannot designate a save directory, it will return + * NULL to indicate that the core should attempt to operate without a + * save directory set. + * + * NOTE: early libretro cores used the system directory for save + * files. Cores that need to be backwards-compatible can still check + * GET_SYSTEM_DIRECTORY. */ #define RETRO_ENVIRONMENT_SET_SYSTEM_AV_INFO 32 /* const struct retro_system_av_info * -- - * Sets a new av_info structure. This can only be called from + * Sets a new av_info structure. This can only be called from * within retro_run(). - * This should *only* be used if the core is completely altering the + * This should *only* be used if the core is completely altering the * internal resolutions, aspect ratios, timings, sampling rate, etc. - * Calling this can require a full reinitialization of video/audio + * Calling this can require a full reinitialization of video/audio * drivers in the frontend, * - * so it is important to call it very sparingly, and usually only with + * so it is important to call it very sparingly, and usually only with * the users explicit consent. - * An eventual driver reinitialize will happen so that video and + * An eventual driver reinitialize will happen so that video and * audio callbacks - * happening after this call within the same retro_run() call will + * happening after this call within the same retro_run() call will * target the newly initialized driver. * - * This callback makes it possible to support configurable resolutions + * This callback makes it possible to support configurable resolutions * in games, which can be useful to * avoid setting the "worst case" in max_width/max_height. * - * ***HIGHLY RECOMMENDED*** Do not call this callback every time + * ***HIGHLY RECOMMENDED*** Do not call this callback every time * resolution changes in an emulator core if it's - * expected to be a temporary change, for the reasons of possible + * expected to be a temporary change, for the reasons of possible * driver reinitialization. - * This call is not a free pass for not trying to provide - * correct values in retro_get_system_av_info(). If you need to change - * things like aspect ratio or nominal width/height, - * use RETRO_ENVIRONMENT_SET_GEOMETRY, which is a softer variant + * This call is not a free pass for not trying to provide + * correct values in retro_get_system_av_info(). If you need to change + * things like aspect ratio or nominal width/height, + * use RETRO_ENVIRONMENT_SET_GEOMETRY, which is a softer variant * of SET_SYSTEM_AV_INFO. * - * If this returns false, the frontend does not acknowledge a + * If this returns false, the frontend does not acknowledge a * changed av_info struct. */ #define RETRO_ENVIRONMENT_SET_PROC_ADDRESS_CALLBACK 33 /* const struct retro_get_proc_address_interface * -- - * Allows a libretro core to announce support for the + * Allows a libretro core to announce support for the * get_proc_address() interface. - * This interface allows for a standard way to extend libretro where + * This interface allows for a standard way to extend libretro where * use of environment calls are too indirect, * e.g. for cases where the frontend wants to call directly into the core. * - * If a core wants to expose this interface, SET_PROC_ADDRESS_CALLBACK + * If a core wants to expose this interface, SET_PROC_ADDRESS_CALLBACK * **MUST** be called from within retro_set_environment(). */ #define RETRO_ENVIRONMENT_SET_SUBSYSTEM_INFO 34 /* const struct retro_subsystem_info * -- * This environment call introduces the concept of libretro "subsystems". - * A subsystem is a variant of a libretro core which supports + * A subsystem is a variant of a libretro core which supports * different kinds of games. - * The purpose of this is to support e.g. emulators which might + * The purpose of this is to support e.g. emulators which might * have special needs, e.g. Super Nintendo's Super GameBoy, Sufami Turbo. - * It can also be used to pick among subsystems in an explicit way + * It can also be used to pick among subsystems in an explicit way * if the libretro implementation is a multi-system emulator itself. * * Loading a game via a subsystem is done with retro_load_game_special(), - * and this environment call allows a libretro core to expose which + * and this environment call allows a libretro core to expose which * subsystems are supported for use with retro_load_game_special(). - * A core passes an array of retro_game_special_info which is terminated + * A core passes an array of retro_game_special_info which is terminated * with a zeroed out retro_game_special_info struct. * * If a core wants to use this functionality, SET_SUBSYSTEM_INFO@@ -821,68 +866,81 @@ * **MUST** be called from within retro_set_environment().
*/ #define RETRO_ENVIRONMENT_SET_CONTROLLER_INFO 35 /* const struct retro_controller_info * -- - * This environment call lets a libretro core tell the frontend - * which controller types are recognized in calls to + * This environment call lets a libretro core tell the frontend + * which controller subclasses are recognized in calls to * retro_set_controller_port_device(). * - * Some emulators such as Super Nintendo - * support multiple lightgun types which must be specifically - * selected from. - * It is therefore sometimes necessary for a frontend to be able - * to tell the core about a special kind of input device which is - * not covered by the libretro input API. + * Some emulators such as Super Nintendo support multiple lightgun + * types which must be specifically selected from. It is therefore + * sometimes necessary for a frontend to be able to tell the core + * about a special kind of input device which is not specifcally + * provided by the Libretro API. * - * In order for a frontend to understand the workings of an input device, - * it must be a specialized type - * of the generic device types already defined in the libretro API. + * In order for a frontend to understand the workings of those devices, + * they must be defined as a specialized subclass of the generic device + * types already defined in the libretro API. * - * Which devices are supported can vary per input port. - * The core must pass an array of const struct retro_controller_info which - * is terminated with a blanked out struct. Each element of the struct - * corresponds to an ascending port index to - * retro_set_controller_port_device(). - * Even if special device types are set in the libretro core, + * The core must pass an array of const struct retro_controller_info which + * is terminated with a blanked out struct. Each element of the + * retro_controller_info struct corresponds to the ascending port index + * that is passed to retro_set_controller_port_device() when that function + * is called to indicate to the core that the frontend has changed the + * active device subclass. SEE ALSO: retro_set_controller_port_device() + * + * The ascending input port indexes provided by the core in the struct + * are generally presented by frontends as ascending User # or Player #, + * such as Player 1, Player 2, Player 3, etc. Which device subclasses are + * supported can vary per input port. + * + * The first inner element of each entry in the retro_controller_info array + * is a retro_controller_description struct that specifies the names and + * codes of all device subclasses that are available for the corresponding + * User or Player, beginning with the generic Libretro device that the + * subclasses are derived from. The second inner element of each entry is the + * total number of subclasses that are listed in the retro_controller_description. + * + * NOTE: Even if special device types are set in the libretro core, * libretro should only poll input based on the base input device types. */ #define RETRO_ENVIRONMENT_SET_MEMORY_MAPS (36 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* const struct retro_memory_map * -- - * This environment call lets a libretro core tell the frontend + * This environment call lets a libretro core tell the frontend * about the memory maps this core emulates. * This can be used to implement, for example, cheats in a core-agnostic way. * - * Should only be used by emulators; it doesn't make much sense for + * Should only be used by emulators; it doesn't make much sense for * anything else. - * It is recommended to expose all relevant pointers through + * It is recommended to expose all relevant pointers through * retro_get_memory_* as well. * * Can be called from retro_init and retro_load_game. */ #define RETRO_ENVIRONMENT_SET_GEOMETRY 37 /* const struct retro_game_geometry * -- - * This environment call is similar to SET_SYSTEM_AV_INFO for changing - * video parameters, but provides a guarantee that drivers will not be + * This environment call is similar to SET_SYSTEM_AV_INFO for changing + * video parameters, but provides a guarantee that drivers will not be * reinitialized. * This can only be called from within retro_run(). * - * The purpose of this call is to allow a core to alter nominal - * width/heights as well as aspect ratios on-the-fly, which can be + * The purpose of this call is to allow a core to alter nominal + * width/heights as well as aspect ratios on-the-fly, which can be * useful for some emulators to change in run-time. * * max_width/max_height arguments are ignored and cannot be changed - * with this call as this could potentially require a reinitialization or a + * with this call as this could potentially require a reinitialization or a * non-constant time operation. * If max_width/max_height are to be changed, SET_SYSTEM_AV_INFO is required. * - * A frontend must guarantee that this environment call completes in + * A frontend must guarantee that this environment call completes in * constant time. */ -#define RETRO_ENVIRONMENT_GET_USERNAME 38 +#define RETRO_ENVIRONMENT_GET_USERNAME 38 /* const char ** * Returns the specified username of the frontend, if specified by the user. - * This username can be used as a nickname for a core that has online facilities + * This username can be used as a nickname for a core that has online facilities * or any other mode where personalization of the user is desirable. * The returned value can be NULL. - * If this environ callback is used by a core that requires a valid username, + * If this environ callback is used by a core that requires a valid username, * a default username should be specified by the core. */ #define RETRO_ENVIRONMENT_GET_LANGUAGE 39@@ -920,20 +978,6 @@ *
* A frontend must make sure that the pointer obtained from this function is * writeable (and readable). */ - -enum retro_hw_render_interface_type -{ - RETRO_HW_RENDER_INTERFACE_VULKAN = 0, - RETRO_HW_RENDER_INTERFACE_DUMMY = INT_MAX -}; - -/* Base struct. All retro_hw_render_interface_* types - * contain at least these fields. */ -struct retro_hw_render_interface -{ - enum retro_hw_render_interface_type interface_type; - unsigned interface_version; -}; #define RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE (41 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* const struct retro_hw_render_interface ** -- * Returns an API specific rendering interface for accessing API specific data.@@ -945,7 +989,6 @@ * GET_HW_RENDER_INTERFACE cannot be called before context_reset has been called.
* Similarly, after context_destroyed callback returns, * the contents of the HW_RENDER_INTERFACE are invalidated. */ - #define RETRO_ENVIRONMENT_SET_SUPPORT_ACHIEVEMENTS (42 | RETRO_ENVIRONMENT_EXPERIMENTAL) /* const bool * -- * If true, the libretro implementation supports achievements@@ -954,6 +997,481 @@ * or via retro_get_memory_data/retro_get_memory_size.
* * This must be called before the first call to retro_run. */ +#define RETRO_ENVIRONMENT_SET_HW_RENDER_CONTEXT_NEGOTIATION_INTERFACE (43 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* const struct retro_hw_render_context_negotiation_interface * -- + * Sets an interface which lets the libretro core negotiate with frontend how a context is created. + * The semantics of this interface depends on which API is used in SET_HW_RENDER earlier. + * This interface will be used when the frontend is trying to create a HW rendering context, + * so it will be used after SET_HW_RENDER, but before the context_reset callback. + */ +#define RETRO_ENVIRONMENT_SET_SERIALIZATION_QUIRKS 44 + /* uint64_t * -- + * Sets quirk flags associated with serialization. The frontend will zero any flags it doesn't + * recognize or support. Should be set in either retro_init or retro_load_game, but not both. + */ +#define RETRO_ENVIRONMENT_SET_HW_SHARED_CONTEXT (44 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* N/A (null) * -- + * The frontend will try to use a 'shared' hardware context (mostly applicable + * to OpenGL) when a hardware context is being set up. + * + * Returns true if the frontend supports shared hardware contexts and false + * if the frontend does not support shared hardware contexts. + * + * This will do nothing on its own until SET_HW_RENDER env callbacks are + * being used. + */ +#define RETRO_ENVIRONMENT_GET_VFS_INTERFACE (45 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* struct retro_vfs_interface_info * -- + * Gets access to the VFS interface. + * VFS presence needs to be queried prior to load_game or any + * get_system/save/other_directory being called to let front end know + * core supports VFS before it starts handing out paths. + * It is recomended to do so in retro_set_environment + */ +#define RETRO_ENVIRONMENT_GET_LED_INTERFACE (46 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* struct retro_led_interface * -- + * Gets an interface which is used by a libretro core to set + * state of LEDs. + */ +#define RETRO_ENVIRONMENT_GET_AUDIO_VIDEO_ENABLE (47 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* int * -- + * Tells the core if the frontend wants audio or video. + * If disabled, the frontend will discard the audio or video, + * so the core may decide to skip generating a frame or generating audio. + * This is mainly used for increasing performance. + * Bit 0 (value 1): Enable Video + * Bit 1 (value 2): Enable Audio + * Bit 2 (value 4): Use Fast Savestates. + * Bit 3 (value 8): Hard Disable Audio + * Other bits are reserved for future use and will default to zero. + * If video is disabled: + * * The frontend wants the core to not generate any video, + * including presenting frames via hardware acceleration. + * * The frontend's video frame callback will do nothing. + * * After running the frame, the video output of the next frame should be + * no different than if video was enabled, and saving and loading state + * should have no issues. + * If audio is disabled: + * * The frontend wants the core to not generate any audio. + * * The frontend's audio callbacks will do nothing. + * * After running the frame, the audio output of the next frame should be + * no different than if audio was enabled, and saving and loading state + * should have no issues. + * Fast Savestates: + * * Guaranteed to be created by the same binary that will load them. + * * Will not be written to or read from the disk. + * * Suggest that the core assumes loading state will succeed. + * * Suggest that the core updates its memory buffers in-place if possible. + * * Suggest that the core skips clearing memory. + * * Suggest that the core skips resetting the system. + * * Suggest that the core may skip validation steps. + * Hard Disable Audio: + * * Used for a secondary core when running ahead. + * * Indicates that the frontend will never need audio from the core. + * * Suggests that the core may stop synthesizing audio, but this should not + * compromise emulation accuracy. + * * Audio output for the next frame does not matter, and the frontend will + * never need an accurate audio state in the future. + * * State will never be saved when using Hard Disable Audio. + */ +#define RETRO_ENVIRONMENT_GET_MIDI_INTERFACE (48 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* struct retro_midi_interface ** -- + * Returns a MIDI interface that can be used for raw data I/O. + */ + +#define RETRO_ENVIRONMENT_GET_FASTFORWARDING (49 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* bool * -- + * Boolean value that indicates whether or not the frontend is in + * fastforwarding mode. + */ + +#define RETRO_ENVIRONMENT_GET_TARGET_REFRESH_RATE (50 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* float * -- + * Float value that lets us know what target refresh rate + * is curently in use by the frontend. + * + * The core can use the returned value to set an ideal + * refresh rate/framerate. + */ + +#define RETRO_ENVIRONMENT_GET_INPUT_BITMASKS (51 | RETRO_ENVIRONMENT_EXPERIMENTAL) + /* bool * -- + * Boolean value that indicates whether or not the frontend supports + * input bitmasks being returned by retro_input_state_t. The advantage + * of this is that retro_input_state_t has to be only called once to + * grab all button states instead of multiple times. + * + * If it returns true, you can pass RETRO_DEVICE_ID_JOYPAD_MASK as 'id' + * to retro_input_state_t (make sure 'device' is set to RETRO_DEVICE_JOYPAD). + * It will return a bitmask of all the digital buttons. + */ + +#define RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION 52 + /* unsigned * -- + * Unsigned value is the API version number of the core options + * interface supported by the frontend. If callback return false, + * API version is assumed to be 0. + * + * In legacy code, core options are set by passing an array of + * retro_variable structs to RETRO_ENVIRONMENT_SET_VARIABLES. + * This may be still be done regardless of the core options + * interface version. + * + * If version is >= 1 however, core options may instead be set by + * passing an array of retro_core_option_definition structs to + * RETRO_ENVIRONMENT_SET_CORE_OPTIONS, or a 2D array of + * retro_core_option_definition structs to RETRO_ENVIRONMENT_SET_CORE_OPTIONS_INTL. + * This allows the core to additionally set option sublabel information + * and/or provide localisation support. + */ + +#define RETRO_ENVIRONMENT_SET_CORE_OPTIONS 53 + /* const struct retro_core_option_definition ** -- + * Allows an implementation to signal the environment + * which variables it might want to check for later using + * GET_VARIABLE. + * This allows the frontend to present these variables to + * a user dynamically. + * This should only be called if RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION + * returns an API version of >= 1. + * This should be called instead of RETRO_ENVIRONMENT_SET_VARIABLES. + * This should be called the first time as early as + * possible (ideally in retro_set_environment). + * Afterwards it may be called again for the core to communicate + * updated options to the frontend, but the number of core + * options must not change from the number in the initial call. + * + * 'data' points to an array of retro_core_option_definition structs + * terminated by a { NULL, NULL, NULL, {{0}}, NULL } element. + * retro_core_option_definition::key should be namespaced to not collide + * with other implementations' keys. e.g. A core called + * 'foo' should use keys named as 'foo_option'. + * retro_core_option_definition::desc should contain a human readable + * description of the key. + * retro_core_option_definition::info should contain any additional human + * readable information text that a typical user may need to + * understand the functionality of the option. + * retro_core_option_definition::values is an array of retro_core_option_value + * structs terminated by a { NULL, NULL } element. + * > retro_core_option_definition::values[index].value is an expected option + * value. + * > retro_core_option_definition::values[index].label is a human readable + * label used when displaying the value on screen. If NULL, + * the value itself is used. + * retro_core_option_definition::default_value is the default core option + * setting. It must match one of the expected option values in the + * retro_core_option_definition::values array. If it does not, or the + * default value is NULL, the first entry in the + * retro_core_option_definition::values array is treated as the default. + * + * The number of possible options should be very limited, + * and must be less than RETRO_NUM_CORE_OPTION_VALUES_MAX. + * i.e. it should be feasible to cycle through options + * without a keyboard. + * + * Example entry: + * { + * "foo_option", + * "Speed hack coprocessor X", + * "Provides increased performance at the expense of reduced accuracy", + * { + * { "false", NULL }, + * { "true", NULL }, + * { "unstable", "Turbo (Unstable)" }, + * { NULL, NULL }, + * }, + * "false" + * } + * + * Only strings are operated on. The possible values will + * generally be displayed and stored as-is by the frontend. + */ + +#define RETRO_ENVIRONMENT_SET_CORE_OPTIONS_INTL 54 + /* const struct retro_core_options_intl * -- + * Allows an implementation to signal the environment + * which variables it might want to check for later using + * GET_VARIABLE. + * This allows the frontend to present these variables to + * a user dynamically. + * This should only be called if RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION + * returns an API version of >= 1. + * This should be called instead of RETRO_ENVIRONMENT_SET_VARIABLES. + * This should be called the first time as early as + * possible (ideally in retro_set_environment). + * Afterwards it may be called again for the core to communicate + * updated options to the frontend, but the number of core + * options must not change from the number in the initial call. + * + * This is fundamentally the same as RETRO_ENVIRONMENT_SET_CORE_OPTIONS, + * with the addition of localisation support. The description of the + * RETRO_ENVIRONMENT_SET_CORE_OPTIONS callback should be consulted + * for further details. + * + * 'data' points to a retro_core_options_intl struct. + * + * retro_core_options_intl::us is a pointer to an array of + * retro_core_option_definition structs defining the US English + * core options implementation. It must point to a valid array. + * + * retro_core_options_intl::local is a pointer to an array of + * retro_core_option_definition structs defining core options for + * the current frontend language. It may be NULL (in which case + * retro_core_options_intl::us is used by the frontend). Any items + * missing from this array will be read from retro_core_options_intl::us + * instead. + * + * NOTE: Default core option values are always taken from the + * retro_core_options_intl::us array. Any default values in + * retro_core_options_intl::local array will be ignored. + */ + +#define RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY 55 + /* struct retro_core_option_display * -- + * + * Allows an implementation to signal the environment to show + * or hide a variable when displaying core options. This is + * considered a *suggestion*. The frontend is free to ignore + * this callback, and its implementation not considered mandatory. + * + * 'data' points to a retro_core_option_display struct + * + * retro_core_option_display::key is a variable identifier + * which has already been set by SET_VARIABLES/SET_CORE_OPTIONS. + * + * retro_core_option_display::visible is a boolean, specifying + * whether variable should be displayed + * + * Note that all core option variables will be set visible by + * default when calling SET_VARIABLES/SET_CORE_OPTIONS. + */ + +/* VFS functionality */ + +/* File paths: + * File paths passed as parameters when using this API shall be well formed UNIX-style, + * using "/" (unquoted forward slash) as directory separator regardless of the platform's native separator. + * Paths shall also include at least one forward slash ("game.bin" is an invalid path, use "./game.bin" instead). + * Other than the directory separator, cores shall not make assumptions about path format: + * "C:/path/game.bin", "http://example.com/game.bin", "#game/game.bin", "./game.bin" (without quotes) are all valid paths. + * Cores may replace the basename or remove path components from the end, and/or add new components; + * however, cores shall not append "./", "../" or multiple consecutive forward slashes ("//") to paths they request to front end. + * The frontend is encouraged to make such paths work as well as it can, but is allowed to give up if the core alters paths too much. + * Frontends are encouraged, but not required, to support native file system paths (modulo replacing the directory separator, if applicable). + * Cores are allowed to try using them, but must remain functional if the front rejects such requests. + * Cores are encouraged to use the libretro-common filestream functions for file I/O, + * as they seamlessly integrate with VFS, deal with directory separator replacement as appropriate + * and provide platform-specific fallbacks in cases where front ends do not support VFS. */ + +/* Opaque file handle + * Introduced in VFS API v1 */ +struct retro_vfs_file_handle; + +/* Opaque directory handle + * Introduced in VFS API v3 */ +struct retro_vfs_dir_handle; + +/* File open flags + * Introduced in VFS API v1 */ +#define RETRO_VFS_FILE_ACCESS_READ (1 << 0) /* Read only mode */ +#define RETRO_VFS_FILE_ACCESS_WRITE (1 << 1) /* Write only mode, discard contents and overwrites existing file unless RETRO_VFS_FILE_ACCESS_UPDATE is also specified */ +#define RETRO_VFS_FILE_ACCESS_READ_WRITE (RETRO_VFS_FILE_ACCESS_READ | RETRO_VFS_FILE_ACCESS_WRITE) /* Read-write mode, discard contents and overwrites existing file unless RETRO_VFS_FILE_ACCESS_UPDATE is also specified*/ +#define RETRO_VFS_FILE_ACCESS_UPDATE_EXISTING (1 << 2) /* Prevents discarding content of existing files opened for writing */ + +/* These are only hints. The frontend may choose to ignore them. Other than RAM/CPU/etc use, + and how they react to unlikely external interference (for example someone else writing to that file, + or the file's server going down), behavior will not change. */ +#define RETRO_VFS_FILE_ACCESS_HINT_NONE (0) +/* Indicate that the file will be accessed many times. The frontend should aggressively cache everything. */ +#define RETRO_VFS_FILE_ACCESS_HINT_FREQUENT_ACCESS (1 << 0) + +/* Seek positions */ +#define RETRO_VFS_SEEK_POSITION_START 0 +#define RETRO_VFS_SEEK_POSITION_CURRENT 1 +#define RETRO_VFS_SEEK_POSITION_END 2 + +/* stat() result flags + * Introduced in VFS API v3 */ +#define RETRO_VFS_STAT_IS_VALID (1 << 0) +#define RETRO_VFS_STAT_IS_DIRECTORY (1 << 1) +#define RETRO_VFS_STAT_IS_CHARACTER_SPECIAL (1 << 2) + +/* Get path from opaque handle. Returns the exact same path passed to file_open when getting the handle + * Introduced in VFS API v1 */ +typedef const char *(RETRO_CALLCONV *retro_vfs_get_path_t)(struct retro_vfs_file_handle *stream); + +/* Open a file for reading or writing. If path points to a directory, this will + * fail. Returns the opaque file handle, or NULL for error. + * Introduced in VFS API v1 */ +typedef struct retro_vfs_file_handle *(RETRO_CALLCONV *retro_vfs_open_t)(const char *path, unsigned mode, unsigned hints); + +/* Close the file and release its resources. Must be called if open_file returns non-NULL. Returns 0 on success, -1 on failure. + * Whether the call succeeds ot not, the handle passed as parameter becomes invalid and should no longer be used. + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_close_t)(struct retro_vfs_file_handle *stream); + +/* Return the size of the file in bytes, or -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_size_t)(struct retro_vfs_file_handle *stream); + +/* Truncate file to specified size. Returns 0 on success or -1 on error + * Introduced in VFS API v2 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_truncate_t)(struct retro_vfs_file_handle *stream, int64_t length); + +/* Get the current read / write position for the file. Returns -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_tell_t)(struct retro_vfs_file_handle *stream); + +/* Set the current read/write position for the file. Returns the new position, -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_seek_t)(struct retro_vfs_file_handle *stream, int64_t offset, int seek_position); + +/* Read data from a file. Returns the number of bytes read, or -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_read_t)(struct retro_vfs_file_handle *stream, void *s, uint64_t len); + +/* Write data to a file. Returns the number of bytes written, or -1 for error. + * Introduced in VFS API v1 */ +typedef int64_t (RETRO_CALLCONV *retro_vfs_write_t)(struct retro_vfs_file_handle *stream, const void *s, uint64_t len); + +/* Flush pending writes to file, if using buffered IO. Returns 0 on sucess, or -1 on failure. + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_flush_t)(struct retro_vfs_file_handle *stream); + +/* Delete the specified file. Returns 0 on success, -1 on failure + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_remove_t)(const char *path); + +/* Rename the specified file. Returns 0 on success, -1 on failure + * Introduced in VFS API v1 */ +typedef int (RETRO_CALLCONV *retro_vfs_rename_t)(const char *old_path, const char *new_path); + +/* Stat the specified file. Retruns a bitmask of RETRO_VFS_STAT_* flags, none are set if path was not valid. + * Additionally stores file size in given variable, unless NULL is given. + * Introduced in VFS API v3 */ +typedef int (RETRO_CALLCONV *retro_vfs_stat_t)(const char *path, int32_t *size); + +/* Create the specified directory. Returns 0 on success, -1 on unknown failure, -2 if already exists. + * Introduced in VFS API v3 */ +typedef int (RETRO_CALLCONV *retro_vfs_mkdir_t)(const char *dir); + +/* Open the specified directory for listing. Returns the opaque dir handle, or NULL for error. + * Support for the include_hidden argument may vary depending on the platform. + * Introduced in VFS API v3 */ +typedef struct retro_vfs_dir_handle *(RETRO_CALLCONV *retro_vfs_opendir_t)(const char *dir, bool include_hidden); + +/* Read the directory entry at the current position, and move the read pointer to the next position. + * Returns true on success, false if already on the last entry. + * Introduced in VFS API v3 */ +typedef bool (RETRO_CALLCONV *retro_vfs_readdir_t)(struct retro_vfs_dir_handle *dirstream); + +/* Get the name of the last entry read. Returns a string on success, or NULL for error. + * The returned string pointer is valid until the next call to readdir or closedir. + * Introduced in VFS API v3 */ +typedef const char *(RETRO_CALLCONV *retro_vfs_dirent_get_name_t)(struct retro_vfs_dir_handle *dirstream); + +/* Check if the last entry read was a directory. Returns true if it was, false otherwise (or on error). + * Introduced in VFS API v3 */ +typedef bool (RETRO_CALLCONV *retro_vfs_dirent_is_dir_t)(struct retro_vfs_dir_handle *dirstream); + +/* Close the directory and release its resources. Must be called if opendir returns non-NULL. Returns 0 on success, -1 on failure. + * Whether the call succeeds ot not, the handle passed as parameter becomes invalid and should no longer be used. + * Introduced in VFS API v3 */ +typedef int (RETRO_CALLCONV *retro_vfs_closedir_t)(struct retro_vfs_dir_handle *dirstream); + +struct retro_vfs_interface +{ + /* VFS API v1 */ + retro_vfs_get_path_t get_path; + retro_vfs_open_t open; + retro_vfs_close_t close; + retro_vfs_size_t size; + retro_vfs_tell_t tell; + retro_vfs_seek_t seek; + retro_vfs_read_t read; + retro_vfs_write_t write; + retro_vfs_flush_t flush; + retro_vfs_remove_t remove; + retro_vfs_rename_t rename; + /* VFS API v2 */ + retro_vfs_truncate_t truncate; + /* VFS API v3 */ + retro_vfs_stat_t stat; + retro_vfs_mkdir_t mkdir; + retro_vfs_opendir_t opendir; + retro_vfs_readdir_t readdir; + retro_vfs_dirent_get_name_t dirent_get_name; + retro_vfs_dirent_is_dir_t dirent_is_dir; + retro_vfs_closedir_t closedir; +}; + +struct retro_vfs_interface_info +{ + /* Set by core: should this be higher than the version the front end supports, + * front end will return false in the RETRO_ENVIRONMENT_GET_VFS_INTERFACE call + * Introduced in VFS API v1 */ + uint32_t required_interface_version; + + /* Frontend writes interface pointer here. The frontend also sets the actual + * version, must be at least required_interface_version. + * Introduced in VFS API v1 */ + struct retro_vfs_interface *iface; +}; + +enum retro_hw_render_interface_type +{ + RETRO_HW_RENDER_INTERFACE_VULKAN = 0, + RETRO_HW_RENDER_INTERFACE_D3D9 = 1, + RETRO_HW_RENDER_INTERFACE_D3D10 = 2, + RETRO_HW_RENDER_INTERFACE_D3D11 = 3, + RETRO_HW_RENDER_INTERFACE_D3D12 = 4, + RETRO_HW_RENDER_INTERFACE_GSKIT_PS2 = 5, + RETRO_HW_RENDER_INTERFACE_DUMMY = INT_MAX +}; + +/* Base struct. All retro_hw_render_interface_* types + * contain at least these fields. */ +struct retro_hw_render_interface +{ + enum retro_hw_render_interface_type interface_type; + unsigned interface_version; +}; + +typedef void (RETRO_CALLCONV *retro_set_led_state_t)(int led, int state); +struct retro_led_interface +{ + retro_set_led_state_t set_led_state; +}; + +/* Retrieves the current state of the MIDI input. + * Returns true if it's enabled, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_input_enabled_t)(void); + +/* Retrieves the current state of the MIDI output. + * Returns true if it's enabled, false otherwise */ +typedef bool (RETRO_CALLCONV *retro_midi_output_enabled_t)(void); + +/* Reads next byte from the input stream. + * Returns true if byte is read, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_read_t)(uint8_t *byte); + +/* Writes byte to the output stream. + * 'delta_time' is in microseconds and represent time elapsed since previous write. + * Returns true if byte is written, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_write_t)(uint8_t byte, uint32_t delta_time); + +/* Flushes previously written data. + * Returns true if successful, false otherwise. */ +typedef bool (RETRO_CALLCONV *retro_midi_flush_t)(void); + +struct retro_midi_interface +{ + retro_midi_input_enabled_t input_enabled; + retro_midi_output_enabled_t output_enabled; + retro_midi_read_t read; + retro_midi_write_t write; + retro_midi_flush_t flush; +}; enum retro_hw_render_context_negotiation_interface_type {@@ -968,77 +1486,97 @@ {
enum retro_hw_render_context_negotiation_interface_type interface_type; unsigned interface_version; }; -#define RETRO_ENVIRONMENT_SET_HW_RENDER_CONTEXT_NEGOTIATION_INTERFACE (43 | RETRO_ENVIRONMENT_EXPERIMENTAL) - /* const struct retro_hw_render_context_negotiation_interface * -- - * Sets an interface which lets the libretro core negotiate with frontend how a context is created. - * The semantics of this interface depends on which API is used in SET_HW_RENDER earlier. - * This interface will be used when the frontend is trying to create a HW rendering context, - * so it will be used after SET_HW_RENDER, but before the context_reset callback. - */ + +/* Serialized state is incomplete in some way. Set if serialization is + * usable in typical end-user cases but should not be relied upon to + * implement frame-sensitive frontend features such as netplay or + * rerecording. */ +#define RETRO_SERIALIZATION_QUIRK_INCOMPLETE (1 << 0) +/* The core must spend some time initializing before serialization is + * supported. retro_serialize() will initially fail; retro_unserialize() + * and retro_serialize_size() may or may not work correctly either. */ +#define RETRO_SERIALIZATION_QUIRK_MUST_INITIALIZE (1 << 1) +/* Serialization size may change within a session. */ +#define RETRO_SERIALIZATION_QUIRK_CORE_VARIABLE_SIZE (1 << 2) +/* Set by the frontend to acknowledge that it supports variable-sized + * states. */ +#define RETRO_SERIALIZATION_QUIRK_FRONT_VARIABLE_SIZE (1 << 3) +/* Serialized state can only be loaded during the same session. */ +#define RETRO_SERIALIZATION_QUIRK_SINGLE_SESSION (1 << 4) +/* Serialized state cannot be loaded on an architecture with a different + * endianness from the one it was saved on. */ +#define RETRO_SERIALIZATION_QUIRK_ENDIAN_DEPENDENT (1 << 5) +/* Serialized state cannot be loaded on a different platform from the one it + * was saved on for reasons other than endianness, such as word size + * dependence */ +#define RETRO_SERIALIZATION_QUIRK_PLATFORM_DEPENDENT (1 << 6) -#define RETRO_MEMDESC_CONST (1 << 0) /* The frontend will never change this memory area once retro_load_game has returned. */ -#define RETRO_MEMDESC_BIGENDIAN (1 << 1) /* The memory area contains big endian data. Default is little endian. */ -#define RETRO_MEMDESC_ALIGN_2 (1 << 16) /* All memory access in this area is aligned to their own size, or 2, whichever is smaller. */ -#define RETRO_MEMDESC_ALIGN_4 (2 << 16) -#define RETRO_MEMDESC_ALIGN_8 (3 << 16) -#define RETRO_MEMDESC_MINSIZE_2 (1 << 24) /* All memory in this region is accessed at least 2 bytes at the time. */ -#define RETRO_MEMDESC_MINSIZE_4 (2 << 24) -#define RETRO_MEMDESC_MINSIZE_8 (3 << 24) +#define RETRO_MEMDESC_CONST (1 << 0) /* The frontend will never change this memory area once retro_load_game has returned. */ +#define RETRO_MEMDESC_BIGENDIAN (1 << 1) /* The memory area contains big endian data. Default is little endian. */ +#define RETRO_MEMDESC_SYSTEM_RAM (1 << 2) /* The memory area is system RAM. This is main RAM of the gaming system. */ +#define RETRO_MEMDESC_SAVE_RAM (1 << 3) /* The memory area is save RAM. This RAM is usually found on a game cartridge, backed up by a battery. */ +#define RETRO_MEMDESC_VIDEO_RAM (1 << 4) /* The memory area is video RAM (VRAM) */ +#define RETRO_MEMDESC_ALIGN_2 (1 << 16) /* All memory access in this area is aligned to their own size, or 2, whichever is smaller. */ +#define RETRO_MEMDESC_ALIGN_4 (2 << 16) +#define RETRO_MEMDESC_ALIGN_8 (3 << 16) +#define RETRO_MEMDESC_MINSIZE_2 (1 << 24) /* All memory in this region is accessed at least 2 bytes at the time. */ +#define RETRO_MEMDESC_MINSIZE_4 (2 << 24) +#define RETRO_MEMDESC_MINSIZE_8 (3 << 24) struct retro_memory_descriptor { uint64_t flags; /* Pointer to the start of the relevant ROM or RAM chip. - * It's strongly recommended to use 'offset' if possible, rather than + * It's strongly recommended to use 'offset' if possible, rather than * doing math on the pointer. * - * If the same byte is mapped my multiple descriptors, their descriptors + * If the same byte is mapped my multiple descriptors, their descriptors * must have the same pointer. - * If 'start' does not point to the first byte in the pointer, put the + * If 'start' does not point to the first byte in the pointer, put the * difference in 'offset' instead. * - * May be NULL if there's nothing usable here (e.g. hardware registers and + * May be NULL if there's nothing usable here (e.g. hardware registers and * open bus). No flags should be set if the pointer is NULL. * It's recommended to minimize the number of descriptors if possible, * but not mandatory. */ void *ptr; size_t offset; - /* This is the location in the emulated address space + /* This is the location in the emulated address space * where the mapping starts. */ size_t start; /* Which bits must be same as in 'start' for this mapping to apply. - * The first memory descriptor to claim a certain byte is the one + * The first memory descriptor to claim a certain byte is the one * that applies. * A bit which is set in 'start' must also be set in this. - * Can be zero, in which case each byte is assumed mapped exactly once. + * Can be zero, in which case each byte is assumed mapped exactly once. * In this case, 'len' must be a power of two. */ size_t select; - /* If this is nonzero, the set bits are assumed not connected to the + /* If this is nonzero, the set bits are assumed not connected to the * memory chip's address pins. */ size_t disconnect; /* This one tells the size of the current memory area. - * If, after start+disconnect are applied, the address is higher than + * If, after start+disconnect are applied, the address is higher than * this, the highest bit of the address is cleared. * * If the address is still too high, the next highest bit is cleared. - * Can be zero, in which case it's assumed to be infinite (as limited + * Can be zero, in which case it's assumed to be infinite (as limited * by 'select' and 'disconnect'). */ size_t len; - /* To go from emulated address to physical address, the following + /* To go from emulated address to physical address, the following * order applies: * Subtract 'start', pick off 'disconnect', apply 'len', add 'offset'. */ - /* The address space name must consist of only a-zA-Z0-9_-, + /* The address space name must consist of only a-zA-Z0-9_-, * should be as short as feasible (maximum length is 8 plus the NUL), - * and may not be any other address space plus one or more 0-9A-F + * and may not be any other address space plus one or more 0-9A-F * at the end. - * However, multiple memory descriptors for the same address space is - * allowed, and the address space name can be empty. NULL is treated + * However, multiple memory descriptors for the same address space is + * allowed, and the address space name can be empty. NULL is treated * as empty. * * Address space names are case sensitive, but avoid lowercase if possible.@@ -1052,11 +1590,11 @@ * 'S'+blank - valid ('S' is not in 0-9A-F)
* 'a'+blank - valid ('a' is not in 0-9A-F) * 'a'+'A' - valid (neither is a prefix of each other) * 'AR'+blank - valid ('R' is not in 0-9A-F) - * 'ARB'+blank - valid (the B can't be part of the address either, because + * 'ARB'+blank - valid (the B can't be part of the address either, because * there is no namespace 'AR') - * blank+'B' - not valid, because it's ambigous which address space B1234 + * blank+'B' - not valid, because it's ambigous which address space B1234 * would refer to. - * The length can't be used for that purpose; the frontend may want + * The length can't be used for that purpose; the frontend may want * to append arbitrary data to an address, without a separator. */ const char *addrspace;@@ -1078,32 +1616,32 @@ * Algorithm: Take bits highest to lowest, but if it goes above len, clear
* the most recent addition and continue on the next bit. * TODO: Can the above be optimized? Is "remove the lowest bit set in both * pointer and 'len'" equivalent? */ - + /* TODO: Some emulators (MAME?) emulate big endian systems by only accessing * the emulated memory in 32-bit chunks, native endian. But that's nothing * compared to Darek Mihocka <http://www.emulators.com/docs/nx07_vm101.htm> * (section Emulation 103 - Nearly Free Byte Reversal) - he flips the ENTIRE * RAM backwards! I'll want to represent both of those, via some flags. - * + * * I suspect MAME either didn't think of that idea, or don't want the #ifdef. * Not sure which, nor do I really care. */ - + /* TODO: Some of those flags are unused and/or don't really make sense. Clean * them up. */ }; -/* The frontend may use the largest value of 'start'+'select' in a +/* The frontend may use the largest value of 'start'+'select' in a * certain namespace to infer the size of the address space. * - * If the address space is larger than that, a mapping with .ptr=NULL - * should be at the end of the array, with .select set to all ones for + * If the address space is larger than that, a mapping with .ptr=NULL + * should be at the end of the array, with .select set to all ones for * as long as the address space is big. * * Sample descriptors (minus .ptr, and RETRO_MEMFLAG_ on the flags): * SNES WRAM: * .start=0x7E0000, .len=0x20000 - * (Note that this must be mapped before the ROM in most cases; some of the - * ROM mappers + * (Note that this must be mapped before the ROM in most cases; some of the + * ROM mappers * try to claim $7E0000, or at least $7E8000.) * SNES SPC700 RAM: * .addrspace="S", .len=0x10000@@ -1112,7 +1650,7 @@ * .flags=MIRROR, .start=0x000000, .select=0xC0E000, .len=0x2000
* .flags=MIRROR, .start=0x800000, .select=0xC0E000, .len=0x2000 * SNES WRAM mirrors, alternate equivalent descriptor: * .flags=MIRROR, .select=0x40E000, .disconnect=~0x1FFF - * (Various similar constructions can be created by combining parts of + * (Various similar constructions can be created by combining parts of * the above two.) * SNES LoROM (512KB, mirrored a couple of times): * .flags=CONST, .start=0x008000, .select=0x408000, .disconnect=0x8000, .len=512*1024@@ -1138,13 +1676,13 @@ };
struct retro_controller_description { - /* Human-readable description of the controller. Even if using a generic - * input device type, this can be set to the particular device type the + /* Human-readable description of the controller. Even if using a generic + * input device type, this can be set to the particular device type the * core uses. */ const char *desc; - /* Device type passed to retro_set_controller_port_device(). If the device - * type is a sub-class of a generic input device type, use the + /* Device type passed to retro_set_controller_port_device(). If the device + * type is a sub-class of a generic input device type, use the * RETRO_DEVICE_SUBCLASS macro to create an ID. * * E.g. RETRO_DEVICE_SUBCLASS(RETRO_DEVICE_JOYPAD, 1). */@@ -1162,8 +1700,8 @@ {
/* The extension associated with a memory type, e.g. "psram". */ const char *extension; - /* The memory type for retro_get_memory(). This should be at - * least 0x100 to avoid conflict with standardized + /* The memory type for retro_get_memory(). This should be at + * least 0x100 to avoid conflict with standardized * libretro memory types. */ unsigned type; };@@ -1182,11 +1720,11 @@
/* Same definition as retro_get_system_info(). */ bool block_extract; - /* This is set if the content is required to load a game. + /* This is set if the content is required to load a game. * If this is set to false, a zeroed-out retro_game_info can be passed. */ bool required; - /* Content can have multiple associated persistent + /* Content can have multiple associated persistent * memory types (retro_get_memory()). */ const struct retro_subsystem_memory_info *memory; unsigned num_memory;@@ -1204,17 +1742,17 @@ * This identifier can be used for command-line interfaces, etc.
*/ const char *ident; - /* Infos for each content file. The first entry is assumed to be the + /* Infos for each content file. The first entry is assumed to be the * "most significant" content for frontend purposes. - * E.g. with Super GameBoy, the first content should be the GameBoy ROM, + * E.g. with Super GameBoy, the first content should be the GameBoy ROM, * as it is the most "significant" content to a user. - * If a frontend creates new file paths based on the content used + * If a frontend creates new file paths based on the content used * (e.g. savestates), it should use the path for the first ROM to do so. */ const struct retro_subsystem_rom_info *roms; /* Number of content files associated with a subsystem. */ unsigned num_roms; - + /* The type passed to retro_load_game_special(). */ unsigned id; };@@ -1225,13 +1763,13 @@ /* libretro API extension functions:
* (None here so far). * * Get a symbol from a libretro core. - * Cores should only return symbols which are actual + * Cores should only return symbols which are actual * extensions to the libretro API. * - * Frontends should not use this to obtain symbols to standard + * Frontends should not use this to obtain symbols to standard * libretro entry points (static linking or dlsym). * - * The symbol name must be equal to the function name, + * The symbol name must be equal to the function name, * e.g. if void retro_foo(void); exists, the symbol must be called "retro_foo". * The returned function pointer must be cast to the corresponding type. */@@ -1284,6 +1822,8 @@ #define RETRO_SIMD_VFPV3 (1 << 16)
#define RETRO_SIMD_VFPV4 (1 << 17) #define RETRO_SIMD_POPCNT (1 << 18) #define RETRO_SIMD_MOVBE (1 << 19) +#define RETRO_SIMD_CMOV (1 << 20) +#define RETRO_SIMD_ASIMD (1 << 21) typedef uint64_t retro_perf_tick_t; typedef int64_t retro_time_t;@@ -1304,7 +1844,7 @@ */
typedef retro_time_t (RETRO_CALLCONV *retro_perf_get_time_usec_t)(void); /* A simple counter. Usually nanoseconds, but can also be CPU cycles. - * Can be used directly if desired (when creating a more sophisticated + * Can be used directly if desired (when creating a more sophisticated * performance counter system). * */ typedef retro_perf_tick_t (RETRO_CALLCONV *retro_perf_get_counter_t)(void);@@ -1318,9 +1858,9 @@ */
typedef void (RETRO_CALLCONV *retro_perf_log_t)(void); /* Register a performance counter. - * ident field must be set with a discrete value and other values in + * ident field must be set with a discrete value and other values in * retro_perf_counter must be 0. - * Registering can be called multiple times. To avoid calling to + * Registering can be called multiple times. To avoid calling to * frontend redundantly, you can check registered field first. */ typedef void (RETRO_CALLCONV *retro_perf_register_t)(struct retro_perf_counter *counter);@@ -1391,7 +1931,7 @@ #define RETRO_SENSOR_ACCELEROMETER_X 0
#define RETRO_SENSOR_ACCELEROMETER_Y 1 #define RETRO_SENSOR_ACCELEROMETER_Z 2 -typedef bool (RETRO_CALLCONV *retro_set_sensor_state_t)(unsigned port, +typedef bool (RETRO_CALLCONV *retro_set_sensor_state_t)(unsigned port, enum retro_sensor_action action, unsigned rate); typedef float (RETRO_CALLCONV *retro_sensor_get_input_t)(unsigned port, unsigned id);@@ -1416,7 +1956,7 @@
/* Stops the camera driver. Can only be called in retro_run(). */ typedef void (RETRO_CALLCONV *retro_camera_stop_t)(void); -/* Callback which signals when the camera driver is initialized +/* Callback which signals when the camera driver is initialized * and/or deinitialized. * retro_camera_start_t can be called in initialized callback. */@@ -1426,36 +1966,36 @@ /* A callback for raw framebuffer data. buffer points to an XRGB8888 buffer.
* Width, height and pitch are similar to retro_video_refresh_t. * First pixel is top-left origin. */ -typedef void (RETRO_CALLCONV *retro_camera_frame_raw_framebuffer_t)(const uint32_t *buffer, +typedef void (RETRO_CALLCONV *retro_camera_frame_raw_framebuffer_t)(const uint32_t *buffer, unsigned width, unsigned height, size_t pitch); /* A callback for when OpenGL textures are used. * * texture_id is a texture owned by camera driver. - * Its state or content should be considered immutable, except for things like + * Its state or content should be considered immutable, except for things like * texture filtering and clamping. * * texture_target is the texture target for the GL texture. - * These can include e.g. GL_TEXTURE_2D, GL_TEXTURE_RECTANGLE, and possibly + * These can include e.g. GL_TEXTURE_2D, GL_TEXTURE_RECTANGLE, and possibly * more depending on extensions. * - * affine points to a packed 3x3 column-major matrix used to apply an affine + * affine points to a packed 3x3 column-major matrix used to apply an affine * transform to texture coordinates. (affine_matrix * vec3(coord_x, coord_y, 1.0)) - * After transform, normalized texture coord (0, 0) should be bottom-left + * After transform, normalized texture coord (0, 0) should be bottom-left * and (1, 1) should be top-right (or (width, height) for RECTANGLE). * - * GL-specific typedefs are avoided here to avoid relying on gl.h in + * GL-specific typedefs are avoided here to avoid relying on gl.h in * the API definition. */ -typedef void (RETRO_CALLCONV *retro_camera_frame_opengl_texture_t)(unsigned texture_id, +typedef void (RETRO_CALLCONV *retro_camera_frame_opengl_texture_t)(unsigned texture_id, unsigned texture_target, const float *affine); struct retro_camera_callback { - /* Set by libretro core. + /* Set by libretro core. * Example bitmask: caps = (1 << RETRO_CAMERA_BUFFER_OPENGL_TEXTURE) | (1 << RETRO_CAMERA_BUFFER_RAW_FRAMEBUFFER). */ - uint64_t caps; + uint64_t caps; /* Desired resolution for camera. Is only used as a hint. */ unsigned width;@@ -1469,22 +2009,22 @@ /* Set by libretro core if raw framebuffer callbacks will be used. */
retro_camera_frame_raw_framebuffer_t frame_raw_framebuffer; /* Set by libretro core if OpenGL texture callbacks will be used. */ - retro_camera_frame_opengl_texture_t frame_opengl_texture; + retro_camera_frame_opengl_texture_t frame_opengl_texture; - /* Set by libretro core. Called after camera driver is initialized and + /* Set by libretro core. Called after camera driver is initialized and * ready to be started. * Can be NULL, in which this callback is not called. */ retro_camera_lifetime_status_t initialized; - /* Set by libretro core. Called right before camera driver is + /* Set by libretro core. Called right before camera driver is * deinitialized. * Can be NULL, in which this callback is not called. */ retro_camera_lifetime_status_t deinitialized; }; -/* Sets the interval of time and/or distance at which to update/poll +/* Sets the interval of time and/or distance at which to update/poll * location-based data. * * To ensure compatibility with all location-based implementations,@@ -1497,20 +2037,20 @@ typedef void (RETRO_CALLCONV *retro_location_set_interval_t)(unsigned interval_ms,
unsigned interval_distance); /* Start location services. The device will start listening for changes to the - * current location at regular intervals (which are defined with + * current location at regular intervals (which are defined with * retro_location_set_interval_t). */ typedef bool (RETRO_CALLCONV *retro_location_start_t)(void); -/* Stop location services. The device will stop listening for changes +/* Stop location services. The device will stop listening for changes * to the current location. */ typedef void (RETRO_CALLCONV *retro_location_stop_t)(void); -/* Get the position of the current location. Will set parameters to +/* Get the position of the current location. Will set parameters to * 0 if no new location update has happened since the last time. */ typedef bool (RETRO_CALLCONV *retro_location_get_position_t)(double *lat, double *lon, double *horiz_accuracy, double *vert_accuracy); -/* Callback which signals when the location driver is initialized +/* Callback which signals when the location driver is initialized * and/or deinitialized. * retro_location_start_t can be called in initialized callback. */@@ -1535,14 +2075,14 @@
RETRO_RUMBLE_DUMMY = INT_MAX }; -/* Sets rumble state for joypad plugged in port 'port'. +/* Sets rumble state for joypad plugged in port 'port'. * Rumble effects are controlled independently, * and setting e.g. strong rumble does not override weak rumble. * Strength has a range of [0, 0xffff]. * - * Returns true if rumble state request was honored. + * Returns true if rumble state request was honored. * Calling this before first retro_run() is likely to return false. */ -typedef bool (RETRO_CALLCONV *retro_set_rumble_state_t)(unsigned port, +typedef bool (RETRO_CALLCONV *retro_set_rumble_state_t)(unsigned port, enum retro_rumble_effect effect, uint16_t strength); struct retro_rumble_interface@@ -1553,10 +2093,10 @@
/* Notifies libretro that audio data should be written. */ typedef void (RETRO_CALLCONV *retro_audio_callback_t)(void); -/* True: Audio driver in frontend is active, and callback is +/* True: Audio driver in frontend is active, and callback is * expected to be called regularily. - * False: Audio driver in frontend is paused or inactive. - * Audio callback will not be called until set_state has been + * False: Audio driver in frontend is paused or inactive. + * Audio callback will not be called until set_state has been * called with true. * Initial state is false (inactive). */@@ -1568,11 +2108,11 @@ retro_audio_callback_t callback;
retro_audio_set_state_callback_t set_state; }; -/* Notifies a libretro core of time spent since last invocation +/* Notifies a libretro core of time spent since last invocation * of retro_run() in microseconds. * * It will be called right before retro_run() every frame. - * The frontend can tamper with timing to support cases like + * The frontend can tamper with timing to support cases like * fast-forward, slow-motion and framestepping. * * In those scenarios the reference frame time value will be used. */@@ -1581,8 +2121,8 @@ typedef void (RETRO_CALLCONV *retro_frame_time_callback_t)(retro_usec_t usec);
struct retro_frame_time_callback { retro_frame_time_callback_t callback; - /* Represents the time of one frame. It is computed as - * 1000000 / fps, but the implementation will resolve the + /* Represents the time of one frame. It is computed as + * 1000000 / fps, but the implementation will resolve the * rounding to ensure that framestepping, etc is exact. */ retro_usec_t reference; };@@ -1598,7 +2138,7 @@ * If explicit deinitialization is desired by the libretro core,
* it should implement context_destroy callback. * If called, all GPU resources must be reinitialized. * Usually called when frontend reinits video driver. - * Also called first time video driver is initialized, + * Also called first time video driver is initialized, * allowing libretro core to initialize resources. */ typedef void (RETRO_CALLCONV *retro_hw_context_reset_t)(void);@@ -1615,7 +2155,7 @@ enum retro_hw_context_type
{ RETRO_HW_CONTEXT_NONE = 0, /* OpenGL 2.x. Driver can choose to use latest compatibility context. */ - RETRO_HW_CONTEXT_OPENGL = 1, + RETRO_HW_CONTEXT_OPENGL = 1, /* OpenGL ES 2.0. */ RETRO_HW_CONTEXT_OPENGLES2 = 2, /* Modern desktop core GL context. Use version_major/@@ -1630,6 +2170,10 @@
/* Vulkan, see RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE. */ RETRO_HW_CONTEXT_VULKAN = 6, + /* Direct3D, set version_major to select the type of interface + * returned by RETRO_ENVIRONMENT_GET_HW_RENDER_INTERFACE */ + RETRO_HW_CONTEXT_DIRECT3D = 7, + RETRO_HW_CONTEXT_DUMMY = INT_MAX };@@ -1641,10 +2185,10 @@
/* Called when a context has been created or when it has been reset. * An OpenGL context is only valid after context_reset() has been called. * - * When context_reset is called, OpenGL resources in the libretro + * When context_reset is called, OpenGL resources in the libretro * implementation are guaranteed to be invalid. * - * It is possible that context_reset is called multiple times during an + * It is possible that context_reset is called multiple times during an * application lifecycle. * If context_reset is called without any notification (context_destroy), * the OpenGL context was lost and resources should just be recreated@@ -1657,7 +2201,8 @@ * TODO: This is rather obsolete. The frontend should not
* be providing preallocated framebuffers. */ retro_hw_get_current_framebuffer_t get_current_framebuffer; - /* Set by frontend. */ + /* Set by frontend. + * Can return all relevant functions, including glClear on Windows. */ retro_hw_get_proc_address_t get_proc_address; /* Set if render buffers should have depth component attached.@@ -1668,48 +2213,48 @@ /* Set if stencil buffers should be attached.
* TODO: Obsolete. */ bool stencil; - /* If depth and stencil are true, a packed 24/8 buffer will be added. + /* If depth and stencil are true, a packed 24/8 buffer will be added. * Only attaching stencil is invalid and will be ignored. */ - /* Use conventional bottom-left origin convention. If false, + /* Use conventional bottom-left origin convention. If false, * standard libretro top-left origin semantics are used. * TODO: Move to GL specific interface. */ bool bottom_left_origin; - + /* Major version number for core GL context or GLES 3.1+. */ unsigned version_major; /* Minor version number for core GL context or GLES 3.1+. */ unsigned version_minor; - /* If this is true, the frontend will go very far to avoid + /* If this is true, the frontend will go very far to avoid * resetting context in scenarios like toggling fullscreen, etc. * TODO: Obsolete? Maybe frontend should just always assume this ... */ bool cache_context; - /* The reset callback might still be called in extreme situations + /* The reset callback might still be called in extreme situations * such as if the context is lost beyond recovery. * - * For optimal stability, set this to false, and allow context to be + * For optimal stability, set this to false, and allow context to be * reset at any time. */ - - /* A callback to be called before the context is destroyed in a + + /* A callback to be called before the context is destroyed in a * controlled way by the frontend. */ retro_hw_context_reset_t context_destroy; /* OpenGL resources can be deinitialized cleanly at this step. - * context_destroy can be set to NULL, in which resources will + * context_destroy can be set to NULL, in which resources will * just be destroyed without any notification. * - * Even when context_destroy is non-NULL, it is possible that + * Even when context_destroy is non-NULL, it is possible that * context_reset is called without any destroy notification. - * This happens if context is lost by external factors (such as + * This happens if context is lost by external factors (such as * notified by GL_ARB_robustness). * * In this case, the context is assumed to be already dead, - * and the libretro implementation must not try to free any OpenGL + * and the libretro implementation must not try to free any OpenGL * resources in the subsequent context_reset. */@@ -1717,7 +2262,7 @@ /* Creates a debug context. */
bool debug_context; }; -/* Callback type passed in RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK. +/* Callback type passed in RETRO_ENVIRONMENT_SET_KEYBOARD_CALLBACK. * Called by the frontend in response to keyboard events. * down is set if the key is being pressed, or false if it is being released. * keycode is the RETROK value of the char.@@ -1725,16 +2270,16 @@ * character is the text character of the pressed key. (UTF-32).
* key_modifiers is a set of RETROKMOD values or'ed together. * * The pressed/keycode state can be indepedent of the character. - * It is also possible that multiple characters are generated from a + * It is also possible that multiple characters are generated from a * single keypress. * Keycode events should be treated separately from character events. * However, when possible, the frontend should try to synchronize these. * If only a character is posted, keycode should be RETROK_UNKNOWN. * - * Similarily if only a keycode event is generated with no corresponding + * Similarily if only a keycode event is generated with no corresponding * character, character should be 0. */ -typedef void (RETRO_CALLCONV *retro_keyboard_event_t)(bool down, unsigned keycode, +typedef void (RETRO_CALLCONV *retro_keyboard_event_t)(bool down, unsigned keycode, uint32_t character, uint16_t key_modifiers); struct retro_keyboard_callback@@ -1743,15 +2288,15 @@ retro_keyboard_event_t callback;
}; /* Callbacks for RETRO_ENVIRONMENT_SET_DISK_CONTROL_INTERFACE. - * Should be set for implementations which can swap out multiple disk + * Should be set for implementations which can swap out multiple disk * images in runtime. * * If the implementation can do this automatically, it should strive to do so. * However, there are cases where the user must manually do so. * - * Overview: To swap a disk image, eject the disk image with + * Overview: To swap a disk image, eject the disk image with * set_eject_state(true). - * Set the disk index with set_image_index(index). Insert the disk again + * Set the disk index with set_image_index(index). Insert the disk again * with set_eject_state(false). */@@ -1769,7 +2314,7 @@ */
typedef unsigned (RETRO_CALLCONV *retro_get_image_index_t)(void); /* Sets image index. Can only be called when disk is ejected. - * The implementation supports setting "no disk" by using an + * The implementation supports setting "no disk" by using an * index >= get_num_images(). */ typedef bool (RETRO_CALLCONV *retro_set_image_index_t)(unsigned index);@@ -1783,11 +2328,11 @@ /* Replaces the disk image associated with index.
* Arguments to pass in info have same requirements as retro_load_game(). * Virtual disk tray must be ejected when calling this. * - * Replacing a disk image with info = NULL will remove the disk image + * Replacing a disk image with info = NULL will remove the disk image * from the internal list. * As a result, calls to get_image_index() can change. * - * E.g. replace_image_index(1, NULL), and previous get_image_index() + * E.g. replace_image_index(1, NULL), and previous get_image_index() * returned 4 before. * Index 1 will be removed, and the new index is 3. */@@ -1796,7 +2341,7 @@ const struct retro_game_info *info);
/* Adds a new valid index (get_num_images()) to the internal disk list. * This will increment subsequent return values from get_num_images() by 1. - * This image index cannot be used until a disk image has been set + * This image index cannot be used until a disk image has been set * with replace_image_index. */ typedef bool (RETRO_CALLCONV *retro_add_image_index_t)(void);@@ -1827,7 +2372,7 @@ RETRO_PIXEL_FORMAT_XRGB8888 = 1,
/* RGB565, native endian. * This pixel format is the recommended format to use if a 15/16-bit - * format is desired as it is the pixel format that is typically + * format is desired as it is the pixel format that is typically * available on a wide range of low-power devices. * * It is also natively supported in APIs like OpenGL ES. */@@ -1857,43 +2402,52 @@
/* Human readable description for parameters. * The pointer must remain valid until * retro_unload_game() is called. */ - const char *description; + const char *description; }; struct retro_system_info { - /* All pointers are owned by libretro implementation, and pointers must + /* All pointers are owned by libretro implementation, and pointers must * remain valid until retro_deinit() is called. */ - const char *library_name; /* Descriptive name of library. Should not + const char *library_name; /* Descriptive name of library. Should not * contain any version numbers, etc. */ const char *library_version; /* Descriptive version of core. */ - const char *valid_extensions; /* A string listing probably content - * extensions the core will be able to + const char *valid_extensions; /* A string listing probably content + * extensions the core will be able to * load, separated with pipe. * I.e. "bin|rom|iso". - * Typically used for a GUI to filter + * Typically used for a GUI to filter * out extensions. */ - /* If true, retro_load_game() is guaranteed to provide a valid pathname - * in retro_game_info::path. - * ::data and ::size are both invalid. + /* Libretro cores that need to have direct access to their content + * files, including cores which use the path of the content files to + * determine the paths of other files, should set need_fullpath to true. + * + * Cores should strive for setting need_fullpath to false, + * as it allows the frontend to perform patching, etc. + * + * If need_fullpath is true and retro_load_game() is called: + * - retro_game_info::path is guaranteed to have a valid path + * - retro_game_info::data and retro_game_info::size are invalid * - * If false, ::data and ::size are guaranteed to be valid, but ::path - * might not be valid. + * If need_fullpath is false and retro_load_game() is called: + * - retro_game_info::path may be NULL + * - retro_game_info::data and retro_game_info::size are guaranteed + * to be valid * - * This is typically set to true for libretro implementations that must - * load from file. - * Implementations should strive for setting this to false, as it allows - * the frontend to perform patching, etc. */ - bool need_fullpath; + * See also: + * - RETRO_ENVIRONMENT_GET_SYSTEM_DIRECTORY + * - RETRO_ENVIRONMENT_GET_SAVE_DIRECTORY + */ + bool need_fullpath; - /* If true, the frontend is not allowed to extract any archives before + /* If true, the frontend is not allowed to extract any archives before * loading the real content. - * Necessary for certain libretro implementations that load games + * Necessary for certain libretro implementations that load games * from zipped archives. */ - bool block_extract; + bool block_extract; }; struct retro_game_geometry@@ -1925,27 +2479,99 @@
struct retro_variable { /* Variable to query in RETRO_ENVIRONMENT_GET_VARIABLE. - * If NULL, obtains the complete environment string if more + * If NULL, obtains the complete environment string if more * complex parsing is necessary. - * The environment string is formatted as key-value pairs + * The environment string is formatted as key-value pairs * delimited by semicolons as so: * "key1=value1;key2=value2;..." */ const char *key; - + /* Value to be obtained. If key does not exist, it is set to NULL. */ const char *value; }; +struct retro_core_option_display +{ + /* Variable to configure in RETRO_ENVIRONMENT_SET_CORE_OPTIONS_DISPLAY */ + const char *key; + + /* Specifies whether variable should be displayed + * when presenting core options to the user */ + bool visible; +}; + +/* Maximum number of values permitted for a core option + * > Note: We have to set a maximum value due the limitations + * of the C language - i.e. it is not possible to create an + * array of structs each containing a variable sized array, + * so the retro_core_option_definition values array must + * have a fixed size. The size limit of 128 is a balancing + * act - it needs to be large enough to support all 'sane' + * core options, but setting it too large may impact low memory + * platforms. In practise, if a core option has more than + * 128 values then the implementation is likely flawed. + * To quote the above API reference: + * "The number of possible options should be very limited + * i.e. it should be feasible to cycle through options + * without a keyboard." + */ +#define RETRO_NUM_CORE_OPTION_VALUES_MAX 128 + +struct retro_core_option_value +{ + /* Expected option value */ + const char *value; + + /* Human-readable value label. If NULL, value itself + * will be displayed by the frontend */ + const char *label; +}; + +struct retro_core_option_definition +{ + /* Variable to query in RETRO_ENVIRONMENT_GET_VARIABLE. */ + const char *key; + + /* Human-readable core option description (used as menu label) */ + const char *desc; + + /* Human-readable core option information (used as menu sublabel) */ + const char *info; + + /* Array of retro_core_option_value structs, terminated by NULL */ + struct retro_core_option_value values[RETRO_NUM_CORE_OPTION_VALUES_MAX]; + + /* Default core option value. Must match one of the values + * in the retro_core_option_value array, otherwise will be + * ignored */ + const char *default_value; +}; + +struct retro_core_options_intl +{ + /* Pointer to an array of retro_core_option_definition structs + * - US English implementation + * - Must point to a valid array */ + struct retro_core_option_definition *us; + + /* Pointer to an array of retro_core_option_definition structs + * - Implementation for current frontend language + * - May be NULL */ + struct retro_core_option_definition *local; +}; + struct retro_game_info { const char *path; /* Path to game, UTF-8 encoded. - * Usually used as a reference. - * May be NULL if rom was loaded from stdin - * or similar. - * retro_system_info::need_fullpath guaranteed + * Sometimes used as a reference for building other paths. + * May be NULL if game was loaded from stdin or similar, + * but in this case some cores will be unable to load `data`. + * So, it is preferable to fabricate something here instead + * of passing NULL, which will help more cores to succeed. + * retro_system_info::need_fullpath requires * that this path is valid. */ - const void *data; /* Memory buffer of loaded game. Will be NULL + const void *data; /* Memory buffer of loaded game. Will be NULL * if need_fullpath was set. */ size_t size; /* Size of memory buffer. */ const char *meta; /* String of implementation specific meta-data. */@@ -1983,25 +2609,25 @@ };
/* Callbacks */ -/* Environment callback. Gives implementations a way of performing +/* Environment callback. Gives implementations a way of performing * uncommon tasks. Extensible. */ typedef bool (RETRO_CALLCONV *retro_environment_t)(unsigned cmd, void *data); -/* Render a frame. Pixel format is 15-bit 0RGB1555 native endian +/* Render a frame. Pixel format is 15-bit 0RGB1555 native endian * unless changed (see RETRO_ENVIRONMENT_SET_PIXEL_FORMAT). * * Width and height specify dimensions of buffer. * Pitch specifices length in bytes between two lines in buffer. * - * For performance reasons, it is highly recommended to have a frame + * For performance reasons, it is highly recommended to have a frame * that is packed in memory, i.e. pitch == width * byte_per_pixel. - * Certain graphic APIs, such as OpenGL ES, do not like textures + * Certain graphic APIs, such as OpenGL ES, do not like textures * that are not packed in memory. */ typedef void (RETRO_CALLCONV *retro_video_refresh_t)(const void *data, unsigned width, unsigned height, size_t pitch); -/* Renders a single audio frame. Should only be used if implementation +/* Renders a single audio frame. Should only be used if implementation * generates a single sample at a time. * Format is signed 16-bit native endian. */@@ -2019,20 +2645,20 @@
/* Polls input. */ typedef void (RETRO_CALLCONV *retro_input_poll_t)(void); -/* Queries for input for player 'port'. device will be masked with +/* Queries for input for player 'port'. device will be masked with * RETRO_DEVICE_MASK. * - * Specialization of devices such as RETRO_DEVICE_JOYPAD_MULTITAP that + * Specialization of devices such as RETRO_DEVICE_JOYPAD_MULTITAP that * have been set with retro_set_controller_port_device() * will still use the higher level RETRO_DEVICE_JOYPAD to request input. */ -typedef int16_t (RETRO_CALLCONV *retro_input_state_t)(unsigned port, unsigned device, +typedef int16_t (RETRO_CALLCONV *retro_input_state_t)(unsigned port, unsigned device, unsigned index, unsigned id); -/* Sets callbacks. retro_set_environment() is guaranteed to be called +/* Sets callbacks. retro_set_environment() is guaranteed to be called * before retro_init(). * - * The rest of the set_* functions are guaranteed to have been called + * The rest of the set_* functions are guaranteed to have been called * before the first call to retro_run() is made. */ RETRO_API void retro_set_environment(retro_environment_t); RETRO_API void retro_set_video_refresh(retro_video_refresh_t);@@ -2049,27 +2675,33 @@ /* Must return RETRO_API_VERSION. Used to validate ABI compatibility
* when the API is revised. */ RETRO_API unsigned retro_api_version(void); -/* Gets statically known system info. Pointers provided in *info +/* Gets statically known system info. Pointers provided in *info * must be statically allocated. * Can be called at any time, even before retro_init(). */ RETRO_API void retro_get_system_info(struct retro_system_info *info); /* Gets information about system audio/video timings and geometry. * Can be called only after retro_load_game() has successfully completed. - * NOTE: The implementation of this function might not initialize every + * NOTE: The implementation of this function might not initialize every * variable if needed. - * E.g. geom.aspect_ratio might not be initialized if core doesn't + * E.g. geom.aspect_ratio might not be initialized if core doesn't * desire a particular aspect ratio. */ RETRO_API void retro_get_system_av_info(struct retro_system_av_info *info); /* Sets device to be used for player 'port'. - * By default, RETRO_DEVICE_JOYPAD is assumed to be plugged into all + * By default, RETRO_DEVICE_JOYPAD is assumed to be plugged into all * available ports. - * Setting a particular device type is not a guarantee that libretro cores - * will only poll input based on that particular device type. It is only a - * hint to the libretro core when a core cannot automatically detect the - * appropriate input device type on its own. It is also relevant when a - * core can change its behavior depending on device type. */ + * Setting a particular device type is not a guarantee that libretro cores + * will only poll input based on that particular device type. It is only a + * hint to the libretro core when a core cannot automatically detect the + * appropriate input device type on its own. It is also relevant when a + * core can change its behavior depending on device type. + * + * As part of the core's implementation of retro_set_controller_port_device, + * the core should call RETRO_ENVIRONMENT_SET_INPUT_DESCRIPTORS to notify the + * frontend if the descriptions for any controls have changed as a + * result of changing the device type. + */ RETRO_API void retro_set_controller_port_device(unsigned port, unsigned device); /* Resets the current game. */@@ -2077,18 +2709,18 @@ RETRO_API void retro_reset(void);
/* Runs the game for one video frame. * During retro_run(), input_poll callback must be called at least once. - * + * * If a frame is not rendered for reasons where a game "dropped" a frame, - * this still counts as a frame, and retro_run() should explicitly dupe + * this still counts as a frame, and retro_run() should explicitly dupe * a frame if GET_CAN_DUPE returns true. * In this case, the video callback can take a NULL argument for data. */ RETRO_API void retro_run(void); -/* Returns the amount of data the implementation requires to serialize +/* Returns the amount of data the implementation requires to serialize * internal state (save states). - * Between calls to retro_load_game() and retro_unload_game(), the - * returned size is never allowed to be larger than a previous returned + * Between calls to retro_load_game() and retro_unload_game(), the + * returned size is never allowed to be larger than a previous returned * value, to ensure that the frontend can allocate a save state buffer once. */ RETRO_API size_t retro_serialize_size(void);@@ -2101,7 +2733,9 @@
RETRO_API void retro_cheat_reset(void); RETRO_API void retro_cheat_set(unsigned index, bool enabled, const char *code); -/* Loads a game. */ +/* Loads a game. + * Return true to indicate successful loading and false to indicate load failure. + */ RETRO_API bool retro_load_game(const struct retro_game_info *game); /* Loads a "special" kind of game. Should not be used,@@ -2111,7 +2745,7 @@ unsigned game_type,
const struct retro_game_info *info, size_t num_info ); -/* Unloads a currently loaded game. */ +/* Unloads the currently loaded game. Called before retro_deinit(void). */ RETRO_API void retro_unload_game(void); /* Gets region of game. */
A
src/platform/libretro/libretro_core_options.h
@@ -0,0 +1,372 @@
+#ifndef LIBRETRO_CORE_OPTIONS_H__ +#define LIBRETRO_CORE_OPTIONS_H__ + +#include <stdlib.h> +#include <string.h> + +#include "libretro.h" +#include "retro_inline.h" + +#ifndef HAVE_NO_LANGEXTRA +#include "libretro_core_options_intl.h" +#endif + +/* + ******************************** + * VERSION: 1.3 + ******************************** + * + * - 1.3: Move translations to libretro_core_options_intl.h + * - libretro_core_options_intl.h includes BOM and utf-8 + * fix for MSVC 2010-2013 + * - Added HAVE_NO_LANGEXTRA flag to disable translations + * on platforms/compilers without BOM support + * - 1.2: Use core options v1 interface when + * RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION is >= 1 + * (previously required RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION == 1) + * - 1.1: Support generation of core options v0 retro_core_option_value + * arrays containing options with a single value + * - 1.0: First commit +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + ******************************** + * Core Option Definitions + ******************************** +*/ + +/* RETRO_LANGUAGE_ENGLISH */ + +/* Default language: + * - All other languages must include the same keys and values + * - Will be used as a fallback in the event that frontend language + * is not available + * - Will be used as a fallback for any missing entries in + * frontend language definition */ + +struct retro_core_option_definition option_defs_us[] = { + { + "mgba_solar_sensor_level", + "Solar Sensor Level", + "Sets ambient sunlight intensity. Can be used by games that included a solar sensor in their cartridges, e.g: the Boktai series.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, + { + "mgba_allow_opposing_directions", + "Allow Opposing Directional Input", + "Enabling this will allow pressing / quickly alternating / holding both left and right (or up and down) directions at the same time. This may cause movement-based glitches.", + { + { "no", "disabled" }, + { "yes", "enabled" }, + { NULL, NULL }, + }, + "no" + }, + { + "mgba_gb_model", + "Game Boy Model (requires restart)", + "Runs loaded content with a specific Game Boy model. 'Autodetect' will select the most appropriate model for the current game.", + { + { "Autodetect", NULL }, + { "Game Boy", NULL }, + { "Super Game Boy", NULL }, + { "Game Boy Color", NULL }, + { "Game Boy Advance", NULL }, + { NULL, NULL }, + }, + "Autodetect" + }, + { + "mgba_use_bios", + "Use BIOS File if Found (requires restart)", + "Use official BIOS/bootloader for emulated hardware, if present in RetroArch's system directory.", + { + { "ON", NULL }, + { "OFF", NULL }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_skip_bios", + "Skip BIOS Intro (requires restart)", + "When using an official BIOS/bootloader, skip the start-up logo animation. This setting is ignored when 'Use BIOS File if Found' is disabled.", + { + { "OFF", NULL }, + { "ON", NULL }, + { NULL, NULL }, + }, + "OFF" + }, + { + "mgba_sgb_borders", + "Use Super Game Boy Borders (requires restart)", + "Display Super Game Boy borders when running Super Game Boy enhanced games.", + { + { "ON", NULL }, + { "OFF", NULL }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_idle_optimization", + "Idle Loop Removal", + "Reduce system load by optimizing so-called 'idle-loops' - sections in the code where nothing happens, but the CPU runs at full speed (like a car revving in neutral). Improves performance, and should be enabled on low-end hardware.", + { + { "Remove Known", NULL }, + { "Detect and Remove", NULL }, + { "Don't Remove", NULL }, + { NULL, NULL }, + }, + "Remove Known" + }, + { + "mgba_frameskip", + "Frameskip", + "Skip frames to improve performance at the expense of visual smoothness. Value set here is the number of frames omitted after a frame is rendered - i.e. '0' = 60fps, '1' = 30fps, '2' = 15fps, etc.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, +#if defined(COLOR_16_BIT) && defined(COLOR_5_6_5) + { + "mgba_color_correction", + "Color Correction", + "Adjusts output colors to match the display of real GBA/GBC hardware.", + { + { "OFF", NULL }, + { "GBA", "Game Boy Advance" }, + { "GBC", "Game Boy Color" }, + { "Auto", NULL }, + { NULL, NULL }, + }, + "OFF" + }, +#endif + { NULL, NULL, NULL, {{0}}, NULL }, +}; + +/* + ******************************** + * Language Mapping + ******************************** +*/ + +#ifndef HAVE_NO_LANGEXTRA +struct retro_core_option_definition *option_defs_intl[RETRO_LANGUAGE_LAST] = { + option_defs_us, /* RETRO_LANGUAGE_ENGLISH */ + NULL, /* RETRO_LANGUAGE_JAPANESE */ + NULL, /* RETRO_LANGUAGE_FRENCH */ + NULL, /* RETRO_LANGUAGE_SPANISH */ + NULL, /* RETRO_LANGUAGE_GERMAN */ + NULL, /* RETRO_LANGUAGE_ITALIAN */ + NULL, /* RETRO_LANGUAGE_DUTCH */ + NULL, /* RETRO_LANGUAGE_PORTUGUESE_BRAZIL */ + NULL, /* RETRO_LANGUAGE_PORTUGUESE_PORTUGAL */ + NULL, /* RETRO_LANGUAGE_RUSSIAN */ + NULL, /* RETRO_LANGUAGE_KOREAN */ + NULL, /* RETRO_LANGUAGE_CHINESE_TRADITIONAL */ + NULL, /* RETRO_LANGUAGE_CHINESE_SIMPLIFIED */ + NULL, /* RETRO_LANGUAGE_ESPERANTO */ + NULL, /* RETRO_LANGUAGE_POLISH */ + NULL, /* RETRO_LANGUAGE_VIETNAMESE */ + NULL, /* RETRO_LANGUAGE_ARABIC */ + NULL, /* RETRO_LANGUAGE_GREEK */ + option_defs_tr, /* RETRO_LANGUAGE_TURKISH */ +}; +#endif + +/* + ******************************** + * Functions + ******************************** +*/ + +/* Handles configuration/setting of core options. + * Should be called as early as possible - ideally inside + * retro_set_environment(), and no later than retro_load_game() + * > We place the function body in the header to avoid the + * necessity of adding more .c files (i.e. want this to + * be as painless as possible for core devs) + */ + +static INLINE void libretro_set_core_options(retro_environment_t environ_cb) +{ + unsigned version = 0; + + if (!environ_cb) + return; + + if (environ_cb(RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION, &version) && (version >= 1)) + { +#ifndef HAVE_NO_LANGEXTRA + struct retro_core_options_intl core_options_intl; + unsigned language = 0; + + core_options_intl.us = option_defs_us; + core_options_intl.local = NULL; + + if (environ_cb(RETRO_ENVIRONMENT_GET_LANGUAGE, &language) && + (language < RETRO_LANGUAGE_LAST) && (language != RETRO_LANGUAGE_ENGLISH)) + core_options_intl.local = option_defs_intl[language]; + + environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS_INTL, &core_options_intl); +#else + environ_cb(RETRO_ENVIRONMENT_SET_CORE_OPTIONS, &option_defs_us); +#endif + } + else + { + size_t i; + size_t num_options = 0; + struct retro_variable *variables = NULL; + char **values_buf = NULL; + + /* Determine number of options */ + while (true) + { + if (option_defs_us[num_options].key) + num_options++; + else + break; + } + + /* Allocate arrays */ + variables = (struct retro_variable *)calloc(num_options + 1, sizeof(struct retro_variable)); + values_buf = (char **)calloc(num_options, sizeof(char *)); + + if (!variables || !values_buf) + goto error; + + /* Copy parameters from option_defs_us array */ + for (i = 0; i < num_options; i++) + { + const char *key = option_defs_us[i].key; + const char *desc = option_defs_us[i].desc; + const char *default_value = option_defs_us[i].default_value; + struct retro_core_option_value *values = option_defs_us[i].values; + size_t buf_len = 3; + size_t default_index = 0; + + values_buf[i] = NULL; + + if (desc) + { + size_t num_values = 0; + + /* Determine number of values */ + while (true) + { + if (values[num_values].value) + { + /* Check if this is the default value */ + if (default_value) + if (strcmp(values[num_values].value, default_value) == 0) + default_index = num_values; + + buf_len += strlen(values[num_values].value); + num_values++; + } + else + break; + } + + /* Build values string */ + if (num_values > 0) + { + size_t j; + + buf_len += num_values - 1; + buf_len += strlen(desc); + + values_buf[i] = (char *)calloc(buf_len, sizeof(char)); + if (!values_buf[i]) + goto error; + + strcpy(values_buf[i], desc); + strcat(values_buf[i], "; "); + + /* Default value goes first */ + strcat(values_buf[i], values[default_index].value); + + /* Add remaining values */ + for (j = 0; j < num_values; j++) + { + if (j != default_index) + { + strcat(values_buf[i], "|"); + strcat(values_buf[i], values[j].value); + } + } + } + } + + variables[i].key = key; + variables[i].value = values_buf[i]; + } + + /* Set variables */ + environ_cb(RETRO_ENVIRONMENT_SET_VARIABLES, variables); + +error: + + /* Clean up */ + if (values_buf) + { + for (i = 0; i < num_options; i++) + { + if (values_buf[i]) + { + free(values_buf[i]); + values_buf[i] = NULL; + } + } + + free(values_buf); + values_buf = NULL; + } + + if (variables) + { + free(variables); + variables = NULL; + } + } +} + +#ifdef __cplusplus +} +#endif + +#endif
A
src/platform/libretro/libretro_core_options_intl.h
@@ -0,0 +1,209 @@
+#ifndef LIBRETRO_CORE_OPTIONS_INTL_H__ +#define LIBRETRO_CORE_OPTIONS_INTL_H__ + +#if defined(_MSC_VER) && (_MSC_VER >= 1500 && _MSC_VER < 1900) +/* https://support.microsoft.com/en-us/kb/980263 */ +#pragma execution_character_set("utf-8") +#pragma warning(disable:4566) +#endif + +#include "libretro.h" + +/* + ******************************** + * VERSION: 1.3 + ******************************** + * + * - 1.3: Move translations to libretro_core_options_intl.h + * - libretro_core_options_intl.h includes BOM and utf-8 + * fix for MSVC 2010-2013 + * - Added HAVE_NO_LANGEXTRA flag to disable translations + * on platforms/compilers without BOM support + * - 1.2: Use core options v1 interface when + * RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION is >= 1 + * (previously required RETRO_ENVIRONMENT_GET_CORE_OPTIONS_VERSION == 1) + * - 1.1: Support generation of core options v0 retro_core_option_value + * arrays containing options with a single value + * - 1.0: First commit +*/ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + ******************************** + * Core Option Definitions + ******************************** +*/ + +/* RETRO_LANGUAGE_JAPANESE */ + +/* RETRO_LANGUAGE_FRENCH */ + +/* RETRO_LANGUAGE_SPANISH */ + +/* RETRO_LANGUAGE_GERMAN */ + +/* RETRO_LANGUAGE_ITALIAN */ + +/* RETRO_LANGUAGE_DUTCH */ + +/* RETRO_LANGUAGE_PORTUGUESE_BRAZIL */ + +/* RETRO_LANGUAGE_PORTUGUESE_PORTUGAL */ + +/* RETRO_LANGUAGE_RUSSIAN */ + +/* RETRO_LANGUAGE_KOREAN */ + +/* RETRO_LANGUAGE_CHINESE_TRADITIONAL */ + +/* RETRO_LANGUAGE_CHINESE_SIMPLIFIED */ + +/* RETRO_LANGUAGE_ESPERANTO */ + +/* RETRO_LANGUAGE_POLISH */ + +/* RETRO_LANGUAGE_VIETNAMESE */ + +/* RETRO_LANGUAGE_ARABIC */ + +/* RETRO_LANGUAGE_GREEK */ + +/* RETRO_LANGUAGE_TURKISH */ + +struct retro_core_option_definition option_defs_tr[] = { + { + "mgba_solar_sensor_level", + "Güneş Sensörü Seviyesi", + "Ortam güneş ışığının yoğunluğunu ayarlar. Boktai serisi, kartuşlarına güneş sensörü içeren oyunlar tarafından kullanılabilir.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, + { + "mgba_allow_opposing_directions", + "Karşı Yönlü Girdiye Çıkmaya İzin Ver", + "Bunu etkinleştirmek aynı anda hem sola hem de sağa (veya yukarı ve aşağı) yönlere basma / hızlı değiştirme / tutma imkanı sağlar. Bu harekete dayalı hatalara neden olabilir.", + { + { "no", "Devre dışı bırak" }, + { "yes", "Etkileştir" }, + { NULL, NULL }, + }, + "no" + }, + { + "mgba_gb_model", + "Game Boy Modeli (yeniden başlatma gerektirir)", + "Yüklenen içeriği belirli bir Game Boy modeliyle çalıştırır. 'Otomatik Tespit' mevcut oyun için en uygun modeli seçecektir.", + { + { "Autodetect", "Otomatik Tespit" }, + { "Game Boy", NULL }, + { "Super Game Boy", NULL }, + { "Game Boy Color", NULL }, + { "Game Boy Advance", NULL }, + { NULL, NULL }, + }, + "Autodetect" + }, + { + "mgba_use_bios", + "Bulunursa BIOS Dosyasını kullanın (yeniden başlatma gerektirir)", + "RetroArch'ın sistem dizininde varsa, öykünülmüş donanım için resmi BIOS/önyükleyici kullanır.", + { + { "ON", "AÇIK" }, + { "OFF", "KAPALI" }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_skip_bios", + "BIOS Girişini Atla (yeniden başlatma gerektirir)", + "Resmi bir BIOS / önyükleyici kullanırken, başlangıç logosu animasyonunu atlayın. Bu ayar, 'Bulunursa BIOS Dosyasını Kullan' devre dışı bırakıldığında geçersiz sayılır.", + { + { "OFF", "HAYIR" }, + { "ON", "AÇIK" }, + { NULL, NULL }, + }, + "OFF" + }, + { + "mgba_sgb_borders", + "Super Game Boy Sınırlarını kullanın (yeniden başlatma gerekir)", + "Super Game Boy gelişmiş oyunlarını çalıştırırken Super Game Boy sınırlarını görüntüleR.", + { + { "ON", "AÇIK" }, + { "OFF", "KAPALI" }, + { NULL, NULL }, + }, + "ON" + }, + { + "mgba_idle_optimization", + "Boşta Döngü Kaldırma", + "'Boşta döngüler' denilen sistemi optimize ederek sistem yükünü azaltın - hiçbir şeyin olmadığı koddaki bölümler için, CPU tam hızda çalıştırır (boşa dönen bir araba gibi). Performansı arttırır ve düşük kaliteli donanımlarda etkinleştirilmesi gerekir.", + { + { "Remove Known", "Bilinenleri Kaldır" }, + { "Detect and Remove", "Algıla ve Kaldır" }, + { "Don't Remove", "Kaldırma" }, + { NULL, NULL }, + }, + "Remove Known" + }, + { + "mgba_frameskip", + "Kare atlama", + "Görsel pürüzsüzlük pahasına performansı artırmak için çerçeveleri atlayın. Burada ayarlanan değer, bir kare oluşturulduktan sonra atlanan kare sayısıdır - yani '0' = 60fps, '1' = 30fps, '2' = 15fps, vb.", + { + { "0", NULL }, + { "1", NULL }, + { "2", NULL }, + { "3", NULL }, + { "4", NULL }, + { "5", NULL }, + { "6", NULL }, + { "7", NULL }, + { "8", NULL }, + { "9", NULL }, + { "10", NULL }, + { NULL, NULL }, + }, + "0" + }, +#if defined(COLOR_16_BIT) && defined(COLOR_5_6_5) + { + "mgba_color_correction", + "Renk Düzeltmesi", + "Çıktı renklerini gerçek GBA / GBC donanımının görüntüsüyle eşleşecek şekilde ayarlar.", + { + { "OFF", "KAPALI" }, + { "GBA", "Game Boy Advance" }, + { "GBC", "Game Boy Color" }, + { "Auto", "Otomatik" }, + { NULL, NULL }, + }, + "OFF" + }, +#endif + { NULL, NULL, NULL, {{0}}, NULL }, +}; + +#ifdef __cplusplus +} +#endif + +#endif
A
src/platform/libretro/retro_inline.h
@@ -0,0 +1,39 @@
+/* Copyright (C) 2010-2018 The RetroArch team + * + * --------------------------------------------------------------------------------------- + * The following license statement only applies to this file (retro_inline.h). + * --------------------------------------------------------------------------------------- + * + * Permission is hereby granted, free of charge, + * to any person obtaining a copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, + * and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __LIBRETRO_SDK_INLINE_H +#define __LIBRETRO_SDK_INLINE_H + +#ifndef INLINE + +#if defined(_WIN32) || defined(__INTEL_COMPILER) +#define INLINE __inline +#elif defined(__STDC_VERSION__) && __STDC_VERSION__>=199901L +#define INLINE inline +#elif defined(__GNUC__) +#define INLINE __inline__ +#else +#define INLINE +#endif + +#endif +#endif
M
src/platform/python/mgba/debugger.py
→
src/platform/python/mgba/debugger.py
@@ -56,7 +56,7 @@ return self._native.state == lib.DEBUGGER_RUNNING
@property def paused(self): - return self._native.state in (lib.DEBUGGER_PAUSED, lib.DEBUGGER_CUSTOM) + return self._native.state in (lib.DEBUGGER_PAUSED, lib.DEBUGGER_CALLBACK) def use_core(self): return DebuggerCoreOwner(self)
M
src/platform/qt/CheatsModel.cpp
→
src/platform/qt/CheatsModel.cpp
@@ -184,7 +184,7 @@ QList<int> order = setOrder.keys();
std::sort(order.begin(), order.end()); for (int i : order) { mCheatSet* set = setOrder[i]; - QList<size_t> indexOrdex = setIndices[set].toList(); + QList<size_t> indexOrdex = setIndices[set].values(); std::sort(indexOrdex.begin(), indexOrdex.end()); for (size_t j : indexOrdex) { strings.append(*StringListGetPointer(&set->lines, j));
M
src/platform/qt/ShortcutController.cpp
→
src/platform/qt/ShortcutController.cpp
@@ -45,8 +45,11 @@ }
void ShortcutController::updateKey(std::shared_ptr<Shortcut> item, int keySequence) { int oldShortcut = item->shortcut(); - if (m_actions->isHeld(item->name())) { + if (oldShortcut != keySequence && m_actions->isHeld(item->name())) { if (oldShortcut > 0) { + if (item->action() && item->action()->booleanAction()) { + item->action()->booleanAction()(false); + } m_heldKeys.take(oldShortcut); } if (keySequence > 0) {
M
src/platform/test/perf-main.c
→
src/platform/test/perf-main.c
@@ -63,12 +63,13 @@ static void _mPerfShutdown(int signal);
static bool _parsePerfOpts(struct mSubParser* parser, int option, const char* arg); static void _log(struct mLogger*, int, enum mLogLevel, const char*, va_list); static bool _mPerfRunCore(const char* fname, const struct mArguments*, const struct PerfOpts*); -static bool _mPerfRunServer(const char* listen, const struct mArguments*, const struct PerfOpts*); +static bool _mPerfRunServer(const struct mArguments*, const struct PerfOpts*); static bool _dispatchExiting = false; static struct VFile* _savestate = 0; static void* _outputBuffer = NULL; static Socket _socket = INVALID_SOCKET; +static Socket _server = INVALID_SOCKET; int main(int argc, char** argv) { #ifdef _3DS@@ -100,7 +101,7 @@ };
struct mArguments args = {}; bool parsed = parseArguments(&args, argc, argv, &subparser); - if (!args.fname) { + if (!args.fname && !perfOpts.server) { parsed = false; } if (!parsed || args.showHelp) {@@ -127,7 +128,7 @@ consoleUpdate(NULL);
#endif } if (perfOpts.server) { - didFail = !_mPerfRunServer(args.fname, &args, &perfOpts); + didFail = !_mPerfRunServer(&args, &perfOpts); } else { didFail = !_mPerfRunCore(args.fname, &args, &perfOpts); }@@ -266,19 +267,24 @@ printf("\033[2K\r");
} } -static bool _mPerfRunServer(const char* listen, const struct mArguments* args, const struct PerfOpts* perfOpts) { +static bool _mPerfRunServer(const struct mArguments* args, const struct PerfOpts* perfOpts) { SocketSubsystemInit(); - Socket server = SocketOpenTCP(7216, NULL); - if (SOCKET_FAILED(server)) { + _server = SocketOpenTCP(7216, NULL); + if (SOCKET_FAILED(_server)) { SocketSubsystemDeinit(); return false; } - if (SOCKET_FAILED(SocketListen(server, 0))) { - SocketClose(server); + if (SOCKET_FAILED(SocketListen(_server, 0))) { + SocketClose(_server); SocketSubsystemDeinit(); return false; } - _socket = SocketAccept(server, NULL); + _socket = SocketAccept(_server, NULL); + if (SOCKET_FAILED(_socket)) { + SocketClose(_server); + SocketSubsystemDeinit(); + return false; + } if (perfOpts->csv) { const char* header = "game_code,frames,duration,renderer\n"; SocketSend(_socket, header, strlen(header));@@ -301,7 +307,7 @@ }
memset(path, 0, sizeof(path)); } SocketClose(_socket); - SocketClose(server); + SocketClose(_server); SocketSubsystemDeinit(); return true; }@@ -310,6 +316,7 @@ static void _mPerfShutdown(int signal) {
UNUSED(signal); _dispatchExiting = true; SocketClose(_socket); + SocketClose(_server); } static bool _parsePerfOpts(struct mSubParser* parser, int option, const char* arg) {
M
src/platform/wii/main.c
→
src/platform/wii/main.c
@@ -64,7 +64,8 @@ // TODO: PAL support
VM_MAX } videoMode = VM_AUTODETECT; -#define SAMPLES 1024 +#define SAMPLES 512 +#define BUFFERS 8 #define GUI_SCALE 1.35f #define GUI_SCALE_240p 2.0f@@ -131,9 +132,12 @@
static void* framebuffer[2] = { 0, 0 }; static int whichFb = 0; -static struct GBAStereoSample audioBuffer[3][SAMPLES] __attribute__((__aligned__(32))); -static volatile size_t audioBufferSize = 0; +static struct AudioBuffer { + struct GBAStereoSample samples[SAMPLES] __attribute__((__aligned__(32))); + volatile size_t size; +} audioBuffer[BUFFERS] = {0}; static volatile int currentAudioBuffer = 0; +static volatile int nextAudioBuffer = 0; static double audioSampleRate = 60.0 / 1.001; static struct GUIFont* font;@@ -610,32 +614,43 @@ return 0;
} static void _audioDMA(void) { - if (!audioBufferSize) { + struct AudioBuffer* buffer = &audioBuffer[currentAudioBuffer]; + if (buffer->size != SAMPLES) { return; } - DCFlushRange(audioBuffer[currentAudioBuffer], audioBufferSize * sizeof(struct GBAStereoSample)); - AUDIO_InitDMA((u32) audioBuffer[currentAudioBuffer], audioBufferSize * sizeof(struct GBAStereoSample)); - currentAudioBuffer = (currentAudioBuffer + 1) % 3; - audioBufferSize = 0; + DCFlushRange(buffer->samples, SAMPLES * sizeof(struct GBAStereoSample)); + AUDIO_InitDMA((u32) buffer->samples, SAMPLES * sizeof(struct GBAStereoSample)); + buffer->size = 0; + currentAudioBuffer = (currentAudioBuffer + 1) % BUFFERS; } static void _postAudioBuffer(struct mAVStream* stream, blip_t* left, blip_t* right) { UNUSED(stream); + + u32 level = 0; + _CPU_ISR_Disable(level); + struct AudioBuffer* buffer = &audioBuffer[nextAudioBuffer]; int available = blip_samples_avail(left); - if (available + audioBufferSize > SAMPLES) { - available = SAMPLES - audioBufferSize; + if (available + buffer->size > SAMPLES) { + available = SAMPLES - buffer->size; } - available &= ~((32 / sizeof(struct GBAStereoSample)) - 1); // Force align to 32 bytes if (available > 0) { // These appear to be reversed for AUDIO_InitDMA - blip_read_samples(left, &audioBuffer[currentAudioBuffer][audioBufferSize].right, available, true); - blip_read_samples(right, &audioBuffer[currentAudioBuffer][audioBufferSize].left, available, true); - audioBufferSize += available; + blip_read_samples(left, &buffer->samples[buffer->size].right, available, true); + blip_read_samples(right, &buffer->samples[buffer->size].left, available, true); + buffer->size += available; } - if (audioBufferSize == SAMPLES && !AUDIO_GetDMAEnableFlag()) { - _audioDMA(); - AUDIO_StartDMA(); + if (buffer->size == SAMPLES) { + int next = (nextAudioBuffer + 1) % BUFFERS; + if ((currentAudioBuffer + BUFFERS - next) % BUFFERS != 1) { + nextAudioBuffer = next; + } + if (!AUDIO_GetDMAEnableFlag()) { + _audioDMA(); + AUDIO_StartDMA(); + } } + _CPU_ISR_Restore(level); } static void _drawStart(void) {@@ -648,6 +663,8 @@ if (frameLimiter) {
VIDEO_WaitVSync(); } referenceRetraceCount = retraceCount; + } else if (frameLimiter && referenceRetraceCount < retraceCount - 1) { + referenceRetraceCount = retraceCount - 1; } _CPU_ISR_Restore(level);@@ -796,6 +813,12 @@
outputBuffer = memalign(32, TEX_W * TEX_H * BYTES_PER_PIXEL); runner->core->setVideoBuffer(runner->core, outputBuffer, TEX_W); + nextAudioBuffer = 0; + currentAudioBuffer = 0; + int i; + for (i = 0; i < BUFFERS; ++i) { + audioBuffer[i].size = 0; + } runner->core->setAudioBufferSize(runner->core, SAMPLES); double ratio = GBAAudioCalculateRatio(1, audioSampleRate, 1);
M
src/sm83/debugger/cli-debugger.c
→
src/sm83/debugger/cli-debugger.c
@@ -46,7 +46,7 @@ if (!dv || dv->type != CLIDV_INT_TYPE) {
size = 1; } else { size = dv->intValue; - dv = dv->next; // TODO: Check for excess args + // TODO: Check for excess args } size_t i;@@ -57,7 +57,7 @@ }
static inline uint16_t _printLine(struct CLIDebugger* debugger, uint16_t address, int segment) { struct CLIDebuggerBackend* be = debugger->backend; - struct SM83InstructionInfo info = {0}; + struct SM83InstructionInfo info = {{0}, 0}; char disassembly[48]; char* disPtr = disassembly; if (segment >= 0) {
M
src/third-party/lzma/7z.h
→
src/third-party/lzma/7z.h
@@ -1,5 +1,5 @@
/* 7z.h -- 7z interface -2015-11-18 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_H #define __7Z_H@@ -98,7 +98,7 @@
SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, ILookInStream *stream, UInt64 startPos, Byte *outBuffer, size_t outSize, - ISzAlloc *allocMain); + ISzAllocPtr allocMain); typedef struct {@@ -131,7 +131,7 @@
#define SzArEx_GetFileSize(p, i) ((p)->UnpackPositions[(i) + 1] - (p)->UnpackPositions[i]) void SzArEx_Init(CSzArEx *p); -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc); +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc); UInt64 SzArEx_GetFolderStreamPos(const CSzArEx *p, UInt32 folderIndex, UInt32 indexInFolder); int SzArEx_GetFolderFullPackSize(const CSzArEx *p, UInt32 folderIndex, UInt64 *resSize);@@ -179,8 +179,8 @@ Byte **outBuffer, /* pointer to pointer to output buffer (allocated with allocMain) */
size_t *outBufferSize, /* buffer size for output buffer */ size_t *offset, /* offset of stream for required file in *outBuffer */ size_t *outSizeProcessed, /* size of file in *outBuffer */ - ISzAlloc *allocMain, - ISzAlloc *allocTemp); + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp); /*@@ -195,7 +195,7 @@ SZ_ERROR_FAIL
*/ SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, ISzAlloc *allocTemp); + ISzAllocPtr allocMain, ISzAllocPtr allocTemp); EXTERN_C_END
M
src/third-party/lzma/7zAlloc.c
→
src/third-party/lzma/7zAlloc.c
@@ -1,7 +1,9 @@
/* 7zAlloc.c -- Allocation functions -2015-11-09 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h" + +#include <stdlib.h> #include "7zAlloc.h"@@ -20,7 +22,7 @@ int g_allocCountTemp = 0;
#endif -void *SzAlloc(void *p, size_t size) +void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); if (size == 0)@@ -32,7 +34,7 @@ #endif
return malloc(size); } -void SzFree(void *p, void *address) +void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); #ifdef _SZ_ALLOC_DEBUG@@ -45,7 +47,7 @@ #endif
free(address); } -void *SzAllocTemp(void *p, size_t size) +void *SzAllocTemp(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); if (size == 0)@@ -60,7 +62,7 @@ #endif
return malloc(size); } -void SzFreeTemp(void *p, void *address) +void SzFreeTemp(ISzAllocPtr p, void *address) { UNUSED_VAR(p); #ifdef _SZ_ALLOC_DEBUG
M
src/third-party/lzma/7zAlloc.h
→
src/third-party/lzma/7zAlloc.h
@@ -1,23 +1,19 @@
/* 7zAlloc.h -- Allocation functions -2013-03-25 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_ALLOC_H #define __7Z_ALLOC_H -#include <stdlib.h> +#include "7zTypes.h" -#ifdef __cplusplus -extern "C" { -#endif +EXTERN_C_BEGIN -void *SzAlloc(void *p, size_t size); -void SzFree(void *p, void *address); +void *SzAlloc(ISzAllocPtr p, size_t size); +void SzFree(ISzAllocPtr p, void *address); -void *SzAllocTemp(void *p, size_t size); -void SzFreeTemp(void *p, void *address); +void *SzAllocTemp(ISzAllocPtr p, size_t size); +void SzFreeTemp(ISzAllocPtr p, void *address); -#ifdef __cplusplus -} -#endif +EXTERN_C_END #endif
M
src/third-party/lzma/7zArcIn.c
→
src/third-party/lzma/7zArcIn.c
@@ -1,5 +1,5 @@
/* 7zArcIn.c -- 7z Input functions -2015-11-18 : Igor Pavlov : Public domain */ +2018-12-31 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -11,7 +11,7 @@ #include "7zCrc.h"
#include "CpuArch.h" #define MY_ALLOC(T, p, size, alloc) { \ - if ((p = (T *)IAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } + if ((p = (T *)ISzAlloc_Alloc(alloc, (size) * sizeof(T))) == NULL) return SZ_ERROR_MEM; } #define MY_ALLOC_ZE(T, p, size, alloc) { if ((size) == 0) p = NULL; else MY_ALLOC(T, p, size, alloc) }@@ -19,7 +19,7 @@ #define MY_ALLOC_AND_CPY(to, size, from, alloc) \
{ MY_ALLOC(Byte, to, size, alloc); memcpy(to, from, size); } #define MY_ALLOC_ZE_AND_CPY(to, size, from, alloc) \ - { if ((size) == 0) p = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } + { if ((size) == 0) to = NULL; else { MY_ALLOC_AND_CPY(to, size, from, alloc) } } #define k7zMajorVersion 0@@ -60,7 +60,7 @@ const Byte k7zSignature[k7zSignatureSize] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C};
#define SzBitUi32s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } -static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAlloc *alloc) +static SRes SzBitUi32s_Alloc(CSzBitUi32s *p, size_t num, ISzAllocPtr alloc) { if (num == 0) {@@ -75,18 +75,18 @@ }
return SZ_OK; } -void SzBitUi32s_Free(CSzBitUi32s *p, ISzAlloc *alloc) +void SzBitUi32s_Free(CSzBitUi32s *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->Defs); p->Defs = NULL; - IAlloc_Free(alloc, p->Vals); p->Vals = NULL; + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; } #define SzBitUi64s_Init(p) { (p)->Defs = NULL; (p)->Vals = NULL; } -void SzBitUi64s_Free(CSzBitUi64s *p, ISzAlloc *alloc) +void SzBitUi64s_Free(CSzBitUi64s *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->Defs); p->Defs = NULL; - IAlloc_Free(alloc, p->Vals); p->Vals = NULL; + ISzAlloc_Free(alloc, p->Defs); p->Defs = NULL; + ISzAlloc_Free(alloc, p->Vals); p->Vals = NULL; }@@ -107,18 +107,18 @@
p->CodersData = NULL; } -static void SzAr_Free(CSzAr *p, ISzAlloc *alloc) +static void SzAr_Free(CSzAr *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->PackPositions); + ISzAlloc_Free(alloc, p->PackPositions); SzBitUi32s_Free(&p->FolderCRCs, alloc); - IAlloc_Free(alloc, p->FoCodersOffsets); - IAlloc_Free(alloc, p->FoStartPackStreamIndex); - IAlloc_Free(alloc, p->FoToCoderUnpackSizes); - IAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); - IAlloc_Free(alloc, p->CoderUnpackSizes); + ISzAlloc_Free(alloc, p->FoCodersOffsets); + ISzAlloc_Free(alloc, p->FoStartPackStreamIndex); + ISzAlloc_Free(alloc, p->FoToCoderUnpackSizes); + ISzAlloc_Free(alloc, p->FoToMainUnpackSizeIndex); + ISzAlloc_Free(alloc, p->CoderUnpackSizes); - IAlloc_Free(alloc, p->CodersData); + ISzAlloc_Free(alloc, p->CodersData); SzAr_Init(p); }@@ -147,16 +147,16 @@ SzBitUi64s_Init(&p->MTime);
SzBitUi64s_Init(&p->CTime); } -void SzArEx_Free(CSzArEx *p, ISzAlloc *alloc) +void SzArEx_Free(CSzArEx *p, ISzAllocPtr alloc) { - IAlloc_Free(alloc, p->UnpackPositions); - IAlloc_Free(alloc, p->IsDirs); + ISzAlloc_Free(alloc, p->UnpackPositions); + ISzAlloc_Free(alloc, p->IsDirs); - IAlloc_Free(alloc, p->FolderToFile); - IAlloc_Free(alloc, p->FileToFolder); + ISzAlloc_Free(alloc, p->FolderToFile); + ISzAlloc_Free(alloc, p->FileToFolder); - IAlloc_Free(alloc, p->FileNameOffsets); - IAlloc_Free(alloc, p->FileNames); + ISzAlloc_Free(alloc, p->FileNameOffsets); + ISzAlloc_Free(alloc, p->FileNames); SzBitUi32s_Free(&p->CRCs, alloc); SzBitUi32s_Free(&p->Attribs, alloc);@@ -305,7 +305,7 @@ }
return sum; } -static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAlloc *alloc) +static MY_NO_INLINE SRes ReadBitVector(CSzData *sd, UInt32 numItems, Byte **v, ISzAllocPtr alloc) { Byte allAreDefined; Byte *v2;@@ -328,12 +328,12 @@ memset(v2, 0xFF, (size_t)numBytes);
{ unsigned numBits = (unsigned)numItems & 7; if (numBits != 0) - v2[numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); + v2[(size_t)numBytes - 1] = (Byte)((((UInt32)1 << numBits) - 1) << (8 - numBits)); } return SZ_OK; } -static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAlloc *alloc) +static MY_NO_INLINE SRes ReadUi32s(CSzData *sd2, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { UInt32 i; CSzData sd;@@ -354,7 +354,7 @@ *sd2 = sd;
return SZ_OK; } -static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAlloc *alloc) +static SRes ReadBitUi32s(CSzData *sd, UInt32 numItems, CSzBitUi32s *crcs, ISzAllocPtr alloc) { SzBitUi32s_Free(crcs, alloc); RINOK(ReadBitVector(sd, numItems, &crcs->Defs, alloc));@@ -380,7 +380,7 @@ SKIP_DATA(sd, (size_t)numDefined * 4);
return SZ_OK; } -static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAlloc *alloc) +static SRes ReadPackInfo(CSzAr *p, CSzData *sd, ISzAllocPtr alloc) { RINOK(SzReadNumber32(sd, &p->NumPackStreams));@@ -635,7 +635,7 @@ static SRes ReadUnpackInfo(CSzAr *p,
CSzData *sd2, UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, - ISzAlloc *alloc) + ISzAllocPtr alloc) { CSzData sd;@@ -666,7 +666,7 @@
MY_ALLOC(size_t, p->FoCodersOffsets, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoStartPackStreamIndex, (size_t)numFolders + 1, alloc); MY_ALLOC(UInt32, p->FoToCoderUnpackSizes, (size_t)numFolders + 1, alloc); - MY_ALLOC(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); + MY_ALLOC_ZE(Byte, p->FoToMainUnpackSizeIndex, (size_t)numFolders, alloc); startBufPtr = sd.Data;@@ -788,13 +788,9 @@ p->FoToMainUnpackSizeIndex[fo] = (Byte)indexOfMainStream;
numCodersOutStreams += numCoders; if (numCodersOutStreams < numCoders) return SZ_ERROR_UNSUPPORTED; - + if (numPackStreams > p->NumPackStreams - packStreamIndex) + return SZ_ERROR_ARCHIVE; packStreamIndex += numPackStreams; - if (packStreamIndex < numPackStreams) - return SZ_ERROR_UNSUPPORTED; - - if (packStreamIndex > p->NumPackStreams) - return SZ_ERROR_ARCHIVE; } }@@ -938,7 +934,7 @@ CSzData *sd,
UInt32 numFoldersMax, const CBuf *tempBufs, UInt32 numTempBufs, UInt64 *dataOffset, CSubStreamInfo *ssi, - ISzAlloc *alloc) + ISzAllocPtr alloc) { UInt64 type;@@ -980,7 +976,7 @@ CBuf *tempBufs,
UInt32 numFoldersMax, UInt64 baseOffset, CSzAr *p, - ISzAlloc *allocTemp) + ISzAllocPtr allocTemp) { UInt64 dataStartPos; UInt32 fo;@@ -1047,7 +1043,7 @@
static MY_NO_INLINE SRes ReadTime(CSzBitUi64s *p, UInt32 num, CSzData *sd2, const CBuf *tempBufs, UInt32 numTempBufs, - ISzAlloc *alloc) + ISzAllocPtr alloc) { CSzData sd; UInt32 i;@@ -1100,17 +1096,15 @@ CSzArEx *p, /* allocMain */
CSzData *sd, ILookInStream *inStream, CBuf *tempBufs, UInt32 *numTempBufs, - ISzAlloc *allocMain, - ISzAlloc *allocTemp + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp ) { - UInt64 type; - UInt32 numFiles = 0; - UInt32 numEmptyStreams = 0; CSubStreamInfo ssi; - const Byte *emptyStreams = NULL; - const Byte *emptyFiles = NULL; +{ + UInt64 type; + SzData_Clear(&ssi.sdSizes); SzData_Clear(&ssi.sdCRCs); SzData_Clear(&ssi.sdNumSubStreams);@@ -1124,9 +1118,9 @@ if (type == k7zIdArchiveProperties)
{ for (;;) { - UInt64 type; - RINOK(ReadID(sd, &type)); - if (type == k7zIdEnd) + UInt64 type2; + RINOK(ReadID(sd, &type2)); + if (type2 == k7zIdEnd) break; RINOK(SkipData(sd)); }@@ -1164,6 +1158,13 @@ }
if (type != k7zIdFilesInfo) return SZ_ERROR_ARCHIVE; +} + +{ + UInt32 numFiles = 0; + UInt32 numEmptyStreams = 0; + const Byte *emptyStreams = NULL; + const Byte *emptyFiles = NULL; RINOK(SzReadNumber32(sd, &numFiles)); p->NumFiles = numFiles;@@ -1462,7 +1463,7 @@
if (ssi.sdNumSubStreams.Data && ssi.sdNumSubStreams.Size != 0) return SZ_ERROR_ARCHIVE; } - +} return SZ_OK; }@@ -1471,8 +1472,8 @@ static SRes SzReadHeader(
CSzArEx *p, CSzData *sd, ILookInStream *inStream, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { UInt32 i; UInt32 numTempBufs = 0;@@ -1500,8 +1501,8 @@
static SRes SzArEx_Open2( CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { Byte header[k7zStartHeaderSize]; Int64 startArcPos;@@ -1512,7 +1513,7 @@ CBuf buf;
SRes res; startArcPos = 0; - RINOK(inStream->Seek(inStream, &startArcPos, SZ_SEEK_CUR)); + RINOK(ILookInStream_Seek(inStream, &startArcPos, SZ_SEEK_CUR)); RINOK(LookInStream_Read2(inStream, header, k7zStartHeaderSize, SZ_ERROR_NO_ARCHIVE));@@ -1541,7 +1542,7 @@ return SZ_ERROR_NO_ARCHIVE;
{ Int64 pos = 0; - RINOK(inStream->Seek(inStream, &pos, SZ_SEEK_END)); + RINOK(ILookInStream_Seek(inStream, &pos, SZ_SEEK_END)); if ((UInt64)pos < startArcPos + nextHeaderOffset || (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset || (UInt64)pos < startArcPos + k7zStartHeaderSize + nextHeaderOffset + nextHeaderSize)@@ -1622,7 +1623,7 @@ }
SRes SzArEx_Open(CSzArEx *p, ILookInStream *inStream, - ISzAlloc *allocMain, ISzAlloc *allocTemp) + ISzAllocPtr allocMain, ISzAllocPtr allocTemp) { SRes res = SzArEx_Open2(p, inStream, allocMain, allocTemp); if (res != SZ_OK)@@ -1640,8 +1641,8 @@ Byte **tempBuf,
size_t *outBufferSize, size_t *offset, size_t *outSizeProcessed, - ISzAlloc *allocMain, - ISzAlloc *allocTemp) + ISzAllocPtr allocMain, + ISzAllocPtr allocTemp) { UInt32 folderIndex = p->FileToFolder[fileIndex]; SRes res = SZ_OK;@@ -1651,7 +1652,7 @@ *outSizeProcessed = 0;
if (folderIndex == (UInt32)-1) { - IAlloc_Free(allocMain, *tempBuf); + ISzAlloc_Free(allocMain, *tempBuf); *blockIndex = folderIndex; *tempBuf = NULL; *outBufferSize = 0;@@ -1663,7 +1664,7 @@ {
UInt64 unpackSizeSpec = SzAr_GetFolderUnpackSize(&p->db, folderIndex); /* UInt64 unpackSizeSpec = - p->UnpackPositions[p->FolderToFile[folderIndex + 1]] - + p->UnpackPositions[p->FolderToFile[(size_t)folderIndex + 1]] - p->UnpackPositions[p->FolderToFile[folderIndex]]; */ size_t unpackSize = (size_t)unpackSizeSpec;@@ -1671,7 +1672,7 @@
if (unpackSize != unpackSizeSpec) return SZ_ERROR_MEM; *blockIndex = folderIndex; - IAlloc_Free(allocMain, *tempBuf); + ISzAlloc_Free(allocMain, *tempBuf); *tempBuf = NULL; if (res == SZ_OK)@@ -1679,7 +1680,7 @@ {
*outBufferSize = unpackSize; if (unpackSize != 0) { - *tempBuf = (Byte *)IAlloc_Alloc(allocMain, unpackSize); + *tempBuf = (Byte *)ISzAlloc_Alloc(allocMain, unpackSize); if (*tempBuf == NULL) res = SZ_ERROR_MEM; }@@ -1696,7 +1697,7 @@ if (res == SZ_OK)
{ UInt64 unpackPos = p->UnpackPositions[fileIndex]; *offset = (size_t)(unpackPos - p->UnpackPositions[p->FolderToFile[folderIndex]]); - *outSizeProcessed = (size_t)(p->UnpackPositions[fileIndex + 1] - unpackPos); + *outSizeProcessed = (size_t)(p->UnpackPositions[(size_t)fileIndex + 1] - unpackPos); if (*offset + *outSizeProcessed > *outBufferSize) return SZ_ERROR_FAIL; if (SzBitWithVals_Check(&p->CRCs, fileIndex))@@ -1743,7 +1744,7 @@ }
UInt16 *SzArEx_GetFullNameUtf16_Back(const CSzArEx *p, size_t fileIndex, UInt16 *dest) { - Bool needSlash; + BoolInt needSlash; if (!p->FileNameOffsets) { *(--dest) = 0;
M
src/third-party/lzma/7zBuf.c
→
src/third-party/lzma/7zBuf.c
@@ -1,5 +1,5 @@
/* 7zBuf.c -- Byte Buffer -2013-01-21 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -11,7 +11,7 @@ p->data = 0;
p->size = 0; } -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc) +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc) { p->size = 0; if (size == 0)@@ -19,8 +19,8 @@ {
p->data = 0; return 1; } - p->data = (Byte *)alloc->Alloc(alloc, size); - if (p->data != 0) + p->data = (Byte *)ISzAlloc_Alloc(alloc, size); + if (p->data) { p->size = size; return 1;@@ -28,9 +28,9 @@ }
return 0; } -void Buf_Free(CBuf *p, ISzAlloc *alloc) +void Buf_Free(CBuf *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->data); + ISzAlloc_Free(alloc, p->data); p->data = 0; p->size = 0; }
M
src/third-party/lzma/7zBuf.h
→
src/third-party/lzma/7zBuf.h
@@ -1,5 +1,5 @@
/* 7zBuf.h -- Byte Buffer -2013-01-18 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_BUF_H #define __7Z_BUF_H@@ -15,8 +15,8 @@ size_t size;
} CBuf; void Buf_Init(CBuf *p); -int Buf_Create(CBuf *p, size_t size, ISzAlloc *alloc); -void Buf_Free(CBuf *p, ISzAlloc *alloc); +int Buf_Create(CBuf *p, size_t size, ISzAllocPtr alloc); +void Buf_Free(CBuf *p, ISzAllocPtr alloc); typedef struct {@@ -27,8 +27,8 @@ } CDynBuf;
void DynBuf_Construct(CDynBuf *p); void DynBuf_SeekToBeg(CDynBuf *p); -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc); -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc); +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc); +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc); EXTERN_C_END
M
src/third-party/lzma/7zBuf2.c
→
src/third-party/lzma/7zBuf2.c
@@ -1,5 +1,5 @@
/* 7zBuf2.c -- Byte Buffer -2014-08-22 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -19,19 +19,20 @@ {
p->pos = 0; } -int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAlloc *alloc) +int DynBuf_Write(CDynBuf *p, const Byte *buf, size_t size, ISzAllocPtr alloc) { if (size > p->size - p->pos) { size_t newSize = p->pos + size; Byte *data; newSize += newSize / 4; - data = (Byte *)alloc->Alloc(alloc, newSize); - if (data == 0) + data = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!data) return 0; p->size = newSize; - memcpy(data, p->data, p->pos); - alloc->Free(alloc, p->data); + if (p->pos != 0) + memcpy(data, p->data, p->pos); + ISzAlloc_Free(alloc, p->data); p->data = data; } if (size != 0)@@ -42,9 +43,9 @@ }
return 1; } -void DynBuf_Free(CDynBuf *p, ISzAlloc *alloc) +void DynBuf_Free(CDynBuf *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->data); + ISzAlloc_Free(alloc, p->data); p->data = 0; p->size = 0; p->pos = 0;
M
src/third-party/lzma/7zCrc.c
→
src/third-party/lzma/7zCrc.c
@@ -1,5 +1,5 @@
/* 7zCrc.c -- CRC32 init -2015-03-10 : Igor Pavlov : Public domain */ +2017-06-06 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -61,12 +61,12 @@ {
UInt32 r = i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); g_CrcTable[i] = r; } - for (; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * CRC_NUM_TABLES; i++) { - UInt32 r = g_CrcTable[i - 256]; + UInt32 r = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8); }@@ -86,8 +86,8 @@ g_CrcUpdateT8 = CrcUpdateT8;
#ifdef MY_CPU_X86_OR_AMD64 if (!CPU_Is_InOrder()) - g_CrcUpdate = CrcUpdateT8; #endif + g_CrcUpdate = CrcUpdateT8; #endif #else@@ -101,7 +101,7 @@ g_CrcUpdateT4 = CrcUpdateT4;
g_CrcUpdate = CrcUpdateT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT8; - // g_CrcUpdate = CrcUpdateT8; + g_CrcUpdate = CrcUpdateT8; #endif } else if (p[0] != 1 || p[1] != 2)@@ -111,14 +111,14 @@ #endif
{ for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) { - UInt32 x = g_CrcTable[i - 256]; + UInt32 x = g_CrcTable[(size_t)i - 256]; g_CrcTable[i] = CRC_UINT32_SWAP(x); } g_CrcUpdateT4 = CrcUpdateT1_BeT4; g_CrcUpdate = CrcUpdateT1_BeT4; #if CRC_NUM_TABLES >= 8 g_CrcUpdateT8 = CrcUpdateT1_BeT8; - // g_CrcUpdate = CrcUpdateT1_BeT8; + g_CrcUpdate = CrcUpdateT1_BeT8; #endif } }
M
src/third-party/lzma/7zCrcOpt.c
→
src/third-party/lzma/7zCrcOpt.c
@@ -1,5 +1,5 @@
/* 7zCrcOpt.c -- CRC32 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -18,10 +18,10 @@ for (; size >= 4; size -= 4, p += 4)
{ v ^= *(const UInt32 *)p; v = - table[0x300 + ((v ) & 0xFF)] - ^ table[0x200 + ((v >> 8) & 0xFF)] - ^ table[0x100 + ((v >> 16) & 0xFF)] - ^ table[0x000 + ((v >> 24))]; + (table + 0x300)[((v ) & 0xFF)] + ^ (table + 0x200)[((v >> 8) & 0xFF)] + ^ (table + 0x100)[((v >> 16) & 0xFF)] + ^ (table + 0x000)[((v >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p);@@ -38,16 +38,16 @@ {
UInt32 d; v ^= *(const UInt32 *)p; v = - table[0x700 + ((v ) & 0xFF)] - ^ table[0x600 + ((v >> 8) & 0xFF)] - ^ table[0x500 + ((v >> 16) & 0xFF)] - ^ table[0x400 + ((v >> 24))]; + (table + 0x700)[((v ) & 0xFF)] + ^ (table + 0x600)[((v >> 8) & 0xFF)] + ^ (table + 0x500)[((v >> 16) & 0xFF)] + ^ (table + 0x400)[((v >> 24))]; d = *((const UInt32 *)p + 1); v ^= - table[0x300 + ((d ) & 0xFF)] - ^ table[0x200 + ((d >> 8) & 0xFF)] - ^ table[0x100 + ((d >> 16) & 0xFF)] - ^ table[0x000 + ((d >> 24))]; + (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2(v, *p);@@ -74,10 +74,10 @@ for (; size >= 4; size -= 4, p += 4)
{ v ^= *(const UInt32 *)p; v = - table[0x000 + ((v ) & 0xFF)] - ^ table[0x100 + ((v >> 8) & 0xFF)] - ^ table[0x200 + ((v >> 16) & 0xFF)] - ^ table[0x300 + ((v >> 24))]; + (table + 0x000)[((v ) & 0xFF)] + ^ (table + 0x100)[((v >> 8) & 0xFF)] + ^ (table + 0x200)[((v >> 16) & 0xFF)] + ^ (table + 0x300)[((v >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p);@@ -96,16 +96,16 @@ {
UInt32 d; v ^= *(const UInt32 *)p; v = - table[0x400 + ((v ) & 0xFF)] - ^ table[0x500 + ((v >> 8) & 0xFF)] - ^ table[0x600 + ((v >> 16) & 0xFF)] - ^ table[0x700 + ((v >> 24))]; + (table + 0x400)[((v ) & 0xFF)] + ^ (table + 0x500)[((v >> 8) & 0xFF)] + ^ (table + 0x600)[((v >> 16) & 0xFF)] + ^ (table + 0x700)[((v >> 24))]; d = *((const UInt32 *)p + 1); v ^= - table[0x000 + ((d ) & 0xFF)] - ^ table[0x100 + ((d >> 8) & 0xFF)] - ^ table[0x200 + ((d >> 16) & 0xFF)] - ^ table[0x300 + ((d >> 24))]; + (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; } for (; size > 0; size--, p++) v = CRC_UPDATE_BYTE_2_BE(v, *p);
M
src/third-party/lzma/7zDec.c
→
src/third-party/lzma/7zDec.c
@@ -1,5 +1,5 @@
/* 7zDec.c -- Decoding from 7z folder -2015-11-18 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -39,28 +39,28 @@ #define k_PPMD 0x30401
typedef struct { - IByteIn p; + IByteIn vt; const Byte *cur; const Byte *end; const Byte *begin; UInt64 processed; - Bool extra; + BoolInt extra; SRes res; - ILookInStream *inStream; + const ILookInStream *inStream; } CByteInToLook; -static Byte ReadByte(void *pp) +static Byte ReadByte(const IByteIn *pp) { - CByteInToLook *p = (CByteInToLook *)pp; + CByteInToLook *p = CONTAINER_FROM_VTBL(pp, CByteInToLook, vt); if (p->cur != p->end) return *p->cur++; if (p->res == SZ_OK) { size_t size = p->cur - p->begin; p->processed += size; - p->res = p->inStream->Skip(p->inStream, size); + p->res = ILookInStream_Skip(p->inStream, size); size = (1 << 25); - p->res = p->inStream->Look(p->inStream, (const void **)&p->begin, &size); + p->res = ILookInStream_Look(p->inStream, (const void **)&p->begin, &size); p->cur = p->begin; p->end = p->begin + size; if (size != 0)@@ -70,14 +70,14 @@ p->extra = True;
return 0; } -static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) +static SRes SzDecodePpmd(const Byte *props, unsigned propsSize, UInt64 inSize, const ILookInStream *inStream, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CPpmd7 ppmd; CByteInToLook s; SRes res = SZ_OK; - s.p.Read = ReadByte; + s.vt.Read = ReadByte; s.inStream = inStream; s.begin = s.end = s.cur = NULL; s.extra = False;@@ -103,7 +103,7 @@ }
{ CPpmd7z_RangeDec rc; Ppmd7z_RangeDec_CreateVTable(&rc); - rc.Stream = &s.p; + rc.Stream = &s.vt; if (!Ppmd7z_RangeDec_Init(&rc)) res = SZ_ERROR_DATA; else if (s.extra)@@ -113,7 +113,7 @@ {
SizeT i; for (i = 0; i < outSize; i++) { - int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.p); + int sym = Ppmd7_DecodeSymbol(&ppmd, &rc.vt); if (s.extra || sym < 0) break; outBuffer[i] = (Byte)sym;@@ -132,7 +132,7 @@ #endif
static SRes SzDecodeLzma(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzmaDec state; SRes res = SZ_OK;@@ -149,14 +149,14 @@ const void *inBuf = NULL;
size_t lookahead = (1 << 18); if (lookahead > inSize) lookahead = (size_t)inSize; - res = inStream->Look(inStream, &inBuf, &lookahead); + res = ILookInStream_Look(inStream, &inBuf, &lookahead); if (res != SZ_OK) break; { SizeT inProcessed = (SizeT)lookahead, dicPos = state.dicPos; ELzmaStatus status; - res = LzmaDec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = LzmaDec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK)@@ -178,7 +178,7 @@ res = SZ_ERROR_DATA;
break; } - res = inStream->Skip((void *)inStream, inProcessed); + res = ILookInStream_Skip(inStream, inProcessed); if (res != SZ_OK) break; }@@ -192,7 +192,7 @@
#ifndef _7Z_NO_METHOD_LZMA2 static SRes SzDecodeLzma2(const Byte *props, unsigned propsSize, UInt64 inSize, ILookInStream *inStream, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain) + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain) { CLzma2Dec state; SRes res = SZ_OK;@@ -211,14 +211,14 @@ const void *inBuf = NULL;
size_t lookahead = (1 << 18); if (lookahead > inSize) lookahead = (size_t)inSize; - res = inStream->Look(inStream, &inBuf, &lookahead); + res = ILookInStream_Look(inStream, &inBuf, &lookahead); if (res != SZ_OK) break; { SizeT inProcessed = (SizeT)lookahead, dicPos = state.decoder.dicPos; ELzmaStatus status; - res = Lzma2Dec_DecodeToDic(&state, outSize, inBuf, &inProcessed, LZMA_FINISH_END, &status); + res = Lzma2Dec_DecodeToDic(&state, outSize, (const Byte *)inBuf, &inProcessed, LZMA_FINISH_END, &status); lookahead -= inProcessed; inSize -= inProcessed; if (res != SZ_OK)@@ -237,7 +237,7 @@ res = SZ_ERROR_DATA;
break; } - res = inStream->Skip((void *)inStream, inProcessed); + res = ILookInStream_Skip(inStream, inProcessed); if (res != SZ_OK) break; }@@ -258,18 +258,18 @@ const void *inBuf;
size_t curSize = (1 << 18); if (curSize > inSize) curSize = (size_t)inSize; - RINOK(inStream->Look(inStream, &inBuf, &curSize)); + RINOK(ILookInStream_Look(inStream, &inBuf, &curSize)); if (curSize == 0) return SZ_ERROR_INPUT_EOF; memcpy(outBuffer, inBuf, curSize); outBuffer += curSize; inSize -= curSize; - RINOK(inStream->Skip((void *)inStream, curSize)); + RINOK(ILookInStream_Skip(inStream, curSize)); } return SZ_OK; } -static Bool IS_MAIN_METHOD(UInt32 m) +static BoolInt IS_MAIN_METHOD(UInt32 m) { switch (m) {@@ -286,7 +286,7 @@ }
return False; } -static Bool IS_SUPPORTED_CODER(const CSzCoderInfo *c) +static BoolInt IS_SUPPORTED_CODER(const CSzCoderInfo *c) { return c->NumStreams == 1@@ -372,7 +372,7 @@ const Byte *propsData,
const UInt64 *unpackSizes, const UInt64 *packPositions, ILookInStream *inStream, UInt64 startPos, - Byte *outBuffer, SizeT outSize, ISzAlloc *allocMain, + Byte *outBuffer, SizeT outSize, ISzAllocPtr allocMain, Byte *tempBuf[]) { UInt32 ci;@@ -404,7 +404,7 @@ Byte *temp;
outSizeCur = (SizeT)unpackSize; if (outSizeCur != unpackSize) return SZ_ERROR_MEM; - temp = (Byte *)IAlloc_Alloc(allocMain, outSizeCur); + temp = (Byte *)ISzAlloc_Alloc(allocMain, outSizeCur); if (!temp && outSizeCur != 0) return SZ_ERROR_MEM; outBufCur = tempBuf[1 - ci] = temp;@@ -421,7 +421,7 @@ else
return SZ_ERROR_UNSUPPORTED; } offset = packPositions[si]; - inSize = packPositions[si + 1] - offset; + inSize = packPositions[(size_t)si + 1] - offset; RINOK(LookInStream_SeekTo(inStream, startPos + offset)); if (coder->MethodID == k_Copy)@@ -460,7 +460,7 @@
tempSizes[2] = (SizeT)s3Size; if (tempSizes[2] != s3Size) return SZ_ERROR_MEM; - tempBuf[2] = (Byte *)IAlloc_Alloc(allocMain, tempSizes[2]); + tempBuf[2] = (Byte *)ISzAlloc_Alloc(allocMain, tempSizes[2]); if (!tempBuf[2] && tempSizes[2] != 0) return SZ_ERROR_MEM;@@ -549,7 +549,7 @@
SRes SzAr_DecodeFolder(const CSzAr *p, UInt32 folderIndex, ILookInStream *inStream, UInt64 startPos, Byte *outBuffer, size_t outSize, - ISzAlloc *allocMain) + ISzAllocPtr allocMain) { SRes res; CSzFolder folder;@@ -557,7 +557,7 @@ CSzData sd;
const Byte *data = p->CodersData + p->FoCodersOffsets[folderIndex]; sd.Data = data; - sd.Size = p->FoCodersOffsets[folderIndex + 1] - p->FoCodersOffsets[folderIndex]; + sd.Size = p->FoCodersOffsets[(size_t)folderIndex + 1] - p->FoCodersOffsets[folderIndex]; res = SzGetNextFolderItem(&folder, &sd);@@ -579,7 +579,7 @@ inStream, startPos,
outBuffer, (SizeT)outSize, allocMain, tempBuf); for (i = 0; i < 3; i++) - IAlloc_Free(allocMain, tempBuf[i]); + ISzAlloc_Free(allocMain, tempBuf[i]); if (res == SZ_OK) if (SzBitWithVals_Check(&p->FolderCRCs, folderIndex))
M
src/third-party/lzma/7zFile.c
→
src/third-party/lzma/7zFile.c
@@ -1,5 +1,5 @@
/* 7zFile.c -- File IO -2009-11-24 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -238,49 +238,49 @@
/* ---------- FileSeqInStream ---------- */ -static SRes FileSeqInStream_Read(void *pp, void *buf, size_t *size) +static SRes FileSeqInStream_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CFileSeqInStream *p = (CFileSeqInStream *)pp; + CFileSeqInStream *p = CONTAINER_FROM_VTBL(pp, CFileSeqInStream, vt); return File_Read(&p->file, buf, size) == 0 ? SZ_OK : SZ_ERROR_READ; } void FileSeqInStream_CreateVTable(CFileSeqInStream *p) { - p->s.Read = FileSeqInStream_Read; + p->vt.Read = FileSeqInStream_Read; } /* ---------- FileInStream ---------- */ -static SRes FileInStream_Read(void *pp, void *buf, size_t *size) +static SRes FileInStream_Read(const ISeekInStream *pp, void *buf, size_t *size) { - CFileInStream *p = (CFileInStream *)pp; + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); return (File_Read(&p->file, buf, size) == 0) ? SZ_OK : SZ_ERROR_READ; } -static SRes FileInStream_Seek(void *pp, Int64 *pos, ESzSeek origin) +static SRes FileInStream_Seek(const ISeekInStream *pp, Int64 *pos, ESzSeek origin) { - CFileInStream *p = (CFileInStream *)pp; + CFileInStream *p = CONTAINER_FROM_VTBL(pp, CFileInStream, vt); return File_Seek(&p->file, pos, origin); } void FileInStream_CreateVTable(CFileInStream *p) { - p->s.Read = FileInStream_Read; - p->s.Seek = FileInStream_Seek; + p->vt.Read = FileInStream_Read; + p->vt.Seek = FileInStream_Seek; } /* ---------- FileOutStream ---------- */ -static size_t FileOutStream_Write(void *pp, const void *data, size_t size) +static size_t FileOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CFileOutStream *p = (CFileOutStream *)pp; + CFileOutStream *p = CONTAINER_FROM_VTBL(pp, CFileOutStream, vt); File_Write(&p->file, data, &size); return size; } void FileOutStream_CreateVTable(CFileOutStream *p) { - p->s.Write = FileOutStream_Write; + p->vt.Write = FileOutStream_Write; }
M
src/third-party/lzma/7zFile.h
→
src/third-party/lzma/7zFile.h
@@ -1,5 +1,5 @@
/* 7zFile.h -- File IO -2013-01-18 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_FILE_H #define __7Z_FILE_H@@ -54,7 +54,7 @@ /* ---------- FileInStream ---------- */
typedef struct { - ISeqInStream s; + ISeqInStream vt; CSzFile file; } CFileSeqInStream;@@ -63,7 +63,7 @@
typedef struct { - ISeekInStream s; + ISeekInStream vt; CSzFile file; } CFileInStream;@@ -72,7 +72,7 @@
typedef struct { - ISeqOutStream s; + ISeqOutStream vt; CSzFile file; } CFileOutStream;
M
src/third-party/lzma/7zStream.c
→
src/third-party/lzma/7zStream.c
@@ -1,5 +1,5 @@
/* 7zStream.c -- 7z Stream functions -2013-11-12 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -7,12 +7,12 @@ #include <string.h>
#include "7zTypes.h" -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType) +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)); if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed);@@ -21,40 +21,42 @@ }
return SZ_OK; } -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size) +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size) { return SeqInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf) +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf) { size_t processed = 1; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ISeqInStream_Read(stream, buf, &processed)); return (processed == 1) ? SZ_OK : SZ_ERROR_INPUT_EOF; } -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset) + + +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset) { Int64 t = offset; - return stream->Seek(stream, &t, SZ_SEEK_SET); + return ILookInStream_Seek(stream, &t, SZ_SEEK_SET); } -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size) +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size) { const void *lookBuf; if (*size == 0) return SZ_OK; - RINOK(stream->Look(stream, &lookBuf, size)); + RINOK(ILookInStream_Look(stream, &lookBuf, size)); memcpy(buf, lookBuf, *size); - return stream->Skip(stream, *size); + return ILookInStream_Skip(stream, *size); } -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType) +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType) { while (size != 0) { size_t processed = size; - RINOK(stream->Read(stream, buf, &processed)); + RINOK(ILookInStream_Read(stream, buf, &processed)); if (processed == 0) return errorType; buf = (void *)((Byte *)buf + processed);@@ -63,61 +65,67 @@ }
return SZ_OK; } -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size) +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size) { return LookInStream_Read2(stream, buf, size, SZ_ERROR_INPUT_EOF); } -static SRes LookToRead_Look_Lookahead(void *pp, const void **buf, size_t *size) + + +#define GET_LookToRead2 CLookToRead2 *p = CONTAINER_FROM_VTBL(pp, CLookToRead2, vt); + +static SRes LookToRead2_Look_Lookahead(const ILookInStream *pp, const void **buf, size_t *size) { SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) + if (size2 == 0 && *size != 0) { p->pos = 0; - size2 = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, &size2); + p->size = 0; + size2 = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, &size2); p->size = size2; } - if (size2 < *size) + if (*size > size2) *size = size2; *buf = p->buf + p->pos; return res; } -static SRes LookToRead_Look_Exact(void *pp, const void **buf, size_t *size) +static SRes LookToRead2_Look_Exact(const ILookInStream *pp, const void **buf, size_t *size) { SRes res = SZ_OK; - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t size2 = p->size - p->pos; - if (size2 == 0 && *size > 0) + if (size2 == 0 && *size != 0) { p->pos = 0; - if (*size > LookToRead_BUF_SIZE) - *size = LookToRead_BUF_SIZE; - res = p->realStream->Read(p->realStream, p->buf, size); + p->size = 0; + if (*size > p->bufSize) + *size = p->bufSize; + res = ISeekInStream_Read(p->realStream, p->buf, size); size2 = p->size = *size; } - if (size2 < *size) + if (*size > size2) *size = size2; *buf = p->buf + p->pos; return res; } -static SRes LookToRead_Skip(void *pp, size_t offset) +static SRes LookToRead2_Skip(const ILookInStream *pp, size_t offset) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 p->pos += offset; return SZ_OK; } -static SRes LookToRead_Read(void *pp, void *buf, size_t *size) +static SRes LookToRead2_Read(const ILookInStream *pp, void *buf, size_t *size) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 size_t rem = p->size - p->pos; if (rem == 0) - return p->realStream->Read(p->realStream, buf, size); + return ISeekInStream_Read(p->realStream, buf, size); if (rem > *size) rem = *size; memcpy(buf, p->buf + p->pos, rem);@@ -126,46 +134,43 @@ *size = rem;
return SZ_OK; } -static SRes LookToRead_Seek(void *pp, Int64 *pos, ESzSeek origin) +static SRes LookToRead2_Seek(const ILookInStream *pp, Int64 *pos, ESzSeek origin) { - CLookToRead *p = (CLookToRead *)pp; + GET_LookToRead2 p->pos = p->size = 0; - return p->realStream->Seek(p->realStream, pos, origin); + return ISeekInStream_Seek(p->realStream, pos, origin); } -void LookToRead_CreateVTable(CLookToRead *p, int lookahead) +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead) { - p->s.Look = lookahead ? - LookToRead_Look_Lookahead : - LookToRead_Look_Exact; - p->s.Skip = LookToRead_Skip; - p->s.Read = LookToRead_Read; - p->s.Seek = LookToRead_Seek; + p->vt.Look = lookahead ? + LookToRead2_Look_Lookahead : + LookToRead2_Look_Exact; + p->vt.Skip = LookToRead2_Skip; + p->vt.Read = LookToRead2_Read; + p->vt.Seek = LookToRead2_Seek; } -void LookToRead_Init(CLookToRead *p) -{ - p->pos = p->size = 0; -} + -static SRes SecToLook_Read(void *pp, void *buf, size_t *size) +static SRes SecToLook_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CSecToLook *p = (CSecToLook *)pp; + CSecToLook *p = CONTAINER_FROM_VTBL(pp, CSecToLook, vt); return LookInStream_LookRead(p->realStream, buf, size); } void SecToLook_CreateVTable(CSecToLook *p) { - p->s.Read = SecToLook_Read; + p->vt.Read = SecToLook_Read; } -static SRes SecToRead_Read(void *pp, void *buf, size_t *size) +static SRes SecToRead_Read(const ISeqInStream *pp, void *buf, size_t *size) { - CSecToRead *p = (CSecToRead *)pp; - return p->realStream->Read(p->realStream, buf, size); + CSecToRead *p = CONTAINER_FROM_VTBL(pp, CSecToRead, vt); + return ILookInStream_Read(p->realStream, buf, size); } void SecToRead_CreateVTable(CSecToRead *p) { - p->s.Read = SecToRead_Read; + p->vt.Read = SecToRead_Read; }
M
src/third-party/lzma/7zTypes.h
→
src/third-party/lzma/7zTypes.h
@@ -1,5 +1,5 @@
/* 7zTypes.h -- Basic types -2013-11-12 : Igor Pavlov : Public domain */ +2018-08-04 : Igor Pavlov : Public domain */ #ifndef __7Z_TYPES_H #define __7Z_TYPES_H@@ -42,12 +42,22 @@ #define SZ_ERROR_NO_ARCHIVE 17
typedef int SRes; + #ifdef _WIN32 + /* typedef DWORD WRes; */ typedef unsigned WRes; +#define MY_SRes_HRESULT_FROM_WRes(x) HRESULT_FROM_WIN32(x) + #else + typedef int WRes; +#define MY__FACILITY_WIN32 7 +#define MY__FACILITY__WRes MY__FACILITY_WIN32 +#define MY_SRes_HRESULT_FROM_WRes(x) ((HRESULT)(x) <= 0 ? ((HRESULT)(x)) : ((HRESULT) (((x) & 0x0000FFFF) | (MY__FACILITY__WRes << 16) | 0x80000000))) + #endif + #ifndef RINOK #define RINOK(x) { int __result__ = (x); if (__result__ != 0) return __result__; }@@ -93,7 +103,8 @@ #else
typedef size_t SizeT; #endif -typedef int Bool; +typedef int BoolInt; +/* typedef BoolInt Bool; */ #define True 1 #define False 0@@ -112,48 +123,72 @@ #else
#define MY_NO_INLINE #endif +#define MY_FORCE_INLINE __forceinline + #define MY_CDECL __cdecl #define MY_FAST_CALL __fastcall #else #define MY_NO_INLINE +#define MY_FORCE_INLINE #define MY_CDECL #define MY_FAST_CALL + +/* inline keyword : for C++ / C99 */ + +/* GCC, clang: */ +/* +#if defined (__GNUC__) && (__GNUC__ >= 4) +#define MY_FORCE_INLINE __attribute__((always_inline)) +#define MY_NO_INLINE __attribute__((noinline)) +#endif +*/ #endif /* The following interfaces use first parameter as pointer to structure */ -typedef struct +typedef struct IByteIn IByteIn; +struct IByteIn { - Byte (*Read)(void *p); /* reads one byte, returns 0 in case of EOF or error */ -} IByteIn; + Byte (*Read)(const IByteIn *p); /* reads one byte, returns 0 in case of EOF or error */ +}; +#define IByteIn_Read(p) (p)->Read(p) -typedef struct + +typedef struct IByteOut IByteOut; +struct IByteOut { - void (*Write)(void *p, Byte b); -} IByteOut; + void (*Write)(const IByteOut *p, Byte b); +}; +#define IByteOut_Write(p, b) (p)->Write(p, b) + -typedef struct +typedef struct ISeqInStream ISeqInStream; +struct ISeqInStream { - SRes (*Read)(void *p, void *buf, size_t *size); + SRes (*Read)(const ISeqInStream *p, void *buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) < input(*size)) is allowed */ -} ISeqInStream; +}; +#define ISeqInStream_Read(p, buf, size) (p)->Read(p, buf, size) /* it can return SZ_ERROR_INPUT_EOF */ -SRes SeqInStream_Read(ISeqInStream *stream, void *buf, size_t size); -SRes SeqInStream_Read2(ISeqInStream *stream, void *buf, size_t size, SRes errorType); -SRes SeqInStream_ReadByte(ISeqInStream *stream, Byte *buf); +SRes SeqInStream_Read(const ISeqInStream *stream, void *buf, size_t size); +SRes SeqInStream_Read2(const ISeqInStream *stream, void *buf, size_t size, SRes errorType); +SRes SeqInStream_ReadByte(const ISeqInStream *stream, Byte *buf); + -typedef struct +typedef struct ISeqOutStream ISeqOutStream; +struct ISeqOutStream { - size_t (*Write)(void *p, const void *buf, size_t size); + size_t (*Write)(const ISeqOutStream *p, const void *buf, size_t size); /* Returns: result - the number of actually written bytes. (result < size) means error */ -} ISeqOutStream; +}; +#define ISeqOutStream_Write(p, buf, size) (p)->Write(p, buf, size) typedef enum {@@ -162,78 +197,162 @@ SZ_SEEK_CUR = 1,
SZ_SEEK_END = 2 } ESzSeek; -typedef struct + +typedef struct ISeekInStream ISeekInStream; +struct ISeekInStream { - SRes (*Read)(void *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ISeekInStream; + SRes (*Read)(const ISeekInStream *p, void *buf, size_t *size); /* same as ISeqInStream::Read */ + SRes (*Seek)(const ISeekInStream *p, Int64 *pos, ESzSeek origin); +}; +#define ISeekInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ISeekInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) -typedef struct + +typedef struct ILookInStream ILookInStream; +struct ILookInStream { - SRes (*Look)(void *p, const void **buf, size_t *size); + SRes (*Look)(const ILookInStream *p, const void **buf, size_t *size); /* if (input(*size) != 0 && output(*size) == 0) means end_of_stream. (output(*size) > input(*size)) is not allowed (output(*size) < input(*size)) is allowed */ - SRes (*Skip)(void *p, size_t offset); + SRes (*Skip)(const ILookInStream *p, size_t offset); /* offset must be <= output(*size) of Look */ - SRes (*Read)(void *p, void *buf, size_t *size); + SRes (*Read)(const ILookInStream *p, void *buf, size_t *size); /* reads directly (without buffer). It's same as ISeqInStream::Read */ - SRes (*Seek)(void *p, Int64 *pos, ESzSeek origin); -} ILookInStream; + SRes (*Seek)(const ILookInStream *p, Int64 *pos, ESzSeek origin); +}; -SRes LookInStream_LookRead(ILookInStream *stream, void *buf, size_t *size); -SRes LookInStream_SeekTo(ILookInStream *stream, UInt64 offset); +#define ILookInStream_Look(p, buf, size) (p)->Look(p, buf, size) +#define ILookInStream_Skip(p, offset) (p)->Skip(p, offset) +#define ILookInStream_Read(p, buf, size) (p)->Read(p, buf, size) +#define ILookInStream_Seek(p, pos, origin) (p)->Seek(p, pos, origin) + + +SRes LookInStream_LookRead(const ILookInStream *stream, void *buf, size_t *size); +SRes LookInStream_SeekTo(const ILookInStream *stream, UInt64 offset); /* reads via ILookInStream::Read */ -SRes LookInStream_Read2(ILookInStream *stream, void *buf, size_t size, SRes errorType); -SRes LookInStream_Read(ILookInStream *stream, void *buf, size_t size); +SRes LookInStream_Read2(const ILookInStream *stream, void *buf, size_t size, SRes errorType); +SRes LookInStream_Read(const ILookInStream *stream, void *buf, size_t size); + -#define LookToRead_BUF_SIZE (1 << 14) typedef struct { - ILookInStream s; - ISeekInStream *realStream; + ILookInStream vt; + const ISeekInStream *realStream; + size_t pos; - size_t size; - Byte buf[LookToRead_BUF_SIZE]; -} CLookToRead; + size_t size; /* it's data size */ + + /* the following variables must be set outside */ + Byte *buf; + size_t bufSize; +} CLookToRead2; -void LookToRead_CreateVTable(CLookToRead *p, int lookahead); -void LookToRead_Init(CLookToRead *p); +void LookToRead2_CreateVTable(CLookToRead2 *p, int lookahead); + +#define LookToRead2_Init(p) { (p)->pos = (p)->size = 0; } + typedef struct { - ISeqInStream s; - ILookInStream *realStream; + ISeqInStream vt; + const ILookInStream *realStream; } CSecToLook; void SecToLook_CreateVTable(CSecToLook *p); + + typedef struct { - ISeqInStream s; - ILookInStream *realStream; + ISeqInStream vt; + const ILookInStream *realStream; } CSecToRead; void SecToRead_CreateVTable(CSecToRead *p); -typedef struct + +typedef struct ICompressProgress ICompressProgress; + +struct ICompressProgress { - SRes (*Progress)(void *p, UInt64 inSize, UInt64 outSize); + SRes (*Progress)(const ICompressProgress *p, UInt64 inSize, UInt64 outSize); /* Returns: result. (result != SZ_OK) means break. Value (UInt64)(Int64)-1 for size means unknown value. */ -} ICompressProgress; +}; +#define ICompressProgress_Progress(p, inSize, outSize) (p)->Progress(p, inSize, outSize) + + -typedef struct +typedef struct ISzAlloc ISzAlloc; +typedef const ISzAlloc * ISzAllocPtr; + +struct ISzAlloc { - void *(*Alloc)(void *p, size_t size); - void (*Free)(void *p, void *address); /* address can be 0 */ -} ISzAlloc; + void *(*Alloc)(ISzAllocPtr p, size_t size); + void (*Free)(ISzAllocPtr p, void *address); /* address can be 0 */ +}; + +#define ISzAlloc_Alloc(p, size) (p)->Alloc(p, size) +#define ISzAlloc_Free(p, a) (p)->Free(p, a) + +/* deprecated */ +#define IAlloc_Alloc(p, size) ISzAlloc_Alloc(p, size) +#define IAlloc_Free(p, a) ISzAlloc_Free(p, a) + + + + + +#ifndef MY_offsetof + #ifdef offsetof + #define MY_offsetof(type, m) offsetof(type, m) + /* + #define MY_offsetof(type, m) FIELD_OFFSET(type, m) + */ + #else + #define MY_offsetof(type, m) ((size_t)&(((type *)0)->m)) + #endif +#endif + + + +#ifndef MY_container_of + +/* +#define MY_container_of(ptr, type, m) container_of(ptr, type, m) +#define MY_container_of(ptr, type, m) CONTAINING_RECORD(ptr, type, m) +#define MY_container_of(ptr, type, m) ((type *)((char *)(ptr) - offsetof(type, m))) +#define MY_container_of(ptr, type, m) (&((type *)0)->m == (ptr), ((type *)(((char *)(ptr)) - MY_offsetof(type, m)))) +*/ + +/* + GCC shows warning: "perhaps the 'offsetof' macro was used incorrectly" + GCC 3.4.4 : classes with constructor + GCC 4.8.1 : classes with non-public variable members" +*/ + +#define MY_container_of(ptr, type, m) ((type *)((char *)(1 ? (ptr) : &((type *)0)->m) - MY_offsetof(type, m))) + + +#endif + +#define CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) ((type *)(ptr)) + +/* +#define CONTAINER_FROM_VTBL(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +*/ +#define CONTAINER_FROM_VTBL(ptr, type, m) MY_container_of(ptr, type, m) + +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL_SIMPLE(ptr, type, m) +/* +#define CONTAINER_FROM_VTBL_CLS(ptr, type, m) CONTAINER_FROM_VTBL(ptr, type, m) +*/ -#define IAlloc_Alloc(p, size) (p)->Alloc((p), size) -#define IAlloc_Free(p, a) (p)->Free((p), a) + #ifdef _WIN32
M
src/third-party/lzma/7zVersion.h
→
src/third-party/lzma/7zVersion.h
@@ -1,14 +1,21 @@
-#define MY_VER_MAJOR 15 -#define MY_VER_MINOR 14 +#define MY_VER_MAJOR 19 +#define MY_VER_MINOR 00 #define MY_VER_BUILD 0 -#define MY_VERSION_NUMBERS "15.14" -#define MY_VERSION "15.14" -#define MY_DATE "2015-12-31" +#define MY_VERSION_NUMBERS "19.00" +#define MY_VERSION MY_VERSION_NUMBERS + +#ifdef MY_CPU_NAME + #define MY_VERSION_CPU MY_VERSION " (" MY_CPU_NAME ")" +#else + #define MY_VERSION_CPU MY_VERSION +#endif + +#define MY_DATE "2019-02-21" #undef MY_COPYRIGHT #undef MY_VERSION_COPYRIGHT_DATE #define MY_AUTHOR_NAME "Igor Pavlov" #define MY_COPYRIGHT_PD "Igor Pavlov : Public domain" -#define MY_COPYRIGHT_CR "Copyright (c) 1999-2015 Igor Pavlov" +#define MY_COPYRIGHT_CR "Copyright (c) 1999-2018 Igor Pavlov" #ifdef USE_COPYRIGHT_CR #define MY_COPYRIGHT MY_COPYRIGHT_CR@@ -16,4 +23,5 @@ #else
#define MY_COPYRIGHT MY_COPYRIGHT_PD #endif -#define MY_VERSION_COPYRIGHT_DATE MY_VERSION " : " MY_COPYRIGHT " : " MY_DATE +#define MY_COPYRIGHT_DATE MY_COPYRIGHT " : " MY_DATE +#define MY_VERSION_COPYRIGHT_DATE MY_VERSION_CPU " : " MY_COPYRIGHT " : " MY_DATE
M
src/third-party/lzma/Aes.c
→
src/third-party/lzma/Aes.c
@@ -1,5 +1,5 @@
/* Aes.c -- AES encryption / decryption -2015-02-23 : Igor Pavlov : Public domain */ +2017-01-24 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -49,7 +49,13 @@
#define gb0(x) ( (x) & 0xFF) #define gb1(x) (((x) >> ( 8)) & 0xFF) #define gb2(x) (((x) >> (16)) & 0xFF) -#define gb3(x) (((x) >> (24)) & 0xFF) +#define gb3(x) (((x) >> (24))) + +#define gb(n, x) gb ## n(x) + +#define TT(x) (T + (x << 8)) +#define DD(x) (D + (x << 8)) + void AesGenTables(void) {@@ -63,10 +69,10 @@ {
UInt32 a1 = Sbox[i]; UInt32 a2 = xtime(a1); UInt32 a3 = a2 ^ a1; - T[ i] = Ui32(a2, a1, a1, a3); - T[0x100 + i] = Ui32(a3, a2, a1, a1); - T[0x200 + i] = Ui32(a1, a3, a2, a1); - T[0x300 + i] = Ui32(a1, a1, a3, a2); + TT(0)[i] = Ui32(a2, a1, a1, a3); + TT(1)[i] = Ui32(a3, a2, a1, a1); + TT(2)[i] = Ui32(a1, a3, a2, a1); + TT(3)[i] = Ui32(a1, a1, a3, a2); } { UInt32 a1 = InvS[i];@@ -77,10 +83,10 @@ UInt32 a9 = a8 ^ a1;
UInt32 aB = a8 ^ a2 ^ a1; UInt32 aD = a8 ^ a4 ^ a1; UInt32 aE = a8 ^ a4 ^ a2; - D[ i] = Ui32(aE, a9, aD, aB); - D[0x100 + i] = Ui32(aB, aE, a9, aD); - D[0x200 + i] = Ui32(aD, aB, aE, a9); - D[0x300 + i] = Ui32(a9, aD, aB, aE); + DD(0)[i] = Ui32(aE, a9, aD, aB); + DD(1)[i] = Ui32(aB, aE, a9, aD); + DD(2)[i] = Ui32(aD, aB, aE, a9); + DD(3)[i] = Ui32(a9, aD, aB, aE); } }@@ -99,7 +105,7 @@ #endif
} -#define HT(i, x, s) (T + (x << 8))[gb ## x(s[(i + x) & 3])] +#define HT(i, x, s) TT(x)[gb(x, s[(i + x) & 3])] #define HT4(m, i, s, p) m[i] = \ HT(i, 0, s) ^ \@@ -113,11 +119,11 @@ HT4(m, 1, s, p); \
HT4(m, 2, s, p); \ HT4(m, 3, s, p); \ -#define FT(i, x) Sbox[gb ## x(m[(i + x) & 3])] +#define FT(i, x) Sbox[gb(x, m[(i + x) & 3])] #define FT4(i) dest[i] = Ui32(FT(i, 0), FT(i, 1), FT(i, 2), FT(i, 3)) ^ w[i]; -#define HD(i, x, s) (D + (x << 8))[gb ## x(s[(i - x) & 3])] +#define HD(i, x, s) DD(x)[gb(x, s[(i - x) & 3])] #define HD4(m, i, s, p) m[i] = \ HD(i, 0, s) ^ \@@ -131,7 +137,7 @@ HD4(m, 1, s, p); \
HD4(m, 2, s, p); \ HD4(m, 3, s, p); \ -#define FD(i, x) InvS[gb ## x(m[(i - x) & 3])] +#define FD(i, x) InvS[gb(x, m[(i - x) & 3])] #define FD4(i) dest[i] = Ui32(FD(i, 0), FD(i, 1), FD(i, 2), FD(i, 3)) ^ w[i]; void MY_FAST_CALL Aes_SetKey_Enc(UInt32 *w, const Byte *key, unsigned keySize)@@ -147,7 +153,7 @@ w[i] = GetUi32(key);
for (; i < wSize; i++) { - UInt32 t = w[i - 1]; + UInt32 t = w[(size_t)i - 1]; unsigned rem = i % keySize; if (rem == 0) t = Ui32(Sbox[gb1(t)] ^ Rcon[i / keySize], Sbox[gb2(t)], Sbox[gb3(t)], Sbox[gb0(t)]);@@ -167,10 +173,10 @@ for (i = 0; i < num; i++)
{ UInt32 r = w[i]; w[i] = - D[ Sbox[gb0(r)]] ^ - D[0x100 + Sbox[gb1(r)]] ^ - D[0x200 + Sbox[gb2(r)]] ^ - D[0x300 + Sbox[gb3(r)]]; + DD(0)[Sbox[gb0(r)]] ^ + DD(1)[Sbox[gb1(r)]] ^ + DD(2)[Sbox[gb2(r)]] ^ + DD(3)[Sbox[gb3(r)]]; } }@@ -276,20 +282,25 @@ {
for (; numBlocks != 0; numBlocks--) { UInt32 temp[4]; - Byte buf[16]; - int i; + unsigned i; if (++p[0] == 0) p[1]++; Aes_Encode(p + 4, temp, p); - SetUi32(buf, temp[0]); - SetUi32(buf + 4, temp[1]); - SetUi32(buf + 8, temp[2]); - SetUi32(buf + 12, temp[3]); - - for (i = 0; i < 16; i++) - *data++ ^= buf[i]; + for (i = 0; i < 4; i++, data += 4) + { + UInt32 t = temp[i]; + + #ifdef MY_CPU_LE_UNALIGN + *((UInt32 *)data) ^= t; + #else + data[0] ^= (t & 0xFF); + data[1] ^= ((t >> 8) & 0xFF); + data[2] ^= ((t >> 16) & 0xFF); + data[3] ^= ((t >> 24)); + #endif + } } }
M
src/third-party/lzma/AesOpt.c
→
src/third-party/lzma/AesOpt.c
@@ -1,12 +1,12 @@
/* AesOpt.c -- Intel's AES -2013-11-12 : Igor Pavlov : Public domain */ +2017-06-08 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "CpuArch.h" #ifdef MY_CPU_X86_OR_AMD64 -#if _MSC_VER >= 1500 +#if (_MSC_VER > 1500) || (_MSC_FULL_VER >= 150030729) #define USE_INTEL_AES #endif #endif
M
src/third-party/lzma/Alloc.c
→
src/third-party/lzma/Alloc.c
@@ -1,7 +1,9 @@
/* Alloc.c -- Memory allocation functions -2015-02-21 : Igor Pavlov : Public domain */ +2018-04-27 : Igor Pavlov : Public domain */ #include "Precomp.h" + +#include <stdio.h> #ifdef _WIN32 #include <windows.h>@@ -14,20 +16,127 @@ /* #define _SZ_ALLOC_DEBUG */
/* use _SZ_ALLOC_DEBUG to debug alloc/free operations */ #ifdef _SZ_ALLOC_DEBUG + #include <stdio.h> int g_allocCount = 0; int g_allocCountMid = 0; int g_allocCountBig = 0; + + +#define CONVERT_INT_TO_STR(charType, tempSize) \ + unsigned char temp[tempSize]; unsigned i = 0; \ + while (val >= 10) { temp[i++] = (unsigned char)('0' + (unsigned)(val % 10)); val /= 10; } \ + *s++ = (charType)('0' + (unsigned)val); \ + while (i != 0) { i--; *s++ = temp[i]; } \ + *s = 0; + +static void ConvertUInt64ToString(UInt64 val, char *s) +{ + CONVERT_INT_TO_STR(char, 24); +} + +#define GET_HEX_CHAR(t) ((char)(((t < 10) ? ('0' + t) : ('A' + (t - 10))))) + +static void ConvertUInt64ToHex(UInt64 val, char *s) +{ + UInt64 v = val; + unsigned i; + for (i = 1;; i++) + { + v >>= 4; + if (v == 0) + break; + } + s[i] = 0; + do + { + unsigned t = (unsigned)(val & 0xF); + val >>= 4; + s[--i] = GET_HEX_CHAR(t); + } + while (i); +} + +#define DEBUG_OUT_STREAM stderr + +static void Print(const char *s) +{ + fputs(s, DEBUG_OUT_STREAM); +} + +static void PrintAligned(const char *s, size_t align) +{ + size_t len = strlen(s); + for(;;) + { + fputc(' ', DEBUG_OUT_STREAM); + if (len >= align) + break; + ++len; + } + Print(s); +} + +static void PrintLn() +{ + Print("\n"); +} + +static void PrintHex(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToHex(v, s); + PrintAligned(s, align); +} + +static void PrintDec(UInt64 v, size_t align) +{ + char s[32]; + ConvertUInt64ToString(v, s); + PrintAligned(s, align); +} + +static void PrintAddr(void *p) +{ + PrintHex((UInt64)(size_t)(ptrdiff_t)p, 12); +} + + +#define PRINT_ALLOC(name, cnt, size, ptr) \ + Print(name " "); \ + PrintDec(cnt++, 10); \ + PrintHex(size, 10); \ + PrintAddr(ptr); \ + PrintLn(); + +#define PRINT_FREE(name, cnt, ptr) if (ptr) { \ + Print(name " "); \ + PrintDec(--cnt, 10); \ + PrintAddr(ptr); \ + PrintLn(); } + +#else + +#define PRINT_ALLOC(name, cnt, size, ptr) +#define PRINT_FREE(name, cnt, ptr) +#define Print(s) +#define PrintLn() +#define PrintHex(v, align) +#define PrintDec(v, align) +#define PrintAddr(p) + #endif + + void *MyAlloc(size_t size) { if (size == 0) - return 0; + return NULL; #ifdef _SZ_ALLOC_DEBUG { void *p = malloc(size); - fprintf(stderr, "\nAlloc %10d bytes, count = %10d, addr = %8X", size, g_allocCount++, (unsigned)p); + PRINT_ALLOC("Alloc ", g_allocCount, size, p); return p; } #else@@ -37,10 +146,8 @@ }
void MyFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree; count = %10d, addr = %8X", --g_allocCount, (unsigned)address); - #endif + PRINT_FREE("Free ", g_allocCount, address); + free(address); }@@ -49,20 +156,18 @@
void *MidAlloc(size_t size) { if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Mid %10d bytes; count = %10d", size, g_allocCountMid++); - #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + return NULL; + + PRINT_ALLOC("Alloc-Mid", g_allocCountMid, size, NULL); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void MidFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Mid; count = %10d", --g_allocCountMid); - #endif - if (address == 0) + PRINT_FREE("Free-Mid", g_allocCountMid, address); + + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); }@@ -79,10 +184,10 @@
void SetLargePageSize() { #ifdef _7ZIP_LARGE_PAGES - SIZE_T size = 0; + SIZE_T size; GetLargePageMinimumP largePageMinimum = (GetLargePageMinimumP) GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "GetLargePageMinimum"); - if (largePageMinimum == 0) + if (!largePageMinimum) return; size = largePageMinimum(); if (size == 0 || (size & (size - 1)) != 0)@@ -95,31 +200,36 @@
void *BigAlloc(size_t size) { if (size == 0) - return 0; - #ifdef _SZ_ALLOC_DEBUG - fprintf(stderr, "\nAlloc_Big %10d bytes; count = %10d", size, g_allocCountBig++); - #endif + return NULL; + + PRINT_ALLOC("Alloc-Big", g_allocCountBig, size, NULL); #ifdef _7ZIP_LARGE_PAGES - if (g_LargePageSize != 0 && g_LargePageSize <= (1 << 30) && size >= (1 << 18)) { - void *res = VirtualAlloc(0, (size + g_LargePageSize - 1) & (~(g_LargePageSize - 1)), - MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); - if (res != 0) - return res; + SIZE_T ps = g_LargePageSize; + if (ps != 0 && ps <= (1 << 30) && size > (ps / 2)) + { + size_t size2; + ps--; + size2 = (size + ps) & ~ps; + if (size2 >= size) + { + void *res = VirtualAlloc(NULL, size2, MEM_COMMIT | MEM_LARGE_PAGES, PAGE_READWRITE); + if (res) + return res; + } + } } #endif - return VirtualAlloc(0, size, MEM_COMMIT, PAGE_READWRITE); + + return VirtualAlloc(NULL, size, MEM_COMMIT, PAGE_READWRITE); } void BigFree(void *address) { - #ifdef _SZ_ALLOC_DEBUG - if (address != 0) - fprintf(stderr, "\nFree_Big; count = %10d", --g_allocCountBig); - #endif + PRINT_FREE("Free-Big", g_allocCountBig, address); - if (address == 0) + if (!address) return; VirtualFree(address, 0, MEM_RELEASE); }@@ -127,10 +237,219 @@
#endif -static void *SzAlloc(void *p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(void *p, void *address) { UNUSED_VAR(p); MyFree(address); } -ISzAlloc g_Alloc = { SzAlloc, SzFree }; +static void *SzAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } +static void SzFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MyFree(address); } +const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +static void *SzMidAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return MidAlloc(size); } +static void SzMidFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); MidFree(address); } +const ISzAlloc g_MidAlloc = { SzMidAlloc, SzMidFree }; + +static void *SzBigAlloc(ISzAllocPtr p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } +static void SzBigFree(ISzAllocPtr p, void *address) { UNUSED_VAR(p); BigFree(address); } +const ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; + + +/* + uintptr_t : <stdint.h> C99 (optional) + : unsupported in VS6 +*/ + +#ifdef _WIN32 + typedef UINT_PTR UIntPtr; +#else + /* + typedef uintptr_t UIntPtr; + */ + typedef ptrdiff_t UIntPtr; +#endif + + +#define ADJUST_ALLOC_SIZE 0 +/* +#define ADJUST_ALLOC_SIZE (sizeof(void *) - 1) +*/ +/* + Use (ADJUST_ALLOC_SIZE = (sizeof(void *) - 1)), if + MyAlloc() can return address that is NOT multiple of sizeof(void *). +*/ + + +/* +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((char *)(p) - ((size_t)(UIntPtr)(p) & ((align) - 1)))) +*/ +#define MY_ALIGN_PTR_DOWN(p, align) ((void *)((((UIntPtr)(p)) & ~((UIntPtr)(align) - 1)))) + +#define MY_ALIGN_PTR_UP_PLUS(p, align) MY_ALIGN_PTR_DOWN(((char *)(p) + (align) + ADJUST_ALLOC_SIZE), align) + + +#if (_POSIX_C_SOURCE >= 200112L) && !defined(_WIN32) + #define USE_posix_memalign +#endif + +/* + This posix_memalign() is for test purposes only. + We also need special Free() function instead of free(), + if this posix_memalign() is used. +*/ + +/* +static int posix_memalign(void **ptr, size_t align, size_t size) +{ + size_t newSize = size + align; + void *p; + void *pAligned; + *ptr = NULL; + if (newSize < size) + return 12; // ENOMEM + p = MyAlloc(newSize); + if (!p) + return 12; // ENOMEM + pAligned = MY_ALIGN_PTR_UP_PLUS(p, align); + ((void **)pAligned)[-1] = p; + *ptr = pAligned; + return 0; +} +*/ + +/* + ALLOC_ALIGN_SIZE >= sizeof(void *) + ALLOC_ALIGN_SIZE >= cache_line_size +*/ + +#define ALLOC_ALIGN_SIZE ((size_t)1 << 7) + +static void *SzAlignedAlloc(ISzAllocPtr pp, size_t size) +{ + #ifndef USE_posix_memalign + + void *p; + void *pAligned; + size_t newSize; + UNUSED_VAR(pp); + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + + newSize = size + ALLOC_ALIGN_SIZE * 1 + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + p = MyAlloc(newSize); + + if (!p) + return NULL; + pAligned = MY_ALIGN_PTR_UP_PLUS(p, ALLOC_ALIGN_SIZE); + + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(p); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + ((void **)pAligned)[-1] = p; + + return pAligned; + + #else + + void *p; + UNUSED_VAR(pp); + if (posix_memalign(&p, ALLOC_ALIGN_SIZE, size)) + return NULL; + + Print(" posix_memalign="); PrintAddr(p); + PrintLn(); + + return p; + + #endif +} + + +static void SzAlignedFree(ISzAllocPtr pp, void *address) +{ + UNUSED_VAR(pp); + #ifndef USE_posix_memalign + if (address) + MyFree(((void **)address)[-1]); + #else + free(address); + #endif +} + + +const ISzAlloc g_AlignedAlloc = { SzAlignedAlloc, SzAlignedFree }; + + + +#define MY_ALIGN_PTR_DOWN_1(p) MY_ALIGN_PTR_DOWN(p, sizeof(void *)) + +/* we align ptr to support cases where CAlignOffsetAlloc::offset is not multiply of sizeof(void *) */ +#define REAL_BLOCK_PTR_VAR(p) ((void **)MY_ALIGN_PTR_DOWN_1(p))[-1] +/* +#define REAL_BLOCK_PTR_VAR(p) ((void **)(p))[-1] +*/ + +static void *AlignOffsetAlloc_Alloc(ISzAllocPtr pp, size_t size) +{ + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + void *adr; + void *pAligned; + size_t newSize; + size_t extra; + size_t alignSize = (size_t)1 << p->numAlignBits; + + if (alignSize < sizeof(void *)) + alignSize = sizeof(void *); + + if (p->offset >= alignSize) + return NULL; + + /* also we can allocate additional dummy ALLOC_ALIGN_SIZE bytes after aligned + block to prevent cache line sharing with another allocated blocks */ + extra = p->offset & (sizeof(void *) - 1); + newSize = size + alignSize + extra + ADJUST_ALLOC_SIZE; + if (newSize < size) + return NULL; + + adr = ISzAlloc_Alloc(p->baseAlloc, newSize); + + if (!adr) + return NULL; + + pAligned = (char *)MY_ALIGN_PTR_DOWN((char *)adr + + alignSize - p->offset + extra + ADJUST_ALLOC_SIZE, alignSize) + p->offset; + + PrintLn(); + Print("- Aligned: "); + Print(" size="); PrintHex(size, 8); + Print(" a_size="); PrintHex(newSize, 8); + Print(" ptr="); PrintAddr(adr); + Print(" a_ptr="); PrintAddr(pAligned); + PrintLn(); + + REAL_BLOCK_PTR_VAR(pAligned) = adr; + + return pAligned; +} + -static void *SzBigAlloc(void *p, size_t size) { UNUSED_VAR(p); return BigAlloc(size); } -static void SzBigFree(void *p, void *address) { UNUSED_VAR(p); BigFree(address); } -ISzAlloc g_BigAlloc = { SzBigAlloc, SzBigFree }; +static void AlignOffsetAlloc_Free(ISzAllocPtr pp, void *address) +{ + if (address) + { + CAlignOffsetAlloc *p = CONTAINER_FROM_VTBL(pp, CAlignOffsetAlloc, vt); + PrintLn(); + Print("- Aligned Free: "); + PrintLn(); + ISzAlloc_Free(p->baseAlloc, REAL_BLOCK_PTR_VAR(address)); + } +} + + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p) +{ + p->vt.Alloc = AlignOffsetAlloc_Alloc; + p->vt.Free = AlignOffsetAlloc_Free; +}
M
src/third-party/lzma/Alloc.h
→
src/third-party/lzma/Alloc.h
@@ -1,5 +1,5 @@
/* Alloc.h -- Memory allocation functions -2015-02-21 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __COMMON_ALLOC_H #define __COMMON_ALLOC_H@@ -29,8 +29,22 @@ #define BigFree(address) MyFree(address)
#endif -extern ISzAlloc g_Alloc; -extern ISzAlloc g_BigAlloc; +extern const ISzAlloc g_Alloc; +extern const ISzAlloc g_BigAlloc; +extern const ISzAlloc g_MidAlloc; +extern const ISzAlloc g_AlignedAlloc; + + +typedef struct +{ + ISzAlloc vt; + ISzAllocPtr baseAlloc; + unsigned numAlignBits; /* ((1 << numAlignBits) >= sizeof(void *)) */ + size_t offset; /* (offset == (k * sizeof(void *)) && offset < (1 << numAlignBits) */ +} CAlignOffsetAlloc; + +void AlignOffsetAlloc_CreateVTable(CAlignOffsetAlloc *p); + EXTERN_C_END
M
src/third-party/lzma/Bcj2.c
→
src/third-party/lzma/Bcj2.c
@@ -1,5 +1,5 @@
/* Bcj2.c -- BCJ2 Decoder (Converter for x86 code) -2015-08-01 : Igor Pavlov : Public domain */ +2018-04-28 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -61,7 +61,8 @@ {
Byte *dest = p->dest; if (dest == p->destLim) return SZ_OK; - *dest = p->temp[p->state++ - BCJ2_DEC_STATE_ORIG_0]; + *dest = p->temp[(size_t)p->state - BCJ2_DEC_STATE_ORIG_0]; + p->state++; p->dest = dest + 1; } }@@ -231,10 +232,10 @@ rem = p->destLim - dest;
if (rem < 4) { - SizeT i; - SetUi32(p->temp, val); - for (i = 0; i < rem; i++) - dest[i] = p->temp[i]; + p->temp[0] = (Byte)val; if (rem > 0) dest[0] = (Byte)val; val >>= 8; + p->temp[1] = (Byte)val; if (rem > 1) dest[1] = (Byte)val; val >>= 8; + p->temp[2] = (Byte)val; if (rem > 2) dest[2] = (Byte)val; val >>= 8; + p->temp[3] = (Byte)val; p->dest = dest + rem; p->state = BCJ2_DEC_STATE_ORIG_0 + (unsigned)rem; break;
M
src/third-party/lzma/Bcj2Enc.c
→
src/third-party/lzma/Bcj2Enc.c
@@ -1,5 +1,5 @@
/* Bcj2Enc.c -- BCJ2 Encoder (Converter for x86 code) -2014-11-10 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -12,7 +12,6 @@ #else
#define PRF(x) #endif -#include <windows.h> #include <string.h> #include "Bcj2.h"@@ -53,7 +52,7 @@ for (i = 0; i < sizeof(p->probs) / sizeof(p->probs[0]); i++)
p->probs[i] = kBitModelTotal >> 1; } -static Bool MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) +static BoolInt MY_FAST_CALL RangeEnc_ShiftLow(CBcj2Enc *p) { if ((UInt32)p->low < (UInt32)0xFF000000 || (UInt32)(p->low >> 32) != 0) {@@ -166,7 +165,7 @@ }
{ Byte context = (Byte)(num == 0 ? p->prevByte : src[-1]); - Bool needConvert; + BoolInt needConvert; p->bufs[BCJ2_STREAM_MAIN] = dest + 1; p->ip += (UInt32)num + 1;@@ -254,7 +253,7 @@ for (;;)
{ const Byte *src = p->src; const Byte *srcLim = p->srcLim; - unsigned finishMode = p->finishMode; + EBcj2Enc_FinishMode finishMode = p->finishMode; p->src = p->temp; p->srcLim = p->temp + p->tempPos;@@ -271,7 +270,7 @@ unsigned tempPos = p->tempPos - num;
unsigned i; p->tempPos = tempPos; for (i = 0; i < tempPos; i++) - p->temp[i] = p->temp[i + num]; + p->temp[i] = p->temp[(size_t)i + num]; p->src = src; p->srcLim = srcLim;
M
src/third-party/lzma/Bra.c
→
src/third-party/lzma/Bra.c
@@ -1,135 +1,230 @@
/* Bra.c -- Converters for RISC code -2010-04-16 : Igor Pavlov : Public domain */ +2017-04-04 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "CpuArch.h" #include "Bra.h" SizeT ARM_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 8; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip += 4; + p = data; + lim = data + size; + + if (encoding) + + for (;;) { - if (data[i + 3] == 0xEB) + for (;;) { - UInt32 dest; - UInt32 src = ((UInt32)data[i + 2] << 16) | ((UInt32)data[i + 1] << 8) | (data[i + 0]); - src <<= 2; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 2; - data[i + 2] = (Byte)(dest >> 16); - data[i + 1] = (Byte)(dest >> 8); - data[i + 0] = (Byte)dest; + if (p >= lim) + return p - data; + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v += ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); } } - return i; + + for (;;) + { + for (;;) + { + if (p >= lim) + return p - data; + p += 4; + if (p[-1] == 0xEB) + break; + } + { + UInt32 v = GetUi32(p - 4); + v <<= 2; + v -= ip + (UInt32)(p - data); + v >>= 2; + v &= 0x00FFFFFF; + v |= 0xEB000000; + SetUi32(p - 4, v); + } + } } + SizeT ARMT_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - ip += 4; - for (i = 0; i <= size; i += 2) + Byte *p; + const Byte *lim; + size &= ~(size_t)1; + p = data; + lim = data + size - 4; + + if (encoding) + + for (;;) { - if ((data[i + 1] & 0xF8) == 0xF0 && - (data[i + 3] & 0xF8) == 0xF8) + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return p - data; + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } { - UInt32 dest; - UInt32 src = - (((UInt32)data[i + 1] & 0x7) << 19) | - ((UInt32)data[i + 0] << 11) | - (((UInt32)data[i + 3] & 0x7) << 8) | - (data[i + 2]); + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v += cur; + } + + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); + } + } + + for (;;) + { + UInt32 b1; + for (;;) + { + UInt32 b3; + if (p > lim) + return p - data; + b1 = p[1]; + b3 = p[3]; + p += 2; + b1 ^= 8; + if ((b3 & b1) >= 0xF8) + break; + } + { + UInt32 v = + ((UInt32)b1 << 19) + + (((UInt32)p[1] & 0x7) << 8) + + (((UInt32)p[-2] << 11)) + + (p[0]); + + p += 2; + { + UInt32 cur = (ip + (UInt32)(p - data)) >> 1; + v -= cur; + } + + /* + SetUi16(p - 4, (UInt16)(((v >> 11) & 0x7FF) | 0xF000)); + SetUi16(p - 2, (UInt16)(v | 0xF800)); + */ - src <<= 1; - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - dest >>= 1; - - data[i + 1] = (Byte)(0xF0 | ((dest >> 19) & 0x7)); - data[i + 0] = (Byte)(dest >> 11); - data[i + 3] = (Byte)(0xF8 | ((dest >> 8) & 0x7)); - data[i + 2] = (Byte)dest; - i += 2; + p[-4] = (Byte)(v >> 11); + p[-3] = (Byte)(0xF0 | ((v >> 19) & 0x7)); + p[-2] = (Byte)v; + p[-1] = (Byte)(0xF8 | (v >> 8)); } } - return i; } + SizeT PPC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - SizeT i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) { - if ((data[i] >> 2) == 0x12 && (data[i + 3] & 3) == 1) + for (;;) + { + if (p >= lim) + return p - data; + p += 4; + /* if ((v & 0xFC000003) == 0x48000001) */ + if ((p[-4] & 0xFC) == 0x48 && (p[-1] & 3) == 1) + break; + } { - UInt32 src = ((UInt32)(data[i + 0] & 3) << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3] & (~3)); - - UInt32 dest; + UInt32 v = GetBe32(p - 4); if (encoding) - dest = ip + (UInt32)i + src; + v += ip + (UInt32)(p - data); else - dest = src - (ip + (UInt32)i); - data[i + 0] = (Byte)(0x48 | ((dest >> 24) & 0x3)); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] &= 0x3; - data[i + 3] |= dest; + v -= ip + (UInt32)(p - data); + v &= 0x03FFFFFF; + v |= 0x48000000; + SetBe32(p - 4, v); } } - return i; } + SizeT SPARC_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) { - UInt32 i; - if (size < 4) - return 0; - size -= 4; - for (i = 0; i <= size; i += 4) + Byte *p; + const Byte *lim; + size &= ~(size_t)3; + ip -= 4; + p = data; + lim = data + size; + + for (;;) { - if ((data[i] == 0x40 && (data[i + 1] & 0xC0) == 0x00) || - (data[i] == 0x7F && (data[i + 1] & 0xC0) == 0xC0)) + for (;;) + { + if (p >= lim) + return p - data; + /* + v = GetBe32(p); + p += 4; + m = v + ((UInt32)5 << 29); + m ^= (UInt32)7 << 29; + m += (UInt32)1 << 22; + if ((m & ((UInt32)0x1FF << 23)) == 0) + break; + */ + p += 4; + if ((p[-4] == 0x40 && (p[-3] & 0xC0) == 0) || + (p[-4] == 0x7F && (p[-3] >= 0xC0))) + break; + } { - UInt32 src = - ((UInt32)data[i + 0] << 24) | - ((UInt32)data[i + 1] << 16) | - ((UInt32)data[i + 2] << 8) | - ((UInt32)data[i + 3]); - UInt32 dest; - - src <<= 2; + UInt32 v = GetBe32(p - 4); + v <<= 2; if (encoding) - dest = ip + i + src; + v += ip + (UInt32)(p - data); else - dest = src - (ip + i); - dest >>= 2; + v -= ip + (UInt32)(p - data); - dest = (((0 - ((dest >> 22) & 1)) << 22) & 0x3FFFFFFF) | (dest & 0x3FFFFF) | 0x40000000; - - data[i + 0] = (Byte)(dest >> 24); - data[i + 1] = (Byte)(dest >> 16); - data[i + 2] = (Byte)(dest >> 8); - data[i + 3] = (Byte)dest; + v &= 0x01FFFFFF; + v -= (UInt32)1 << 24; + v ^= 0xFF000000; + v >>= 2; + v |= 0x40000000; + SetBe32(p - 4, v); } } - return i; }
M
src/third-party/lzma/Bra86.c
→
src/third-party/lzma/Bra86.c
@@ -1,5 +1,5 @@
/* Bra86.c -- Converter for x86 code (BCJ) -2013-11-12 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -37,7 +37,7 @@ mask = 0;
else { mask >>= (unsigned)d; - if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(mask >> 1) + 1]))) + if (mask != 0 && (mask > 4 || mask == 3 || Test86MSByte(p[(size_t)(mask >> 1) + 1]))) { mask = (mask >> 1) | 4; pos++;
M
src/third-party/lzma/BraIA64.c
→
src/third-party/lzma/BraIA64.c
@@ -1,17 +1,10 @@
/* BraIA64.c -- Converter for IA-64 code -2013-11-12 : Igor Pavlov : Public domain */ +2017-01-26 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include "CpuArch.h" #include "Bra.h" - -static const Byte kBranchTable[32] = -{ - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 4, 4, 6, 6, 0, 0, 7, 7, - 4, 4, 0, 0, 4, 4, 0, 0 -}; SizeT IA64_Convert(Byte *data, SizeT size, UInt32 ip, int encoding) {@@ -19,51 +12,42 @@ SizeT i;
if (size < 16) return 0; size -= 16; - for (i = 0; i <= size; i += 16) + i = 0; + do { - UInt32 instrTemplate = data[i] & 0x1F; - UInt32 mask = kBranchTable[instrTemplate]; - UInt32 bitPos = 5; - int slot; - for (slot = 0; slot < 3; slot++, bitPos += 41) + unsigned m = ((UInt32)0x334B0000 >> (data[i] & 0x1E)) & 3; + if (m) { - UInt32 bytePos, bitRes; - UInt64 instruction, instNorm; - int j; - if (((mask >> slot) & 1) == 0) - continue; - bytePos = (bitPos >> 3); - bitRes = bitPos & 0x7; - instruction = 0; - for (j = 0; j < 6; j++) - instruction += (UInt64)data[i + j + bytePos] << (8 * j); - - instNorm = instruction >> bitRes; - if (((instNorm >> 37) & 0xF) == 0x5 && ((instNorm >> 9) & 0x7) == 0) + m++; + do { - UInt32 src = (UInt32)((instNorm >> 13) & 0xFFFFF); - UInt32 dest; - src |= ((UInt32)(instNorm >> 36) & 1) << 20; - - src <<= 4; - - if (encoding) - dest = ip + (UInt32)i + src; - else - dest = src - (ip + (UInt32)i); - - dest >>= 4; - - instNorm &= ~((UInt64)(0x8FFFFF) << 13); - instNorm |= ((UInt64)(dest & 0xFFFFF) << 13); - instNorm |= ((UInt64)(dest & 0x100000) << (36 - 20)); - - instruction &= (1 << bitRes) - 1; - instruction |= (instNorm << bitRes); - for (j = 0; j < 6; j++) - data[i + j + bytePos] = (Byte)(instruction >> (8 * j)); + Byte *p = data + (i + (size_t)m * 5 - 8); + if (((p[3] >> m) & 15) == 5 + && (((p[-1] | ((UInt32)p[0] << 8)) >> m) & 0x70) == 0) + { + unsigned raw = GetUi32(p); + unsigned v = raw >> m; + v = (v & 0xFFFFF) | ((v & (1 << 23)) >> 3); + + v <<= 4; + if (encoding) + v += ip + (UInt32)i; + else + v -= ip + (UInt32)i; + v >>= 4; + + v &= 0x1FFFFF; + v += 0x700000; + v &= 0x8FFFFF; + raw &= ~((UInt32)0x8FFFFF << m); + raw |= (v << m); + SetUi32(p, raw); + } } + while (++m <= 4); } + i += 16; } + while (i <= size); return i; }
M
src/third-party/lzma/Compiler.h
→
src/third-party/lzma/Compiler.h
@@ -1,5 +1,5 @@
/* Compiler.h -2015-08-02 : Igor Pavlov : Public domain */ +2017-04-03 : Igor Pavlov : Public domain */ #ifndef __7Z_COMPILER_H #define __7Z_COMPILER_H@@ -21,6 +21,7 @@ #pragma warning(disable : 4512) // assignment operator could not be generated
#pragma warning(disable : 4514) // unreferenced inline function has been removed #pragma warning(disable : 4702) // unreachable code #pragma warning(disable : 4710) // not inlined + #pragma warning(disable : 4714) // function marked as __forceinline not inlined #pragma warning(disable : 4786) // identifier was truncated to '255' characters in the debug information #endif
M
src/third-party/lzma/CpuArch.c
→
src/third-party/lzma/CpuArch.c
@@ -1,5 +1,5 @@
/* CpuArch.c -- CPU specific code -2015-03-25: Igor Pavlov : Public domain */ +2018-02-18: Igor Pavlov : Public domain */ #include "Precomp.h"@@ -45,7 +45,8 @@ "xorl %%EDX,%%EAX\n\t"
"push %%EDX\n\t" "popf\n\t" "andl %%EAX, %0\n\t": - "=c" (flag) : "c" (flag)); + "=c" (flag) : "c" (flag) : + "%eax", "%edx"); #endif return flag; }@@ -79,7 +80,13 @@
#else __asm__ __volatile__ ( - #if defined(MY_CPU_X86) && defined(__PIC__) + #if defined(MY_CPU_AMD64) && defined(__PIC__) + "mov %%rbx, %%rdi;" + "cpuid;" + "xchg %%rbx, %%rdi;" + : "=a" (*a) , + "=D" (*b) , + #elif defined(MY_CPU_X86) && defined(__PIC__) "mov %%ebx, %%edi;" "cpuid;" "xchgl %%ebx, %%edi;"@@ -108,7 +115,7 @@
#endif } -Bool x86cpuid_CheckAndRead(Cx86cpuid *p) +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p) { CHECK_CPUID_IS_SUPPORTED MyCPUID(0, &p->maxFunc, &p->vendor[0], &p->vendor[2], &p->vendor[1]);@@ -137,7 +144,7 @@ }
return -1; } -Bool CPU_Is_InOrder() +BoolInt CPU_Is_InOrder() { Cx86cpuid p; int firm;@@ -168,7 +175,7 @@ }
#if !defined(MY_CPU_AMD64) && defined(_WIN32) #include <windows.h> -static Bool CPU_Sys_Is_SSE_Supported() +static BoolInt CPU_Sys_Is_SSE_Supported() { OSVERSIONINFO vi; vi.dwOSVersionInfoSize = sizeof(vi);@@ -181,13 +188,31 @@ #else
#define CHECK_SYS_SSE_SUPPORT #endif -Bool CPU_Is_Aes_Supported() +BoolInt CPU_Is_Aes_Supported() { Cx86cpuid p; CHECK_SYS_SSE_SUPPORT if (!x86cpuid_CheckAndRead(&p)) return False; return (p.c >> 25) & 1; +} + +BoolInt CPU_IsSupported_PageGB() +{ + Cx86cpuid cpuid; + if (!x86cpuid_CheckAndRead(&cpuid)) + return False; + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000000, &d[0], &d[1], &d[2], &d[3]); + if (d[0] < 0x80000001) + return False; + } + { + UInt32 d[4] = { 0 }; + MyCPUID(0x80000001, &d[0], &d[1], &d[2], &d[3]); + return (d[3] >> 26) & 1; + } } #endif
M
src/third-party/lzma/CpuArch.h
→
src/third-party/lzma/CpuArch.h
@@ -1,5 +1,5 @@
/* CpuArch.h -- CPU specific code -2015-12-01: Igor Pavlov : Public domain */ +2018-02-18 : Igor Pavlov : Public domain */ #ifndef __CPU_ARCH_H #define __CPU_ARCH_H@@ -16,48 +16,122 @@
MY_CPU_LE_UNALIGN means that CPU is LITTLE ENDIAN and CPU supports unaligned memory accesses. */ -#if defined(_M_X64) \ - || defined(_M_AMD64) \ - || defined(__x86_64__) \ - || defined(__AMD64__) \ - || defined(__amd64__) +#if defined(_M_X64) \ + || defined(_M_AMD64) \ + || defined(__x86_64__) \ + || defined(__AMD64__) \ + || defined(__amd64__) #define MY_CPU_AMD64 + #ifdef __ILP32__ + #define MY_CPU_NAME "x32" + #else + #define MY_CPU_NAME "x64" + #endif + #define MY_CPU_64BIT #endif -#if defined(MY_CPU_AMD64) \ - || defined(_M_IA64) \ - || defined(__AARCH64EL__) \ - || defined(__AARCH64EB__) + +#if defined(_M_IX86) \ + || defined(__i386__) + #define MY_CPU_X86 + #define MY_CPU_NAME "x86" + #define MY_CPU_32BIT +#endif + + +#if defined(_M_ARM64) \ + || defined(__AARCH64EL__) \ + || defined(__AARCH64EB__) \ + || defined(__aarch64__) + #define MY_CPU_ARM64 + #define MY_CPU_NAME "arm64" #define MY_CPU_64BIT #endif -#if defined(_M_IX86) || defined(__i386__) -#define MY_CPU_X86 + +#if defined(_M_ARM) \ + || defined(_M_ARM_NT) \ + || defined(_M_ARMT) \ + || defined(__arm__) \ + || defined(__thumb__) \ + || defined(__ARMEL__) \ + || defined(__ARMEB__) \ + || defined(__THUMBEL__) \ + || defined(__THUMBEB__) + #define MY_CPU_ARM + #define MY_CPU_NAME "arm" + #define MY_CPU_32BIT #endif -#if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) -#define MY_CPU_X86_OR_AMD64 + +#if defined(_M_IA64) \ + || defined(__ia64__) + #define MY_CPU_IA64 + #define MY_CPU_NAME "ia64" + #define MY_CPU_64BIT +#endif + + +#if defined(__mips64) \ + || defined(__mips64__) \ + || (defined(__mips) && (__mips == 64 || __mips == 4 || __mips == 3)) + #define MY_CPU_NAME "mips64" + #define MY_CPU_64BIT +#elif defined(__mips__) + #define MY_CPU_NAME "mips" + /* #define MY_CPU_32BIT */ #endif -#if defined(MY_CPU_X86) \ - || defined(_M_ARM) \ - || defined(__ARMEL__) \ - || defined(__THUMBEL__) \ - || defined(__ARMEB__) \ - || defined(__THUMBEB__) + +#if defined(__ppc64__) \ + || defined(__powerpc64__) + #ifdef __ILP32__ + #define MY_CPU_NAME "ppc64-32" + #else + #define MY_CPU_NAME "ppc64" + #endif + #define MY_CPU_64BIT +#elif defined(__ppc__) \ + || defined(__powerpc__) + #define MY_CPU_NAME "ppc" #define MY_CPU_32BIT #endif -#if defined(_WIN32) && defined(_M_ARM) -#define MY_CPU_ARM_LE + +#if defined(__sparc64__) + #define MY_CPU_NAME "sparc64" + #define MY_CPU_64BIT +#elif defined(__sparc__) + #define MY_CPU_NAME "sparc" + /* #define MY_CPU_32BIT */ #endif -#if defined(_WIN32) && defined(_M_IA64) -#define MY_CPU_IA64_LE + +#if defined(MY_CPU_X86) || defined(MY_CPU_AMD64) +#define MY_CPU_X86_OR_AMD64 #endif + +#ifdef _WIN32 + + #ifdef MY_CPU_ARM + #define MY_CPU_ARM_LE + #endif + + #ifdef MY_CPU_ARM64 + #define MY_CPU_ARM64_LE + #endif + + #ifdef _M_IA64 + #define MY_CPU_IA64_LE + #endif + +#endif + + #if defined(MY_CPU_X86_OR_AMD64) \ || defined(MY_CPU_ARM_LE) \ + || defined(MY_CPU_ARM64_LE) \ || defined(MY_CPU_IA64_LE) \ || defined(__LITTLE_ENDIAN__) \ || defined(__ARMEL__) \@@ -66,6 +140,7 @@ || defined(__AARCH64EL__) \
|| defined(__MIPSEL__) \ || defined(__MIPSEL) \ || defined(_MIPSEL) \ + || defined(__BFIN__) \ || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) #define MY_CPU_LE #endif@@ -85,14 +160,37 @@ || (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__))
#define MY_CPU_BE #endif + #if defined(MY_CPU_LE) && defined(MY_CPU_BE) -Stop_Compiling_Bad_Endian + #error Stop_Compiling_Bad_Endian +#endif + + +#if defined(MY_CPU_32BIT) && defined(MY_CPU_64BIT) + #error Stop_Compiling_Bad_32_64_BIT +#endif + + +#ifndef MY_CPU_NAME + #ifdef MY_CPU_LE + #define MY_CPU_NAME "LE" + #elif defined(MY_CPU_BE) + #define MY_CPU_NAME "BE" + #else + /* + #define MY_CPU_NAME "" + */ + #endif #endif + + + #ifdef MY_CPU_LE #if defined(MY_CPU_X86_OR_AMD64) \ - /* || defined(__AARCH64EL__) */ + || defined(MY_CPU_ARM64) \ + || defined(__ARM_FEATURE_UNALIGNED) #define MY_CPU_LE_UNALIGN #endif #endif@@ -138,6 +236,11 @@ SetUi32(_ppp2_ + 4, (UInt32)(_vvv2_ >> 32)); }
#endif +#ifdef __has_builtin + #define MY__has_builtin(x) __has_builtin(x) +#else + #define MY__has_builtin(x) 0 +#endif #if defined(MY_CPU_LE_UNALIGN) && /* defined(_WIN64) && */ (_MSC_VER >= 1300)@@ -145,15 +248,21 @@ /* Note: we use bswap instruction, that is unsupported in 386 cpu */
#include <stdlib.h> +#pragma intrinsic(_byteswap_ushort) #pragma intrinsic(_byteswap_ulong) #pragma intrinsic(_byteswap_uint64) + +/* #define GetBe16(p) _byteswap_ushort(*(const UInt16 *)(const Byte *)(p)) */ #define GetBe32(p) _byteswap_ulong(*(const UInt32 *)(const Byte *)(p)) #define GetBe64(p) _byteswap_uint64(*(const UInt64 *)(const Byte *)(p)) #define SetBe32(p, v) (*(UInt32 *)(void *)(p)) = _byteswap_ulong(v) -#elif defined(MY_CPU_LE_UNALIGN) && defined (__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +#elif defined(MY_CPU_LE_UNALIGN) && ( \ + (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3))) \ + || (defined(__clang__) && MY__has_builtin(__builtin_bswap16)) ) +/* #define GetBe16(p) __builtin_bswap16(*(const UInt16 *)(const Byte *)(p)) */ #define GetBe32(p) __builtin_bswap32(*(const UInt32 *)(const Byte *)(p)) #define GetBe64(p) __builtin_bswap64(*(const UInt64 *)(const Byte *)(p))@@ -178,9 +287,13 @@
#endif +#ifndef GetBe16 + #define GetBe16(p) ( (UInt16) ( \ ((UInt16)((const Byte *)(p))[0] << 8) | \ ((const Byte *)(p))[1] )) + +#endif@@ -205,15 +318,16 @@ };
void MyCPUID(UInt32 function, UInt32 *a, UInt32 *b, UInt32 *c, UInt32 *d); -Bool x86cpuid_CheckAndRead(Cx86cpuid *p); +BoolInt x86cpuid_CheckAndRead(Cx86cpuid *p); int x86cpuid_GetFirm(const Cx86cpuid *p); #define x86cpuid_GetFamily(ver) (((ver >> 16) & 0xFF0) | ((ver >> 8) & 0xF)) #define x86cpuid_GetModel(ver) (((ver >> 12) & 0xF0) | ((ver >> 4) & 0xF)) #define x86cpuid_GetStepping(ver) (ver & 0xF) -Bool CPU_Is_InOrder(); -Bool CPU_Is_Aes_Supported(); +BoolInt CPU_Is_InOrder(); +BoolInt CPU_Is_Aes_Supported(); +BoolInt CPU_IsSupported_PageGB(); #endif
A
src/third-party/lzma/DllSecur.c
@@ -0,0 +1,108 @@
+/* DllSecur.c -- DLL loading security +2018-02-21 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +#ifdef _WIN32 + +#include <windows.h> + +#include "DllSecur.h" + +#ifndef UNDER_CE + +typedef BOOL (WINAPI *Func_SetDefaultDllDirectories)(DWORD DirectoryFlags); + +#define MY_LOAD_LIBRARY_SEARCH_USER_DIRS 0x400 +#define MY_LOAD_LIBRARY_SEARCH_SYSTEM32 0x800 + +static const char * const g_Dlls = + #ifndef _CONSOLE + "UXTHEME\0" + #endif + "USERENV\0" + "SETUPAPI\0" + "APPHELP\0" + "PROPSYS\0" + "DWMAPI\0" + "CRYPTBASE\0" + "OLEACC\0" + "CLBCATQ\0" + "VERSION\0" + ; + +#endif + +void My_SetDefaultDllDirectories() +{ + #ifndef UNDER_CE + + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + GetVersionEx(&vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + + #endif +} + + +void LoadSecurityDlls() +{ + #ifndef UNDER_CE + + wchar_t buf[MAX_PATH + 100]; + + { + // at Vista (ver 6.0) : CoCreateInstance(CLSID_ShellLink, ...) doesn't work after SetDefaultDllDirectories() : Check it ??? + OSVERSIONINFO vi; + vi.dwOSVersionInfoSize = sizeof(vi); + if (!GetVersionEx(&vi) || vi.dwMajorVersion != 6 || vi.dwMinorVersion != 0) + { + Func_SetDefaultDllDirectories setDllDirs = (Func_SetDefaultDllDirectories) + GetProcAddress(GetModuleHandle(TEXT("kernel32.dll")), "SetDefaultDllDirectories"); + if (setDllDirs) + if (setDllDirs(MY_LOAD_LIBRARY_SEARCH_SYSTEM32 | MY_LOAD_LIBRARY_SEARCH_USER_DIRS)) + return; + } + } + + { + unsigned len = GetSystemDirectoryW(buf, MAX_PATH + 2); + if (len == 0 || len > MAX_PATH) + return; + } + { + const char *dll; + unsigned pos = (unsigned)lstrlenW(buf); + + if (buf[pos - 1] != '\\') + buf[pos++] = '\\'; + + for (dll = g_Dlls; dll[0] != 0;) + { + unsigned k = 0; + for (;;) + { + char c = *dll++; + buf[pos + k] = (Byte)c; + k++; + if (c == 0) + break; + } + + lstrcatW(buf, L".dll"); + LoadLibraryExW(buf, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + } + } + + #endif +} + +#endif
A
src/third-party/lzma/DllSecur.h
@@ -0,0 +1,20 @@
+/* DllSecur.h -- DLL loading for security +2018-02-19 : Igor Pavlov : Public domain */ + +#ifndef __DLL_SECUR_H +#define __DLL_SECUR_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +#ifdef _WIN32 + +void My_SetDefaultDllDirectories(); +void LoadSecurityDlls(); + +#endif + +EXTERN_C_END + +#endif
M
src/third-party/lzma/LzFind.c
→
src/third-party/lzma/LzFind.c
@@ -1,5 +1,5 @@
/* LzFind.c -- Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2018-07-08 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -16,18 +16,18 @@ #define kMaxHistorySize ((UInt32)7 << 29)
#define kStartMaxLen 3 -static void LzInWindow_Free(CMatchFinder *p, ISzAlloc *alloc) +static void LzInWindow_Free(CMatchFinder *p, ISzAllocPtr alloc) { if (!p->directInput) { - alloc->Free(alloc, p->bufferBase); + ISzAlloc_Free(alloc, p->bufferBase); p->bufferBase = NULL; } } /* keepSizeBefore + keepSizeAfter + keepSizeReserv must be < 4G) */ -static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAlloc *alloc) +static int LzInWindow_Create(CMatchFinder *p, UInt32 keepSizeReserv, ISzAllocPtr alloc) { UInt32 blockSize = p->keepSizeBefore + p->keepSizeAfter + keepSizeReserv; if (p->directInput)@@ -39,7 +39,7 @@ if (!p->bufferBase || p->blockSize != blockSize)
{ LzInWindow_Free(p, alloc); p->blockSize = blockSize; - p->bufferBase = (Byte *)alloc->Alloc(alloc, (size_t)blockSize); + p->bufferBase = (Byte *)ISzAlloc_Alloc(alloc, (size_t)blockSize); } return (p->bufferBase != NULL); }@@ -81,7 +81,7 @@ size_t size = (p->bufferBase + p->blockSize - dest);
if (size == 0) return; - p->result = p->stream->Read(p->stream, dest, &size); + p->result = ISeqInStream_Read(p->stream, dest, &size); if (p->result != SZ_OK) return; if (size == 0)@@ -138,45 +138,46 @@ #define kCrcPoly 0xEDB88320
void MatchFinder_Construct(CMatchFinder *p) { - UInt32 i; + unsigned i; p->bufferBase = NULL; p->directInput = 0; p->hash = NULL; + p->expectedDataSize = (UInt64)(Int64)-1; MatchFinder_SetDefaultSettings(p); for (i = 0; i < 256; i++) { - UInt32 r = i; + UInt32 r = (UInt32)i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrcPoly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); p->crc[i] = r; } } -static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAlloc *alloc) +static void MatchFinder_FreeThisClassMemory(CMatchFinder *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->hash); + ISzAlloc_Free(alloc, p->hash); p->hash = NULL; } -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc) +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc) { MatchFinder_FreeThisClassMemory(p, alloc); LzInWindow_Free(p, alloc); } -static CLzRef* AllocRefs(size_t num, ISzAlloc *alloc) +static CLzRef* AllocRefs(size_t num, ISzAllocPtr alloc) { size_t sizeInBytes = (size_t)num * sizeof(CLzRef); if (sizeInBytes / sizeof(CLzRef) != num) return NULL; - return (CLzRef *)alloc->Alloc(alloc, sizeInBytes); + return (CLzRef *)ISzAlloc_Alloc(alloc, sizeInBytes); } int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc) + ISzAllocPtr alloc) { UInt32 sizeReserv;@@ -208,7 +209,11 @@ if (p->numHashBytes == 2)
hs = (1 << 16) - 1; else { - hs = historySize - 1; + hs = historySize; + if (hs > p->expectedDataSize) + hs = (UInt32)p->expectedDataSize; + if (hs != 0) + hs--; hs |= (hs >> 1); hs |= (hs >> 2); hs |= (hs >> 4);@@ -292,17 +297,33 @@ }
p->posLimit = p->pos + limit; } -void MatchFinder_Init_2(CMatchFinder *p, int readData) + +void MatchFinder_Init_LowHash(CMatchFinder *p) +{ + size_t i; + CLzRef *items = p->hash; + size_t numItems = p->fixedHashSize; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_HighHash(CMatchFinder *p) { - UInt32 i; - UInt32 *hash = p->hash; - UInt32 num = p->hashSizeSum; - for (i = 0; i < num; i++) - hash[i] = kEmptyHashValue; - + size_t i; + CLzRef *items = p->hash + p->fixedHashSize; + size_t numItems = (size_t)p->hashMask + 1; + for (i = 0; i < numItems; i++) + items[i] = kEmptyHashValue; +} + + +void MatchFinder_Init_3(CMatchFinder *p, int readData) +{ p->cyclicBufferPos = 0; p->buffer = p->bufferBase; - p->pos = p->streamPos = p->cyclicBufferSize; + p->pos = + p->streamPos = p->cyclicBufferSize; p->result = SZ_OK; p->streamEndWasReached = 0;@@ -312,10 +333,14 @@
MatchFinder_SetLimits(p); } + void MatchFinder_Init(CMatchFinder *p) { - MatchFinder_Init_2(p, True); + MatchFinder_Init_HighHash(p); + MatchFinder_Init_LowHash(p); + MatchFinder_Init_3(p, True); } + static UInt32 MatchFinder_GetSubValue(CMatchFinder *p) {@@ -343,6 +368,8 @@ MatchFinder_Normalize3(subValue, p->hash, p->numRefs);
MatchFinder_ReduceOffsets(p, subValue); } + +MY_NO_INLINE static void MatchFinder_CheckLimits(CMatchFinder *p) { if (p->pos == kMaxValForNormalize)@@ -354,10 +381,16 @@ p->cyclicBufferPos = 0;
MatchFinder_SetLimits(p); } -static UInt32 * Hc_GetMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, + +/* + (lenLimit > maxLen) +*/ +MY_FORCE_INLINE +static UInt32 * Hc_GetMatchesSpec(unsigned lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, - UInt32 *distances, UInt32 maxLen) + UInt32 *distances, unsigned maxLen) { + /* son[_cyclicBufferPos] = curMatch; for (;;) {@@ -375,7 +408,8 @@ if (pb[len] != cur[len])
break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = len; *distances++ = delta - 1; if (len == lenLimit) return distances;@@ -383,15 +417,58 @@ }
} } } + */ + + const Byte *lim = cur + lenLimit; + son[_cyclicBufferPos] = curMatch; + do + { + UInt32 delta = pos - curMatch; + if (delta >= _cyclicBufferSize) + break; + { + ptrdiff_t diff; + curMatch = son[_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)]; + diff = (ptrdiff_t)0 - delta; + if (cur[maxLen] == cur[maxLen + diff]) + { + const Byte *c = cur; + while (*c == c[diff]) + { + if (++c == lim) + { + distances[0] = (UInt32)(lim - cur); + distances[1] = delta - 1; + return distances + 2; + } + } + { + unsigned len = (unsigned)(c - cur); + if (maxLen < len) + { + maxLen = len; + distances[0] = (UInt32)len; + distances[1] = delta - 1; + distances += 2; + } + } + } + } + } + while (--cutValue); + + return distances; } + +MY_FORCE_INLINE UInt32 * GetMatchesSpec1(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue, UInt32 *distances, UInt32 maxLen) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch;@@ -401,9 +478,10 @@ *ptr0 = *ptr1 = kEmptyHashValue;
return distances; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = pair[0]; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len])@@ -412,11 +490,12 @@ if (pb[len] != cur[len])
break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = (UInt32)len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; + *ptr1 = pair0; *ptr0 = pair[1]; return distances; }@@ -443,9 +522,9 @@
static void SkipMatchesSpec(UInt32 lenLimit, UInt32 curMatch, UInt32 pos, const Byte *cur, CLzRef *son, UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 cutValue) { - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; for (;;) { UInt32 delta = pos - curMatch;@@ -455,9 +534,9 @@ *ptr0 = *ptr1 = kEmptyHashValue;
return; } { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); if (pb[len] == cur[len]) { while (++len != lenLimit)@@ -495,13 +574,13 @@ ++p->cyclicBufferPos; \
p->buffer++; \ if (++p->pos == p->posLimit) MatchFinder_CheckLimits(p); -#define MOVE_POS_RET MOVE_POS return offset; +#define MOVE_POS_RET MOVE_POS return (UInt32)offset; static void MatchFinder_MovePos(CMatchFinder *p) { MOVE_POS; } #define GET_MATCHES_HEADER2(minLen, ret_op) \ - UInt32 lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ - lenLimit = p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ + unsigned lenLimit; UInt32 hv; const Byte *cur; UInt32 curMatch; \ + lenLimit = (unsigned)p->lenLimit; { if (lenLimit < minLen) { MatchFinder_MovePos(p); ret_op; }} \ cur = p->buffer; #define GET_MATCHES_HEADER(minLen) GET_MATCHES_HEADER2(minLen, return 0)@@ -510,22 +589,22 @@
#define MF_PARAMS(p) p->pos, p->buffer, p->son, p->cyclicBufferPos, p->cyclicBufferSize, p->cutValue #define GET_MATCHES_FOOTER(offset, maxLen) \ - offset = (UInt32)(GetMatchesSpec1(lenLimit, curMatch, MF_PARAMS(p), \ - distances + offset, maxLen) - distances); MOVE_POS_RET; + offset = (unsigned)(GetMatchesSpec1((UInt32)lenLimit, curMatch, MF_PARAMS(p), \ + distances + offset, (UInt32)maxLen) - distances); MOVE_POS_RET; #define SKIP_FOOTER \ - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS; #define UPDATE_maxLen { \ ptrdiff_t diff = (ptrdiff_t)0 - d2; \ const Byte *c = cur + maxLen; \ const Byte *lim = cur + lenLimit; \ for (; c != lim; c++) if (*(c + diff) != *c) break; \ - maxLen = (UInt32)(c - cur); } + maxLen = (unsigned)(c - cur); } static UInt32 Bt2_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(2) HASH2_CALC; curMatch = p->hash[hv];@@ -536,7 +615,7 @@ }
UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv];@@ -547,7 +626,8 @@ }
static UInt32 Bt3_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, d2, maxLen, offset, pos; + UInt32 h2, d2, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(3)@@ -558,10 +638,10 @@ pos = p->pos;
d2 = pos - hash[h2]; - curMatch = hash[kFix3HashSize + hv]; + curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = pos; - hash[kFix3HashSize + hv] = pos; + (hash + kFix3HashSize)[hv] = pos; maxLen = 2; offset = 0;@@ -569,12 +649,12 @@
if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { UPDATE_maxLen - distances[0] = maxLen; + distances[0] = (UInt32)maxLen; distances[1] = d2 - 1; offset = 2; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } }@@ -584,7 +664,8 @@ }
static UInt32 Bt4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4)@@ -593,21 +674,22 @@
hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; - curMatch = hash[kFix4HashSize + hv]; + curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; maxLen = 0; offset = 0; if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; }@@ -615,7 +697,7 @@
if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) { maxLen = 3; - distances[offset + 1] = d3 - 1; + distances[(size_t)offset + 1] = d3 - 1; offset += 2; d2 = d3; }@@ -623,10 +705,10 @@
if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { - SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p)); + SkipMatchesSpec((UInt32)lenLimit, curMatch, MF_PARAMS(p)); MOVE_POS_RET; } }@@ -649,16 +731,16 @@
hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - d4 = pos - hash[kFix4HashSize + h4]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + d4 = pos - (hash + kFix4HashSize)[h4]; - curMatch = hash[kFix5HashSize + hv]; + curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + h4] = pos; - hash[kFix5HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; maxLen = 0; offset = 0;@@ -691,7 +773,7 @@ && *(cur - d4) == *cur
&& *(cur - d4 + 3) == *(cur + 3)) { maxLen = 4; - distances[offset + 1] = d4 - 1; + distances[(size_t)offset + 1] = d4 - 1; offset += 2; d2 = d4; }@@ -699,7 +781,7 @@
if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = maxLen; if (maxLen == lenLimit) { SkipMatchesSpec(lenLimit, curMatch, MF_PARAMS(p));@@ -716,7 +798,8 @@ */
static UInt32 Hc4_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 h2, h3, d2, d3, maxLen, offset, pos; + UInt32 h2, h3, d2, d3, pos; + unsigned maxLen, offset; UInt32 *hash; GET_MATCHES_HEADER(4)@@ -725,21 +808,21 @@
hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - - curMatch = hash[kFix4HashSize + hv]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + curMatch = (hash + kFix4HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[hv] = pos; maxLen = 0; offset = 0; if (d2 < p->cyclicBufferSize && *(cur - d2) == *cur) { - distances[0] = maxLen = 2; + maxLen = 2; + distances[0] = 2; distances[1] = d2 - 1; offset = 2; }@@ -747,7 +830,7 @@
if (d2 != d3 && d3 < p->cyclicBufferSize && *(cur - d3) == *cur) { maxLen = 3; - distances[offset + 1] = d3 - 1; + distances[(size_t)offset + 1] = d3 - 1; offset += 2; d2 = d3; }@@ -755,7 +838,7 @@
if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = (UInt32)maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch;@@ -766,7 +849,7 @@
if (maxLen < 3) maxLen = 3; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances + offset, maxLen) - (distances)); MOVE_POS_RET }@@ -783,16 +866,16 @@
hash = p->hash; pos = p->pos; - d2 = pos - hash[ h2]; - d3 = pos - hash[kFix3HashSize + h3]; - d4 = pos - hash[kFix4HashSize + h4]; + d2 = pos - hash [h2]; + d3 = pos - (hash + kFix3HashSize)[h3]; + d4 = pos - (hash + kFix4HashSize)[h4]; - curMatch = hash[kFix5HashSize + hv]; + curMatch = (hash + kFix5HashSize)[hv]; - hash[ h2] = pos; - hash[kFix3HashSize + h3] = pos; - hash[kFix4HashSize + h4] = pos; - hash[kFix5HashSize + hv] = pos; + hash [h2] = pos; + (hash + kFix3HashSize)[h3] = pos; + (hash + kFix4HashSize)[h4] = pos; + (hash + kFix5HashSize)[hv] = pos; maxLen = 0; offset = 0;@@ -825,7 +908,7 @@ && *(cur - d4) == *cur
&& *(cur - d4 + 3) == *(cur + 3)) { maxLen = 4; - distances[offset + 1] = d4 - 1; + distances[(size_t)offset + 1] = d4 - 1; offset += 2; d2 = d4; }@@ -833,7 +916,7 @@
if (offset != 0) { UPDATE_maxLen - distances[offset - 2] = maxLen; + distances[(size_t)offset - 2] = maxLen; if (maxLen == lenLimit) { p->son[p->cyclicBufferPos] = curMatch;@@ -852,12 +935,12 @@ */
UInt32 Hc3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances) { - UInt32 offset; + unsigned offset; GET_MATCHES_HEADER(3) HASH_ZIP_CALC; curMatch = p->hash[hv]; p->hash[hv] = p->pos; - offset = (UInt32)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), + offset = (unsigned)(Hc_GetMatchesSpec(lenLimit, curMatch, MF_PARAMS(p), distances, 2) - (distances)); MOVE_POS_RET }@@ -897,9 +980,9 @@ UInt32 *hash;
SKIP_HEADER(3) HASH3_CALC; hash = p->hash; - curMatch = hash[kFix3HashSize + hv]; + curMatch = (hash + kFix3HashSize)[hv]; hash[h2] = - hash[kFix3HashSize + hv] = p->pos; + (hash + kFix3HashSize)[hv] = p->pos; SKIP_FOOTER } while (--num != 0);@@ -914,10 +997,10 @@ UInt32 *hash;
SKIP_HEADER(4) HASH4_CALC; hash = p->hash; - curMatch = hash[kFix4HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + hv] = p->pos; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; SKIP_FOOTER } while (--num != 0);@@ -933,11 +1016,11 @@ UInt32 *hash;
SKIP_HEADER(5) HASH5_CALC; hash = p->hash; - curMatch = hash[kFix5HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + h4] = - hash[kFix5HashSize + hv] = p->pos; + curMatch = (hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; SKIP_FOOTER } while (--num != 0);@@ -953,10 +1036,10 @@ UInt32 *hash;
SKIP_HEADER(4) HASH4_CALC; hash = p->hash; - curMatch = hash[kFix4HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + hv] = p->pos; + curMatch = (hash + kFix4HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[hv] = p->pos; p->son[p->cyclicBufferPos] = curMatch; MOVE_POS }@@ -973,11 +1056,11 @@ UInt32 *hash;
SKIP_HEADER(5) HASH5_CALC; hash = p->hash; - curMatch = p->hash[kFix5HashSize + hv]; - hash[ h2] = - hash[kFix3HashSize + h3] = - hash[kFix4HashSize + h4] = - hash[kFix5HashSize + hv] = p->pos; + curMatch = hash + kFix5HashSize)[hv]; + hash [h2] = + (hash + kFix3HashSize)[h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix5HashSize)[hv] = p->pos; p->son[p->cyclicBufferPos] = curMatch; MOVE_POS }
M
src/third-party/lzma/LzFind.h
→
src/third-party/lzma/LzFind.h
@@ -1,5 +1,5 @@
/* LzFind.h -- Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2017-06-10 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_H #define __LZ_FIND_H@@ -47,6 +47,8 @@ UInt32 hashSizeSum;
SRes result; UInt32 crc[256]; size_t numRefs; + + UInt64 expectedDataSize; } CMatchFinder; #define Inline_MatchFinder_GetPointerToCurrentPos(p) ((p)->buffer)@@ -71,8 +73,8 @@ keepAddBufferBefore + matchMaxLen + keepAddBufferAfter < 511MB
*/ int MatchFinder_Create(CMatchFinder *p, UInt32 historySize, UInt32 keepAddBufferBefore, UInt32 matchMaxLen, UInt32 keepAddBufferAfter, - ISzAlloc *alloc); -void MatchFinder_Free(CMatchFinder *p, ISzAlloc *alloc); + ISzAllocPtr alloc); +void MatchFinder_Free(CMatchFinder *p, ISzAllocPtr alloc); void MatchFinder_Normalize3(UInt32 subValue, CLzRef *items, size_t numItems); void MatchFinder_ReduceOffsets(CMatchFinder *p, UInt32 subValue);@@ -103,7 +105,9 @@ } IMatchFinder;
void MatchFinder_CreateVTable(CMatchFinder *p, IMatchFinder *vTable); -void MatchFinder_Init_2(CMatchFinder *p, int readData); +void MatchFinder_Init_LowHash(CMatchFinder *p); +void MatchFinder_Init_HighHash(CMatchFinder *p); +void MatchFinder_Init_3(CMatchFinder *p, int readData); void MatchFinder_Init(CMatchFinder *p); UInt32 Bt3Zip_MatchFinder_GetMatches(CMatchFinder *p, UInt32 *distances);
M
src/third-party/lzma/LzFindMt.c
→
src/third-party/lzma/LzFindMt.c
@@ -1,5 +1,5 @@
/* LzFindMt.c -- multithreaded Match finder for LZ algorithms -2015-10-15 : Igor Pavlov : Public domain */ +2018-12-29 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -33,6 +33,8 @@ Event_Reset(&p->wasStopped);
Event_Set(&p->canStart); Event_Wait(&p->wasStarted); + + // if (mt) MatchFinder_Init_LowHash(mt->MatchFinder); } else {@@ -155,6 +157,9 @@ {
UInt32 numProcessedBlocks = 0; Event_Wait(&p->canStart); Event_Set(&p->wasStarted); + + MatchFinder_Init_HighHash(mt->MatchFinder); + for (;;) { if (p->exit)@@ -205,7 +210,7 @@ num = num - mf->numHashBytes + 1;
if (num > kMtHashBlockSize - 2) num = kMtHashBlockSize - 2; mt->GetHeadsFunc(mf->buffer, mf->pos, mf->hash + mf->fixedHashSize, mf->hashMask, heads + 2, num, mf->crc); - heads[0] += num; + heads[0] = 2 + num; } mf->pos += num; mf->buffer += num;@@ -227,38 +232,57 @@ }
#define kEmptyHashValue 0 -/* #define MFMT_GM_INLINE */ +#define MFMT_GM_INLINE #ifdef MFMT_GM_INLINE -#define NO_INLINE MY_FAST_CALL +/* + we use size_t for _cyclicBufferPos instead of UInt32 + to eliminate "movsx" BUG in old MSVC x64 compiler. +*/ -static Int32 NO_INLINE GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, - UInt32 _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, - UInt32 *_distances, UInt32 _maxLen, const UInt32 *hash, Int32 limit, UInt32 size, UInt32 *posRes) +MY_NO_INLINE +static UInt32 *GetMatchesSpecN(UInt32 lenLimit, UInt32 pos, const Byte *cur, CLzRef *son, + size_t _cyclicBufferPos, UInt32 _cyclicBufferSize, UInt32 _cutValue, + UInt32 *distances, UInt32 _maxLen, const UInt32 *hash, const UInt32 *limit, UInt32 size, UInt32 *posRes) { do { - UInt32 *distances = _distances + 1; - UInt32 curMatch = pos - *hash++; + UInt32 *_distances = ++distances; + UInt32 delta = *hash++; - CLzRef *ptr0 = son + (_cyclicBufferPos << 1) + 1; - CLzRef *ptr1 = son + (_cyclicBufferPos << 1); - UInt32 len0 = 0, len1 = 0; + CLzRef *ptr0 = son + ((size_t)_cyclicBufferPos << 1) + 1; + CLzRef *ptr1 = son + ((size_t)_cyclicBufferPos << 1); + unsigned len0 = 0, len1 = 0; UInt32 cutValue = _cutValue; - UInt32 maxLen = _maxLen; - for (;;) + unsigned maxLen = (unsigned)_maxLen; + + /* + if (size > 1) { - UInt32 delta = pos - curMatch; - if (cutValue-- == 0 || delta >= _cyclicBufferSize) + UInt32 delta = *hash; + if (delta < _cyclicBufferSize) { - *ptr0 = *ptr1 = kEmptyHashValue; - break; + UInt32 cyc1 = _cyclicBufferPos + 1; + CLzRef *pair = son + ((size_t)(cyc1 - delta + ((delta > cyc1) ? _cyclicBufferSize : 0)) << 1); + Byte b = *(cur + 1 - delta); + _distances[0] = pair[0]; + _distances[1] = b; } + } + */ + if (cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + } + else + for(;;) + { { - CLzRef *pair = son + ((_cyclicBufferPos - delta + ((delta > _cyclicBufferPos) ? _cyclicBufferSize : 0)) << 1); + CLzRef *pair = son + ((size_t)(_cyclicBufferPos - delta + ((_cyclicBufferPos < delta) ? _cyclicBufferSize : 0)) << 1); const Byte *pb = cur - delta; - UInt32 len = (len0 < len1 ? len0 : len1); + unsigned len = (len0 < len1 ? len0 : len1); + UInt32 pair0 = *pair; if (pb[len] == cur[len]) { if (++len != lenLimit && pb[len] == cur[len])@@ -267,54 +291,66 @@ if (pb[len] != cur[len])
break; if (maxLen < len) { - *distances++ = maxLen = len; + maxLen = len; + *distances++ = (UInt32)len; *distances++ = delta - 1; if (len == lenLimit) { - *ptr1 = pair[0]; - *ptr0 = pair[1]; + UInt32 pair1 = pair[1]; + *ptr1 = pair0; + *ptr0 = pair1; break; } } } - if (pb[len] < cur[len]) - { - *ptr1 = curMatch; - ptr1 = pair + 1; - curMatch = *ptr1; - len1 = len; - } - else { - *ptr0 = curMatch; - ptr0 = pair; - curMatch = *ptr0; - len0 = len; + UInt32 curMatch = pos - delta; + // delta = pos - *pair; + // delta = pos - pair[((UInt32)pb[len] - (UInt32)cur[len]) >> 31]; + if (pb[len] < cur[len]) + { + delta = pos - pair[1]; + *ptr1 = curMatch; + ptr1 = pair + 1; + len1 = len; + } + else + { + delta = pos - *pair; + *ptr0 = curMatch; + ptr0 = pair; + len0 = len; + } } } + if (--cutValue == 0 || delta >= _cyclicBufferSize) + { + *ptr0 = *ptr1 = kEmptyHashValue; + break; + } } pos++; _cyclicBufferPos++; cur++; { UInt32 num = (UInt32)(distances - _distances); - *_distances = num - 1; - _distances += num; - limit -= num; + _distances[-1] = num; } } - while (limit > 0 && --size != 0); + while (distances < limit && --size != 0); *posRes = pos; - return limit; + return distances; } #endif + + static void BtGetMatches(CMatchFinderMt *p, UInt32 *distances) { UInt32 numProcessed = 0; UInt32 curPos = 2; - UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); + UInt32 limit = kMtBtBlockSize - (p->matchMaxLen * 2); // * 2 distances[1] = p->hashNumAvail;@@ -364,8 +400,10 @@ }
#else { UInt32 posRes; - curPos = limit - GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, - distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, (Int32)(limit - curPos), size, &posRes); + curPos = (UInt32)(GetMatchesSpecN(lenLimit, pos, p->buffer, p->son, cyclicBufferPos, p->cyclicBufferSize, p->cutValue, + distances + curPos, p->numHashBytes - 1, p->hashBuf + p->hashBufPos, + distances + limit, + size, &posRes) - distances); p->hashBufPos += posRes - pos; cyclicBufferPos += posRes - pos; p->buffer += posRes - pos;@@ -443,13 +481,13 @@ MtSync_Construct(&p->hashSync);
MtSync_Construct(&p->btSync); } -static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAlloc *alloc) +static void MatchFinderMt_FreeMem(CMatchFinderMt *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->hashBuf); + ISzAlloc_Free(alloc, p->hashBuf); p->hashBuf = NULL; } -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc) +void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc) { MtSync_Destruct(&p->hashSync); MtSync_Destruct(&p->btSync);@@ -472,7 +510,7 @@ return 0;
} SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc) + UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc) { CMatchFinder *mf = p->MatchFinder; p->historySize = historySize;@@ -480,7 +518,7 @@ if (kMtBtBlockSize <= matchMaxLen * 4)
return SZ_ERROR_PARAM; if (!p->hashBuf) { - p->hashBuf = (UInt32 *)alloc->Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32)); + p->hashBuf = (UInt32 *)ISzAlloc_Alloc(alloc, (kHashBufferSize + kBtBufferSize) * sizeof(UInt32)); if (!p->hashBuf) return SZ_ERROR_MEM; p->btBuf = p->hashBuf + kHashBufferSize;@@ -496,14 +534,18 @@ return SZ_OK;
} /* Call it after ReleaseStream / SetStream */ -void MatchFinderMt_Init(CMatchFinderMt *p) +static void MatchFinderMt_Init(CMatchFinderMt *p) { CMatchFinder *mf = p->MatchFinder; - p->btBufPos = p->btBufPosLimit = 0; - p->hashBufPos = p->hashBufPosLimit = 0; + + p->btBufPos = + p->btBufPosLimit = 0; + p->hashBufPos = + p->hashBufPosLimit = 0; /* Init without data reading. We don't want to read data in this thread */ - MatchFinder_Init_2(mf, False); + MatchFinder_Init_3(mf, False); + MatchFinder_Init_LowHash(mf); p->pointerToCurPos = Inline_MatchFinder_GetPointerToCurrentPos(mf); p->btNumAvailBytes = 0;@@ -591,10 +633,10 @@ UInt32 lzPos = p->lzPos;
MT_HASH3_CALC curMatch2 = hash[ h2]; - curMatch3 = hash[kFix3HashSize + h3]; + curMatch3 = (hash + kFix3HashSize)[h3]; hash[ h2] = lzPos; - hash[kFix3HashSize + h3] = lzPos; + (hash + kFix3HashSize)[h3] = lzPos; if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) {@@ -627,12 +669,12 @@ UInt32 lzPos = p->lzPos;
MT_HASH4_CALC curMatch2 = hash[ h2]; - curMatch3 = hash[kFix3HashSize + h3]; - curMatch4 = hash[kFix4HashSize + h4]; + curMatch3 = (hash + kFix3HashSize)[h3]; + curMatch4 = (hash + kFix4HashSize)[h4]; hash[ h2] = lzPos; - hash[kFix3HashSize + h3] = lzPos; - hash[kFix4HashSize + h4] = lzPos; + (hash + kFix3HashSize)[h3] = lzPos; + (hash + kFix4HashSize)[h4] = lzPos; if (curMatch2 >= matchMinPos && cur[(ptrdiff_t)curMatch2 - lzPos] == cur[0]) {@@ -684,8 +726,12 @@ {
UInt32 i; for (i = 0; i < len; i += 2) { - *distances++ = *btBuf++; - *distances++ = *btBuf++; + UInt32 v0 = btBuf[0]; + UInt32 v1 = btBuf[1]; + btBuf += 2; + distances[0] = v0; + distances[1] = v1; + distances += 2; } } INCREASE_LZ_POS@@ -712,8 +758,12 @@ p->btNumAvailBytes--;
distances2 = p->MixMatchesFunc(p, p->lzPos - btBuf[1], distances); do { - *distances2++ = *btBuf++; - *distances2++ = *btBuf++; + UInt32 v0 = btBuf[0]; + UInt32 v1 = btBuf[1]; + btBuf += 2; + distances2[0] = v0; + distances2[1] = v1; + distances2 += 2; } while ((len -= 2) != 0); len = (UInt32)(distances2 - (distances));@@ -746,7 +796,7 @@ {
SKIP_HEADER_MT(3) UInt32 h2, h3; MT_HASH3_CALC - hash[kFix3HashSize + h3] = + (hash + kFix3HashSize)[h3] = hash[ h2] = p->lzPos; SKIP_FOOTER_MT@@ -758,8 +808,8 @@ {
SKIP_HEADER_MT(4) UInt32 h2, h3, h4; MT_HASH4_CALC - hash[kFix4HashSize + h4] = - hash[kFix3HashSize + h3] = + (hash + kFix4HashSize)[h4] = + (hash + kFix3HashSize)[h3] = hash[ h2] = p->lzPos; SKIP_FOOTER_MT@@ -777,7 +827,7 @@ switch (p->MatchFinder->numHashBytes)
{ case 2: p->GetHeadsFunc = GetHeads2; - p->MixMatchesFunc = (Mf_Mix_Matches)0; + p->MixMatchesFunc = (Mf_Mix_Matches)NULL; vTable->Skip = (Mf_Skip_Func)MatchFinderMt0_Skip; vTable->GetMatches = (Mf_GetMatches_Func)MatchFinderMt2_GetMatches; break;
M
src/third-party/lzma/LzFindMt.h
→
src/third-party/lzma/LzFindMt.h
@@ -1,5 +1,5 @@
/* LzFindMt.h -- multithreaded Match finder for LZ algorithms -2015-05-03 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __LZ_FIND_MT_H #define __LZ_FIND_MT_H@@ -19,10 +19,10 @@ #define kMtBtNumBlocksMask (kMtBtNumBlocks - 1)
typedef struct _CMtSync { - Bool wasCreated; - Bool needStart; - Bool exit; - Bool stopWriting; + BoolInt wasCreated; + BoolInt needStart; + BoolInt exit; + BoolInt stopWriting; CThread thread; CAutoResetEvent canStart;@@ -30,8 +30,8 @@ CAutoResetEvent wasStarted;
CAutoResetEvent wasStopped; CSemaphore freeSemaphore; CSemaphore filledSemaphore; - Bool csWasInitialized; - Bool csWasEntered; + BoolInt csWasInitialized; + BoolInt csWasEntered; CCriticalSection cs; UInt32 numProcessedBlocks; } CMtSync;@@ -90,9 +90,9 @@ CMatchFinder *MatchFinder;
} CMatchFinderMt; void MatchFinderMt_Construct(CMatchFinderMt *p); -void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAlloc *alloc); +void MatchFinderMt_Destruct(CMatchFinderMt *p, ISzAllocPtr alloc); SRes MatchFinderMt_Create(CMatchFinderMt *p, UInt32 historySize, UInt32 keepAddBufferBefore, - UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAlloc *alloc); + UInt32 matchMaxLen, UInt32 keepAddBufferAfter, ISzAllocPtr alloc); void MatchFinderMt_CreateVTable(CMatchFinderMt *p, IMatchFinder *vTable); void MatchFinderMt_ReleaseStream(CMatchFinderMt *p);
M
src/third-party/lzma/Lzma2Dec.c
→
src/third-party/lzma/Lzma2Dec.c
@@ -1,5 +1,5 @@
/* Lzma2Dec.c -- LZMA2 Decoder -2015-11-09 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ /* #define SHOW_DEBUG_INFO */@@ -14,28 +14,22 @@
#include "Lzma2Dec.h" /* -00000000 - EOS -00000001 U U - Uncompressed Reset Dic -00000010 U U - Uncompressed No Reset -100uuuuu U U P P - LZMA no reset -101uuuuu U U P P - LZMA reset state -110uuuuu U U P P S - LZMA reset state + new prop -111uuuuu U U P P S - LZMA reset state + new prop + reset dic +00000000 - End of data +00000001 U U - Uncompressed, reset dic, need reset state and set new prop +00000010 U U - Uncompressed, no reset +100uuuuu U U P P - LZMA, no reset +101uuuuu U U P P - LZMA, reset state +110uuuuu U U P P S - LZMA, reset state + set new prop +111uuuuu U U P P S - LZMA, reset state + set new prop, reset dic u, U - Unpack Size P - Pack Size S - Props */ -#define LZMA2_CONTROL_LZMA (1 << 7) -#define LZMA2_CONTROL_COPY_NO_RESET 2 #define LZMA2_CONTROL_COPY_RESET_DIC 1 -#define LZMA2_CONTROL_EOF 0 -#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & LZMA2_CONTROL_LZMA) == 0) - -#define LZMA2_GET_LZMA_MODE(p) (((p)->control >> 5) & 3) -#define LZMA2_IS_THERE_PROP(mode) ((mode) >= 2) +#define LZMA2_IS_UNCOMPRESSED_STATE(p) (((p)->control & (1 << 7)) == 0) #define LZMA2_LCLP_MAX 4 #define LZMA2_DIC_SIZE_FROM_PROP(p) (((UInt32)2 | ((p) & 1)) << ((p) / 2 + 11))@@ -74,14 +68,14 @@ props[4] = (Byte)(dicSize >> 24);
return SZ_OK; } -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; RINOK(Lzma2Dec_GetOldProps(prop, props)); return LzmaDec_AllocateProbs(&p->decoder, props, LZMA_PROPS_SIZE, alloc); } -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc) +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc) { Byte props[LZMA_PROPS_SIZE]; RINOK(Lzma2Dec_GetOldProps(prop, props));@@ -91,9 +85,11 @@
void Lzma2Dec_Init(CLzma2Dec *p) { p->state = LZMA2_STATE_CONTROL; - p->needInitDic = True; - p->needInitState = True; - p->needInitProp = True; + p->needInitLevel = 0xE0; + p->isExtraMode = False; + p->unpackSize = 0; + + // p->decoder.dicPos = 0; // we can use it instead of full init LzmaDec_Init(&p->decoder); }@@ -102,19 +98,26 @@ {
switch (p->state) { case LZMA2_STATE_CONTROL: + p->isExtraMode = False; p->control = b; - PRF(printf("\n %4X ", (unsigned)p->decoder.dicPos)); - PRF(printf(" %2X", (unsigned)b)); - if (p->control == 0) + PRF(printf("\n %8X", (unsigned)p->decoder.dicPos)); + PRF(printf(" %02X", (unsigned)b)); + if (b == 0) return LZMA2_STATE_FINISHED; if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if ((p->control & 0x7F) > 2) + if (b == LZMA2_CONTROL_COPY_RESET_DIC) + p->needInitLevel = 0xC0; + else if (b > 2 || p->needInitLevel == 0xE0) return LZMA2_STATE_ERROR; - p->unpackSize = 0; } else - p->unpackSize = (UInt32)(p->control & 0x1F) << 16; + { + if (b < p->needInitLevel) + return LZMA2_STATE_ERROR; + p->needInitLevel = 0; + p->unpackSize = (UInt32)(b & 0x1F) << 16; + } return LZMA2_STATE_UNPACK0; case LZMA2_STATE_UNPACK0:@@ -124,8 +127,8 @@
case LZMA2_STATE_UNPACK1: p->unpackSize |= (UInt32)b; p->unpackSize++; - PRF(printf(" %8u", (unsigned)p->unpackSize)); - return (LZMA2_IS_UNCOMPRESSED_STATE(p)) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; + PRF(printf(" %7u", (unsigned)p->unpackSize)); + return LZMA2_IS_UNCOMPRESSED_STATE(p) ? LZMA2_STATE_DATA : LZMA2_STATE_PACK0; case LZMA2_STATE_PACK0: p->packSize = (UInt32)b << 8;@@ -134,9 +137,9 @@
case LZMA2_STATE_PACK1: p->packSize |= (UInt32)b; p->packSize++; - PRF(printf(" %8u", (unsigned)p->packSize)); - return LZMA2_IS_THERE_PROP(LZMA2_GET_LZMA_MODE(p)) ? LZMA2_STATE_PROP: - (p->needInitProp ? LZMA2_STATE_ERROR : LZMA2_STATE_DATA); + // if (p->packSize < 5) return LZMA2_STATE_ERROR; + PRF(printf(" %5u", (unsigned)p->packSize)); + return (p->control & 0x40) ? LZMA2_STATE_PROP : LZMA2_STATE_DATA; case LZMA2_STATE_PROP: {@@ -145,13 +148,12 @@ if (b >= (9 * 5 * 5))
return LZMA2_STATE_ERROR; lc = b % 9; b /= 9; - p->decoder.prop.pb = b / 5; + p->decoder.prop.pb = (Byte)(b / 5); lp = b % 5; if (lc + lp > LZMA2_LCLP_MAX) return LZMA2_STATE_ERROR; - p->decoder.prop.lc = lc; - p->decoder.prop.lp = lp; - p->needInitProp = False; + p->decoder.prop.lc = (Byte)lc; + p->decoder.prop.lp = (Byte)lp; return LZMA2_STATE_DATA; } }@@ -167,7 +169,8 @@ p->checkDicSize = p->prop.dicSize;
p->processedPos += (UInt32)size; } -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState); +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState); + SRes Lzma2Dec_DecodeToDic(CLzma2Dec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status)@@ -176,12 +179,17 @@ SizeT inSize = *srcLen;
*srcLen = 0; *status = LZMA_STATUS_NOT_SPECIFIED; - while (p->state != LZMA2_STATE_FINISHED) + while (p->state != LZMA2_STATE_ERROR) { - SizeT dicPos = p->decoder.dicPos; + SizeT dicPos; + + if (p->state == LZMA2_STATE_FINISHED) + { + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } - if (p->state == LZMA2_STATE_ERROR) - return SZ_ERROR_DATA; + dicPos = p->decoder.dicPos; if (dicPos == dicLimit && finishMode == LZMA_FINISH_ANY) {@@ -198,29 +206,25 @@ return SZ_OK;
} (*srcLen)++; p->state = Lzma2Dec_UpdateState(p, *src++); - if (dicPos == dicLimit && p->state != LZMA2_STATE_FINISHED) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + break; continue; } { - SizeT destSizeCur = dicLimit - dicPos; - SizeT srcSizeCur = inSize - *srcLen; + SizeT inCur = inSize - *srcLen; + SizeT outCur = dicLimit - dicPos; ELzmaFinishMode curFinishMode = LZMA_FINISH_ANY; - if (p->unpackSize <= destSizeCur) + if (outCur >= p->unpackSize) { - destSizeCur = (SizeT)p->unpackSize; + outCur = (SizeT)p->unpackSize; curFinishMode = LZMA_FINISH_END; } if (LZMA2_IS_UNCOMPRESSED_STATE(p)) { - if (*srcLen == inSize) + if (inCur == 0) { *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK;@@ -228,135 +232,241 @@ }
if (p->state == LZMA2_STATE_DATA) { - Bool initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); - if (initDic) - p->needInitProp = p->needInitState = True; - else if (p->needInitDic) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } - p->needInitDic = False; + BoolInt initDic = (p->control == LZMA2_CONTROL_COPY_RESET_DIC); LzmaDec_InitDicAndState(&p->decoder, initDic, False); } - if (srcSizeCur > destSizeCur) - srcSizeCur = destSizeCur; - - if (srcSizeCur == 0) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } + if (inCur > outCur) + inCur = outCur; + if (inCur == 0) + break; - LzmaDec_UpdateWithUncompressed(&p->decoder, src, srcSizeCur); + LzmaDec_UpdateWithUncompressed(&p->decoder, src, inCur); - src += srcSizeCur; - *srcLen += srcSizeCur; - p->unpackSize -= (UInt32)srcSizeCur; + src += inCur; + *srcLen += inCur; + p->unpackSize -= (UInt32)inCur; p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; } else { - SizeT outSizeProcessed; SRes res; if (p->state == LZMA2_STATE_DATA) { - unsigned mode = LZMA2_GET_LZMA_MODE(p); - Bool initDic = (mode == 3); - Bool initState = (mode != 0); - if ((!initDic && p->needInitDic) || (!initState && p->needInitState)) - { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; - } - + BoolInt initDic = (p->control >= 0xE0); + BoolInt initState = (p->control >= 0xA0); LzmaDec_InitDicAndState(&p->decoder, initDic, initState); - p->needInitDic = False; - p->needInitState = False; p->state = LZMA2_STATE_DATA_CONT; } - if (srcSizeCur > p->packSize) - srcSizeCur = (SizeT)p->packSize; - - res = LzmaDec_DecodeToDic(&p->decoder, dicPos + destSizeCur, src, &srcSizeCur, curFinishMode, status); + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; - src += srcSizeCur; - *srcLen += srcSizeCur; - p->packSize -= (UInt32)srcSizeCur; + res = LzmaDec_DecodeToDic(&p->decoder, dicPos + outCur, src, &inCur, curFinishMode, status); - outSizeProcessed = p->decoder.dicPos - dicPos; - p->unpackSize -= (UInt32)outSizeProcessed; + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + outCur = p->decoder.dicPos - dicPos; + p->unpackSize -= (UInt32)outCur; - RINOK(res); + if (res != 0) + break; + if (*status == LZMA_STATUS_NEEDS_MORE_INPUT) - return res; + { + if (p->packSize == 0) + break; + return SZ_OK; + } - if (srcSizeCur == 0 && outSizeProcessed == 0) + if (inCur == 0 && outCur == 0) { if (*status != LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK || p->unpackSize != 0 || p->packSize != 0) + break; + p->state = LZMA2_STATE_CONTROL; + } + + *status = LZMA_STATUS_NOT_SPECIFIED; + } + } + } + + *status = LZMA_STATUS_NOT_SPECIFIED; + p->state = LZMA2_STATE_ERROR; + return SZ_ERROR_DATA; +} + + + + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, + const Byte *src, SizeT *srcLen, + int checkFinishBlock) +{ + SizeT inSize = *srcLen; + *srcLen = 0; + + while (p->state != LZMA2_STATE_ERROR) + { + if (p->state == LZMA2_STATE_FINISHED) + return (ELzma2ParseStatus)LZMA_STATUS_FINISHED_WITH_MARK; + + if (outSize == 0 && !checkFinishBlock) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + if (p->state != LZMA2_STATE_DATA && p->state != LZMA2_STATE_DATA_CONT) + { + if (*srcLen == inSize) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + (*srcLen)++; + + p->state = Lzma2Dec_UpdateState(p, *src++); + + if (p->state == LZMA2_STATE_UNPACK0) + { + // if (p->decoder.dicPos != 0) + if (p->control == LZMA2_CONTROL_COPY_RESET_DIC || p->control >= 0xE0) + return LZMA2_PARSE_STATUS_NEW_BLOCK; + // if (outSize == 0) return LZMA_STATUS_NOT_FINISHED; + } + + // The following code can be commented. + // It's not big problem, if we read additional input bytes. + // It will be stopped later in LZMA2_STATE_DATA / LZMA2_STATE_DATA_CONT state. + + if (outSize == 0 && p->state != LZMA2_STATE_FINISHED) + { + // checkFinishBlock is true. So we expect that block must be finished, + // We can return LZMA_STATUS_NOT_SPECIFIED or LZMA_STATUS_NOT_FINISHED here + // break; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + } + + if (p->state == LZMA2_STATE_DATA) + return LZMA2_PARSE_STATUS_NEW_CHUNK; + + continue; + } + + if (outSize == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NOT_FINISHED; + + { + SizeT inCur = inSize - *srcLen; + + if (LZMA2_IS_UNCOMPRESSED_STATE(p)) + { + if (inCur == 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + if (inCur > p->unpackSize) + inCur = p->unpackSize; + if (inCur > outSize) + inCur = outSize; + p->decoder.dicPos += inCur; + src += inCur; + *srcLen += inCur; + outSize -= inCur; + p->unpackSize -= (UInt32)inCur; + p->state = (p->unpackSize == 0) ? LZMA2_STATE_CONTROL : LZMA2_STATE_DATA_CONT; + } + else + { + p->isExtraMode = True; + + if (inCur == 0) + { + if (p->packSize != 0) + return (ELzma2ParseStatus)LZMA_STATUS_NEEDS_MORE_INPUT; + } + else if (p->state == LZMA2_STATE_DATA) + { + p->state = LZMA2_STATE_DATA_CONT; + if (*src != 0) { - p->state = LZMA2_STATE_ERROR; - return SZ_ERROR_DATA; + // first byte of lzma chunk must be Zero + *srcLen += 1; + p->packSize--; + break; } - p->state = LZMA2_STATE_CONTROL; } - - if (*status == LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK) - *status = LZMA_STATUS_NOT_FINISHED; + + if (inCur > p->packSize) + inCur = (SizeT)p->packSize; + + src += inCur; + *srcLen += inCur; + p->packSize -= (UInt32)inCur; + + if (p->packSize == 0) + { + SizeT rem = outSize; + if (rem > p->unpackSize) + rem = p->unpackSize; + p->decoder.dicPos += rem; + p->unpackSize -= (UInt32)rem; + outSize -= rem; + if (p->unpackSize == 0) + p->state = LZMA2_STATE_CONTROL; + } } } } - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return SZ_OK; + p->state = LZMA2_STATE_ERROR; + return (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; } + + + SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT outSize = *destLen, inSize = *srcLen; *srcLen = *destLen = 0; + for (;;) { - SizeT srcSizeCur = inSize, outSizeCur, dicPos; + SizeT inCur = inSize, outCur, dicPos; ELzmaFinishMode curFinishMode; SRes res; + if (p->decoder.dicPos == p->decoder.dicBufSize) p->decoder.dicPos = 0; dicPos = p->decoder.dicPos; - if (outSize > p->decoder.dicBufSize - dicPos) - { - outSizeCur = p->decoder.dicBufSize; - curFinishMode = LZMA_FINISH_ANY; - } - else + curFinishMode = LZMA_FINISH_ANY; + outCur = p->decoder.dicBufSize - dicPos; + + if (outCur >= outSize) { - outSizeCur = dicPos + outSize; + outCur = outSize; curFinishMode = finishMode; } - res = Lzma2Dec_DecodeToDic(p, outSizeCur, src, &srcSizeCur, curFinishMode, status); - src += srcSizeCur; - inSize -= srcSizeCur; - *srcLen += srcSizeCur; - outSizeCur = p->decoder.dicPos - dicPos; - memcpy(dest, p->decoder.dic + dicPos, outSizeCur); - dest += outSizeCur; - outSize -= outSizeCur; - *destLen += outSizeCur; + res = Lzma2Dec_DecodeToDic(p, dicPos + outCur, src, &inCur, curFinishMode, status); + + src += inCur; + inSize -= inCur; + *srcLen += inCur; + outCur = p->decoder.dicPos - dicPos; + memcpy(dest, p->decoder.dic + dicPos, outCur); + dest += outCur; + outSize -= outCur; + *destLen += outCur; if (res != 0) return res; - if (outSizeCur == 0 || outSize == 0) + if (outCur == 0 || outSize == 0) return SZ_OK; } } + SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc) + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc) { CLzma2Dec p; SRes res;
M
src/third-party/lzma/Lzma2Dec.h
→
src/third-party/lzma/Lzma2Dec.h
@@ -1,5 +1,5 @@
/* Lzma2Dec.h -- LZMA2 Decoder -2015-05-13 : Igor Pavlov : Public domain */ +2018-02-19 : Igor Pavlov : Public domain */ #ifndef __LZMA2_DEC_H #define __LZMA2_DEC_H@@ -12,24 +12,23 @@ /* ---------- State Interface ---------- */
typedef struct { - CLzmaDec decoder; + unsigned state; + Byte control; + Byte needInitLevel; + Byte isExtraMode; + Byte _pad_; UInt32 packSize; UInt32 unpackSize; - unsigned state; - Byte control; - Bool needInitDic; - Bool needInitState; - Bool needInitProp; + CLzmaDec decoder; } CLzma2Dec; #define Lzma2Dec_Construct(p) LzmaDec_Construct(&(p)->decoder) -#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc); -#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc); +#define Lzma2Dec_FreeProbs(p, alloc) LzmaDec_FreeProbs(&(p)->decoder, alloc) +#define Lzma2Dec_Free(p, alloc) LzmaDec_Free(&(p)->decoder, alloc) -SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAlloc *alloc); -SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAlloc *alloc); +SRes Lzma2Dec_AllocateProbs(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); +SRes Lzma2Dec_Allocate(CLzma2Dec *p, Byte prop, ISzAllocPtr alloc); void Lzma2Dec_Init(CLzma2Dec *p); - /* finishMode:@@ -53,6 +52,47 @@ SRes Lzma2Dec_DecodeToBuf(CLzma2Dec *p, Byte *dest, SizeT *destLen,
const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status); +/* ---------- LZMA2 block and chunk parsing ---------- */ + +/* +Lzma2Dec_Parse() parses compressed data stream up to next independent block or next chunk data. +It can return LZMA_STATUS_* code or LZMA2_PARSE_STATUS_* code: + - LZMA2_PARSE_STATUS_NEW_BLOCK - there is new block, and 1 additional byte (control byte of next block header) was read from input. + - LZMA2_PARSE_STATUS_NEW_CHUNK - there is new chunk, and only lzma2 header of new chunk was read. + CLzma2Dec::unpackSize contains unpack size of that chunk +*/ + +typedef enum +{ +/* + LZMA_STATUS_NOT_SPECIFIED // data error + LZMA_STATUS_FINISHED_WITH_MARK + LZMA_STATUS_NOT_FINISHED // + LZMA_STATUS_NEEDS_MORE_INPUT + LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK // unused +*/ + LZMA2_PARSE_STATUS_NEW_BLOCK = LZMA_STATUS_MAYBE_FINISHED_WITHOUT_MARK + 1, + LZMA2_PARSE_STATUS_NEW_CHUNK +} ELzma2ParseStatus; + +ELzma2ParseStatus Lzma2Dec_Parse(CLzma2Dec *p, + SizeT outSize, // output size + const Byte *src, SizeT *srcLen, + int checkFinishBlock // set (checkFinishBlock = 1), if it must read full input data, if decoder.dicPos reaches blockMax position. + ); + +/* +LZMA2 parser doesn't decode LZMA chunks, so we must read + full input LZMA chunk to decode some part of LZMA chunk. + +Lzma2Dec_GetUnpackExtra() returns the value that shows + max possible number of output bytes that can be output by decoder + at current input positon. +*/ + +#define Lzma2Dec_GetUnpackExtra(p) ((p)->isExtraMode ? (p)->unpackSize : 0); + + /* ---------- One Call Interface ---------- */ /*@@ -73,7 +113,7 @@ SZ_ERROR_INPUT_EOF - It needs more bytes in input buffer (src).
*/ SRes Lzma2Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAlloc *alloc); + Byte prop, ELzmaFinishMode finishMode, ELzmaStatus *status, ISzAllocPtr alloc); EXTERN_C_END
A
src/third-party/lzma/Lzma2DecMt.c
@@ -0,0 +1,1082 @@
+/* Lzma2DecMt.c -- LZMA2 Decoder Multi-thread +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include <stdio.h> +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) +#define PRF_STR_INT_2(s, d1, d2) PRF(printf("\n" s " %d %d\n", (unsigned)d1, (unsigned)d2)) + +// #define _7ZIP_ST + +#include "Alloc.h" + +#include "Lzma2Dec.h" +#include "Lzma2DecMt.h" + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +#define LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT (1 << 28) + +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p) +{ + p->inBufSize_ST = 1 << 20; + p->outStep_ST = 1 << 20; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->outBlockMax = LZMA2DECMT_OUT_BLOCK_MAX_DEFAULT; + p->inBlockMax = p->outBlockMax + p->outBlockMax / 16; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CLzma2DecMtThread ---------- */ + +typedef struct +{ + CLzma2Dec dec; + Byte dec_created; + Byte needInit; + + Byte *outBuf; + size_t outBufSize; + + EMtDecParseState state; + ELzma2ParseStatus parseStatus; + + size_t inPreSize; + size_t outPreSize; + + size_t inCodeSize; + size_t outCodeSize; + SRes codeRes; + + CAlignOffsetAlloc alloc; + + Byte mtPad[1 << 7]; +} CLzma2DecMtThread; + +#endif + + +/* ---------- CLzma2DecMt ---------- */ + +typedef struct +{ + // ISzAllocPtr alloc; + ISzAllocPtr allocMid; + + CAlignOffsetAlloc alignOffsetAlloc; + CLzma2DecMtProps props; + Byte prop; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + BoolInt readWasFinished; + SRes readRes; + + Byte *inBuf; + size_t inBufSize; + Byte dec_created; + CLzma2Dec dec; + + size_t inPos; + size_t inLim; + + #ifndef _7ZIP_ST + UInt64 outProcessed_Parse; + BoolInt mtc_WasConstructed; + CMtDec mtc; + CLzma2DecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CLzma2DecMt; + + + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CLzma2DecMt *p = (CLzma2DecMt *)ISzAlloc_Alloc(alloc, sizeof(CLzma2DecMt)); + if (!p) + return NULL; + + // p->alloc = alloc; + p->allocMid = allocMid; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + p->alignOffsetAlloc.baseAlloc = alloc; + + p->inBuf = NULL; + p->inBufSize = 0; + p->dec_created = False; + + // Lzma2DecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + t->dec_created = False; + t->outBuf = NULL; + t->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_FreeOutBufs(CLzma2DecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->outBuf) + { + ISzAlloc_Free(p->allocMid, t->outBuf); + t->outBuf = NULL; + t->outBufSize = 0; + } + } +} + +#endif + + +static void Lzma2DecMt_FreeSt(CLzma2DecMt *p) +{ + if (p->dec_created) + { + Lzma2Dec_Free(&p->dec, &p->alignOffsetAlloc.vt); + p->dec_created = False; + } + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void Lzma2DecMt_Destroy(CLzma2DecMtHandle pp) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + Lzma2DecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CLzma2DecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + Lzma2Dec_FreeProbs(&t->dec, &t->alloc.vt); // p->alloc !!! + t->dec_created = False; + } + } + } + Lzma2DecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void Lzma2DecMt_MtCallback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CLzma2DecMt *me = (CLzma2DecMt *)obj; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + PRF_STR_INT_2("Parse", coderIndex, cc->srcSize); + + cc->state = MTDEC_PARSE_CONTINUE; + + if (cc->startCall) + { + if (!t->dec_created) + { + Lzma2Dec_Construct(&t->dec); + t->dec_created = True; + AlignOffsetAlloc_CreateVTable(&t->alloc); + { + /* (1 << 12) is expected size of one way in data cache. + We optimize alignment for cache line size of 128 bytes and smaller */ + const unsigned kNumAlignBits = 12; + const unsigned kNumCacheLineBits = 7; /* <= kNumAlignBits */ + t->alloc.numAlignBits = kNumAlignBits; + t->alloc.offset = ((UInt32)coderIndex * ((1 << 11) + (1 << 8) + (1 << 6))) & ((1 << kNumAlignBits) - (1 << kNumCacheLineBits)); + t->alloc.baseAlloc = me->alignOffsetAlloc.baseAlloc; + } + } + Lzma2Dec_Init(&t->dec); + + t->inPreSize = 0; + t->outPreSize = 0; + // t->blockWasFinished = False; + // t->finishedWithMark = False; + t->parseStatus = (ELzma2ParseStatus)LZMA_STATUS_NOT_SPECIFIED; + t->state = MTDEC_PARSE_CONTINUE; + + t->inCodeSize = 0; + t->outCodeSize = 0; + t->codeRes = SZ_OK; + + // (cc->srcSize == 0) is allowed + } + + { + ELzma2ParseStatus status; + BoolInt overflow; + UInt32 unpackRem = 0; + + int checkFinishBlock = True; + size_t limit = me->props.outBlockMax; + if (me->outSize_Defined) + { + UInt64 rem = me->outSize - me->outProcessed_Parse; + if (limit >= rem) + { + limit = (size_t)rem; + if (!me->finishMode) + checkFinishBlock = False; + } + } + + // checkFinishBlock = False, if we want to decode partial data + // that must be finished at position <= outBlockMax. + + { + const SizeT srcOrig = cc->srcSize; + SizeT srcSize_Point = 0; + SizeT dicPos_Point = 0; + + cc->srcSize = 0; + overflow = False; + + for (;;) + { + SizeT srcCur = srcOrig - cc->srcSize; + + status = Lzma2Dec_Parse(&t->dec, + limit - t->dec.decoder.dicPos, + cc->src + cc->srcSize, &srcCur, + checkFinishBlock); + + cc->srcSize += srcCur; + + if (status == LZMA2_PARSE_STATUS_NEW_CHUNK) + { + if (t->dec.unpackSize > me->props.outBlockMax - t->dec.decoder.dicPos) + { + overflow = True; + break; + } + continue; + } + + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + if (t->dec.decoder.dicPos == 0) + continue; + // we decode small blocks in one thread + if (t->dec.decoder.dicPos >= (1 << 14)) + break; + dicPos_Point = t->dec.decoder.dicPos; + srcSize_Point = cc->srcSize; + continue; + } + + if ((int)status == LZMA_STATUS_NOT_FINISHED && checkFinishBlock + // && limit == t->dec.decoder.dicPos + // && limit == me->props.outBlockMax + ) + { + overflow = True; + break; + } + + unpackRem = Lzma2Dec_GetUnpackExtra(&t->dec); + break; + } + + if (dicPos_Point != 0 + && (int)status != LZMA2_PARSE_STATUS_NEW_BLOCK + && (int)status != LZMA_STATUS_FINISHED_WITH_MARK + && (int)status != LZMA_STATUS_NOT_SPECIFIED) + { + // we revert to latest newBlock state + status = LZMA2_PARSE_STATUS_NEW_BLOCK; + unpackRem = 0; + t->dec.decoder.dicPos = dicPos_Point; + cc->srcSize = srcSize_Point; + overflow = False; + } + } + + t->inPreSize += cc->srcSize; + t->parseStatus = status; + + if (overflow) + cc->state = MTDEC_PARSE_OVERFLOW; + else + { + size_t dicPos = t->dec.decoder.dicPos; + + if ((int)status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (status == LZMA2_PARSE_STATUS_NEW_BLOCK) + { + cc->state = MTDEC_PARSE_NEW; + cc->srcSize--; // we don't need control byte of next block + t->inPreSize--; + } + else + { + cc->state = MTDEC_PARSE_END; + if ((int)status != LZMA_STATUS_FINISHED_WITH_MARK) + { + // (status == LZMA_STATUS_NOT_SPECIFIED) + // (status == LZMA_STATUS_NOT_FINISHED) + if (unpackRem != 0) + { + /* we also reserve space for max possible number of output bytes of current LZMA chunk */ + SizeT rem = limit - dicPos; + if (rem > unpackRem) + rem = unpackRem; + dicPos += rem; + } + } + } + + me->outProcessed_Parse += dicPos; + } + + cc->outPos = dicPos; + t->outPreSize = (size_t)dicPos; + } + + t->state = cc->state; + return; + } +} + + +static SRes Lzma2DecMt_MtCallback_PreCode(void *pp, unsigned coderIndex) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + Byte *dest = t->outBuf; + + if (t->inPreSize == 0) + { + t->codeRes = SZ_ERROR_DATA; + return t->codeRes; + } + + if (!dest || t->outBufSize < t->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + t->outBuf = NULL; + t->outBufSize = 0; + } + + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, t->outPreSize + // + (1 << 28) + ); + // Sleep(200); + if (!dest) + return SZ_ERROR_MEM; + t->outBuf = dest; + t->outBufSize = t->outPreSize; + } + + t->dec.decoder.dic = dest; + t->dec.decoder.dicBufSize = t->outPreSize; + + t->needInit = True; + + return Lzma2Dec_AllocateProbs(&t->dec, me->prop, &t->alloc.vt); // alloc.vt +} + + +static SRes Lzma2DecMt_MtCallback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + CLzma2DecMtThread *t = &me->coders[coderIndex]; + + UNUSED_VAR(srcFinished) + + PRF_STR_INT_2("Code", coderIndex, srcSize); + + *inCodePos = t->inCodeSize; + *outCodePos = 0; + *stop = True; + + if (t->needInit) + { + Lzma2Dec_Init(&t->dec); + t->needInit = False; + } + + { + ELzmaStatus status; + size_t srcProcessed = srcSize; + BoolInt blockWasFinished = + ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK); + + SRes res = Lzma2Dec_DecodeToDic(&t->dec, + t->outPreSize, + src, &srcProcessed, + blockWasFinished ? LZMA_FINISH_END : LZMA_FINISH_ANY, + &status); + + t->codeRes = res; + + t->inCodeSize += srcProcessed; + *inCodePos = t->inCodeSize; + t->outCodeSize = t->dec.decoder.dicPos; + *outCodePos = t->dec.decoder.dicPos; + + if (res != SZ_OK) + return res; + + if (srcProcessed == srcSize) + *stop = False; + + if (blockWasFinished) + { + if (srcSize != srcProcessed) + return SZ_ERROR_FAIL; + + if (t->inPreSize == t->inCodeSize) + { + if (t->outPreSize != t->outCodeSize) + return SZ_ERROR_FAIL; + *stop = True; + } + } + else + { + if (t->outPreSize == t->outCodeSize) + *stop = True; + } + + return SZ_OK; + } +} + + +#define LZMA2DECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes Lzma2DecMt_MtCallback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + BoolInt *needContinue, BoolInt *canRecode) +{ + CLzma2DecMt *me = (CLzma2DecMt *)pp; + const CLzma2DecMtThread *t = &me->coders[coderIndex]; + size_t size = t->outCodeSize; + const Byte *data = t->outBuf; + BoolInt needContinue2 = True; + + PRF_STR_INT_2("Write", coderIndex, srcSize); + + *needContinue = False; + *canRecode = True; + UNUSED_VAR(src) + UNUSED_VAR(srcSize) + + if ( + // t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + t->state == MTDEC_PARSE_OVERFLOW + || t->state == MTDEC_PARSE_END) + needContinue2 = False; + + + if (!needWriteToStream) + return SZ_OK; + + me->mtc.inProcessed += t->inCodeSize; + + if (t->codeRes == SZ_OK) + if ((int)t->parseStatus == LZMA_STATUS_FINISHED_WITH_MARK + || t->parseStatus == LZMA2_PARSE_STATUS_NEW_BLOCK) + if (t->outPreSize != t->outCodeSize + || t->inPreSize != t->inCodeSize) + return SZ_ERROR_FAIL; + + *canRecode = False; + + if (me->outStream) + { + for (;;) + { + size_t cur = size; + size_t written; + if (cur > LZMA2DECMT_STREAM_WRITE_STEP) + cur = LZMA2DECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + me->outProcessed += written; + // me->mtc.writtenTotal += written; + if (written != cur) + return SZ_ERROR_WRITE; + data += cur; + size -= cur; + if (size == 0) + { + *needContinue = needContinue2; + return SZ_OK; + } + RINOK(MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0)); + } + } + + return SZ_ERROR_FAIL; + /* + if (size > me->outBufSize) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBufSize -= size; + me->outBuf += size; + *needContinue = needContinue2; + return SZ_OK; + */ +} + +#endif + + +static SRes Lzma2Dec_Prepare_ST(CLzma2DecMt *p) +{ + if (!p->dec_created) + { + Lzma2Dec_Construct(&p->dec); + p->dec_created = True; + } + + RINOK(Lzma2Dec_Allocate(&p->dec, p->prop, &p->alignOffsetAlloc.vt)); + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + Lzma2Dec_Init(&p->dec); + + return SZ_OK; +} + + +static SRes Lzma2Dec_Decode_ST(CLzma2DecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + ) +{ + SizeT wrPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CLzma2Dec *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + Lzma2DecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + RINOK(Lzma2Dec_Prepare_ST(p)); + + dec = &p->dec; + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + wrPos = dec->decoder.dicPos; + + for (;;) + { + SizeT dicPos; + SizeT size; + ELzmaFinishMode finishMode; + SizeT inProcessed; + ELzmaStatus status; + SRes res; + + SizeT outProcessed; + BoolInt outFinished; + BoolInt needStop; + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + // p->readProcessed += inLim; + // inLim -= 5; p->readWasFinished = True; // for test + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + dicPos = dec->decoder.dicPos; + { + SizeT next = dec->decoder.dicBufSize; + if (next - wrPos > p->props.outStep_ST) + next = wrPos + p->props.outStep_ST; + size = next - dicPos; + } + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (SizeT)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + inProcessed = inLim - inPos; + + res = Lzma2Dec_DecodeToDic(dec, dicPos + size, inData + inPos, &inProcessed, finishMode, &status); + + inPos += inProcessed; + p->inProcessed += inProcessed; + outProcessed = dec->decoder.dicPos - dicPos; + p->outProcessed += outProcessed; + + outFinished = (p->outSize_Defined && p->outSize <= p->outProcessed); + + needStop = (res != SZ_OK + || (inProcessed == 0 && outProcessed == 0) + || status == LZMA_STATUS_FINISHED_WITH_MARK + || (!p->finishMode && outFinished)); + + if (needStop || outProcessed >= size) + { + SRes res2; + { + size_t writeSize = dec->decoder.dicPos - wrPos; + size_t written = ISeqOutStream_Write(p->outStream, dec->decoder.dic + wrPos, writeSize); + res2 = (written == writeSize) ? SZ_OK : SZ_ERROR_WRITE; + } + + if (dec->decoder.dicPos == dec->decoder.dicBufSize) + dec->decoder.dicPos = 0; + wrPos = dec->decoder.dicPos; + + RINOK(res2); + + if (needStop) + { + if (res != SZ_OK) + return res; + + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + { + if (p->finishMode) + { + if (p->outSize_Defined && p->outSize != p->outProcessed) + return SZ_ERROR_DATA; + } + return SZ_OK; + } + + if (!p->finishMode && outFinished) + return SZ_OK; + + if (status == LZMA_STATUS_NEEDS_MORE_INPUT) + return SZ_ERROR_INPUT_EOF; + + return SZ_ERROR_DATA; + } + } + + if (p->progress) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + RINOK(ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed)); + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + } +} + + + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, const UInt64 *outDataSize, int finishMode, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + UInt64 *inProcessed, + // UInt64 *outProcessed, + int *isMT, + ICompressProgress *progress) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + *inProcessed = 0; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->readWasFinished = False; + + *isMT = False; + + + #ifndef _7ZIP_ST + + tMode = False; + + // p->mtc.parseRes = SZ_OK; + + // p->mtc.numFilledThreads = 0; + // p->mtc.crossStart = 0; + // p->mtc.crossEnd = 0; + // p->mtc.allocError_for_Read_BlockIndex = 0; + // p->mtc.isAllocError = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + Lzma2DecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + + // p->outBuf = NULL; + // p->outBufSize = 0; + /* + if (!outStream) + { + // p->outBuf = outBuf; + // p->outBufSize = *outBufSize; + // *outBufSize = 0; + return SZ_ERROR_PARAM; + } + */ + + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->alignOffsetAlloc.baseAlloc; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.inBufSize = p->props.inBufSize_MT; + + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = Lzma2DecMt_MtCallback_Parse; + vt.PreCode = Lzma2DecMt_MtCallback_PreCode; + vt.Code = Lzma2DecMt_MtCallback_Code; + vt.Write = Lzma2DecMt_MtCallback_Write; + + { + BoolInt needContinue = False; + + SRes res = MtDec_Code(&p->mtc); + + /* + if (!outStream) + *outBufSize = p->outBuf - outBuf; + */ + + *inProcessed = p->mtc.inProcessed; + + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + res = p->mtc.mtProgress.res; + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + if (res == SZ_OK) + return p->mtc.readRes; + return res; + } + + tMode = True; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->inProcessed = p->mtc.inProcessed; + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = Lzma2Dec_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + ); + + *inProcessed = p->inProcessed; + + // res = SZ_OK; // for test + if (res == SZ_OK && p->readRes != SZ_OK) + res = p->readRes; + + /* + #ifndef _7ZIP_ST + if (res == SZ_OK && tMode && p->mtc.parseRes != SZ_OK) + res = p->mtc.parseRes; + #endif + */ + + return res; + } +} + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + + if (prop > 40) + return SZ_ERROR_UNSUPPORTED; + + p->prop = prop; + p->props = *props; + + p->inStream = inStream; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + p->finishMode = finishMode; + + p->outProcessed = 0; + p->inProcessed = 0; + + p->inPos = 0; + p->inLim = 0; + + return Lzma2Dec_Prepare_ST(p); +} + + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed) +{ + CLzma2DecMt *p = (CLzma2DecMt *)pp; + ELzmaFinishMode finishMode; + SRes readRes; + size_t size = *outSize; + + *outSize = 0; + *inStreamProcessed = 0; + + finishMode = LZMA_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (size >= rem) + { + size = (size_t)rem; + if (p->finishMode) + finishMode = LZMA_FINISH_END; + } + } + + readRes = SZ_OK; + + for (;;) + { + SizeT inCur; + SizeT outCur; + ELzmaStatus status; + SRes res; + + if (p->inPos == p->inLim && readRes == SZ_OK) + { + p->inPos = 0; + p->inLim = p->props.inBufSize_ST; + readRes = ISeqInStream_Read(p->inStream, p->inBuf, &p->inLim); + } + + inCur = p->inLim - p->inPos; + outCur = size; + + res = Lzma2Dec_DecodeToBuf(&p->dec, data, &outCur, + p->inBuf + p->inPos, &inCur, finishMode, &status); + + p->inPos += inCur; + p->inProcessed += inCur; + *inStreamProcessed += inCur; + p->outProcessed += outCur; + *outSize += outCur; + size -= outCur; + data += outCur; + + if (res != 0) + return res; + + /* + if (status == LZMA_STATUS_FINISHED_WITH_MARK) + return readRes; + + if (size == 0 && status != LZMA_STATUS_NEEDS_MORE_INPUT) + { + if (p->finishMode && p->outSize_Defined && p->outProcessed >= p->outSize) + return SZ_ERROR_DATA; + return readRes; + } + */ + + if (inCur == 0 && outCur == 0) + return readRes; + } +}
A
src/third-party/lzma/Lzma2DecMt.h
@@ -0,0 +1,79 @@
+/* Lzma2DecMt.h -- LZMA2 Decoder Multi-thread +2018-02-17 : Igor Pavlov : Public domain */ + +#ifndef __LZMA2_DEC_MT_H +#define __LZMA2_DEC_MT_H + +#include "7zTypes.h" + +EXTERN_C_BEGIN + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t outBlockMax; + size_t inBlockMax; + #endif +} CLzma2DecMtProps; + +/* init to single-thread mode */ +void Lzma2DecMtProps_Init(CLzma2DecMtProps *p); + + +/* ---------- CLzma2DecMtHandle Interface ---------- */ + +/* Lzma2DecMt_ * functions can return the following exit codes: +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_PARAM - Incorrect paramater in props + SZ_ERROR_WRITE - ISeqOutStream write callback error + // SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output + SZ_ERROR_PROGRESS - some break from progress callback + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) +*/ + +typedef void * CLzma2DecMtHandle; + +CLzma2DecMtHandle Lzma2DecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void Lzma2DecMt_Destroy(CLzma2DecMtHandle p); + +SRes Lzma2DecMt_Decode(CLzma2DecMtHandle p, + Byte prop, + const CLzma2DecMtProps *props, + ISeqOutStream *outStream, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - if lzma2 stream must be finished + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + + // out variables: + UInt64 *inProcessed, + int *isMT, /* out: (*isMT == 0), if single thread decoding was used */ + + // UInt64 *outProcessed, + ICompressProgress *progress); + + +/* ---------- Read from CLzma2DecMtHandle Interface ---------- */ + +SRes Lzma2DecMt_Init(CLzma2DecMtHandle pp, + Byte prop, + const CLzma2DecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqInStream *inStream); + +SRes Lzma2DecMt_Read(CLzma2DecMtHandle pp, + Byte *data, size_t *outSize, + UInt64 *inStreamProcessed); + + +EXTERN_C_END + +#endif
M
src/third-party/lzma/Lzma2Enc.c
→
src/third-party/lzma/Lzma2Enc.c
@@ -1,9 +1,8 @@
/* Lzma2Enc.c -- LZMA2 Encoder -2015-10-04 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" -/* #include <stdio.h> */ #include <string.h> /* #define _7ZIP_ST */@@ -13,7 +12,7 @@
#ifndef _7ZIP_ST #include "MtCoder.h" #else -#define NUM_MT_CODER_THREADS_MAX 1 +#define MTCODER__THREADS_MAX 1 #endif #define LZMA2_CONTROL_LZMA (1 << 7)@@ -35,41 +34,97 @@
#define PRF(x) /* x */ + +/* ---------- CLimitedSeqInStream ---------- */ + +typedef struct +{ + ISeqInStream vt; + ISeqInStream *realStream; + UInt64 limit; + UInt64 processed; + int finished; +} CLimitedSeqInStream; + +static void LimitedSeqInStream_Init(CLimitedSeqInStream *p) +{ + p->limit = (UInt64)(Int64)-1; + p->processed = 0; + p->finished = 0; +} + +static SRes LimitedSeqInStream_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CLimitedSeqInStream *p = CONTAINER_FROM_VTBL(pp, CLimitedSeqInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->finished = (size2 == 0 ? 1 : 0); + p->processed += size2; + } + *size = size2; + return res; +} + + /* ---------- CLzma2EncInt ---------- */ typedef struct { CLzmaEncHandle enc; + Byte propsAreSet; + Byte propsByte; + Byte needInitState; + Byte needInitProp; UInt64 srcPos; - Byte props; - Bool needInitState; - Bool needInitProp; } CLzma2EncInt; -static SRes Lzma2EncInt_Init(CLzma2EncInt *p, const CLzma2EncProps *props) + +static SRes Lzma2EncInt_InitStream(CLzma2EncInt *p, const CLzma2EncProps *props) +{ + if (!p->propsAreSet) + { + SizeT propsSize = LZMA_PROPS_SIZE; + Byte propsEncoded[LZMA_PROPS_SIZE]; + RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); + RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); + p->propsByte = propsEncoded[0]; + p->propsAreSet = True; + } + return SZ_OK; +} + +static void Lzma2EncInt_InitBlock(CLzma2EncInt *p) { - Byte propsEncoded[LZMA_PROPS_SIZE]; - SizeT propsSize = LZMA_PROPS_SIZE; - RINOK(LzmaEnc_SetProps(p->enc, &props->lzmaProps)); - RINOK(LzmaEnc_WriteProperties(p->enc, propsEncoded, &propsSize)); p->srcPos = 0; - p->props = propsEncoded[0]; p->needInitState = True; p->needInitProp = True; - return SZ_OK; } + SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig); + ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig); -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig); +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize); const Byte *LzmaEnc_GetCurBuf(CLzmaEncHandle pp); void LzmaEnc_Finish(CLzmaEncHandle pp); void LzmaEnc_SaveState(CLzmaEncHandle pp); void LzmaEnc_RestoreState(CLzmaEncHandle pp); +/* +UInt32 LzmaEnc_GetNumAvailableBytes(CLzmaEncHandle pp); +*/ static SRes Lzma2EncInt_EncodeSubblock(CLzma2EncInt *p, Byte *outBuf, size_t *packSizeRes, ISeqOutStream *outStream)@@ -78,7 +133,7 @@ size_t packSizeLimit = *packSizeRes;
size_t packSize = packSizeLimit; UInt32 unpackSize = LZMA2_UNPACK_SIZE_MAX; unsigned lzHeaderSize = 5 + (p->needInitProp ? 1 : 0); - Bool useCopyBlock; + BoolInt useCopyBlock; SRes res; *packSizeRes = 0;@@ -126,7 +181,7 @@
if (outStream) { *packSizeRes += destPos; - if (outStream->Write(outStream, outBuf, destPos) != destPos) + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) return SZ_ERROR_WRITE; destPos = 0; }@@ -154,7 +209,7 @@ outBuf[destPos++] = (Byte)(pm >> 8);
outBuf[destPos++] = (Byte)pm; if (p->needInitProp) - outBuf[destPos++] = p->props; + outBuf[destPos++] = p->propsByte; p->needInitProp = False; p->needInitState = False;@@ -162,7 +217,7 @@ destPos += packSize;
p->srcPos += unpackSize; if (outStream) - if (outStream->Write(outStream, outBuf, destPos) != destPos) + if (ISeqOutStream_Write(outStream, outBuf, destPos) != destPos) return SZ_ERROR_WRITE; *packSizeRes = destPos;@@ -176,14 +231,16 @@
void Lzma2EncProps_Init(CLzma2EncProps *p) { LzmaEncProps_Init(&p->lzmaProps); + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; p->numTotalThreads = -1; - p->numBlockThreads = -1; - p->blockSize = 0; } void Lzma2EncProps_Normalize(CLzma2EncProps *p) { - int t1, t1n, t2, t3; + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; { CLzmaEncProps lzmaProps = p->lzmaProps; LzmaEncProps_Normalize(&lzmaProps);@@ -191,11 +248,11 @@ t1n = lzmaProps.numThreads;
} t1 = p->lzmaProps.numThreads; - t2 = p->numBlockThreads; + t2 = p->numBlockThreads_Max; t3 = p->numTotalThreads; - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; if (t3 <= 0) {@@ -211,8 +268,8 @@ {
t1 = 1; t2 = t3; } - if (t2 > NUM_MT_CODER_THREADS_MAX) - t2 = NUM_MT_CODER_THREADS_MAX; + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; } else if (t1 <= 0) {@@ -225,46 +282,71 @@ t3 = t1n * t2;
p->lzmaProps.numThreads = t1; + t2r = t2; + + fileSize = p->lzmaProps.reduceSize; + + if ( p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && p->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO + && (p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzmaProps.reduceSize = p->blockSize; + LzmaEncProps_Normalize(&p->lzmaProps); + + p->lzmaProps.reduceSize = fileSize; t1 = p->lzmaProps.numThreads; - if (p->blockSize == 0) + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + { + t2r = t2 = 1; + t3 = t1; + } + else if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO && t2 <= 1) { - UInt32 dictSize = p->lzmaProps.dictSize; - UInt64 blockSize = (UInt64)dictSize << 2; - const UInt32 kMinSize = (UInt32)1 << 20; - const UInt32 kMaxSize = (UInt32)1 << 28; - if (blockSize < kMinSize) blockSize = kMinSize; - if (blockSize > kMaxSize) blockSize = kMaxSize; - if (blockSize < dictSize) blockSize = dictSize; - p->blockSize = (size_t)blockSize; + /* if there is no block multi-threading, we use SOLID block */ + p->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; } - - if (t2 > 1 && p->lzmaProps.reduceSize != (UInt64)(Int64)-1) + else { - UInt64 temp = p->lzmaProps.reduceSize + p->blockSize - 1; - if (temp > p->lzmaProps.reduceSize) + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) { - UInt64 numBlocks = temp / p->blockSize; + const UInt32 kMinSize = (UInt32)1 << 20; + const UInt32 kMaxSize = (UInt32)1 << 28; + const UInt32 dictSize = p->lzmaProps.dictSize; + UInt64 blockSize = (UInt64)dictSize << 2; + if (blockSize < kMinSize) blockSize = kMinSize; + if (blockSize > kMaxSize) blockSize = kMaxSize; + if (blockSize < dictSize) blockSize = dictSize; + blockSize += (kMinSize - 1); + blockSize &= ~(UInt64)(kMinSize - 1); + p->blockSize = blockSize; + } + + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; if (numBlocks < (unsigned)t2) { - t2 = (unsigned)numBlocks; - if (t2 == 0) - t2 = 1; - t3 = t1 * t2; + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; } } } - p->numBlockThreads = t2; + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; p->numTotalThreads = t3; } static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) { - return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; + return (p && ICompressProgress_Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; }@@ -274,179 +356,111 @@ typedef struct
{ Byte propEncoded; CLzma2EncProps props; + UInt64 expectedDataSize; - Byte *outBuf; + Byte *tempBufLzma; - ISzAlloc *alloc; - ISzAlloc *allocBig; + ISzAllocPtr alloc; + ISzAllocPtr allocBig; - CLzma2EncInt coders[NUM_MT_CODER_THREADS_MAX]; + CLzma2EncInt coders[MTCODER__THREADS_MAX]; #ifndef _7ZIP_ST + + ISeqOutStream *outStream; + Byte *outBuf; + size_t outBuf_Rem; /* remainder in outBuf */ + + size_t outBufSize; /* size of allocated outBufs[i] */ + size_t outBufsDataSizes[MTCODER__BLOCKS_MAX]; + BoolInt mtCoder_WasConstructed; CMtCoder mtCoder; + Byte *outBufs[MTCODER__BLOCKS_MAX]; + #endif } CLzma2Enc; -/* ---------- Lzma2EncThread ---------- */ -static SRes Lzma2Enc_EncodeMt1(CLzma2EncInt *p, CLzma2Enc *mainEncoder, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt64 packTotal = 0; - SRes res = SZ_OK; - - if (!mainEncoder->outBuf) + CLzma2Enc *p = (CLzma2Enc *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Enc)); + if (!p) + return NULL; + Lzma2EncProps_Init(&p->props); + Lzma2EncProps_Normalize(&p->props); + p->expectedDataSize = (UInt64)(Int64)-1; + p->tempBufLzma = NULL; + p->alloc = alloc; + p->allocBig = allocBig; { - mainEncoder->outBuf = (Byte *)IAlloc_Alloc(mainEncoder->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); - if (!mainEncoder->outBuf) - return SZ_ERROR_MEM; + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].enc = NULL; } - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - RINOK(LzmaEnc_PrepareForLzma2(p->enc, inStream, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - for (;;) + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; { - size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; - res = Lzma2EncInt_EncodeSubblock(p, mainEncoder->outBuf, &packSize, outStream); - if (res != SZ_OK) - break; - packTotal += packSize; - res = Progress(progress, p->srcPos, packTotal); - if (res != SZ_OK) - break; - if (packSize == 0) - break; + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; } - - LzmaEnc_Finish(p->enc); + #endif - if (res == SZ_OK) - { - Byte b = 0; - if (outStream->Write(outStream, &b, 1) != 1) - return SZ_ERROR_WRITE; - } - - return res; + return p; } #ifndef _7ZIP_ST -typedef struct -{ - IMtCoderCallback funcTable; - CLzma2Enc *lzma2Enc; -} CMtCallbackImp; - -static SRes MtCallbackImp_Code(void *pp, unsigned index, Byte *dest, size_t *destSize, - const Byte *src, size_t srcSize, int finished) +static void Lzma2Enc_FreeOutBufs(CLzma2Enc *p) { - CMtCallbackImp *imp = (CMtCallbackImp *)pp; - CLzma2Enc *mainEncoder = imp->lzma2Enc; - CLzma2EncInt *p = &mainEncoder->coders[index]; - - SRes res = SZ_OK; - { - size_t destLim = *destSize; - *destSize = 0; - - if (srcSize != 0) + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) { - RINOK(Lzma2EncInt_Init(p, &mainEncoder->props)); - - RINOK(LzmaEnc_MemPrepare(p->enc, src, srcSize, LZMA2_KEEP_WINDOW_SIZE, - mainEncoder->alloc, mainEncoder->allocBig)); - - while (p->srcPos < srcSize) - { - size_t packSize = destLim - *destSize; - res = Lzma2EncInt_EncodeSubblock(p, dest + *destSize, &packSize, NULL); - if (res != SZ_OK) - break; - *destSize += packSize; - - if (packSize == 0) - { - res = SZ_ERROR_FAIL; - break; - } - - if (MtProgress_Set(&mainEncoder->mtCoder.mtProgress, index, p->srcPos, *destSize) != SZ_OK) - { - res = SZ_ERROR_PROGRESS; - break; - } - } - - LzmaEnc_Finish(p->enc); - if (res != SZ_OK) - return res; + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; } - - if (finished) - { - if (*destSize == destLim) - return SZ_ERROR_OUTPUT_EOF; - dest[(*destSize)++] = 0; - } - } - return res; + p->outBufSize = 0; } #endif - -/* ---------- Lzma2Enc ---------- */ - -CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig) -{ - CLzma2Enc *p = (CLzma2Enc *)alloc->Alloc(alloc, sizeof(CLzma2Enc)); - if (!p) - return NULL; - Lzma2EncProps_Init(&p->props); - Lzma2EncProps_Normalize(&p->props); - p->outBuf = 0; - p->alloc = alloc; - p->allocBig = allocBig; - { - unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->coders[i].enc = 0; - } - - #ifndef _7ZIP_ST - MtCoder_Construct(&p->mtCoder); - #endif - - return p; -} void Lzma2Enc_Destroy(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp; unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + for (i = 0; i < MTCODER__THREADS_MAX; i++) { CLzma2EncInt *t = &p->coders[i]; if (t->enc) { LzmaEnc_Destroy(t->enc, p->alloc, p->allocBig); - t->enc = 0; + t->enc = NULL; } } + #ifndef _7ZIP_ST - MtCoder_Destruct(&p->mtCoder); + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + Lzma2Enc_FreeOutBufs(p); #endif - IAlloc_Free(p->alloc, p->outBuf); - IAlloc_Free(p->alloc, pp); + ISzAlloc_Free(p->alloc, p->tempBufLzma); + p->tempBufLzma = NULL; + + ISzAlloc_Free(p->alloc, pp); } + SRes Lzma2Enc_SetProps(CLzma2EncHandle pp, const CLzma2EncProps *props) {@@ -460,6 +474,14 @@ Lzma2EncProps_Normalize(&p->props);
return SZ_OK; } + +void Lzma2Enc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + Byte Lzma2Enc_WriteProperties(CLzma2EncHandle pp) { CLzma2Enc *p = (CLzma2Enc *)pp;@@ -471,50 +493,311 @@ break;
return (Byte)i; } -SRes Lzma2Enc_Encode(CLzma2EncHandle pp, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) + +static SRes Lzma2Enc_EncodeMt1( + CLzma2Enc *me, + CLzma2EncInt *p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + int finished, + ICompressProgress *progress) { - CLzma2Enc *p = (CLzma2Enc *)pp; - int i; + UInt64 unpackTotal = 0; + UInt64 packTotal = 0; + size_t outLim = 0; + CLimitedSeqInStream limitedInStream; + + if (outBuf) + { + outLim = *outBufSize; + *outBufSize = 0; + } + + if (!p->enc) + { + p->propsAreSet = False; + p->enc = LzmaEnc_Create(me->alloc); + if (!p->enc) + return SZ_ERROR_MEM; + } - for (i = 0; i < p->props.numBlockThreads; i++) + limitedInStream.realStream = inStream; + if (inStream) { - CLzma2EncInt *t = &p->coders[(unsigned)i]; - if (!t->enc) + limitedInStream.vt.Read = LimitedSeqInStream_Read; + } + + if (!outBuf) + { + // outStream version works only in one thread. So we use CLzma2Enc::tempBufLzma + if (!me->tempBufLzma) { - t->enc = LzmaEnc_Create(p->alloc); - if (!t->enc) + me->tempBufLzma = (Byte *)ISzAlloc_Alloc(me->alloc, LZMA2_CHUNK_SIZE_COMPRESSED_MAX); + if (!me->tempBufLzma) return SZ_ERROR_MEM; } } - #ifndef _7ZIP_ST - if (p->props.numBlockThreads > 1) + RINOK(Lzma2EncInt_InitStream(p, &me->props)); + + for (;;) { - CMtCallbackImp mtCallback; + SRes res = SZ_OK; + size_t inSizeCur = 0; - mtCallback.funcTable.Code = MtCallbackImp_Code; - mtCallback.lzma2Enc = p; + Lzma2EncInt_InitBlock(p); + LimitedSeqInStream_Init(&limitedInStream); + limitedInStream.limit = me->props.blockSize; + + if (inStream) + { + UInt64 expected = (UInt64)(Int64)-1; + // inStream version works only in one thread. So we use CLzma2Enc::expectedDataSize + if (me->expectedDataSize != (UInt64)(Int64)-1 + && me->expectedDataSize >= unpackTotal) + expected = me->expectedDataSize - unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && expected > me->props.blockSize) + expected = (size_t)me->props.blockSize; + + LzmaEnc_SetDataSize(p->enc, expected); + + RINOK(LzmaEnc_PrepareForLzma2(p->enc, + &limitedInStream.vt, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + else + { + inSizeCur = inDataSize - (size_t)unpackTotal; + if (me->props.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + && inSizeCur > me->props.blockSize) + inSizeCur = (size_t)me->props.blockSize; + + // LzmaEnc_SetDataSize(p->enc, inSizeCur); + + RINOK(LzmaEnc_MemPrepare(p->enc, + inData + (size_t)unpackTotal, inSizeCur, + LZMA2_KEEP_WINDOW_SIZE, + me->alloc, + me->allocBig)); + } + + for (;;) + { + size_t packSize = LZMA2_CHUNK_SIZE_COMPRESSED_MAX; + if (outBuf) + packSize = outLim - (size_t)packTotal; + + res = Lzma2EncInt_EncodeSubblock(p, + outBuf ? outBuf + (size_t)packTotal : me->tempBufLzma, &packSize, + outBuf ? NULL : outStream); + + if (res != SZ_OK) + break; + + packTotal += packSize; + if (outBuf) + *outBufSize = (size_t)packTotal; + + res = Progress(progress, unpackTotal + p->srcPos, packTotal); + if (res != SZ_OK) + break; + + /* + if (LzmaEnc_GetNumAvailableBytes(p->enc) == 0) + break; + */ + + if (packSize == 0) + break; + } + + LzmaEnc_Finish(p->enc); + + unpackTotal += p->srcPos; + + RINOK(res); + + if (p->srcPos != (inStream ? limitedInStream.processed : inSizeCur)) + return SZ_ERROR_FAIL; + + if (inStream ? limitedInStream.finished : (unpackTotal == inDataSize)) + { + if (finished) + { + if (outBuf) + { + size_t destPos = *outBufSize; + if (destPos >= outLim) + return SZ_ERROR_OUTPUT_EOF; + outBuf[destPos] = 0; + *outBufSize = destPos + 1; + } + else + { + Byte b = 0; + if (ISeqOutStream_Write(outStream, &b, 1) != 1) + return SZ_ERROR_WRITE; + } + } + return SZ_OK; + } + } +} + + + +#ifndef _7ZIP_ST + +static SRes Lzma2Enc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t destSize = me->outBufSize; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + me->outBufsDataSizes[outBufIndex] = 0; + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + progressThunk.inSize = 0; + progressThunk.outSize = 0; + + res = Lzma2Enc_EncodeMt1(me, + &me->coders[coderIndex], + NULL, dest, &destSize, + NULL, src, srcSize, + finished, + &progressThunk.vt); + + me->outBufsDataSizes[outBufIndex] = destSize; + + return res; +} + + +static SRes Lzma2Enc_MtCallback_Write(void *pp, unsigned outBufIndex) +{ + CLzma2Enc *me = (CLzma2Enc *)pp; + size_t size = me->outBufsDataSizes[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + if (me->outStream) + return ISeqOutStream_Write(me->outStream, data, size) == size ? SZ_OK : SZ_ERROR_WRITE; + + if (size > me->outBuf_Rem) + return SZ_ERROR_OUTPUT_EOF; + memcpy(me->outBuf, data, size); + me->outBuf_Rem -= size; + me->outBuf += size; + return SZ_OK; +} + +#endif + + + +SRes Lzma2Enc_Encode2(CLzma2EncHandle pp, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress) +{ + CLzma2Enc *p = (CLzma2Enc *)pp; + + if (inStream && inData) + return SZ_ERROR_PARAM; + + if (outStream && outBuf) + return SZ_ERROR_PARAM; + + { + unsigned i; + for (i = 0; i < MTCODER__THREADS_MAX; i++) + p->coders[i].propsAreSet = False; + } + + #ifndef _7ZIP_ST + + if (p->props.numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) + { + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); + } + + vt.Code = Lzma2Enc_MtCallback_Code; + vt.Write = Lzma2Enc_MtCallback_Write; + + p->outStream = outStream; + p->outBuf = NULL; + p->outBuf_Rem = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBuf_Rem = *outBufSize; + *outBufSize = 0; + } + + p->mtCoder.allocBig = p->allocBig; p->mtCoder.progress = progress; p->mtCoder.inStream = inStream; - p->mtCoder.outStream = outStream; - p->mtCoder.alloc = p->alloc; - p->mtCoder.mtCallback = &mtCallback.funcTable; + p->mtCoder.inData = inData; + p->mtCoder.inDataSize = inDataSize; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + p->mtCoder.blockSize = (size_t)p->props.blockSize; + if (p->mtCoder.blockSize != p->props.blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ - p->mtCoder.blockSize = p->props.blockSize; - p->mtCoder.destBlockSize = p->props.blockSize + (p->props.blockSize >> 10) + 16; - if (p->mtCoder.destBlockSize < p->props.blockSize) { - p->mtCoder.destBlockSize = (size_t)0 - 1; - if (p->mtCoder.destBlockSize < p->props.blockSize) - return SZ_ERROR_FAIL; + size_t destBlockSize = p->mtCoder.blockSize + (p->mtCoder.blockSize >> 10) + 16; + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + Lzma2Enc_FreeOutBufs(p); + p->outBufSize = destBlockSize; } - p->mtCoder.numThreads = p->props.numBlockThreads; + + p->mtCoder.numThreadsMax = p->props.numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; - return MtCoder_Code(&p->mtCoder); + { + SRes res = MtCoder_Code(&p->mtCoder); + if (!outStream) + *outBufSize = p->outBuf - outBuf; + return res; + } } + #endif - return Lzma2Enc_EncodeMt1(&p->coders[0], p, outStream, inStream, progress); + + return Lzma2Enc_EncodeMt1(p, + &p->coders[0], + outStream, outBuf, outBufSize, + inStream, inData, inDataSize, + True, /* finished */ + progress); }
M
src/third-party/lzma/Lzma2Enc.h
→
src/third-party/lzma/Lzma2Enc.h
@@ -1,5 +1,5 @@
/* Lzma2Enc.h -- LZMA2 Encoder -2013-01-18 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __LZMA2_ENC_H #define __LZMA2_ENC_H@@ -8,11 +8,15 @@ #include "LzmaEnc.h"
EXTERN_C_BEGIN +#define LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO 0 +#define LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID ((UInt64)(Int64)-1) + typedef struct { CLzmaEncProps lzmaProps; - size_t blockSize; - int numBlockThreads; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; int numTotalThreads; } CLzma2EncProps;@@ -22,40 +26,29 @@
/* ---------- CLzmaEnc2Handle Interface ---------- */ /* Lzma2Enc_* functions can return the following exit codes: -Returns: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzma2EncHandle; -CLzma2EncHandle Lzma2Enc_Create(ISzAlloc *alloc, ISzAlloc *allocBig); +CLzma2EncHandle Lzma2Enc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); void Lzma2Enc_Destroy(CLzma2EncHandle p); SRes Lzma2Enc_SetProps(CLzma2EncHandle p, const CLzma2EncProps *props); +void Lzma2Enc_SetDataSize(CLzma2EncHandle p, UInt64 expectedDataSiize); Byte Lzma2Enc_WriteProperties(CLzma2EncHandle p); -SRes Lzma2Enc_Encode(CLzma2EncHandle p, - ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); - -/* ---------- One Call Interface ---------- */ - -/* Lzma2Encode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ - -/* -SRes Lzma2Encode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - const CLzmaEncProps *props, Byte *propsEncoded, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); -*/ +SRes Lzma2Enc_Encode2(CLzma2EncHandle p, + ISeqOutStream *outStream, + Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + const Byte *inData, size_t inDataSize, + ICompressProgress *progress); EXTERN_C_END
M
src/third-party/lzma/Lzma86Dec.c
→
src/third-party/lzma/Lzma86Dec.c
@@ -1,5 +1,5 @@
/* Lzma86Dec.c -- LZMA + x86 (BCJ) Filter Decoder -2015-11-08 : Igor Pavlov : Public domain */ +2016-05-16 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -8,9 +8,6 @@
#include "Alloc.h" #include "Bra.h" #include "LzmaDec.h" - -static void *SzAlloc(void *p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(void *p, void *address) { UNUSED_VAR(p); MyFree(address); } SRes Lzma86_GetUnpackSize(const Byte *src, SizeT srcLen, UInt64 *unpackSize) {@@ -25,7 +22,6 @@ }
SRes Lzma86_Decode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen) { - ISzAlloc g_Alloc = { SzAlloc, SzFree }; SRes res; int useFilter; SizeT inSizePure;
M
src/third-party/lzma/Lzma86Enc.c
→
src/third-party/lzma/Lzma86Enc.c
@@ -1,5 +1,5 @@
/* Lzma86Enc.c -- LZMA + x86 (BCJ) Filter Encoder -2015-11-08 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -13,16 +13,12 @@ #include "LzmaEnc.h"
#define SZE_OUT_OVERFLOW SZE_DATA_ERROR -static void *SzAlloc(void *p, size_t size) { UNUSED_VAR(p); return MyAlloc(size); } -static void SzFree(void *p, void *address) { UNUSED_VAR(p); MyFree(address); } - int Lzma86_Encode(Byte *dest, size_t *destLen, const Byte *src, size_t srcLen, int level, UInt32 dictSize, int filterMode) { - ISzAlloc g_Alloc = { SzAlloc, SzFree }; size_t outSize2 = *destLen; Byte *filteredStream; - Bool useFilter; + BoolInt useFilter; int mainResult = SZ_ERROR_OUTPUT_EOF; CLzmaEncProps props; LzmaEncProps_Init(&props);@@ -60,7 +56,7 @@ }
{ size_t minSize = 0; - Bool bestIsFiltered = False; + BoolInt bestIsFiltered = False; /* passes for SZ_FILTER_AUTO: 0 - BCJ + LZMA@@ -75,7 +71,7 @@ {
size_t outSizeProcessed = outSize2 - LZMA86_HEADER_SIZE; size_t outPropsSize = 5; SRes curRes; - Bool curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); + BoolInt curModeIsFiltered = (numPasses > 1 && i == numPasses - 1); if (curModeIsFiltered && !bestIsFiltered) break; if (useFilter && i == 0)
M
src/third-party/lzma/LzmaDec.c
→
src/third-party/lzma/LzmaDec.c
@@ -1,11 +1,12 @@
/* LzmaDec.c -- LZMA Decoder -2015-06-23 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" +#include <string.h> + +/* #include "CpuArch.h" */ #include "LzmaDec.h" - -#include <string.h> #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits)@@ -18,15 +19,22 @@ #define RC_INIT_SIZE 5
#define NORMALIZE if (range < kTopValue) { range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0(p) ttt = *(p); NORMALIZE; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0(p) range = bound; *(p) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); #define UPDATE_1(p) range -= bound; code -= bound; *(p) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); #define GET_BIT2(p, i, A0, A1) IF_BIT_0(p) \ { UPDATE_0(p); i = (i + i); A0; } else \ { UPDATE_1(p); i = (i + i) + 1; A1; } -#define GET_BIT(p, i) GET_BIT2(p, i, ; , ;) + +#define TREE_GET_BIT(probs, i) { GET_BIT2(probs + i, i, ;, ;); } -#define TREE_GET_BIT(probs, i) { GET_BIT((probs + i), i); } +#define REV_BIT(p, i, A0, A1) IF_BIT_0(p + i) \ + { UPDATE_0(p + i); A0; } else \ + { UPDATE_1(p + i); A1; } +#define REV_BIT_VAR( p, i, m) REV_BIT(p, i, i += m; m += m, m += m; i += m; ) +#define REV_BIT_CONST(p, i, m) REV_BIT(p, i, i += m; , i += m * 2; ) +#define REV_BIT_LAST( p, i, m) REV_BIT(p, i, i -= m , ; ) + #define TREE_DECODE(probs, limit, i) \ { i = 1; do { TREE_GET_BIT(probs, i); } while (i < limit); i -= limit; }@@ -46,16 +54,19 @@ TREE_GET_BIT(probs, i); \
i -= 0x40; } #endif -#define NORMAL_LITER_DEC GET_BIT(prob + symbol, symbol) +#define NORMAL_LITER_DEC TREE_GET_BIT(prob, symbol) #define MATCHED_LITER_DEC \ - matchByte <<= 1; \ - bit = (matchByte & offs); \ - probLit = prob + offs + bit + symbol; \ - GET_BIT2(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; \ + bit = offs; \ + offs &= matchByte; \ + probLit = prob + (offs + bit + symbol); \ + GET_BIT2(probLit, symbol, offs ^= bit; , ;) + + #define NORMALIZE_CHECK if (range < kTopValue) { if (buf >= bufLimit) return DUMMY_ERROR; range <<= 8; code = (code << 8) | (*buf++); } -#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * ttt; if (code < bound) +#define IF_BIT_0_CHECK(p) ttt = *(p); NORMALIZE_CHECK; bound = (range >> kNumBitModelTotalBits) * (UInt32)ttt; if (code < bound) #define UPDATE_0_CHECK range = bound; #define UPDATE_1_CHECK range -= bound; code -= bound; #define GET_BIT2_CHECK(p, i, A0, A1) IF_BIT_0_CHECK(p) \@@ -66,25 +77,28 @@ #define TREE_DECODE_CHECK(probs, limit, i) \
{ i = 1; do { GET_BIT_CHECK(probs + i, i) } while (i < limit); i -= limit; } +#define REV_BIT_CHECK(p, i, m) IF_BIT_0_CHECK(p + i) \ + { UPDATE_0_CHECK; i += m; m += m; } else \ + { UPDATE_1_CHECK; m += m; i += m; } + + #define kNumPosBitsMax 4 #define kNumPosStatesMax (1 << kNumPosBitsMax) #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) -#define LenChoice 0 -#define LenChoice2 (LenChoice + 1) -#define LenLow (LenChoice2 + 1) -#define LenMid (LenLow + (kNumPosStatesMax << kLenNumLowBits)) -#define LenHigh (LenMid + (kNumPosStatesMax << kLenNumMidBits)) +#define LenLow 0 +#define LenHigh (LenLow + 2 * (kNumPosStatesMax << kLenNumLowBits)) #define kNumLenProbs (LenHigh + kLenNumHighSymbols) +#define LenChoice LenLow +#define LenChoice2 (LenLow + (1 << kLenNumLowBits)) #define kNumStates 12 +#define kNumStates2 16 #define kNumLitStates 7 #define kStartPosModelIndex 4@@ -98,54 +112,117 @@ #define kNumAlignBits 4
#define kAlignTableSize (1 << kNumAlignBits) #define kMatchMinLen 2 -#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kMatchSpecLenStart (kMatchMinLen + kLenNumLowSymbols * 2 + kLenNumHighSymbols) -#define IsMatch 0 -#define IsRep (IsMatch + (kNumStates << kNumPosBitsMax)) +/* External ASM code needs same CLzmaProb array layout. So don't change it. */ + +/* (probs_1664) is faster and better for code size at some platforms */ +/* +#ifdef MY_CPU_X86_OR_AMD64 +*/ +#define kStartOffset 1664 +#define GET_PROBS p->probs_1664 +/* +#define GET_PROBS p->probs + kStartOffset +#else +#define kStartOffset 0 +#define GET_PROBS p->probs +#endif +*/ + +#define SpecPos (-kStartOffset) +#define IsRep0Long (SpecPos + kNumFullDistances) +#define RepLenCoder (IsRep0Long + (kNumStates2 << kNumPosBitsMax)) +#define LenCoder (RepLenCoder + kNumLenProbs) +#define IsMatch (LenCoder + kNumLenProbs) +#define Align (IsMatch + (kNumStates2 << kNumPosBitsMax)) +#define IsRep (Align + kAlignTableSize) #define IsRepG0 (IsRep + kNumStates) #define IsRepG1 (IsRepG0 + kNumStates) #define IsRepG2 (IsRepG1 + kNumStates) -#define IsRep0Long (IsRepG2 + kNumStates) -#define PosSlot (IsRep0Long + (kNumStates << kNumPosBitsMax)) -#define SpecPos (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) -#define Align (SpecPos + kNumFullDistances - kEndPosModelIndex) -#define LenCoder (Align + kAlignTableSize) -#define RepLenCoder (LenCoder + kNumLenProbs) -#define Literal (RepLenCoder + kNumLenProbs) +#define PosSlot (IsRepG2 + kNumStates) +#define Literal (PosSlot + (kNumLenToPosStates << kNumPosSlotBits)) +#define NUM_BASE_PROBS (Literal + kStartOffset) + +#if Align != 0 && kStartOffset != 0 + #error Stop_Compiling_Bad_LZMA_kAlign +#endif + +#if NUM_BASE_PROBS != 1984 + #error Stop_Compiling_Bad_LZMA_PROBS +#endif + -#define LZMA_BASE_SIZE 1846 #define LZMA_LIT_SIZE 0x300 -#if Literal != LZMA_BASE_SIZE -StopCompilingDueBUG -#endif +#define LzmaProps_GetNumProbs(p) (NUM_BASE_PROBS + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) + -#define LzmaProps_GetNumProbs(p) (Literal + ((UInt32)LZMA_LIT_SIZE << ((p)->lc + (p)->lp))) +#define CALC_POS_STATE(processedPos, pbMask) (((processedPos) & (pbMask)) << 4) +#define COMBINED_PS_STATE (posState + state) +#define GET_LEN_STATE (posState) #define LZMA_DIC_MIN (1 << 12) -/* First LZMA-symbol is always decoded. -And it decodes new LZMA-symbols while (buf < bufLimit), but "buf" is without last normalization +/* +p->remainLen : shows status of LZMA decoder: + < kMatchSpecLenStart : normal remain + = kMatchSpecLenStart : finished + = kMatchSpecLenStart + 1 : need init range coder + = kMatchSpecLenStart + 2 : need init range coder and state +*/ + +/* ---------- LZMA_DECODE_REAL ---------- */ +/* +LzmaDec_DecodeReal_3() can be implemented in external ASM file. +3 - is the code compatibility version of that function for check at link time. +*/ + +#define LZMA_DECODE_REAL LzmaDec_DecodeReal_3 + +/* +LZMA_DECODE_REAL() +In: + RangeCoder is normalized + if (p->dicPos == limit) + { + LzmaDec_TryDummy() was called before to exclude LITERAL and MATCH-REP cases. + So first symbol can be only MATCH-NON-REP. And if that MATCH-NON-REP symbol + is not END_OF_PAYALOAD_MARKER, then function returns error code. + } + +Processing: + first LZMA symbol will be decoded in any case + All checks for limits are at the end of main loop, + It will decode new LZMA-symbols while (p->buf < bufLimit && dicPos < limit), + RangeCoder is still without last normalization when (p->buf < bufLimit) is being checked. + Out: + RangeCoder is normalized Result: SZ_OK - OK SZ_ERROR_DATA - Error p->remainLen: < kMatchSpecLenStart : normal remain = kMatchSpecLenStart : finished - = kMatchSpecLenStart + 1 : Flush marker (unused now) - = kMatchSpecLenStart + 2 : State Init Marker (unused now) */ -static int MY_FAST_CALL LzmaDec_DecodeReal(CLzmaDec *p, SizeT limit, const Byte *bufLimit) + +#ifdef _LZMA_DEC_OPT + +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit); + +#else + +static +int MY_FAST_CALL LZMA_DECODE_REAL(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { - CLzmaProb *probs = p->probs; - - unsigned state = p->state; + CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; UInt32 rep0 = p->reps[0], rep1 = p->reps[1], rep2 = p->reps[2], rep3 = p->reps[3]; unsigned pbMask = ((unsigned)1 << (p->prop.pb)) - 1; - unsigned lpMask = ((unsigned)1 << (p->prop.lp)) - 1; unsigned lc = p->prop.lc; + unsigned lpMask = ((unsigned)0x100 << p->prop.lp) - ((unsigned)0x100 >> lc); Byte *dic = p->dic; SizeT dicBufSize = p->dicBufSize;@@ -164,17 +241,16 @@ {
CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = processedPos & pbMask; + unsigned posState = CALC_POS_STATE(processedPos, pbMask); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0(prob) { unsigned symbol; UPDATE_0(prob); prob = probs + Literal; if (processedPos != 0 || checkDicSize != 0) - prob += ((UInt32)LZMA_LIT_SIZE * (((processedPos & lpMask) << lc) + - (dic[(dicPos == 0 ? dicBufSize : dicPos) - 1] >> (8 - lc)))); + prob += (UInt32)3 * ((((processedPos << 8) + dic[(dicPos == 0 ? dicBufSize : dicPos) - 1]) & lpMask) << lc); processedPos++; if (state < kNumLitStates)@@ -240,13 +316,16 @@ }
else { UPDATE_1(prob); + /* + // that case was checked before with kBadRepCode if (checkDicSize == 0 && processedPos == 0) return SZ_ERROR_DATA; + */ prob = probs + IsRepG0 + state; IF_BIT_0(prob) { UPDATE_0(prob); - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0(prob) { UPDATE_0(prob);@@ -294,14 +373,14 @@ }
#ifdef _LZMA_SIZE_OPT { - unsigned limit, offset; + unsigned lim, offset; CLzmaProb *probLen = prob + LenChoice; IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; - limit = (1 << kLenNumLowBits); + lim = (1 << kLenNumLowBits); } else {@@ -310,19 +389,19 @@ probLen = prob + LenChoice2;
IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = (1 << kLenNumMidBits); + lim = (1 << kLenNumLowBits); } else { UPDATE_1(probLen); probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; - limit = (1 << kLenNumHighBits); + offset = kLenNumLowSymbols * 2; + lim = (1 << kLenNumHighBits); } } - TREE_DECODE(probLen, limit, len); + TREE_DECODE(probLen, lim, len); len += offset; } #else@@ -331,7 +410,7 @@ CLzmaProb *probLen = prob + LenChoice;
IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len);@@ -345,7 +424,7 @@ probLen = prob + LenChoice2;
IF_BIT_0(probLen) { UPDATE_0(probLen); - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); len = 1; TREE_GET_BIT(probLen, len); TREE_GET_BIT(probLen, len);@@ -356,7 +435,7 @@ {
UPDATE_1(probLen); probLen = prob + LenHigh; TREE_DECODE(probLen, (1 << kLenNumHighBits), len); - len += kLenNumLowSymbols + kLenNumMidSymbols; + len += kLenNumLowSymbols * 2; } } }@@ -376,16 +455,16 @@ distance = (2 | (distance & 1));
if (posSlot < kEndPosModelIndex) { distance <<= numDirectBits; - prob = probs + SpecPos + distance - posSlot - 1; + prob = probs + SpecPos; { - UInt32 mask = 1; - unsigned i = 1; + UInt32 m = 1; + distance++; do { - GET_BIT2(prob + i, i, ; , distance |= mask); - mask <<= 1; + REV_BIT_VAR(prob, distance, m); } - while (--numDirectBits != 0); + while (--numDirectBits); + distance -= m; } } else@@ -412,19 +491,20 @@ distance |= 1;
} */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; distance <<= kNumAlignBits; { unsigned i = 1; - GET_BIT2(prob + i, i, ; , distance |= 1); - GET_BIT2(prob + i, i, ; , distance |= 2); - GET_BIT2(prob + i, i, ; , distance |= 4); - GET_BIT2(prob + i, i, ; , distance |= 8); + REV_BIT_CONST(prob, i, 1); + REV_BIT_CONST(prob, i, 2); + REV_BIT_CONST(prob, i, 4); + REV_BIT_LAST (prob, i, 8); + distance |= i; } if (distance == (UInt32)0xFFFFFFFF) { - len += kMatchSpecLenStart; + len = kMatchSpecLenStart; state -= kNumStates; break; }@@ -435,20 +515,12 @@ rep3 = rep2;
rep2 = rep1; rep1 = rep0; rep0 = distance + 1; - if (checkDicSize == 0) - { - if (distance >= processedPos) - { - p->dicPos = dicPos; - return SZ_ERROR_DATA; - } - } - else if (distance >= checkDicSize) + state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; + if (distance >= (checkDicSize == 0 ? processedPos: checkDicSize)) { p->dicPos = dicPos; return SZ_ERROR_DATA; } - state = (state < kNumStates + kNumLitStates) ? kNumLitStates : kNumLitStates + 3; } len += kMatchMinLen;@@ -467,7 +539,7 @@
curLen = ((rem < len) ? (unsigned)rem : len); pos = dicPos - rep0 + (dicPos < rep0 ? dicBufSize : 0); - processedPos += curLen; + processedPos += (UInt32)curLen; len -= curLen; if (curLen <= dicBufSize - pos)@@ -475,7 +547,7 @@ {
Byte *dest = dic + dicPos; ptrdiff_t src = (ptrdiff_t)pos - (ptrdiff_t)dicPos; const Byte *lim = dest + curLen; - dicPos += curLen; + dicPos += (SizeT)curLen; do *(dest) = (Byte)*(dest + src); while (++dest != lim);@@ -500,17 +572,18 @@
p->buf = buf; p->range = range; p->code = code; - p->remainLen = len; + p->remainLen = (UInt32)len; p->dicPos = dicPos; p->processedPos = processedPos; p->reps[0] = rep0; p->reps[1] = rep1; p->reps[2] = rep2; p->reps[3] = rep3; - p->state = state; + p->state = (UInt32)state; return SZ_OK; } +#endif static void MY_FAST_CALL LzmaDec_WriteRem(CLzmaDec *p, SizeT limit) {@@ -519,7 +592,7 @@ {
Byte *dic = p->dic; SizeT dicPos = p->dicPos; SizeT dicBufSize = p->dicBufSize; - unsigned len = p->remainLen; + unsigned len = (unsigned)p->remainLen; SizeT rep0 = p->reps[0]; /* we use SizeT to avoid the BUG of VC14 for AMD64 */ SizeT rem = limit - dicPos; if (rem < len)@@ -528,8 +601,8 @@
if (p->checkDicSize == 0 && p->prop.dicSize - p->processedPos <= len) p->checkDicSize = p->prop.dicSize; - p->processedPos += len; - p->remainLen -= len; + p->processedPos += (UInt32)len; + p->remainLen -= (UInt32)len; while (len != 0) { len--;@@ -540,6 +613,14 @@ p->dicPos = dicPos;
} } + +#define kRange0 0xFFFFFFFF +#define kBound0 ((kRange0 >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1)) +#define kBadRepCode (kBound0 + (((kRange0 - kBound0) >> kNumBitModelTotalBits) << (kNumBitModelTotalBits - 1))) +#if kBadRepCode != (0xC0000000 - 0x400) + #error Stop_Compiling_Bad_LZMA_Check +#endif + static int MY_FAST_CALL LzmaDec_DecodeReal2(CLzmaDec *p, SizeT limit, const Byte *bufLimit) { do@@ -550,9 +631,13 @@ {
UInt32 rem = p->prop.dicSize - p->processedPos; if (limit - p->dicPos > rem) limit2 = p->dicPos + rem; + + if (p->processedPos == 0) + if (p->code >= kBadRepCode) + return SZ_ERROR_DATA; } - - RINOK(LzmaDec_DecodeReal(p, limit2, bufLimit)); + + RINOK(LZMA_DECODE_REAL(p, limit2, bufLimit)); if (p->checkDicSize == 0 && p->processedPos >= p->prop.dicSize) p->checkDicSize = p->prop.dicSize;@@ -560,9 +645,6 @@
LzmaDec_WriteRem(p, limit); } while (p->dicPos < limit && p->buf < bufLimit && p->remainLen < kMatchSpecLenStart); - - if (p->remainLen > kMatchSpecLenStart) - p->remainLen = kMatchSpecLenStart; return 0; }@@ -580,17 +662,17 @@ {
UInt32 range = p->range; UInt32 code = p->code; const Byte *bufLimit = buf + inSize; - const CLzmaProb *probs = p->probs; - unsigned state = p->state; + const CLzmaProb *probs = GET_PROBS; + unsigned state = (unsigned)p->state; ELzmaDummy res; { const CLzmaProb *prob; UInt32 bound; unsigned ttt; - unsigned posState = (p->processedPos) & ((1 << p->prop.pb) - 1); + unsigned posState = CALC_POS_STATE(p->processedPos, (1 << p->prop.pb) - 1); - prob = probs + IsMatch + (state << kNumPosBitsMax) + posState; + prob = probs + IsMatch + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK@@ -618,10 +700,11 @@ do
{ unsigned bit; const CLzmaProb *probLit; - matchByte <<= 1; - bit = (matchByte & offs); - probLit = prob + offs + bit + symbol; - GET_BIT2_CHECK(probLit, symbol, offs &= ~bit, offs &= bit) + matchByte += matchByte; + bit = offs; + offs &= matchByte; + probLit = prob + (offs + bit + symbol); + GET_BIT2_CHECK(probLit, symbol, offs ^= bit; , ; ) } while (symbol < 0x100); }@@ -648,7 +731,7 @@ prob = probs + IsRepG0 + state;
IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK; - prob = probs + IsRep0Long + (state << kNumPosBitsMax) + posState; + prob = probs + IsRep0Long + COMBINED_PS_STATE; IF_BIT_0_CHECK(prob) { UPDATE_0_CHECK;@@ -691,7 +774,7 @@ const CLzmaProb *probLen = prob + LenChoice;
IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenLow + (posState << kLenNumLowBits); + probLen = prob + LenLow + GET_LEN_STATE; offset = 0; limit = 1 << kLenNumLowBits; }@@ -702,15 +785,15 @@ probLen = prob + LenChoice2;
IF_BIT_0_CHECK(probLen) { UPDATE_0_CHECK; - probLen = prob + LenMid + (posState << kLenNumMidBits); + probLen = prob + LenLow + GET_LEN_STATE + (1 << kLenNumLowBits); offset = kLenNumLowSymbols; - limit = 1 << kLenNumMidBits; + limit = 1 << kLenNumLowBits; } else { UPDATE_1_CHECK; probLen = prob + LenHigh; - offset = kLenNumLowSymbols + kLenNumMidSymbols; + offset = kLenNumLowSymbols * 2; limit = 1 << kLenNumHighBits; } }@@ -722,7 +805,7 @@ if (state < 4)
{ unsigned posSlot; prob = probs + PosSlot + - ((len < kNumLenToPosStates ? len : kNumLenToPosStates - 1) << + ((len < kNumLenToPosStates - 1 ? len : kNumLenToPosStates - 1) << kNumPosSlotBits); TREE_DECODE_CHECK(prob, 1 << kNumPosSlotBits, posSlot); if (posSlot >= kStartPosModelIndex)@@ -733,7 +816,7 @@ /* if (bufLimit - buf >= 8) return DUMMY_MATCH; */
if (posSlot < kEndPosModelIndex) { - prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits) - posSlot - 1; + prob = probs + SpecPos + ((2 | (posSlot & 1)) << numDirectBits); } else {@@ -745,17 +828,18 @@ range >>= 1;
code -= range & (((code - range) >> 31) - 1); /* if (code >= range) code -= range; */ } - while (--numDirectBits != 0); + while (--numDirectBits); prob = probs + Align; numDirectBits = kNumAlignBits; } { unsigned i = 1; + unsigned m = 1; do { - GET_BIT_CHECK(prob + i, i); + REV_BIT_CHECK(prob, i, m); } - while (--numDirectBits != 0); + while (--numDirectBits); } } }@@ -766,20 +850,19 @@ return res;
} -void LzmaDec_InitDicAndState(CLzmaDec *p, Bool initDic, Bool initState) +void LzmaDec_InitDicAndState(CLzmaDec *p, BoolInt initDic, BoolInt initState) { - p->needFlush = 1; - p->remainLen = 0; + p->remainLen = kMatchSpecLenStart + 1; p->tempBufSize = 0; if (initDic) { p->processedPos = 0; p->checkDicSize = 0; - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } if (initState) - p->needInitState = 1; + p->remainLen = kMatchSpecLenStart + 2; } void LzmaDec_Init(CLzmaDec *p)@@ -788,53 +871,54 @@ p->dicPos = 0;
LzmaDec_InitDicAndState(p, True, True); } -static void LzmaDec_InitStateReal(CLzmaDec *p) -{ - SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); - SizeT i; - CLzmaProb *probs = p->probs; - for (i = 0; i < numProbs; i++) - probs[i] = kBitModelTotal >> 1; - p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; - p->state = 0; - p->needInitState = 0; -} SRes LzmaDec_DecodeToDic(CLzmaDec *p, SizeT dicLimit, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) { SizeT inSize = *srcLen; (*srcLen) = 0; - LzmaDec_WriteRem(p, dicLimit); *status = LZMA_STATUS_NOT_SPECIFIED; - while (p->remainLen != kMatchSpecLenStart) + if (p->remainLen > kMatchSpecLenStart) { - int checkEndMarkNow; + for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) + p->tempBuf[p->tempBufSize++] = *src++; + if (p->tempBufSize != 0 && p->tempBuf[0] != 0) + return SZ_ERROR_DATA; + if (p->tempBufSize < RC_INIT_SIZE) + { + *status = LZMA_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } + p->code = + ((UInt32)p->tempBuf[1] << 24) + | ((UInt32)p->tempBuf[2] << 16) + | ((UInt32)p->tempBuf[3] << 8) + | ((UInt32)p->tempBuf[4]); + p->range = 0xFFFFFFFF; + p->tempBufSize = 0; + + if (p->remainLen > kMatchSpecLenStart + 1) + { + SizeT numProbs = LzmaProps_GetNumProbs(&p->prop); + SizeT i; + CLzmaProb *probs = p->probs; + for (i = 0; i < numProbs; i++) + probs[i] = kBitModelTotal >> 1; + p->reps[0] = p->reps[1] = p->reps[2] = p->reps[3] = 1; + p->state = 0; + } - if (p->needFlush) - { - for (; inSize > 0 && p->tempBufSize < RC_INIT_SIZE; (*srcLen)++, inSize--) - p->tempBuf[p->tempBufSize++] = *src++; - if (p->tempBufSize < RC_INIT_SIZE) - { - *status = LZMA_STATUS_NEEDS_MORE_INPUT; - return SZ_OK; - } - if (p->tempBuf[0] != 0) - return SZ_ERROR_DATA; - p->code = - ((UInt32)p->tempBuf[1] << 24) - | ((UInt32)p->tempBuf[2] << 16) - | ((UInt32)p->tempBuf[3] << 8) - | ((UInt32)p->tempBuf[4]); - p->range = 0xFFFFFFFF; - p->needFlush = 0; - p->tempBufSize = 0; - } + p->remainLen = 0; + } + + LzmaDec_WriteRem(p, dicLimit); + + while (p->remainLen != kMatchSpecLenStart) + { + int checkEndMarkNow = 0; - checkEndMarkNow = 0; if (p->dicPos >= dicLimit) { if (p->remainLen == 0 && p->code == 0)@@ -855,9 +939,6 @@ }
checkEndMarkNow = 1; } - if (p->needInitState) - LzmaDec_InitStateReal(p); - if (p->tempBufSize == 0) { SizeT processed;@@ -898,10 +979,10 @@ p->tempBuf[rem++] = src[lookAhead++];
p->tempBufSize = rem; if (rem < LZMA_REQUIRED_INPUT_MAX || checkEndMarkNow) { - int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, rem); + int dummyRes = LzmaDec_TryDummy(p, p->tempBuf, (SizeT)rem); if (dummyRes == DUMMY_ERROR) { - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; *status = LZMA_STATUS_NEEDS_MORE_INPUT; return SZ_OK; }@@ -924,16 +1005,19 @@ if (lookAhead < rem)
return SZ_ERROR_FAIL; /* some internal error */ lookAhead -= rem; } - (*srcLen) += lookAhead; + (*srcLen) += (SizeT)lookAhead; src += lookAhead; - inSize -= lookAhead; + inSize -= (SizeT)lookAhead; p->tempBufSize = 0; } } - if (p->code == 0) - *status = LZMA_STATUS_FINISHED_WITH_MARK; - return (p->code == 0) ? SZ_OK : SZ_ERROR_DATA; + + if (p->code != 0) + return SZ_ERROR_DATA; + *status = LZMA_STATUS_FINISHED_WITH_MARK; + return SZ_OK; } + SRes LzmaDec_DecodeToBuf(CLzmaDec *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, ELzmaFinishMode finishMode, ELzmaStatus *status) {@@ -975,19 +1059,19 @@ return SZ_OK;
} } -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->probs); + ISzAlloc_Free(alloc, p->probs); p->probs = NULL; } -static void LzmaDec_FreeDict(CLzmaDec *p, ISzAlloc *alloc) +static void LzmaDec_FreeDict(CLzmaDec *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->dic); + ISzAlloc_Free(alloc, p->dic); p->dic = NULL; } -void LzmaDec_Free(CLzmaDec *p, ISzAlloc *alloc) +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc) { LzmaDec_FreeProbs(p, alloc); LzmaDec_FreeDict(p, alloc);@@ -1011,29 +1095,30 @@ d = data[0];
if (d >= (9 * 5 * 5)) return SZ_ERROR_UNSUPPORTED; - p->lc = d % 9; + p->lc = (Byte)(d % 9); d /= 9; - p->pb = d / 5; - p->lp = d % 5; + p->pb = (Byte)(d / 5); + p->lp = (Byte)(d % 5); return SZ_OK; } -static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAlloc *alloc) +static SRes LzmaDec_AllocateProbs2(CLzmaDec *p, const CLzmaProps *propNew, ISzAllocPtr alloc) { UInt32 numProbs = LzmaProps_GetNumProbs(propNew); if (!p->probs || numProbs != p->numProbs) { LzmaDec_FreeProbs(p, alloc); - p->probs = (CLzmaProb *)alloc->Alloc(alloc, numProbs * sizeof(CLzmaProb)); - p->numProbs = numProbs; + p->probs = (CLzmaProb *)ISzAlloc_Alloc(alloc, numProbs * sizeof(CLzmaProb)); if (!p->probs) return SZ_ERROR_MEM; + p->probs_1664 = p->probs + 1664; + p->numProbs = numProbs; } return SZ_OK; } -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) { CLzmaProps propNew; RINOK(LzmaProps_Decode(&propNew, props, propsSize));@@ -1042,7 +1127,7 @@ p->prop = propNew;
return SZ_OK; } -SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc) +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc) { CLzmaProps propNew; SizeT dicBufSize;@@ -1062,7 +1147,7 @@
if (!p->dic || dicBufSize != p->dicBufSize) { LzmaDec_FreeDict(p, alloc); - p->dic = (Byte *)alloc->Alloc(alloc, dicBufSize); + p->dic = (Byte *)ISzAlloc_Alloc(alloc, dicBufSize); if (!p->dic) { LzmaDec_FreeProbs(p, alloc);@@ -1076,7 +1161,7 @@ }
SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc) + ELzmaStatus *status, ISzAllocPtr alloc) { CLzmaDec p; SRes res;
M
src/third-party/lzma/LzmaDec.h
→
src/third-party/lzma/LzmaDec.h
@@ -1,5 +1,5 @@
/* LzmaDec.h -- LZMA Decoder -2013-01-18 : Igor Pavlov : Public domain */ +2018-04-21 : Igor Pavlov : Public domain */ #ifndef __LZMA_DEC_H #define __LZMA_DEC_H@@ -12,11 +12,13 @@ /* #define _LZMA_PROB32 */
/* _LZMA_PROB32 can increase the speed on some CPUs, but memory usage for CLzmaDec::probs will be doubled in that case */ +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; /* ---------- LZMA Properties ---------- */@@ -25,7 +27,10 @@ #define LZMA_PROPS_SIZE 5
typedef struct _CLzmaProps { - unsigned lc, lp, pb; + Byte lc; + Byte lp; + Byte pb; + Byte _pad_; UInt32 dicSize; } CLzmaProps;@@ -47,32 +52,34 @@ #define LZMA_REQUIRED_INPUT_MAX 20
typedef struct { + /* Don't change this structure. ASM code can use it. */ CLzmaProps prop; CLzmaProb *probs; + CLzmaProb *probs_1664; Byte *dic; + SizeT dicBufSize; + SizeT dicPos; const Byte *buf; - UInt32 range, code; - SizeT dicPos; - SizeT dicBufSize; + UInt32 range; + UInt32 code; UInt32 processedPos; UInt32 checkDicSize; - unsigned state; UInt32 reps[4]; - unsigned remainLen; - int needFlush; - int needInitState; + UInt32 state; + UInt32 remainLen; + UInt32 numProbs; unsigned tempBufSize; Byte tempBuf[LZMA_REQUIRED_INPUT_MAX]; } CLzmaDec; -#define LzmaDec_Construct(p) { (p)->dic = 0; (p)->probs = 0; } +#define LzmaDec_Construct(p) { (p)->dic = NULL; (p)->probs = NULL; } void LzmaDec_Init(CLzmaDec *p); /* There are two types of LZMA streams: - 0) Stream with end mark. That end mark adds about 6 bytes to compressed size. - 1) Stream without end mark. You must know exact uncompressed size to decompress such stream. */ + - Stream with end mark. That end mark adds about 6 bytes to compressed size. + - Stream without end mark. You must know exact uncompressed size to decompress such stream. */ typedef enum {@@ -129,11 +136,11 @@ SZ_ERROR_MEM - Memory allocation error
SZ_ERROR_UNSUPPORTED - Unsupported properties */ -SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_FreeProbs(CLzmaDec *p, ISzAlloc *alloc); +SRes LzmaDec_AllocateProbs(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_FreeProbs(CLzmaDec *p, ISzAllocPtr alloc); -SRes LzmaDec_Allocate(CLzmaDec *state, const Byte *prop, unsigned propsSize, ISzAlloc *alloc); -void LzmaDec_Free(CLzmaDec *state, ISzAlloc *alloc); +SRes LzmaDec_Allocate(CLzmaDec *p, const Byte *props, unsigned propsSize, ISzAllocPtr alloc); +void LzmaDec_Free(CLzmaDec *p, ISzAllocPtr alloc); /* ---------- Dictionary Interface ---------- */@@ -142,7 +149,7 @@ dictionary to some other external buffer.
You must work with CLzmaDec variables directly in this interface. STEPS: - LzmaDec_Constr() + LzmaDec_Construct() LzmaDec_Allocate() for (each new stream) {@@ -220,7 +227,7 @@ */
SRes LzmaDecode(Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, const Byte *propData, unsigned propSize, ELzmaFinishMode finishMode, - ELzmaStatus *status, ISzAlloc *alloc); + ELzmaStatus *status, ISzAllocPtr alloc); EXTERN_C_END
M
src/third-party/lzma/LzmaEnc.c
→
src/third-party/lzma/LzmaEnc.c
@@ -1,5 +1,5 @@
/* LzmaEnc.c -- LZMA Encoder -2015-11-08 : Igor Pavlov : Public domain */ +2019-01-10: Igor Pavlov : Public domain */ #include "Precomp.h"@@ -23,17 +23,8 @@ #ifdef SHOW_STAT
static unsigned g_STAT_OFFSET = 0; #endif -#define kMaxHistorySize ((UInt32)3 << 29) -/* #define kMaxHistorySize ((UInt32)7 << 29) */ - -#define kBlockSizeMax ((1 << LZMA_NUM_BLOCK_SIZE_BITS) - 1) - -#define kBlockSize (9 << 10) -#define kUnpackBlockSize (1 << 18) -#define kMatchArraySize (1 << 21) -#define kMatchRecordMaxSize ((LZMA_MATCH_LEN_MAX * 2 + 3) * LZMA_MATCH_LEN_MAX) - -#define kNumMaxDirectBits (31) +#define kLzmaMaxHistorySize ((UInt32)3 << 29) +/* #define kLzmaMaxHistorySize ((UInt32)7 << 29) */ #define kNumTopBits 24 #define kTopValue ((UInt32)1 << kNumTopBits)@@ -47,6 +38,8 @@ #define kNumMoveReducingBits 4
#define kNumBitPriceShiftBits 4 #define kBitPrice (1 << kNumBitPriceShiftBits) +#define REP_LEN_COUNT 64 + void LzmaEncProps_Init(CLzmaEncProps *p) { p->level = 5;@@ -62,14 +55,15 @@ int level = p->level;
if (level < 0) level = 5; p->level = level; - if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level == 6 ? (1 << 25) : (1 << 26))); + if (p->dictSize == 0) p->dictSize = (level <= 5 ? (1 << (level * 2 + 14)) : (level <= 7 ? (1 << 25) : (1 << 26))); if (p->dictSize > p->reduceSize) { unsigned i; + UInt32 reduceSize = (UInt32)p->reduceSize; for (i = 11; i <= 30; i++) { - if ((UInt32)p->reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } - if ((UInt32)p->reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } + if (reduceSize <= ((UInt32)2 << i)) { p->dictSize = ((UInt32)2 << i); break; } + if (reduceSize <= ((UInt32)3 << i)) { p->dictSize = ((UInt32)3 << i); break; } } }@@ -108,11 +102,11 @@ #ifdef LZMA_LOG_BSR
#define kDicLogSizeMaxCompress 32 -#define BSR2_RET(pos, res) { unsigned long i; _BitScanReverse(&i, (pos)); res = (i + i) + ((pos >> (i - 1)) & 1); } +#define BSR2_RET(pos, res) { unsigned long zz; _BitScanReverse(&zz, (pos)); res = (zz + zz) + ((pos >> (zz - 1)) & 1); } -static UInt32 GetPosSlot1(UInt32 pos) +static unsigned GetPosSlot1(UInt32 pos) { - UInt32 res; + unsigned res; BSR2_RET(pos, res); return res; }@@ -145,19 +139,19 @@ }
/* we can use ((limit - pos) >> 31) only if (pos < ((UInt32)1 << 31)) */ /* -#define BSR2_RET(pos, res) { UInt32 i = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits + 6)) - 1) - pos) >> 31))); \ - res = p->g_FastPos[pos >> i] + (i * 2); } + res = p->g_FastPos[pos >> zz] + (zz * 2); } */ /* -#define BSR2_RET(pos, res) { UInt32 i = 6 + ((kNumLogBits - 1) & \ +#define BSR2_RET(pos, res) { unsigned zz = 6 + ((kNumLogBits - 1) & \ (0 - (((((UInt32)1 << (kNumLogBits)) - 1) - (pos >> 6)) >> 31))); \ - res = p->g_FastPos[pos >> i] + (i * 2); } + res = p->g_FastPos[pos >> zz] + (zz * 2); } */ -#define BSR2_RET(pos, res) { UInt32 i = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ - res = p->g_FastPos[pos >> i] + (i * 2); } +#define BSR2_RET(pos, res) { unsigned zz = (pos < (1 << (kNumLogBits + 6))) ? 6 : 6 + kNumLogBits - 1; \ + res = p->g_FastPos[pos >> zz] + (zz * 2); } /* #define BSR2_RET(pos, res) { res = (pos < (1 << (kNumLogBits + 6))) ? \@@ -167,39 +161,41 @@ */
#define GetPosSlot1(pos) p->g_FastPos[pos] #define GetPosSlot2(pos, res) { BSR2_RET(pos, res); } -#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos]; else BSR2_RET(pos, res); } +#define GetPosSlot(pos, res) { if (pos < kNumFullDistances) res = p->g_FastPos[pos & (kNumFullDistances - 1)]; else BSR2_RET(pos, res); } #endif #define LZMA_NUM_REPS 4 -typedef unsigned CState; +typedef UInt16 CState; +typedef UInt16 CExtra; typedef struct { UInt32 price; - CState state; - int prev1IsChar; - int prev2; + CExtra extra; + // 0 : normal + // 1 : LIT : MATCH + // > 1 : MATCH (extra-1) : LIT : REP0 (len) + UInt32 len; + UInt32 dist; + UInt32 reps[LZMA_NUM_REPS]; +} COptimal; - UInt32 posPrev2; - UInt32 backPrev2; - UInt32 posPrev; - UInt32 backPrev; - UInt32 backs[LZMA_NUM_REPS]; -} COptimal; - -#define kNumOpts (1 << 12) +// 18.06 +#define kNumOpts (1 << 11) +#define kPackReserve (kNumOpts * 8) +// #define kNumOpts (1 << 12) +// #define kPackReserve (1 + kNumOpts * 2) #define kNumLenToPosStates 4 #define kNumPosSlotBits 6 #define kDicLogSizeMin 0 #define kDicLogSizeMax 32 #define kDistTableSizeMax (kDicLogSizeMax * 2) - #define kNumAlignBits 4 #define kAlignTableSize (1 << kNumAlignBits)@@ -207,15 +203,15 @@ #define kAlignMask (kAlignTableSize - 1)
#define kStartPosModelIndex 4 #define kEndPosModelIndex 14 -#define kNumPosModels (kEndPosModelIndex - kStartPosModelIndex) - #define kNumFullDistances (1 << (kEndPosModelIndex >> 1)) +typedef #ifdef _LZMA_PROB32 -#define CLzmaProb UInt32 + UInt32 #else -#define CLzmaProb UInt16 + UInt16 #endif + CLzmaProb; #define LZMA_PB_MAX 4 #define LZMA_LC_MAX 8@@ -223,15 +219,11 @@ #define LZMA_LP_MAX 4
#define LZMA_NUM_PB_STATES_MAX (1 << LZMA_PB_MAX) - #define kLenNumLowBits 3 #define kLenNumLowSymbols (1 << kLenNumLowBits) -#define kLenNumMidBits 3 -#define kLenNumMidSymbols (1 << kLenNumMidBits) #define kLenNumHighBits 8 #define kLenNumHighSymbols (1 << kLenNumHighBits) - -#define kLenNumSymbolsTotal (kLenNumLowSymbols + kLenNumMidSymbols + kLenNumHighSymbols) +#define kLenNumSymbolsTotal (kLenNumLowSymbols * 2 + kLenNumHighSymbols) #define LZMA_MATCH_LEN_MIN 2 #define LZMA_MATCH_LEN_MAX (LZMA_MATCH_LEN_MIN + kLenNumSymbolsTotal - 1)@@ -241,27 +233,31 @@
typedef struct { - CLzmaProb choice; - CLzmaProb choice2; - CLzmaProb low[LZMA_NUM_PB_STATES_MAX << kLenNumLowBits]; - CLzmaProb mid[LZMA_NUM_PB_STATES_MAX << kLenNumMidBits]; + CLzmaProb low[LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)]; CLzmaProb high[kLenNumHighSymbols]; } CLenEnc; typedef struct { - CLenEnc p; - UInt32 tableSize; + unsigned tableSize; UInt32 prices[LZMA_NUM_PB_STATES_MAX][kLenNumSymbolsTotal]; - UInt32 counters[LZMA_NUM_PB_STATES_MAX]; + // UInt32 prices1[LZMA_NUM_PB_STATES_MAX][kLenNumLowSymbols * 2]; + // UInt32 prices2[kLenNumSymbolsTotal]; } CLenPriceEnc; +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices[posState][(size_t)(len) - LZMA_MATCH_LEN_MIN]) + +/* +#define GET_PRICE_LEN(p, posState, len) \ + ((p)->prices2[(size_t)(len) - 2] + ((p)->prices1[posState][((len) - 2) & (kLenNumLowSymbols * 2 - 1)] & (((len) - 2 - kLenNumLowSymbols * 2) >> 9))) +*/ typedef struct { UInt32 range; - Byte cache; + unsigned cache; UInt64 low; UInt64 cacheSize; Byte *buf;@@ -277,23 +273,27 @@ typedef struct
{ CLzmaProb *litProbs; - UInt32 state; + unsigned state; UInt32 reps[LZMA_NUM_REPS]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; - CLenPriceEnc lenEnc; - CLenPriceEnc repLenEnc; + CLenEnc lenProbs; + CLenEnc repLenProbs; + } CSaveState; + + +typedef UInt32 CProbPrice; typedef struct@@ -301,45 +301,49 @@ {
void *matchFinderObj; IMatchFinder matchFinder; - UInt32 optimumEndIndex; - UInt32 optimumCurrentIndex; + unsigned optCur; + unsigned optEnd; - UInt32 longestMatchLength; - UInt32 numPairs; + unsigned longestMatchLen; + unsigned numPairs; UInt32 numAvail; - UInt32 numFastBytes; - UInt32 additionalOffset; + unsigned state; + unsigned numFastBytes; + unsigned additionalOffset; UInt32 reps[LZMA_NUM_REPS]; - UInt32 state; + unsigned lpMask, pbMask; + CLzmaProb *litProbs; + CRangeEnc rc; + + UInt32 backRes; unsigned lc, lp, pb; - unsigned lpMask, pbMask; unsigned lclp; - CLzmaProb *litProbs; - - Bool fastMode; - Bool writeEndMark; - Bool finished; - Bool multiThread; - Bool needInit; + BoolInt fastMode; + BoolInt writeEndMark; + BoolInt finished; + BoolInt multiThread; + BoolInt needInit; + // BoolInt _maxMode; UInt64 nowPos64; - UInt32 matchPriceCount; - UInt32 alignPriceCount; + unsigned matchPriceCount; + // unsigned alignPriceCount; + int repLenEncCounter; - UInt32 distTableSize; + unsigned distTableSize; UInt32 dictSize; SRes result; - - CRangeEnc rc; #ifndef _7ZIP_ST - Bool mtMode; + BoolInt mtMode; + // begin of CMatchFinderMt is used in LZ thread CMatchFinderMt matchFinderMt; + // end of CMatchFinderMt is used in BT and HASH threads #endif CMatchFinder matchFinderBase;@@ -348,32 +352,36 @@ #ifndef _7ZIP_ST
Byte pad[128]; #endif - COptimal opt[kNumOpts]; - - #ifndef LZMA_LOG_BSR - Byte g_FastPos[1 << kNumLogBits]; - #endif + // LZ thread + CProbPrice ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; - UInt32 ProbPrices[kBitModelTotal >> kNumMoveReducingBits]; UInt32 matches[LZMA_MATCH_LEN_MAX * 2 + 2 + 1]; + UInt32 alignPrices[kAlignTableSize]; UInt32 posSlotPrices[kNumLenToPosStates][kDistTableSizeMax]; UInt32 distancesPrices[kNumLenToPosStates][kNumFullDistances]; - UInt32 alignPrices[kAlignTableSize]; - CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; + CLzmaProb posAlignEncoder[1 << kNumAlignBits]; CLzmaProb isRep[kNumStates]; CLzmaProb isRepG0[kNumStates]; CLzmaProb isRepG1[kNumStates]; CLzmaProb isRepG2[kNumStates]; + CLzmaProb isMatch[kNumStates][LZMA_NUM_PB_STATES_MAX]; CLzmaProb isRep0Long[kNumStates][LZMA_NUM_PB_STATES_MAX]; - CLzmaProb posSlotEncoder[kNumLenToPosStates][1 << kNumPosSlotBits]; - CLzmaProb posEncoders[kNumFullDistances - kEndPosModelIndex]; - CLzmaProb posAlignEncoder[1 << kNumAlignBits]; + CLzmaProb posEncoders[kNumFullDistances]; + CLenEnc lenProbs; + CLenEnc repLenProbs; + + #ifndef LZMA_LOG_BSR + Byte g_FastPos[1 << kNumLogBits]; + #endif + CLenPriceEnc lenEnc; CLenPriceEnc repLenEnc; + + COptimal opt[kNumOpts]; CSaveState saveState;@@ -381,59 +389,63 @@ #ifndef _7ZIP_ST
Byte pad2[128]; #endif } CLzmaEnc; + +#define COPY_ARR(dest, src, arr) memcpy(dest->arr, src->arr, sizeof(src->arr)); + void LzmaEnc_SaveState(CLzmaEncHandle pp) { CLzmaEnc *p = (CLzmaEnc *)pp; CSaveState *dest = &p->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; + + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << p->lclp) * sizeof(CLzmaProb)); } + void LzmaEnc_RestoreState(CLzmaEncHandle pp) { CLzmaEnc *dest = (CLzmaEnc *)pp; const CSaveState *p = &dest->saveState; - int i; - dest->lenEnc = p->lenEnc; - dest->repLenEnc = p->repLenEnc; + dest->state = p->state; - for (i = 0; i < kNumStates; i++) - { - memcpy(dest->isMatch[i], p->isMatch[i], sizeof(p->isMatch[i])); - memcpy(dest->isRep0Long[i], p->isRep0Long[i], sizeof(p->isRep0Long[i])); - } - for (i = 0; i < kNumLenToPosStates; i++) - memcpy(dest->posSlotEncoder[i], p->posSlotEncoder[i], sizeof(p->posSlotEncoder[i])); - memcpy(dest->isRep, p->isRep, sizeof(p->isRep)); - memcpy(dest->isRepG0, p->isRepG0, sizeof(p->isRepG0)); - memcpy(dest->isRepG1, p->isRepG1, sizeof(p->isRepG1)); - memcpy(dest->isRepG2, p->isRepG2, sizeof(p->isRepG2)); - memcpy(dest->posEncoders, p->posEncoders, sizeof(p->posEncoders)); - memcpy(dest->posAlignEncoder, p->posAlignEncoder, sizeof(p->posAlignEncoder)); - memcpy(dest->reps, p->reps, sizeof(p->reps)); + dest->lenProbs = p->lenProbs; + dest->repLenProbs = p->repLenProbs; + + COPY_ARR(dest, p, reps); + + COPY_ARR(dest, p, posAlignEncoder); + COPY_ARR(dest, p, isRep); + COPY_ARR(dest, p, isRepG0); + COPY_ARR(dest, p, isRepG1); + COPY_ARR(dest, p, isRepG2); + COPY_ARR(dest, p, isMatch); + COPY_ARR(dest, p, isRep0Long); + COPY_ARR(dest, p, posSlotEncoder); + COPY_ARR(dest, p, posEncoders); + memcpy(dest->litProbs, p->litProbs, ((UInt32)0x300 << dest->lclp) * sizeof(CLzmaProb)); } + + SRes LzmaEnc_SetProps(CLzmaEncHandle pp, const CLzmaEncProps *props2) {@@ -445,7 +457,7 @@ if (props.lc > LZMA_LC_MAX
|| props.lp > LZMA_LP_MAX || props.pb > LZMA_PB_MAX || props.dictSize > ((UInt64)1 << kDicLogSizeMaxCompress) - || props.dictSize > kMaxHistorySize) + || props.dictSize > kLzmaMaxHistorySize) return SZ_ERROR_PARAM; p->dictSize = props.dictSize;@@ -461,9 +473,10 @@ p->lc = props.lc;
p->lp = props.lp; p->pb = props.pb; p->fastMode = (props.algo == 0); + // p->_maxMode = True; p->matchFinderBase.btMode = (Byte)(props.btMode ? 1 : 0); { - UInt32 numHashBytes = 4; + unsigned numHashBytes = 4; if (props.btMode) { if (props.numHashBytes < 2)@@ -492,13 +505,27 @@
return SZ_OK; } -static const int kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; -static const int kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; -static const int kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; -static const int kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; + +void LzmaEnc_SetDataSize(CLzmaEncHandle pp, UInt64 expectedDataSiize) +{ + CLzmaEnc *p = (CLzmaEnc *)pp; + p->matchFinderBase.expectedDataSize = expectedDataSiize; +} + + +#define kState_Start 0 +#define kState_LitAfterMatch 4 +#define kState_LitAfterRep 5 +#define kState_MatchAfterLit 7 +#define kState_RepAfterLit 8 -#define IsCharState(s) ((s) < 7) +static const Byte kLiteralNextStates[kNumStates] = {0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5}; +static const Byte kMatchNextStates[kNumStates] = {7, 7, 7, 7, 7, 7, 7, 10, 10, 10, 10, 10}; +static const Byte kRepNextStates[kNumStates] = {8, 8, 8, 8, 8, 8, 8, 11, 11, 11, 11, 11}; +static const Byte kShortRepNextStates[kNumStates]= {9, 9, 9, 9, 9, 9, 9, 11, 11, 11, 11, 11}; +#define IsLitState(s) ((s) < 7) +#define GetLenToPosState2(len) (((len) < kNumLenToPosStates - 1) ? (len) : kNumLenToPosStates - 1) #define GetLenToPosState(len) (((len) < kNumLenToPosStates + 1) ? (len) - 2 : kNumLenToPosStates - 1) #define kInfinityPrice (1 << 30)@@ -509,14 +536,16 @@ p->outStream = NULL;
p->bufBase = NULL; } -#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed(p) ((p)->processed + ((p)->buf - (p)->bufBase) + (p)->cacheSize) +#define RangeEnc_GetProcessed_sizet(p) ((size_t)(p)->processed + ((p)->buf - (p)->bufBase) + (size_t)(p)->cacheSize) #define RC_BUF_SIZE (1 << 16) -static int RangeEnc_Alloc(CRangeEnc *p, ISzAlloc *alloc) + +static int RangeEnc_Alloc(CRangeEnc *p, ISzAllocPtr alloc) { if (!p->bufBase) { - p->bufBase = (Byte *)alloc->Alloc(alloc, RC_BUF_SIZE); + p->bufBase = (Byte *)ISzAlloc_Alloc(alloc, RC_BUF_SIZE); if (!p->bufBase) return 0; p->bufLim = p->bufBase + RC_BUF_SIZE;@@ -524,19 +553,19 @@ }
return 1; } -static void RangeEnc_Free(CRangeEnc *p, ISzAlloc *alloc) +static void RangeEnc_Free(CRangeEnc *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->bufBase); + ISzAlloc_Free(alloc, p->bufBase); p->bufBase = 0; } static void RangeEnc_Init(CRangeEnc *p) { /* Stream.Init(); */ - p->low = 0; p->range = 0xFFFFFFFF; - p->cacheSize = 1; p->cache = 0; + p->low = 0; + p->cacheSize = 0; p->buf = p->bufBase;@@ -544,37 +573,48 @@ p->processed = 0;
p->res = SZ_OK; } -static void RangeEnc_FlushStream(CRangeEnc *p) +MY_NO_INLINE static void RangeEnc_FlushStream(CRangeEnc *p) { size_t num; if (p->res != SZ_OK) return; num = p->buf - p->bufBase; - if (num != p->outStream->Write(p->outStream, p->bufBase, num)) + if (num != ISeqOutStream_Write(p->outStream, p->bufBase, num)) p->res = SZ_ERROR_WRITE; p->processed += num; p->buf = p->bufBase; } -static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) +MY_NO_INLINE static void MY_FAST_CALL RangeEnc_ShiftLow(CRangeEnc *p) { - if ((UInt32)p->low < (UInt32)0xFF000000 || (unsigned)(p->low >> 32) != 0) + UInt32 low = (UInt32)p->low; + unsigned high = (unsigned)(p->low >> 32); + p->low = (UInt32)(low << 8); + if (low < (UInt32)0xFF000000 || high != 0) { - Byte temp = p->cache; - do + { + Byte *buf = p->buf; + *buf++ = (Byte)(p->cache + high); + p->cache = (unsigned)(low >> 24); + p->buf = buf; + if (buf == p->bufLim) + RangeEnc_FlushStream(p); + if (p->cacheSize == 0) + return; + } + high += 0xFF; + for (;;) { Byte *buf = p->buf; - *buf++ = (Byte)(temp + (Byte)(p->low >> 32)); + *buf++ = (Byte)(high); p->buf = buf; if (buf == p->bufLim) RangeEnc_FlushStream(p); - temp = 0xFF; + if (--p->cacheSize == 0) + return; } - while (--p->cacheSize != 0); - p->cache = (Byte)((UInt32)p->low >> 24); } p->cacheSize++; - p->low = (UInt32)p->low << 8; } static void RangeEnc_FlushData(CRangeEnc *p)@@ -584,78 +624,121 @@ for (i = 0; i < 5; i++)
RangeEnc_ShiftLow(p); } -static void RangeEnc_EncodeDirectBits(CRangeEnc *p, UInt32 value, unsigned numBits) -{ - do - { - p->range >>= 1; - p->low += p->range & (0 - ((value >> --numBits) & 1)); - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } +#define RC_NORM(p) if (range < kTopValue) { range <<= 8; RangeEnc_ShiftLow(p); } + +#define RC_BIT_PRE(p, prob) \ + ttt = *(prob); \ + newBound = (range >> kNumBitModelTotalBits) * ttt; + +// #define _LZMA_ENC_USE_BRANCH + +#ifdef _LZMA_ENC_USE_BRANCH + +#define RC_BIT(p, prob, bit) { \ + RC_BIT_PRE(p, prob) \ + if (bit == 0) { range = newBound; ttt += (kBitModelTotal - ttt) >> kNumMoveBits; } \ + else { (p)->low += newBound; range -= newBound; ttt -= ttt >> kNumMoveBits; } \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ } - while (numBits != 0); -} + +#else + +#define RC_BIT(p, prob, bit) { \ + UInt32 mask; \ + RC_BIT_PRE(p, prob) \ + mask = 0 - (UInt32)bit; \ + range &= mask; \ + mask &= newBound; \ + range -= mask; \ + (p)->low += mask; \ + mask = (UInt32)bit - 1; \ + range += newBound & mask; \ + mask &= (kBitModelTotal - ((1 << kNumMoveBits) - 1)); \ + mask += ((1 << kNumMoveBits) - 1); \ + ttt += (Int32)(mask - ttt) >> kNumMoveBits; \ + *(prob) = (CLzmaProb)ttt; \ + RC_NORM(p) \ + } + +#endif + + + + +#define RC_BIT_0_BASE(p, prob) \ + range = newBound; *(prob) = (CLzmaProb)(ttt + ((kBitModelTotal - ttt) >> kNumMoveBits)); + +#define RC_BIT_1_BASE(p, prob) \ + range -= newBound; (p)->low += newBound; *(prob) = (CLzmaProb)(ttt - (ttt >> kNumMoveBits)); \ + +#define RC_BIT_0(p, prob) \ + RC_BIT_0_BASE(p, prob) \ + RC_NORM(p) + +#define RC_BIT_1(p, prob) \ + RC_BIT_1_BASE(p, prob) \ + RC_NORM(p) -static void RangeEnc_EncodeBit(CRangeEnc *p, CLzmaProb *prob, UInt32 symbol) +static void RangeEnc_EncodeBit_0(CRangeEnc *p, CLzmaProb *prob) { - UInt32 ttt = *prob; - UInt32 newBound = (p->range >> kNumBitModelTotalBits) * ttt; - if (symbol == 0) - { - p->range = newBound; - ttt += (kBitModelTotal - ttt) >> kNumMoveBits; - } - else - { - p->low += newBound; - p->range -= newBound; - ttt -= ttt >> kNumMoveBits; - } - *prob = (CLzmaProb)ttt; - if (p->range < kTopValue) - { - p->range <<= 8; - RangeEnc_ShiftLow(p); - } + UInt32 range, ttt, newBound; + range = p->range; + RC_BIT_PRE(p, prob) + RC_BIT_0(p, prob) + p->range = range; } -static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol) +static void LitEnc_Encode(CRangeEnc *p, CLzmaProb *probs, UInt32 sym) { - symbol |= 0x100; + UInt32 range = p->range; + sym |= 0x100; do { - RangeEnc_EncodeBit(p, probs + (symbol >> 8), (symbol >> 7) & 1); - symbol <<= 1; + UInt32 ttt, newBound; + // RangeEnc_EncodeBit(p, probs + (sym >> 8), (sym >> 7) & 1); + CLzmaProb *prob = probs + (sym >> 8); + UInt32 bit = (sym >> 7) & 1; + sym <<= 1; + RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); + p->range = range; } -static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 symbol, UInt32 matchByte) +static void LitEnc_EncodeMatched(CRangeEnc *p, CLzmaProb *probs, UInt32 sym, UInt32 matchByte) { + UInt32 range = p->range; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { + UInt32 ttt, newBound; + CLzmaProb *prob; + UInt32 bit; matchByte <<= 1; - RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (symbol >> 8)), (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + // RangeEnc_EncodeBit(p, probs + (offs + (matchByte & offs) + (sym >> 8)), (sym >> 7) & 1); + prob = probs + (offs + (matchByte & offs) + (sym >> 8)); + bit = (sym >> 7) & 1; + sym <<= 1; + offs &= ~(matchByte ^ sym); + RC_BIT(p, prob, bit); } - while (symbol < 0x10000); + while (sym < 0x10000); + p->range = range; } -static void LzmaEnc_InitPriceTables(UInt32 *ProbPrices) + + +static void LzmaEnc_InitPriceTables(CProbPrice *ProbPrices) { UInt32 i; - for (i = (1 << kNumMoveReducingBits) / 2; i < kBitModelTotal; i += (1 << kNumMoveReducingBits)) + for (i = 0; i < (kBitModelTotal >> kNumMoveReducingBits); i++) { - const int kCyclesBits = kNumBitPriceShiftBits; - UInt32 w = i; - UInt32 bitCount = 0; - int j; + const unsigned kCyclesBits = kNumBitPriceShiftBits; + UInt32 w = (i << kNumMoveReducingBits) + (1 << (kNumMoveReducingBits - 1)); + unsigned bitCount = 0; + unsigned j; for (j = 0; j < kCyclesBits; j++) { w = w * w;@@ -666,559 +749,655 @@ w >>= 1;
bitCount++; } } - ProbPrices[i >> kNumMoveReducingBits] = ((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + ProbPrices[i] = (CProbPrice)((kNumBitModelTotalBits << kCyclesBits) - 15 - bitCount); + // printf("\n%3d: %5d", i, ProbPrices[i]); } } -#define GET_PRICE(prob, symbol) \ - p->ProbPrices[((prob) ^ (((-(int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICE(prob, bit) \ + p->ProbPrices[((prob) ^ (unsigned)(((-(int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; -#define GET_PRICEa(prob, symbol) \ - ProbPrices[((prob) ^ ((-((int)(symbol))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; +#define GET_PRICEa(prob, bit) \ + ProbPrices[((prob) ^ (unsigned)((-((int)(bit))) & (kBitModelTotal - 1))) >> kNumMoveReducingBits]; #define GET_PRICE_0(prob) p->ProbPrices[(prob) >> kNumMoveReducingBits] #define GET_PRICE_1(prob) p->ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] -#define GET_PRICE_0a(prob) ProbPrices[(prob) >> kNumMoveReducingBits] -#define GET_PRICE_1a(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] +#define GET_PRICEa_0(prob) ProbPrices[(prob) >> kNumMoveReducingBits] +#define GET_PRICEa_1(prob) ProbPrices[((prob) ^ (kBitModelTotal - 1)) >> kNumMoveReducingBits] + -static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 symbol, const UInt32 *ProbPrices) +static UInt32 LitEnc_GetPrice(const CLzmaProb *probs, UInt32 sym, const CProbPrice *ProbPrices) { UInt32 price = 0; - symbol |= 0x100; + sym |= 0x100; do { - price += GET_PRICEa(probs[symbol >> 8], (symbol >> 7) & 1); - symbol <<= 1; + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); } - while (symbol < 0x10000); + while (sym >= 2); return price; } -static UInt32 LitEnc_GetPriceMatched(const CLzmaProb *probs, UInt32 symbol, UInt32 matchByte, const UInt32 *ProbPrices) + +static UInt32 LitEnc_Matched_GetPrice(const CLzmaProb *probs, UInt32 sym, UInt32 matchByte, const CProbPrice *ProbPrices) { UInt32 price = 0; UInt32 offs = 0x100; - symbol |= 0x100; + sym |= 0x100; do { matchByte <<= 1; - price += GET_PRICEa(probs[offs + (matchByte & offs) + (symbol >> 8)], (symbol >> 7) & 1); - symbol <<= 1; - offs &= ~(matchByte ^ symbol); + price += GET_PRICEa(probs[offs + (matchByte & offs) + (sym >> 8)], (sym >> 7) & 1); + sym <<= 1; + offs &= ~(matchByte ^ sym); } - while (symbol < 0x10000); + while (sym < 0x10000); return price; } -static void RcTree_Encode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) +static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, unsigned numBits, unsigned sym) { - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0;) + UInt32 range = rc->range; + unsigned m = 1; + do { - UInt32 bit; - i--; - bit = (symbol >> i) & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); + UInt32 ttt, newBound; + unsigned bit = sym & 1; + // RangeEnc_EncodeBit(rc, probs + m, bit); + sym >>= 1; + RC_BIT(rc, probs + m, bit); m = (m << 1) | bit; } + while (--numBits); + rc->range = range; } -static void RcTree_ReverseEncode(CRangeEnc *rc, CLzmaProb *probs, int numBitLevels, UInt32 symbol) -{ - UInt32 m = 1; - int i; - for (i = 0; i < numBitLevels; i++) - { - UInt32 bit = symbol & 1; - RangeEnc_EncodeBit(rc, probs + m, bit); - m = (m << 1) | bit; - symbol >>= 1; - } -} - -static UInt32 RcTree_GetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - symbol |= (1 << numBitLevels); - while (symbol != 1) - { - price += GET_PRICEa(probs[symbol >> 1], symbol & 1); - symbol >>= 1; - } - return price; -} - -static UInt32 RcTree_ReverseGetPrice(const CLzmaProb *probs, int numBitLevels, UInt32 symbol, const UInt32 *ProbPrices) -{ - UInt32 price = 0; - UInt32 m = 1; - int i; - for (i = numBitLevels; i != 0; i--) - { - UInt32 bit = symbol & 1; - symbol >>= 1; - price += GET_PRICEa(probs[m], bit); - m = (m << 1) | bit; - } - return price; -} static void LenEnc_Init(CLenEnc *p) { unsigned i; - p->choice = p->choice2 = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumLowBits); i++) + for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << (kLenNumLowBits + 1)); i++) p->low[i] = kProbInitValue; - for (i = 0; i < (LZMA_NUM_PB_STATES_MAX << kLenNumMidBits); i++) - p->mid[i] = kProbInitValue; for (i = 0; i < kLenNumHighSymbols; i++) p->high[i] = kProbInitValue; } -static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState) +static void LenEnc_Encode(CLenEnc *p, CRangeEnc *rc, unsigned sym, unsigned posState) { - if (symbol < kLenNumLowSymbols) + UInt32 range, ttt, newBound; + CLzmaProb *probs = p->low; + range = rc->range; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols) { - RangeEnc_EncodeBit(rc, &p->choice, 0); - RcTree_Encode(rc, p->low + (posState << kLenNumLowBits), kLenNumLowBits, symbol); + RC_BIT_1(rc, probs); + probs += kLenNumLowSymbols; + RC_BIT_PRE(rc, probs); + if (sym >= kLenNumLowSymbols * 2) + { + RC_BIT_1(rc, probs); + rc->range = range; + // RcTree_Encode(rc, p->high, kLenNumHighBits, sym - kLenNumLowSymbols * 2); + LitEnc_Encode(rc, p->high, sym - kLenNumLowSymbols * 2); + return; + } + sym -= kLenNumLowSymbols; } - else + + // RcTree_Encode(rc, probs + (posState << kLenNumLowBits), kLenNumLowBits, sym); { - RangeEnc_EncodeBit(rc, &p->choice, 1); - if (symbol < kLenNumLowSymbols + kLenNumMidSymbols) - { - RangeEnc_EncodeBit(rc, &p->choice2, 0); - RcTree_Encode(rc, p->mid + (posState << kLenNumMidBits), kLenNumMidBits, symbol - kLenNumLowSymbols); - } - else - { - RangeEnc_EncodeBit(rc, &p->choice2, 1); - RcTree_Encode(rc, p->high, kLenNumHighBits, symbol - kLenNumLowSymbols - kLenNumMidSymbols); - } + unsigned m; + unsigned bit; + RC_BIT_0(rc, probs); + probs += (posState << (1 + kLenNumLowBits)); + bit = (sym >> 2) ; RC_BIT(rc, probs + 1, bit); m = (1 << 1) + bit; + bit = (sym >> 1) & 1; RC_BIT(rc, probs + m, bit); m = (m << 1) + bit; + bit = sym & 1; RC_BIT(rc, probs + m, bit); + rc->range = range; } } -static void LenEnc_SetPrices(CLenEnc *p, UInt32 posState, UInt32 numSymbols, UInt32 *prices, const UInt32 *ProbPrices) +static void SetPrices_3(const CLzmaProb *probs, UInt32 startPrice, UInt32 *prices, const CProbPrice *ProbPrices) { - UInt32 a0 = GET_PRICE_0a(p->choice); - UInt32 a1 = GET_PRICE_1a(p->choice); - UInt32 b0 = a1 + GET_PRICE_0a(p->choice2); - UInt32 b1 = a1 + GET_PRICE_1a(p->choice2); - UInt32 i = 0; - for (i = 0; i < kLenNumLowSymbols; i++) + unsigned i; + for (i = 0; i < 8; i += 2) { - if (i >= numSymbols) - return; - prices[i] = a0 + RcTree_GetPrice(p->low + (posState << kLenNumLowBits), kLenNumLowBits, i, ProbPrices); + UInt32 price = startPrice; + UInt32 prob; + price += GET_PRICEa(probs[1 ], (i >> 2)); + price += GET_PRICEa(probs[2 + (i >> 2)], (i >> 1) & 1); + prob = probs[4 + (i >> 1)]; + prices[i ] = price + GET_PRICEa_0(prob); + prices[i + 1] = price + GET_PRICEa_1(prob); } - for (; i < kLenNumLowSymbols + kLenNumMidSymbols; i++) - { - if (i >= numSymbols) - return; - prices[i] = b0 + RcTree_GetPrice(p->mid + (posState << kLenNumMidBits), kLenNumMidBits, i - kLenNumLowSymbols, ProbPrices); - } - for (; i < numSymbols; i++) - prices[i] = b1 + RcTree_GetPrice(p->high, kLenNumHighBits, i - kLenNumLowSymbols - kLenNumMidSymbols, ProbPrices); } -static void MY_FAST_CALL LenPriceEnc_UpdateTable(CLenPriceEnc *p, UInt32 posState, const UInt32 *ProbPrices) -{ - LenEnc_SetPrices(&p->p, posState, p->tableSize, p->prices[posState], ProbPrices); - p->counters[posState] = p->tableSize; -} -static void LenPriceEnc_UpdateTables(CLenPriceEnc *p, UInt32 numPosStates, const UInt32 *ProbPrices) +MY_NO_INLINE static void MY_FAST_CALL LenPriceEnc_UpdateTables( + CLenPriceEnc *p, + unsigned numPosStates, + const CLenEnc *enc, + const CProbPrice *ProbPrices) { - UInt32 posState; - for (posState = 0; posState < numPosStates; posState++) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} + UInt32 b; + + { + unsigned prob = enc->low[0]; + UInt32 a, c; + unsigned posState; + b = GET_PRICEa_1(prob); + a = GET_PRICEa_0(prob); + c = b + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (posState = 0; posState < numPosStates; posState++) + { + UInt32 *prices = p->prices[posState]; + const CLzmaProb *probs = enc->low + (posState << (1 + kLenNumLowBits)); + SetPrices_3(probs, a, prices, ProbPrices); + SetPrices_3(probs + kLenNumLowSymbols, c, prices + kLenNumLowSymbols, ProbPrices); + } + } -static void LenEnc_Encode2(CLenPriceEnc *p, CRangeEnc *rc, UInt32 symbol, UInt32 posState, Bool updatePrice, const UInt32 *ProbPrices) -{ - LenEnc_Encode(&p->p, rc, symbol, posState); - if (updatePrice) - if (--p->counters[posState] == 0) - LenPriceEnc_UpdateTable(p, posState, ProbPrices); -} + /* + { + unsigned i; + UInt32 b; + a = GET_PRICEa_0(enc->low[0]); + for (i = 0; i < kLenNumLowSymbols; i++) + p->prices2[i] = a; + a = GET_PRICEa_1(enc->low[0]); + b = a + GET_PRICEa_0(enc->low[kLenNumLowSymbols]); + for (i = kLenNumLowSymbols; i < kLenNumLowSymbols * 2; i++) + p->prices2[i] = b; + a += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + } + */ + + // p->counter = numSymbols; + // p->counter = 64; + { + unsigned i = p->tableSize; + + if (i > kLenNumLowSymbols * 2) + { + const CLzmaProb *probs = enc->high; + UInt32 *prices = p->prices[0] + kLenNumLowSymbols * 2; + i -= kLenNumLowSymbols * 2 - 1; + i >>= 1; + b += GET_PRICEa_1(enc->low[kLenNumLowSymbols]); + do + { + /* + p->prices2[i] = a + + // RcTree_GetPrice(enc->high, kLenNumHighBits, i - kLenNumLowSymbols * 2, ProbPrices); + LitEnc_GetPrice(probs, i - kLenNumLowSymbols * 2, ProbPrices); + */ + // UInt32 price = a + RcTree_GetPrice(probs, kLenNumHighBits - 1, sym, ProbPrices); + unsigned sym = --i + (1 << (kLenNumHighBits - 1)); + UInt32 price = b; + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[sym], bit); + } + while (sym >= 2); + { + unsigned prob = probs[(size_t)i + (1 << (kLenNumHighBits - 1))]; + prices[(size_t)i * 2 ] = price + GET_PRICEa_0(prob); + prices[(size_t)i * 2 + 1] = price + GET_PRICEa_1(prob); + } + } + while (i); + { + unsigned posState; + size_t num = (p->tableSize - kLenNumLowSymbols * 2) * sizeof(p->prices[0][0]); + for (posState = 1; posState < numPosStates; posState++) + memcpy(p->prices[posState] + kLenNumLowSymbols * 2, p->prices[0] + kLenNumLowSymbols * 2, num); + } + } + } +} -static void MovePos(CLzmaEnc *p, UInt32 num) -{ +/* #ifdef SHOW_STAT g_STAT_OFFSET += num; printf("\n MovePos %u", num); #endif +*/ - if (num != 0) - { - p->additionalOffset += num; - p->matchFinder.Skip(p->matchFinderObj, num); - } -} +#define MOVE_POS(p, num) { \ + p->additionalOffset += (num); \ + p->matchFinder.Skip(p->matchFinderObj, (UInt32)(num)); } -static UInt32 ReadMatchDistances(CLzmaEnc *p, UInt32 *numDistancePairsRes) + +static unsigned ReadMatchDistances(CLzmaEnc *p, unsigned *numPairsRes) { - UInt32 lenRes = 0, numPairs; + unsigned numPairs; + + p->additionalOffset++; p->numAvail = p->matchFinder.GetNumAvailableBytes(p->matchFinderObj); numPairs = p->matchFinder.GetMatches(p->matchFinderObj, p->matches); + *numPairsRes = numPairs; #ifdef SHOW_STAT printf("\n i = %u numPairs = %u ", g_STAT_OFFSET, numPairs / 2); g_STAT_OFFSET++; { - UInt32 i; + unsigned i; for (i = 0; i < numPairs; i += 2) printf("%2u %6u | ", p->matches[i], p->matches[i + 1]); } #endif - if (numPairs > 0) + if (numPairs == 0) + return 0; { - lenRes = p->matches[numPairs - 2]; - if (lenRes == p->numFastBytes) + unsigned len = p->matches[(size_t)numPairs - 2]; + if (len != p->numFastBytes) + return len; { UInt32 numAvail = p->numAvail; if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; { - const Byte *pbyCur = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - const Byte *pby = pbyCur + lenRes; - ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[numPairs - 1]; - const Byte *pbyLim = pbyCur + numAvail; - for (; pby != pbyLim && *pby == pby[dif]; pby++); - lenRes = (UInt32)(pby - pbyCur); + const Byte *p1 = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + const Byte *p2 = p1 + len; + ptrdiff_t dif = (ptrdiff_t)-1 - p->matches[(size_t)numPairs - 1]; + const Byte *lim = p1 + numAvail; + for (; p2 != lim && *p2 == p2[dif]; p2++) + {} + return (unsigned)(p2 - p1); } } } - p->additionalOffset++; - *numDistancePairsRes = numPairs; - return lenRes; } +#define MARK_LIT ((UInt32)(Int32)-1) -#define MakeAsChar(p) (p)->backPrev = (UInt32)(-1); (p)->prev1IsChar = False; -#define MakeAsShortRep(p) (p)->backPrev = 0; (p)->prev1IsChar = False; -#define IsShortRep(p) ((p)->backPrev == 0) +#define MakeAs_Lit(p) { (p)->dist = MARK_LIT; (p)->extra = 0; } +#define MakeAs_ShortRep(p) { (p)->dist = 0; (p)->extra = 0; } +#define IsShortRep(p) ((p)->dist == 0) -static UInt32 GetRepLen1Price(CLzmaEnc *p, UInt32 state, UInt32 posState) -{ - return - GET_PRICE_0(p->isRepG0[state]) + - GET_PRICE_0(p->isRep0Long[state][posState]); -} + +#define GetPrice_ShortRep(p, state, posState) \ + ( GET_PRICE_0(p->isRepG0[state]) + GET_PRICE_0(p->isRep0Long[state][posState])) -static UInt32 GetPureRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 state, UInt32 posState) +#define GetPrice_Rep_0(p, state, posState) ( \ + GET_PRICE_1(p->isMatch[state][posState]) \ + + GET_PRICE_1(p->isRep0Long[state][posState])) \ + + GET_PRICE_1(p->isRep[state]) \ + + GET_PRICE_0(p->isRepG0[state]) + +MY_FORCE_INLINE +static UInt32 GetPrice_PureRep(const CLzmaEnc *p, unsigned repIndex, size_t state, size_t posState) { UInt32 price; + UInt32 prob = p->isRepG0[state]; if (repIndex == 0) { - price = GET_PRICE_0(p->isRepG0[state]); + price = GET_PRICE_0(prob); price += GET_PRICE_1(p->isRep0Long[state][posState]); } else { - price = GET_PRICE_1(p->isRepG0[state]); + price = GET_PRICE_1(prob); + prob = p->isRepG1[state]; if (repIndex == 1) - price += GET_PRICE_0(p->isRepG1[state]); + price += GET_PRICE_0(prob); else { - price += GET_PRICE_1(p->isRepG1[state]); + price += GET_PRICE_1(prob); price += GET_PRICE(p->isRepG2[state], repIndex - 2); } } return price; } -static UInt32 GetRepPrice(CLzmaEnc *p, UInt32 repIndex, UInt32 len, UInt32 state, UInt32 posState) -{ - return p->repLenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN] + - GetPureRepPrice(p, repIndex, state, posState); -} -static UInt32 Backward(CLzmaEnc *p, UInt32 *backRes, UInt32 cur) +static unsigned Backward(CLzmaEnc *p, unsigned cur) { - UInt32 posMem = p->opt[cur].posPrev; - UInt32 backMem = p->opt[cur].backPrev; - p->optimumEndIndex = cur; - do + unsigned wr = cur + 1; + p->optEnd = wr; + + for (;;) { - if (p->opt[cur].prev1IsChar) + UInt32 dist = p->opt[cur].dist; + unsigned len = (unsigned)p->opt[cur].len; + unsigned extra = (unsigned)p->opt[cur].extra; + cur -= len; + + if (extra) { - MakeAsChar(&p->opt[posMem]) - p->opt[posMem].posPrev = posMem - 1; - if (p->opt[cur].prev2) + wr--; + p->opt[wr].len = (UInt32)len; + cur -= extra; + len = extra; + if (extra == 1) + { + p->opt[wr].dist = dist; + dist = MARK_LIT; + } + else { - p->opt[posMem - 1].prev1IsChar = False; - p->opt[posMem - 1].posPrev = p->opt[cur].posPrev2; - p->opt[posMem - 1].backPrev = p->opt[cur].backPrev2; + p->opt[wr].dist = 0; + len--; + wr--; + p->opt[wr].dist = MARK_LIT; + p->opt[wr].len = 1; } } + + if (cur == 0) { - UInt32 posPrev = posMem; - UInt32 backCur = backMem; - - backMem = p->opt[posPrev].backPrev; - posMem = p->opt[posPrev].posPrev; - - p->opt[posPrev].backPrev = backCur; - p->opt[posPrev].posPrev = cur; - cur = posPrev; + p->backRes = dist; + p->optCur = wr; + return len; } + + wr--; + p->opt[wr].dist = dist; + p->opt[wr].len = (UInt32)len; } - while (cur != 0); - *backRes = p->opt[0].backPrev; - p->optimumCurrentIndex = p->opt[0].posPrev; - return p->optimumCurrentIndex; } -#define LIT_PROBS(pos, prevByte) (p->litProbs + ((((pos) & p->lpMask) << p->lc) + ((prevByte) >> (8 - p->lc))) * (UInt32)0x300) + + +#define LIT_PROBS(pos, prevByte) \ + (p->litProbs + (UInt32)3 * (((((pos) << 8) + (prevByte)) & p->lpMask) << p->lc)) -static UInt32 GetOptimum(CLzmaEnc *p, UInt32 position, UInt32 *backRes) + +static unsigned GetOptimum(CLzmaEnc *p, UInt32 position) { - UInt32 numAvail, mainLen, numPairs, repMaxIndex, i, posState, lenEnd, len, cur; - UInt32 matchPrice, repMatchPrice, normalMatchPrice; - UInt32 reps[LZMA_NUM_REPS], repLens[LZMA_NUM_REPS]; + unsigned last, cur; + UInt32 reps[LZMA_NUM_REPS]; + unsigned repLens[LZMA_NUM_REPS]; UInt32 *matches; - const Byte *data; - Byte curByte, matchByte; - if (p->optimumEndIndex != p->optimumCurrentIndex) - { - const COptimal *opt = &p->opt[p->optimumCurrentIndex]; - UInt32 lenRes = opt->posPrev - p->optimumCurrentIndex; - *backRes = opt->backPrev; - p->optimumCurrentIndex = opt->posPrev; - return lenRes; - } - p->optimumCurrentIndex = p->optimumEndIndex = 0; - - if (p->additionalOffset == 0) - mainLen = ReadMatchDistances(p, &numPairs); - else - { - mainLen = p->longestMatchLength; - numPairs = p->numPairs; - } - numAvail = p->numAvail; - if (numAvail < 2) { - *backRes = (UInt32)(-1); - return 1; - } - if (numAvail > LZMA_MATCH_LEN_MAX) - numAvail = LZMA_MATCH_LEN_MAX; - - data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repMaxIndex = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 lenTest; - const Byte *data2; - reps[i] = p->reps[i]; - data2 = data - reps[i] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) + UInt32 numAvail; + unsigned numPairs, mainLen, repMaxIndex, i, posState; + UInt32 matchPrice, repMatchPrice; + const Byte *data; + Byte curByte, matchByte; + + p->optCur = p->optEnd = 0; + + if (p->additionalOffset == 0) + mainLen = ReadMatchDistances(p, &numPairs); + else { - repLens[i] = 0; - continue; + mainLen = p->longestMatchLen; + numPairs = p->numPairs; } - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - repLens[i] = lenTest; - if (lenTest > repLens[repMaxIndex]) - repMaxIndex = i; - } - if (repLens[repMaxIndex] >= p->numFastBytes) - { - UInt32 lenRes; - *backRes = repMaxIndex; - lenRes = repLens[repMaxIndex]; - MovePos(p, lenRes - 1); - return lenRes; - } - - matches = p->matches; - if (mainLen >= p->numFastBytes) - { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); - return mainLen; - } - curByte = *data; - matchByte = *(data - (reps[0] + 1)); - - if (mainLen < 2 && curByte != matchByte && repLens[repMaxIndex] < 2) - { - *backRes = (UInt32)-1; - return 1; - } - - p->opt[0].state = (CState)p->state; - - posState = (position & p->pbMask); - - { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + - (!IsCharState(p->state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); - } - - MakeAsChar(&p->opt[1]); - - matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); - repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); - - if (matchByte == curByte) - { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, p->state, posState); - if (shortRepPrice < p->opt[1].price) + + numAvail = p->numAvail; + if (numAvail < 2) { - p->opt[1].price = shortRepPrice; - MakeAsShortRep(&p->opt[1]); + p->backRes = MARK_LIT; + return 1; } - } - lenEnd = ((mainLen >= repLens[repMaxIndex]) ? mainLen : repLens[repMaxIndex]); - - if (lenEnd < 2) - { - *backRes = p->opt[1].backPrev; - return 1; - } - - p->opt[1].posPrev = 0; - for (i = 0; i < LZMA_NUM_REPS; i++) - p->opt[0].backs[i] = reps[i]; - - len = lenEnd; - do - p->opt[len--].price = kInfinityPrice; - while (len >= 2); - - for (i = 0; i < LZMA_NUM_REPS; i++) - { - UInt32 repLen = repLens[i]; - UInt32 price; - if (repLen < 2) - continue; - price = repMatchPrice + GetPureRepPrice(p, i, p->state, posState); - do + if (numAvail > LZMA_MATCH_LEN_MAX) + numAvail = LZMA_MATCH_LEN_MAX; + + data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + repMaxIndex = 0; + + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][repLen - 2]; - COptimal *opt = &p->opt[repLen]; - if (curAndLenPrice < opt->price) + unsigned len; + const Byte *data2; + reps[i] = p->reps[i]; + data2 = data - reps[i]; + if (data[0] != data2[0] || data[1] != data2[1]) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = i; - opt->prev1IsChar = False; + repLens[i] = 0; + continue; } + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + repLens[i] = len; + if (len > repLens[repMaxIndex]) + repMaxIndex = i; } - while (--repLen >= 2); - } - - normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (repLens[repMaxIndex] >= p->numFastBytes) + { + unsigned len; + p->backRes = (UInt32)repMaxIndex; + len = repLens[repMaxIndex]; + MOVE_POS(p, len - 1) + return len; + } + + matches = p->matches; + + if (mainLen >= p->numFastBytes) + { + p->backRes = matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) + return mainLen; + } + + curByte = *data; + matchByte = *(data - reps[0]); - len = ((repLens[0] >= 2) ? repLens[0] + 1 : 2); - if (len <= mainLen) - { - UInt32 offs = 0; - while (len > matches[offs]) - offs += 2; - for (; ; len++) + last = repLens[repMaxIndex]; + if (last <= mainLen) + last = mainLen; + + if (last < 2 && curByte != matchByte) { - COptimal *opt; - UInt32 distance = matches[offs + 1]; + p->backRes = MARK_LIT; + return 1; + } + + p->opt[0].state = (CState)p->state; + + posState = (position & p->pbMask); + + { + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + p->opt[1].price = GET_PRICE_0(p->isMatch[p->state][posState]) + + (!IsLitState(p->state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + } - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][len - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(len); - if (distance < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][distance]; - else + MakeAs_Lit(&p->opt[1]); + + matchPrice = GET_PRICE_1(p->isMatch[p->state][posState]); + repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[p->state]); + + // 18.06 + if (matchByte == curByte && repLens[0] == 0) + { + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, p->state, posState); + if (shortRepPrice < p->opt[1].price) { - UInt32 slot; - GetPosSlot2(distance, slot); - curAndLenPrice += p->alignPrices[distance & kAlignMask] + p->posSlotPrices[lenToPosState][slot]; + p->opt[1].price = shortRepPrice; + MakeAs_ShortRep(&p->opt[1]); } - opt = &p->opt[len]; - if (curAndLenPrice < opt->price) + if (last < 2) { - opt->price = curAndLenPrice; - opt->posPrev = 0; - opt->backPrev = distance + LZMA_NUM_REPS; - opt->prev1IsChar = False; + p->backRes = p->opt[1].dist; + return 1; } - if (len == matches[offs]) + } + + p->opt[1].len = 1; + + p->opt[0].reps[0] = reps[0]; + p->opt[0].reps[1] = reps[1]; + p->opt[0].reps[2] = reps[2]; + p->opt[0].reps[3] = reps[3]; + + // ---------- REP ---------- + + for (i = 0; i < LZMA_NUM_REPS; i++) + { + unsigned repLen = repLens[i]; + UInt32 price; + if (repLen < 2) + continue; + price = repMatchPrice + GetPrice_PureRep(p, i, p->state, posState); + do { - offs += 2; - if (offs == numPairs) - break; + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, repLen); + COptimal *opt = &p->opt[repLen]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)repLen; + opt->dist = (UInt32)i; + opt->extra = 0; + } + } + while (--repLen >= 2); + } + + + // ---------- MATCH ---------- + { + unsigned len = repLens[0] + 1; + if (len <= mainLen) + { + unsigned offs = 0; + UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[p->state]); + + if (len < 2) + len = 2; + else + while (len > matches[offs]) + offs += 2; + + for (; ; len++) + { + COptimal *opt; + UInt32 dist = matches[(size_t)offs + 1]; + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); + unsigned lenToPosState = GetLenToPosState(len); + + if (dist < kNumFullDistances) + price += p->distancesPrices[lenToPosState][dist & (kNumFullDistances - 1)]; + else + { + unsigned slot; + GetPosSlot2(dist, slot); + price += p->alignPrices[dist & kAlignMask]; + price += p->posSlotPrices[lenToPosState][slot]; + } + + opt = &p->opt[len]; + + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } + + if (len == matches[offs]) + { + offs += 2; + if (offs == numPairs) + break; + } + } } } - } + - cur = 0; + cur = 0; #ifdef SHOW_STAT2 /* if (position >= 0) */ { unsigned i; printf("\n pos = %4X", position); - for (i = cur; i <= lenEnd; i++) + for (i = cur; i <= last; i++) printf("\nprice[%4X] = %u", position - cur + i, p->opt[i].price); } #endif + } + + + + // ---------- Optimal Parsing ---------- for (;;) { - UInt32 numAvailFull, newLen, numPairs, posPrev, state, posState, startLen; - UInt32 curPrice, curAnd1Price, matchPrice, repMatchPrice; - Bool nextIsChar; + unsigned numAvail; + UInt32 numAvailFull; + unsigned newLen, numPairs, prev, state, posState, startLen; + UInt32 litPrice, matchPrice, repMatchPrice; + BoolInt nextIsLit; Byte curByte, matchByte; const Byte *data; - COptimal *curOpt; - COptimal *nextOpt; + COptimal *curOpt, *nextOpt; - cur++; - if (cur == lenEnd) - return Backward(p, backRes, cur); + if (++cur == last) + break; + + // 18.06 + if (cur >= kNumOpts - 64) + { + unsigned j, best; + UInt32 price = p->opt[cur].price; + best = cur; + for (j = cur + 1; j <= last; j++) + { + UInt32 price2 = p->opt[j].price; + if (price >= price2) + { + price = price2; + best = j; + } + } + { + unsigned delta = best - cur; + if (delta != 0) + { + MOVE_POS(p, delta); + } + } + cur = best; + break; + } newLen = ReadMatchDistances(p, &numPairs); + if (newLen >= p->numFastBytes) { p->numPairs = numPairs; - p->longestMatchLength = newLen; - return Backward(p, backRes, cur); + p->longestMatchLen = newLen; + break; } + + curOpt = &p->opt[cur]; + position++; - curOpt = &p->opt[cur]; - posPrev = curOpt->posPrev; - if (curOpt->prev1IsChar) + + // we need that check here, if skip_items in p->opt are possible + /* + if (curOpt->price >= kInfinityPrice) + continue; + */ + + prev = cur - curOpt->len; + + if (curOpt->len == 1) { - posPrev--; - if (curOpt->prev2) - { - state = p->opt[curOpt->posPrev2].state; - if (curOpt->backPrev2 < LZMA_NUM_REPS) - state = kRepNextStates[state]; - else - state = kMatchNextStates[state]; - } - else - state = p->opt[posPrev].state; - state = kLiteralNextStates[state]; - } - else - state = p->opt[posPrev].state; - if (posPrev == cur - 1) - { + state = (unsigned)p->opt[prev].state; if (IsShortRep(curOpt)) state = kShortRepNextStates[state]; else@@ -1226,355 +1405,499 @@ state = kLiteralNextStates[state];
} else { - UInt32 pos; const COptimal *prevOpt; - if (curOpt->prev1IsChar && curOpt->prev2) + UInt32 b0; + UInt32 dist = curOpt->dist; + + if (curOpt->extra) { - posPrev = curOpt->posPrev2; - pos = curOpt->backPrev2; - state = kRepNextStates[state]; + prev -= (unsigned)curOpt->extra; + state = kState_RepAfterLit; + if (curOpt->extra == 1) + state = (dist < LZMA_NUM_REPS ? kState_RepAfterLit : kState_MatchAfterLit); } else { - pos = curOpt->backPrev; - if (pos < LZMA_NUM_REPS) + state = (unsigned)p->opt[prev].state; + if (dist < LZMA_NUM_REPS) state = kRepNextStates[state]; else state = kMatchNextStates[state]; } - prevOpt = &p->opt[posPrev]; - if (pos < LZMA_NUM_REPS) + + prevOpt = &p->opt[prev]; + b0 = prevOpt->reps[0]; + + if (dist < LZMA_NUM_REPS) { - UInt32 i; - reps[0] = prevOpt->backs[pos]; - for (i = 1; i <= pos; i++) - reps[i] = prevOpt->backs[i - 1]; - for (; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i]; + if (dist == 0) + { + reps[0] = b0; + reps[1] = prevOpt->reps[1]; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[1] = b0; + b0 = prevOpt->reps[1]; + if (dist == 1) + { + reps[0] = b0; + reps[2] = prevOpt->reps[2]; + reps[3] = prevOpt->reps[3]; + } + else + { + reps[2] = b0; + reps[0] = prevOpt->reps[dist]; + reps[3] = prevOpt->reps[dist ^ 1]; + } + } } else { - UInt32 i; - reps[0] = (pos - LZMA_NUM_REPS); - for (i = 1; i < LZMA_NUM_REPS; i++) - reps[i] = prevOpt->backs[i - 1]; + reps[0] = (dist - LZMA_NUM_REPS + 1); + reps[1] = b0; + reps[2] = prevOpt->reps[1]; + reps[3] = prevOpt->reps[2]; } } + curOpt->state = (CState)state; - - curOpt->backs[0] = reps[0]; - curOpt->backs[1] = reps[1]; - curOpt->backs[2] = reps[2]; - curOpt->backs[3] = reps[3]; + curOpt->reps[0] = reps[0]; + curOpt->reps[1] = reps[1]; + curOpt->reps[2] = reps[2]; + curOpt->reps[3] = reps[3]; - curPrice = curOpt->price; - nextIsChar = False; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; curByte = *data; - matchByte = *(data - (reps[0] + 1)); + matchByte = *(data - reps[0]); posState = (position & p->pbMask); - curAnd1Price = curPrice + GET_PRICE_0(p->isMatch[state][posState]); + /* + The order of Price checks: + < LIT + <= SHORT_REP + < LIT : REP_0 + < REP [ : LIT : REP_0 ] + < MATCH [ : LIT : REP_0 ] + */ + { - const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); - curAnd1Price += - (!IsCharState(state) ? - LitEnc_GetPriceMatched(probs, curByte, matchByte, p->ProbPrices) : - LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + UInt32 curPrice = curOpt->price; + unsigned prob = p->isMatch[state][posState]; + matchPrice = curPrice + GET_PRICE_1(prob); + litPrice = curPrice + GET_PRICE_0(prob); } - nextOpt = &p->opt[cur + 1]; + nextOpt = &p->opt[(size_t)cur + 1]; + nextIsLit = False; - if (curAnd1Price < nextOpt->price) + // here we can allow skip_items in p->opt, if we don't check (nextOpt->price < kInfinityPrice) + // 18.new.06 + if ((nextOpt->price < kInfinityPrice + // && !IsLitState(state) + && matchByte == curByte) + || litPrice > nextOpt->price + ) + litPrice = 0; + else { - nextOpt->price = curAnd1Price; - nextOpt->posPrev = cur; - MakeAsChar(nextOpt); - nextIsChar = True; + const CLzmaProb *probs = LIT_PROBS(position, *(data - 1)); + litPrice += (!IsLitState(state) ? + LitEnc_Matched_GetPrice(probs, curByte, matchByte, p->ProbPrices) : + LitEnc_GetPrice(probs, curByte, p->ProbPrices)); + + if (litPrice < nextOpt->price) + { + nextOpt->price = litPrice; + nextOpt->len = 1; + MakeAs_Lit(nextOpt); + nextIsLit = True; + } } - matchPrice = curPrice + GET_PRICE_1(p->isMatch[state][posState]); repMatchPrice = matchPrice + GET_PRICE_1(p->isRep[state]); - if (matchByte == curByte && !(nextOpt->posPrev < cur && nextOpt->backPrev == 0)) + numAvailFull = p->numAvail; + { + unsigned temp = kNumOpts - 1 - cur; + if (numAvailFull > temp) + numAvailFull = (UInt32)temp; + } + + // 18.06 + // ---------- SHORT_REP ---------- + if (IsLitState(state)) // 18.new + if (matchByte == curByte) + if (repMatchPrice < nextOpt->price) // 18.new + // if (numAvailFull < 2 || data[1] != *(data - reps[0] + 1)) + if ( + // nextOpt->price >= kInfinityPrice || + nextOpt->len < 2 // we can check nextOpt->len, if skip items are not allowed in p->opt + || (nextOpt->dist != 0 + // && nextOpt->extra <= 1 // 17.old + ) + ) { - UInt32 shortRepPrice = repMatchPrice + GetRepLen1Price(p, state, posState); - if (shortRepPrice <= nextOpt->price) + UInt32 shortRepPrice = repMatchPrice + GetPrice_ShortRep(p, state, posState); + // if (shortRepPrice <= nextOpt->price) // 17.old + if (shortRepPrice < nextOpt->price) // 18.new { nextOpt->price = shortRepPrice; - nextOpt->posPrev = cur; - MakeAsShortRep(nextOpt); - nextIsChar = True; + nextOpt->len = 1; + MakeAs_ShortRep(nextOpt); + nextIsLit = False; } } - numAvailFull = p->numAvail; - { - UInt32 temp = kNumOpts - 1 - cur; - if (temp < numAvailFull) - numAvailFull = temp; - } - + if (numAvailFull < 2) continue; numAvail = (numAvailFull <= p->numFastBytes ? numAvailFull : p->numFastBytes); - if (!nextIsChar && matchByte != curByte) /* speed optimization */ - { - /* try Literal + rep0 */ - UInt32 temp; - UInt32 lenTest2; - const Byte *data2 = data - reps[0] - 1; - UInt32 limit = p->numFastBytes + 1; - if (limit > numAvailFull) - limit = numAvailFull; + // numAvail <= p->numFastBytes + + // ---------- LIT : REP_0 ---------- - for (temp = 1; temp < limit && data[temp] == data2[temp]; temp++); - lenTest2 = temp - 1; - if (lenTest2 >= 2) + if (!nextIsLit + && litPrice != 0 // 18.new + && matchByte != curByte + && numAvailFull > 2) + { + const Byte *data2 = data - reps[0]; + if (data[1] == data2[1] && data[2] == data2[2]) { - UInt32 state2 = kLiteralNextStates[state]; - UInt32 posStateNext = (position + 1) & p->pbMask; - UInt32 nextRepMatchPrice = curAnd1Price + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - /* for (; lenTest2 >= 2; lenTest2--) */ + unsigned len; + unsigned limit = p->numFastBytes + 1; + if (limit > numAvailFull) + limit = numAvailFull; + for (len = 3; len < limit && data[len] == data2[len]; len++) + {} + { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned state2 = kLiteralNextStates[state]; + unsigned posState2 = (position + 1) & p->pbMask; + UInt32 price = litPrice + GetPrice_Rep_0(p, state2, posState2); { - opt->price = curAndLenPrice; - opt->posPrev = cur + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = False; + unsigned offset = cur + len; + + if (last < offset) + last = offset; + + // do + { + UInt32 price2; + COptimal *opt; + len--; + // price2 = price + GetPrice_Len_Rep_0(p, len, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len; + opt->dist = 0; + opt->extra = 1; + } + } + // while (len >= 3); } } } } startLen = 2; /* speed optimization */ - { - UInt32 repIndex; - for (repIndex = 0; repIndex < LZMA_NUM_REPS; repIndex++) + { - UInt32 lenTest; - UInt32 lenTestTemp; - UInt32 price; - const Byte *data2 = data - reps[repIndex] - 1; - if (data[0] != data2[0] || data[1] != data2[1]) - continue; - for (lenTest = 2; lenTest < numAvail && data[lenTest] == data2[lenTest]; lenTest++); - while (lenEnd < cur + lenTest) - p->opt[++lenEnd].price = kInfinityPrice; - lenTestTemp = lenTest; - price = repMatchPrice + GetPureRepPrice(p, repIndex, state, posState); - do + // ---------- REP ---------- + unsigned repIndex = 0; // 17.old + // unsigned repIndex = IsLitState(state) ? 0 : 1; // 18.notused + for (; repIndex < LZMA_NUM_REPS; repIndex++) { - UInt32 curAndLenPrice = price + p->repLenEnc.prices[posState][lenTest - 2]; - COptimal *opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) + unsigned len; + UInt32 price; + const Byte *data2 = data - reps[repIndex]; + if (data[0] != data2[0] || data[1] != data2[1]) + continue; + + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} + + // if (len < startLen) continue; // 18.new: speed optimization + { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = repIndex; - opt->prev1IsChar = False; + unsigned offset = cur + len; + if (last < offset) + last = offset; + } + { + unsigned len2 = len; + price = repMatchPrice + GetPrice_PureRep(p, repIndex, state, posState); + do + { + UInt32 price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState, len2); + COptimal *opt = &p->opt[cur + len2]; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->dist = (UInt32)repIndex; + opt->extra = 0; + } + } + while (--len2 >= 2); } - } - while (--lenTest >= 2); - lenTest = lenTestTemp; - - if (repIndex == 0) - startLen = lenTest + 1; - /* if (_maxMode) */ + if (repIndex == 0) startLen = len + 1; // 17.old + // startLen = len + 1; // 18.new + + /* if (_maxMode) */ { - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // ---------- REP : LIT : REP_0 ---------- + // numFastBytes + 1 + numFastBytes + + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kRepNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = - price + p->repLenEnc.prices[posState][lenTest - 2] + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (position + lenTest + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); + unsigned state2 = kRepNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + price += GET_PRICE_LEN(&p->repLenEnc, posState, len) + + GET_PRICE_0(p->isMatch[state2][posState2]) + + LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); - /* for (; lenTest2 >= 2; lenTest2--) */ + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterRep; + posState2 = (posState2 + 1) & p->pbMask; + + + price += GetPrice_Rep_0(p, state2, posState2); + + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + // if (len2 >= 3) + { { - UInt32 curAndLenPrice; - COptimal *opt; - UInt32 offset = cur + lenTest + 1 + lenTest2; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); - opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + unsigned offset = cur + len + len2; + + if (last < offset) + last = offset; + // do { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = repIndex; + UInt32 price2; + COptimal *opt; + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); + + opt = &p->opt[offset]; + // offset--; + if (price2 < opt->price) + { + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = (UInt32)repIndex; + } } + // while (len2 >= 3); } + } } } + } } - } - /* for (UInt32 lenTest = 2; lenTest <= newLen; lenTest++) */ + + + // ---------- MATCH ---------- + /* for (unsigned len = 2; len <= newLen; len++) */ if (newLen > numAvail) { newLen = numAvail; for (numPairs = 0; newLen > matches[numPairs]; numPairs += 2); - matches[numPairs] = newLen; + matches[numPairs] = (UInt32)newLen; numPairs += 2; } + + // startLen = 2; /* speed optimization */ + if (newLen >= startLen) { UInt32 normalMatchPrice = matchPrice + GET_PRICE_0(p->isRep[state]); - UInt32 offs, curBack, posSlot; - UInt32 lenTest; - while (lenEnd < cur + newLen) - p->opt[++lenEnd].price = kInfinityPrice; + UInt32 dist; + unsigned offs, posSlot, len; + + { + unsigned offset = cur + newLen; + if (last < offset) + last = offset; + } offs = 0; while (startLen > matches[offs]) offs += 2; - curBack = matches[offs + 1]; - GetPosSlot2(curBack, posSlot); - for (lenTest = /*2*/ startLen; ; lenTest++) + dist = matches[(size_t)offs + 1]; + + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); + + for (len = /*2*/ startLen; ; len++) { - UInt32 curAndLenPrice = normalMatchPrice + p->lenEnc.prices[posState][lenTest - LZMA_MATCH_LEN_MIN]; - UInt32 lenToPosState = GetLenToPosState(lenTest); - COptimal *opt; - if (curBack < kNumFullDistances) - curAndLenPrice += p->distancesPrices[lenToPosState][curBack]; - else - curAndLenPrice += p->posSlotPrices[lenToPosState][posSlot] + p->alignPrices[curBack & kAlignMask]; - - opt = &p->opt[cur + lenTest]; - if (curAndLenPrice < opt->price) + UInt32 price = normalMatchPrice + GET_PRICE_LEN(&p->lenEnc, posState, len); { - opt->price = curAndLenPrice; - opt->posPrev = cur; - opt->backPrev = curBack + LZMA_NUM_REPS; - opt->prev1IsChar = False; + COptimal *opt; + unsigned lenNorm = len - 2; + lenNorm = GetLenToPosState2(lenNorm); + if (dist < kNumFullDistances) + price += p->distancesPrices[lenNorm][dist & (kNumFullDistances - 1)]; + else + price += p->posSlotPrices[lenNorm][posSlot] + p->alignPrices[dist & kAlignMask]; + + opt = &p->opt[cur + len]; + if (price < opt->price) + { + opt->price = price; + opt->len = (UInt32)len; + opt->dist = dist + LZMA_NUM_REPS; + opt->extra = 0; + } } - if (/*_maxMode && */lenTest == matches[offs]) + if (len == matches[offs]) { - /* Try Match + Literal + Rep0 */ - const Byte *data2 = data - curBack - 1; - UInt32 lenTest2 = lenTest + 1; - UInt32 limit = lenTest2 + p->numFastBytes; + // if (p->_maxMode) { + // MATCH : LIT : REP_0 + + const Byte *data2 = data - dist - 1; + unsigned len2 = len + 1; + unsigned limit = len2 + p->numFastBytes; if (limit > numAvailFull) limit = numAvailFull; - for (; lenTest2 < limit && data[lenTest2] == data2[lenTest2]; lenTest2++); - lenTest2 -= lenTest + 1; - if (lenTest2 >= 2) + + len2 += 2; + if (len2 <= limit) + if (data[len2 - 2] == data2[len2 - 2]) + if (data[len2 - 1] == data2[len2 - 1]) { - UInt32 nextRepMatchPrice; - UInt32 state2 = kMatchNextStates[state]; - UInt32 posStateNext = (position + lenTest) & p->pbMask; - UInt32 curAndLenCharPrice = curAndLenPrice + - GET_PRICE_0(p->isMatch[state2][posStateNext]) + - LitEnc_GetPriceMatched(LIT_PROBS(position + lenTest, data[lenTest - 1]), - data[lenTest], data2[lenTest], p->ProbPrices); - state2 = kLiteralNextStates[state2]; - posStateNext = (posStateNext + 1) & p->pbMask; - nextRepMatchPrice = curAndLenCharPrice + - GET_PRICE_1(p->isMatch[state2][posStateNext]) + - GET_PRICE_1(p->isRep[state2]); - - /* for (; lenTest2 >= 2; lenTest2--) */ + for (; len2 < limit && data[len2] == data2[len2]; len2++) + {} + + len2 -= len; + + // if (len2 >= 3) + { + unsigned state2 = kMatchNextStates[state]; + unsigned posState2 = (position + len) & p->pbMask; + unsigned offset; + price += GET_PRICE_0(p->isMatch[state2][posState2]); + price += LitEnc_Matched_GetPrice(LIT_PROBS(position + len, data[(size_t)len - 1]), + data[len], data2[len], p->ProbPrices); + + // state2 = kLiteralNextStates[state2]; + state2 = kState_LitAfterMatch; + + posState2 = (posState2 + 1) & p->pbMask; + price += GetPrice_Rep_0(p, state2, posState2); + + offset = cur + len + len2; + + if (last < offset) + last = offset; + // do { - UInt32 offset = cur + lenTest + 1 + lenTest2; - UInt32 curAndLenPrice; + UInt32 price2; COptimal *opt; - while (lenEnd < offset) - p->opt[++lenEnd].price = kInfinityPrice; - curAndLenPrice = nextRepMatchPrice + GetRepPrice(p, 0, lenTest2, state2, posStateNext); + len2--; + // price2 = price + GetPrice_Len_Rep_0(p, len2, state2, posState2); + price2 = price + GET_PRICE_LEN(&p->repLenEnc, posState2, len2); opt = &p->opt[offset]; - if (curAndLenPrice < opt->price) + // offset--; + if (price2 < opt->price) { - opt->price = curAndLenPrice; - opt->posPrev = cur + lenTest + 1; - opt->backPrev = 0; - opt->prev1IsChar = True; - opt->prev2 = True; - opt->posPrev2 = cur; - opt->backPrev2 = curBack + LZMA_NUM_REPS; + opt->price = price2; + opt->len = (UInt32)len2; + opt->extra = (CExtra)(len + 1); + opt->dist = dist + LZMA_NUM_REPS; } } + // while (len2 >= 3); + } + } + offs += 2; if (offs == numPairs) break; - curBack = matches[offs + 1]; - if (curBack >= kNumFullDistances) - GetPosSlot2(curBack, posSlot); + dist = matches[(size_t)offs + 1]; + // if (dist >= kNumFullDistances) + GetPosSlot2(dist, posSlot); } } } } + + do + p->opt[last].price = kInfinityPrice; + while (--last); + + return Backward(p, cur); } + + #define ChangePair(smallDist, bigDist) (((bigDist) >> 7) > (smallDist)) -static UInt32 GetOptimumFast(CLzmaEnc *p, UInt32 *backRes) + + +static unsigned GetOptimumFast(CLzmaEnc *p) { - UInt32 numAvail, mainLen, mainDist, numPairs, repIndex, repLen, i; + UInt32 numAvail, mainDist; + unsigned mainLen, numPairs, repIndex, repLen, i; const Byte *data; - const UInt32 *matches; if (p->additionalOffset == 0) mainLen = ReadMatchDistances(p, &numPairs); else { - mainLen = p->longestMatchLength; + mainLen = p->longestMatchLen; numPairs = p->numPairs; } numAvail = p->numAvail; - *backRes = (UInt32)-1; + p->backRes = MARK_LIT; if (numAvail < 2) return 1; + // if (mainLen < 2 && p->state == 0) return 1; // 18.06.notused if (numAvail > LZMA_MATCH_LEN_MAX) numAvail = LZMA_MATCH_LEN_MAX; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; - repLen = repIndex = 0; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; - for (len = 2; len < numAvail && data[len] == data2[len]; len++); + for (len = 2; len < numAvail && data[len] == data2[len]; len++) + {} if (len >= p->numFastBytes) { - *backRes = i; - MovePos(p, len - 1); + p->backRes = (UInt32)i; + MOVE_POS(p, len - 1) return len; } if (len > repLen)@@ -1584,83 +1907,151 @@ repLen = len;
} } - matches = p->matches; if (mainLen >= p->numFastBytes) { - *backRes = matches[numPairs - 1] + LZMA_NUM_REPS; - MovePos(p, mainLen - 1); + p->backRes = p->matches[(size_t)numPairs - 1] + LZMA_NUM_REPS; + MOVE_POS(p, mainLen - 1) return mainLen; } mainDist = 0; /* for GCC */ + if (mainLen >= 2) { - mainDist = matches[numPairs - 1]; - while (numPairs > 2 && mainLen == matches[numPairs - 4] + 1) + mainDist = p->matches[(size_t)numPairs - 1]; + while (numPairs > 2) { - if (!ChangePair(matches[numPairs - 3], mainDist)) + UInt32 dist2; + if (mainLen != p->matches[(size_t)numPairs - 4] + 1) + break; + dist2 = p->matches[(size_t)numPairs - 3]; + if (!ChangePair(dist2, mainDist)) break; numPairs -= 2; - mainLen = matches[numPairs - 2]; - mainDist = matches[numPairs - 1]; + mainLen--; + mainDist = dist2; } if (mainLen == 2 && mainDist >= 0x80) mainLen = 1; } - if (repLen >= 2 && ( - (repLen + 1 >= mainLen) || - (repLen + 2 >= mainLen && mainDist >= (1 << 9)) || - (repLen + 3 >= mainLen && mainDist >= (1 << 15)))) + if (repLen >= 2) + if ( repLen + 1 >= mainLen + || (repLen + 2 >= mainLen && mainDist >= (1 << 9)) + || (repLen + 3 >= mainLen && mainDist >= (1 << 15))) { - *backRes = repIndex; - MovePos(p, repLen - 1); + p->backRes = (UInt32)repIndex; + MOVE_POS(p, repLen - 1) return repLen; } if (mainLen < 2 || numAvail <= 2) return 1; - p->longestMatchLength = ReadMatchDistances(p, &p->numPairs); - if (p->longestMatchLength >= 2) { - UInt32 newDistance = matches[p->numPairs - 1]; - if ((p->longestMatchLength >= mainLen && newDistance < mainDist) || - (p->longestMatchLength == mainLen + 1 && !ChangePair(mainDist, newDistance)) || - (p->longestMatchLength > mainLen + 1) || - (p->longestMatchLength + 1 >= mainLen && mainLen >= 3 && ChangePair(newDistance, mainDist))) - return 1; + unsigned len1 = ReadMatchDistances(p, &p->numPairs); + p->longestMatchLen = len1; + + if (len1 >= 2) + { + UInt32 newDist = p->matches[(size_t)p->numPairs - 1]; + if ( (len1 >= mainLen && newDist < mainDist) + || (len1 == mainLen + 1 && !ChangePair(mainDist, newDist)) + || (len1 > mainLen + 1) + || (len1 + 1 >= mainLen && mainLen >= 3 && ChangePair(newDist, mainDist))) + return 1; + } } data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - 1; + for (i = 0; i < LZMA_NUM_REPS; i++) { - UInt32 len, limit; - const Byte *data2 = data - p->reps[i] - 1; + unsigned len, limit; + const Byte *data2 = data - p->reps[i]; if (data[0] != data2[0] || data[1] != data2[1]) continue; limit = mainLen - 1; - for (len = 2; len < limit && data[len] == data2[len]; len++); - if (len >= limit) - return 1; + for (len = 2;; len++) + { + if (len >= limit) + return 1; + if (data[len] != data2[len]) + break; + } } - *backRes = mainDist + LZMA_NUM_REPS; - MovePos(p, mainLen - 2); + + p->backRes = mainDist + LZMA_NUM_REPS; + if (mainLen != 2) + { + MOVE_POS(p, mainLen - 2) + } return mainLen; } -static void WriteEndMarker(CLzmaEnc *p, UInt32 posState) + + + +static void WriteEndMarker(CLzmaEnc *p, unsigned posState) { - UInt32 len; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + UInt32 range; + range = p->rc.range; + { + UInt32 ttt, newBound; + CLzmaProb *prob = &p->isMatch[p->state][posState]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_1(&p->rc, prob) + prob = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, prob) + RC_BIT_0(&p->rc, prob) + } p->state = kMatchNextStates[p->state]; - len = LZMA_MATCH_LEN_MIN; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, (1 << kNumPosSlotBits) - 1); - RangeEnc_EncodeDirectBits(&p->rc, (((UInt32)1 << 30) - 1) >> kNumAlignBits, 30 - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + + p->rc.range = range; + LenEnc_Encode(&p->lenProbs, &p->rc, 0, posState); + range = p->rc.range; + + { + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[0], (1 << kNumPosSlotBits) - 1); + CLzmaProb *probs = p->posSlotEncoder[0]; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < (1 << kNumPosSlotBits)); + } + { + // RangeEnc_EncodeDirectBits(&p->rc, ((UInt32)1 << (30 - kNumAlignBits)) - 1, 30 - kNumAlignBits); UInt32 range = p->range; + unsigned numBits = 30 - kNumAlignBits; + do + { + range >>= 1; + p->rc.low += range; + RC_NORM(&p->rc) + } + while (--numBits); + } + + { + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, kAlignMask); + CLzmaProb *probs = p->posAlignEncoder; + unsigned m = 1; + do + { + UInt32 ttt, newBound; + RC_BIT_PRE(p, probs + m) + RC_BIT_1(&p->rc, probs + m); + m = (m << 1) + 1; + } + while (m < kAlignTableSize); + } + p->rc.range = range; } + static SRes CheckErrors(CLzmaEnc *p) {@@ -1675,7 +2066,8 @@ p->finished = True;
return p->result; } -static SRes Flush(CLzmaEnc *p, UInt32 nowPos) + +MY_NO_INLINE static SRes Flush(CLzmaEnc *p, UInt32 nowPos) { /* ReleaseMFStream(); */ p->finished = True;@@ -1686,47 +2078,125 @@ RangeEnc_FlushStream(&p->rc);
return CheckErrors(p); } -static void FillAlignPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillAlignPrices(CLzmaEnc *p) { - UInt32 i; - for (i = 0; i < kAlignTableSize; i++) - p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); - p->alignPriceCount = 0; + unsigned i; + const CProbPrice *ProbPrices = p->ProbPrices; + const CLzmaProb *probs = p->posAlignEncoder; + // p->alignPriceCount = 0; + for (i = 0; i < kAlignTableSize / 2; i++) + { + UInt32 price = 0; + unsigned sym = i; + unsigned m = 1; + unsigned bit; + UInt32 prob; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[m], bit); m = (m << 1) + bit; + prob = probs[m]; + p->alignPrices[i ] = price + GET_PRICEa_0(prob); + p->alignPrices[i + 8] = price + GET_PRICEa_1(prob); + // p->alignPrices[i] = RcTree_ReverseGetPrice(p->posAlignEncoder, kNumAlignBits, i, p->ProbPrices); + } } -static void FillDistancesPrices(CLzmaEnc *p) + +MY_NO_INLINE static void FillDistancesPrices(CLzmaEnc *p) { + // int y; for (y = 0; y < 100; y++) { + UInt32 tempPrices[kNumFullDistances]; - UInt32 i, lenToPosState; - for (i = kStartPosModelIndex; i < kNumFullDistances; i++) + unsigned i, lps; + + const CProbPrice *ProbPrices = p->ProbPrices; + p->matchPriceCount = 0; + + for (i = kStartPosModelIndex / 2; i < kNumFullDistances / 2; i++) { - UInt32 posSlot = GetPosSlot1(i); - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base - posSlot - 1, footerBits, i - base, p->ProbPrices); + unsigned posSlot = GetPosSlot1(i); + unsigned footerBits = (posSlot >> 1) - 1; + unsigned base = ((2 | (posSlot & 1)) << footerBits); + const CLzmaProb *probs = p->posEncoders + (size_t)base * 2; + // tempPrices[i] = RcTree_ReverseGetPrice(p->posEncoders + base, footerBits, i - base, p->ProbPrices); + UInt32 price = 0; + unsigned m = 1; + unsigned sym = i; + unsigned offset = (unsigned)1 << footerBits; + base += i; + + if (footerBits) + do + { + unsigned bit = sym & 1; + sym >>= 1; + price += GET_PRICEa(probs[m], bit); + m = (m << 1) + bit; + } + while (--footerBits); + + { + unsigned prob = probs[m]; + tempPrices[base ] = price + GET_PRICEa_0(prob); + tempPrices[base + offset] = price + GET_PRICEa_1(prob); + } } - for (lenToPosState = 0; lenToPosState < kNumLenToPosStates; lenToPosState++) + for (lps = 0; lps < kNumLenToPosStates; lps++) { - UInt32 posSlot; - const CLzmaProb *encoder = p->posSlotEncoder[lenToPosState]; - UInt32 *posSlotPrices = p->posSlotPrices[lenToPosState]; - for (posSlot = 0; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] = RcTree_GetPrice(encoder, kNumPosSlotBits, posSlot, p->ProbPrices); - for (posSlot = kEndPosModelIndex; posSlot < p->distTableSize; posSlot++) - posSlotPrices[posSlot] += ((((posSlot >> 1) - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + unsigned slot; + unsigned distTableSize2 = (p->distTableSize + 1) >> 1; + UInt32 *posSlotPrices = p->posSlotPrices[lps]; + const CLzmaProb *probs = p->posSlotEncoder[lps]; + + for (slot = 0; slot < distTableSize2; slot++) + { + // posSlotPrices[slot] = RcTree_GetPrice(encoder, kNumPosSlotBits, slot, p->ProbPrices); + UInt32 price; + unsigned bit; + unsigned sym = slot + (1 << (kNumPosSlotBits - 1)); + unsigned prob; + bit = sym & 1; sym >>= 1; price = GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + bit = sym & 1; sym >>= 1; price += GET_PRICEa(probs[sym], bit); + prob = probs[(size_t)slot + (1 << (kNumPosSlotBits - 1))]; + posSlotPrices[(size_t)slot * 2 ] = price + GET_PRICEa_0(prob); + posSlotPrices[(size_t)slot * 2 + 1] = price + GET_PRICEa_1(prob); + } + + { + UInt32 delta = ((UInt32)((kEndPosModelIndex / 2 - 1) - kNumAlignBits) << kNumBitPriceShiftBits); + for (slot = kEndPosModelIndex / 2; slot < distTableSize2; slot++) + { + posSlotPrices[(size_t)slot * 2 ] += delta; + posSlotPrices[(size_t)slot * 2 + 1] += delta; + delta += ((UInt32)1 << kNumBitPriceShiftBits); + } + } { - UInt32 *distancesPrices = p->distancesPrices[lenToPosState]; - UInt32 i; - for (i = 0; i < kStartPosModelIndex; i++) - distancesPrices[i] = posSlotPrices[i]; - for (; i < kNumFullDistances; i++) - distancesPrices[i] = posSlotPrices[GetPosSlot1(i)] + tempPrices[i]; + UInt32 *dp = p->distancesPrices[lps]; + + dp[0] = posSlotPrices[0]; + dp[1] = posSlotPrices[1]; + dp[2] = posSlotPrices[2]; + dp[3] = posSlotPrices[3]; + + for (i = 4; i < kNumFullDistances; i += 2) + { + UInt32 slotPrice = posSlotPrices[GetPosSlot1(i)]; + dp[i ] = slotPrice + tempPrices[i]; + dp[i + 1] = slotPrice + tempPrices[i + 1]; + } } } - p->matchPriceCount = 0; + // } } + + void LzmaEnc_Construct(CLzmaEnc *p) {@@ -1751,26 +2221,27 @@
LzmaEnc_InitPriceTables(p->ProbPrices); p->litProbs = NULL; p->saveState.litProbs = NULL; + } -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc) +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc) { void *p; - p = alloc->Alloc(alloc, sizeof(CLzmaEnc)); + p = ISzAlloc_Alloc(alloc, sizeof(CLzmaEnc)); if (p) LzmaEnc_Construct((CLzmaEnc *)p); return p; } -void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAlloc *alloc) +void LzmaEnc_FreeLits(CLzmaEnc *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->litProbs); - alloc->Free(alloc, p->saveState.litProbs); + ISzAlloc_Free(alloc, p->litProbs); + ISzAlloc_Free(alloc, p->saveState.litProbs); p->litProbs = NULL; p->saveState.litProbs = NULL; } -void LzmaEnc_Destruct(CLzmaEnc *p, ISzAlloc *alloc, ISzAlloc *allocBig) +void LzmaEnc_Destruct(CLzmaEnc *p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { #ifndef _7ZIP_ST MatchFinderMt_Destruct(&p->matchFinderMt, allocBig);@@ -1781,13 +2252,14 @@ LzmaEnc_FreeLits(p, alloc);
RangeEnc_Free(&p->rc, alloc); } -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig) +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig) { LzmaEnc_Destruct((CLzmaEnc *)p, alloc, allocBig); - alloc->Free(alloc, p); + ISzAlloc_Free(alloc, p); } -static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, Bool useLimits, UInt32 maxPackSize, UInt32 maxUnpackSize) + +static SRes LzmaEnc_CodeOneBlock(CLzmaEnc *p, UInt32 maxPackSize, UInt32 maxUnpackSize) { UInt32 nowPos32, startPos32; if (p->needInit)@@ -1805,13 +2277,13 @@ startPos32 = nowPos32;
if (p->nowPos64 == 0) { - UInt32 numPairs; + unsigned numPairs; Byte curByte; if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) return Flush(p, nowPos32); ReadMatchDistances(p, &numPairs); - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][0], 0); - p->state = kLiteralNextStates[p->state]; + RangeEnc_EncodeBit_0(&p->rc, &p->isMatch[kState_Start][0]); + // p->state = kLiteralNextStates[p->state]; curByte = *(p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset); LitEnc_Encode(&p->rc, p->litProbs, curByte); p->additionalOffset--;@@ -1819,123 +2291,253 @@ nowPos32++;
} if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) != 0) + for (;;) { - UInt32 pos, len, posState; - + UInt32 dist; + unsigned len, posState; + UInt32 range, ttt, newBound; + CLzmaProb *probs; + if (p->fastMode) - len = GetOptimumFast(p, &pos); + len = GetOptimumFast(p); else - len = GetOptimum(p, nowPos32, &pos); + { + unsigned oci = p->optCur; + if (p->optEnd == oci) + len = GetOptimum(p, nowPos32); + else + { + const COptimal *opt = &p->opt[oci]; + len = opt->len; + p->backRes = opt->dist; + p->optCur = oci + 1; + } + } + + posState = (unsigned)nowPos32 & p->pbMask; + range = p->rc.range; + probs = &p->isMatch[p->state][posState]; + + RC_BIT_PRE(&p->rc, probs) + + dist = p->backRes; #ifdef SHOW_STAT2 - printf("\n pos = %4X, len = %u pos = %u", nowPos32, len, pos); + printf("\n pos = %6X, len = %3u pos = %6u", nowPos32, len, dist); #endif - posState = nowPos32 & p->pbMask; - if (len == 1 && pos == (UInt32)-1) + if (dist == MARK_LIT) { Byte curByte; - CLzmaProb *probs; const Byte *data; + unsigned state; - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 0); + RC_BIT_0(&p->rc, probs); + p->rc.range = range; data = p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset; - curByte = *data; probs = LIT_PROBS(nowPos32, *(data - 1)); - if (IsCharState(p->state)) + curByte = *data; + state = p->state; + p->state = kLiteralNextStates[state]; + if (IsLitState(state)) LitEnc_Encode(&p->rc, probs, curByte); else - LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0] - 1)); - p->state = kLiteralNextStates[p->state]; + LitEnc_EncodeMatched(&p->rc, probs, curByte, *(data - p->reps[0])); } else { - RangeEnc_EncodeBit(&p->rc, &p->isMatch[p->state][posState], 1); - if (pos < LZMA_NUM_REPS) + RC_BIT_1(&p->rc, probs); + probs = &p->isRep[p->state]; + RC_BIT_PRE(&p->rc, probs) + + if (dist < LZMA_NUM_REPS) { - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 1); - if (pos == 0) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG0[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 0) { - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 0); - RangeEnc_EncodeBit(&p->rc, &p->isRep0Long[p->state][posState], ((len == 1) ? 0 : 1)); + RC_BIT_0(&p->rc, probs); + probs = &p->isRep0Long[p->state][posState]; + RC_BIT_PRE(&p->rc, probs) + if (len != 1) + { + RC_BIT_1_BASE(&p->rc, probs); + } + else + { + RC_BIT_0_BASE(&p->rc, probs); + p->state = kShortRepNextStates[p->state]; + } } else { - UInt32 distance = p->reps[pos]; - RangeEnc_EncodeBit(&p->rc, &p->isRepG0[p->state], 1); - if (pos == 1) - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 0); + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG1[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 1) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[1]; + } else { - RangeEnc_EncodeBit(&p->rc, &p->isRepG1[p->state], 1); - RangeEnc_EncodeBit(&p->rc, &p->isRepG2[p->state], pos - 2); - if (pos == 3) + RC_BIT_1(&p->rc, probs); + probs = &p->isRepG2[p->state]; + RC_BIT_PRE(&p->rc, probs) + if (dist == 2) + { + RC_BIT_0_BASE(&p->rc, probs); + dist = p->reps[2]; + } + else + { + RC_BIT_1_BASE(&p->rc, probs); + dist = p->reps[3]; p->reps[3] = p->reps[2]; + } p->reps[2] = p->reps[1]; } p->reps[1] = p->reps[0]; - p->reps[0] = distance; + p->reps[0] = dist; } - if (len == 1) - p->state = kShortRepNextStates[p->state]; - else + + RC_NORM(&p->rc) + + p->rc.range = range; + + if (len != 1) { - LenEnc_Encode2(&p->repLenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); + LenEnc_Encode(&p->repLenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + --p->repLenEncCounter; p->state = kRepNextStates[p->state]; } } else { - UInt32 posSlot; - RangeEnc_EncodeBit(&p->rc, &p->isRep[p->state], 0); + unsigned posSlot; + RC_BIT_0(&p->rc, probs); + p->rc.range = range; p->state = kMatchNextStates[p->state]; - LenEnc_Encode2(&p->lenEnc, &p->rc, len - LZMA_MATCH_LEN_MIN, posState, !p->fastMode, p->ProbPrices); - pos -= LZMA_NUM_REPS; - GetPosSlot(pos, posSlot); - RcTree_Encode(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], kNumPosSlotBits, posSlot); + + LenEnc_Encode(&p->lenProbs, &p->rc, len - LZMA_MATCH_LEN_MIN, posState); + // --p->lenEnc.counter; + + dist -= LZMA_NUM_REPS; + p->reps[3] = p->reps[2]; + p->reps[2] = p->reps[1]; + p->reps[1] = p->reps[0]; + p->reps[0] = dist + 1; + + p->matchPriceCount++; + GetPosSlot(dist, posSlot); + // RcTree_Encode_PosSlot(&p->rc, p->posSlotEncoder[GetLenToPosState(len)], posSlot); + { + UInt32 sym = (UInt32)posSlot + (1 << kNumPosSlotBits); + range = p->rc.range; + probs = p->posSlotEncoder[GetLenToPosState(len)]; + do + { + CLzmaProb *prob = probs + (sym >> kNumPosSlotBits); + UInt32 bit = (sym >> (kNumPosSlotBits - 1)) & 1; + sym <<= 1; + RC_BIT(&p->rc, prob, bit); + } + while (sym < (1 << kNumPosSlotBits * 2)); + p->rc.range = range; + } - if (posSlot >= kStartPosModelIndex) + if (dist >= kStartPosModelIndex) { - UInt32 footerBits = ((posSlot >> 1) - 1); - UInt32 base = ((2 | (posSlot & 1)) << footerBits); - UInt32 posReduced = pos - base; + unsigned footerBits = ((posSlot >> 1) - 1); - if (posSlot < kEndPosModelIndex) - RcTree_ReverseEncode(&p->rc, p->posEncoders + base - posSlot - 1, footerBits, posReduced); + if (dist < kNumFullDistances) + { + unsigned base = ((2 | (posSlot & 1)) << footerBits); + RcTree_ReverseEncode(&p->rc, p->posEncoders + base, footerBits, (unsigned)(dist /* - base */)); + } else { - RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); - RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); - p->alignPriceCount++; + UInt32 pos2 = (dist | 0xF) << (32 - footerBits); + range = p->rc.range; + // RangeEnc_EncodeDirectBits(&p->rc, posReduced >> kNumAlignBits, footerBits - kNumAlignBits); + /* + do + { + range >>= 1; + p->rc.low += range & (0 - ((dist >> --footerBits) & 1)); + RC_NORM(&p->rc) + } + while (footerBits > kNumAlignBits); + */ + do + { + range >>= 1; + p->rc.low += range & (0 - (pos2 >> 31)); + pos2 += pos2; + RC_NORM(&p->rc) + } + while (pos2 != 0xF0000000); + + + // RcTree_ReverseEncode(&p->rc, p->posAlignEncoder, kNumAlignBits, posReduced & kAlignMask); + + { + unsigned m = 1; + unsigned bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; dist >>= 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); m = (m << 1) + bit; + bit = dist & 1; RC_BIT(&p->rc, p->posAlignEncoder + m, bit); + p->rc.range = range; + // p->alignPriceCount++; + } } } - p->reps[3] = p->reps[2]; - p->reps[2] = p->reps[1]; - p->reps[1] = p->reps[0]; - p->reps[0] = pos; - p->matchPriceCount++; } } + + nowPos32 += (UInt32)len; p->additionalOffset -= len; - nowPos32 += len; + if (p->additionalOffset == 0) { UInt32 processed; + if (!p->fastMode) { - if (p->matchPriceCount >= (1 << 7)) + /* + if (p->alignPriceCount >= 16) // kAlignTableSize + FillAlignPrices(p); + if (p->matchPriceCount >= 128) FillDistancesPrices(p); - if (p->alignPriceCount >= kAlignTableSize) + if (p->lenEnc.counter <= 0) + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + */ + if (p->matchPriceCount >= 64) + { FillAlignPrices(p); + // { int y; for (y = 0; y < 100; y++) { + FillDistancesPrices(p); + // }} + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + } + if (p->repLenEncCounter <= 0) + { + p->repLenEncCounter = REP_LEN_COUNT; + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); + } } + if (p->matchFinder.GetNumAvailableBytes(p->matchFinderObj) == 0) break; processed = nowPos32 - startPos32; - if (useLimits) + + if (maxPackSize) { - if (processed + kNumOpts + 300 >= maxUnpackSize || - RangeEnc_GetProcessed(&p->rc) + kNumOpts * 2 >= maxPackSize) + if (processed + kNumOpts + 300 >= maxUnpackSize + || RangeEnc_GetProcessed_sizet(&p->rc) + kPackReserve >= maxPackSize) break; } else if (processed >= (1 << 17))@@ -1945,13 +2547,16 @@ return CheckErrors(p);
} } } + p->nowPos64 += nowPos32 - startPos32; return Flush(p, nowPos32); } + + #define kBigHashDicLimit ((UInt32)1 << 24) -static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_Alloc(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { UInt32 beforeSize = kNumOpts; if (!RangeEnc_Alloc(&p->rc, alloc))@@ -1966,8 +2571,8 @@ unsigned lclp = p->lc + p->lp;
if (!p->litProbs || !p->saveState.litProbs || p->lclp != lclp) { LzmaEnc_FreeLits(p, alloc); - p->litProbs = (CLzmaProb *)alloc->Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); - p->saveState.litProbs = (CLzmaProb *)alloc->Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); + p->saveState.litProbs = (CLzmaProb *)ISzAlloc_Alloc(alloc, ((UInt32)0x300 << lclp) * sizeof(CLzmaProb)); if (!p->litProbs || !p->saveState.litProbs) { LzmaEnc_FreeLits(p, alloc);@@ -1985,8 +2590,13 @@
#ifndef _7ZIP_ST if (p->mtMode) { - RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, LZMA_MATCH_LEN_MAX, allocBig)); + RINOK(MatchFinderMt_Create(&p->matchFinderMt, p->dictSize, beforeSize, p->numFastBytes, + LZMA_MATCH_LEN_MAX + + 1 /* 18.04 */ + , allocBig)); p->matchFinderObj = &p->matchFinderMt; + p->matchFinderBase.bigHash = (Byte)( + (p->dictSize > kBigHashDicLimit && p->matchFinderBase.hashMask >= 0xFFFFFF) ? 1 : 0); MatchFinderMt_CreateVTable(&p->matchFinderMt, &p->matchFinder); } else@@ -2003,17 +2613,21 @@ }
void LzmaEnc_Init(CLzmaEnc *p) { - UInt32 i; + unsigned i; p->state = 0; - for (i = 0 ; i < LZMA_NUM_REPS; i++) - p->reps[i] = 0; + p->reps[0] = + p->reps[1] = + p->reps[2] = + p->reps[3] = 1; RangeEnc_Init(&p->rc); + for (i = 0; i < (1 << kNumAlignBits); i++) + p->posAlignEncoder[i] = kProbInitValue; for (i = 0; i < kNumStates; i++) { - UInt32 j; + unsigned j; for (j = 0; j < LZMA_NUM_PB_STATES_MAX; j++) { p->isMatch[i][j] = kProbInitValue;@@ -2026,39 +2640,45 @@ p->isRepG2[i] = kProbInitValue;
} { - UInt32 num = (UInt32)0x300 << (p->lp + p->lc); - CLzmaProb *probs = p->litProbs; - for (i = 0; i < num; i++) - probs[i] = kProbInitValue; - } - - { for (i = 0; i < kNumLenToPosStates; i++) { CLzmaProb *probs = p->posSlotEncoder[i]; - UInt32 j; + unsigned j; for (j = 0; j < (1 << kNumPosSlotBits); j++) probs[j] = kProbInitValue; } } { - for (i = 0; i < kNumFullDistances - kEndPosModelIndex; i++) + for (i = 0; i < kNumFullDistances; i++) p->posEncoders[i] = kProbInitValue; } - LenEnc_Init(&p->lenEnc.p); - LenEnc_Init(&p->repLenEnc.p); + { + UInt32 num = (UInt32)0x300 << (p->lp + p->lc); + UInt32 k; + CLzmaProb *probs = p->litProbs; + for (k = 0; k < num; k++) + probs[k] = kProbInitValue; + } + + + LenEnc_Init(&p->lenProbs); + LenEnc_Init(&p->repLenProbs); + + p->optEnd = 0; + p->optCur = 0; - for (i = 0; i < (1 << kNumAlignBits); i++) - p->posAlignEncoder[i] = kProbInitValue; + { + for (i = 0; i < kNumOpts; i++) + p->opt[i].price = kInfinityPrice; + } - p->optimumEndIndex = 0; - p->optimumCurrentIndex = 0; p->additionalOffset = 0; p->pbMask = (1 << p->pb) - 1; - p->lpMask = (1 << p->lp) - 1; + p->lpMask = ((UInt32)0x100 << p->lp) - ((unsigned)0x100 >> p->lc); } + void LzmaEnc_InitPrices(CLzmaEnc *p) {@@ -2071,14 +2691,17 @@
p->lenEnc.tableSize = p->repLenEnc.tableSize = p->numFastBytes + 1 - LZMA_MATCH_LEN_MIN; - LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, p->ProbPrices); - LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, p->ProbPrices); + + p->repLenEncCounter = REP_LEN_COUNT; + + LenPriceEnc_UpdateTables(&p->lenEnc, 1 << p->pb, &p->lenProbs, p->ProbPrices); + LenPriceEnc_UpdateTables(&p->repLenEnc, 1 << p->pb, &p->repLenProbs, p->ProbPrices); } -static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) +static SRes LzmaEnc_AllocAndInit(CLzmaEnc *p, UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { - UInt32 i; - for (i = 0; i < (UInt32)kDicLogSizeMaxCompress; i++) + unsigned i; + for (i = kEndPosModelIndex / 2; i < kDicLogSizeMax; i++) if (p->dictSize <= ((UInt32)1 << i)) break; p->distTableSize = i * 2;@@ -2093,7 +2716,7 @@ return SZ_OK;
} static SRes LzmaEnc_Prepare(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; p->matchFinderBase.stream = inStream;@@ -2104,7 +2727,7 @@ }
SRes LzmaEnc_PrepareForLzma2(CLzmaEncHandle pp, ISeqInStream *inStream, UInt32 keepWindowSize, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; p->matchFinderBase.stream = inStream;@@ -2120,12 +2743,13 @@ p->matchFinderBase.directInputRem = srcLen;
} SRes LzmaEnc_MemPrepare(CLzmaEncHandle pp, const Byte *src, SizeT srcLen, - UInt32 keepWindowSize, ISzAlloc *alloc, ISzAlloc *allocBig) + UInt32 keepWindowSize, ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)pp; LzmaEnc_SetInputBuf(p, src, srcLen); p->needInit = 1; + LzmaEnc_SetDataSize(pp, srcLen); return LzmaEnc_AllocAndInit(p, keepWindowSize, alloc, allocBig); }@@ -2143,15 +2767,15 @@
typedef struct { - ISeqOutStream funcTable; + ISeqOutStream vt; Byte *data; SizeT rem; - Bool overflow; -} CSeqOutStreamBuf; + BoolInt overflow; +} CLzmaEnc_SeqOutStreamBuf; -static size_t MyWrite(void *pp, const void *data, size_t size) +static size_t SeqOutStreamBuf_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqOutStreamBuf *p = (CSeqOutStreamBuf *)pp; + CLzmaEnc_SeqOutStreamBuf *p = CONTAINER_FROM_VTBL(pp, CLzmaEnc_SeqOutStreamBuf, vt); if (p->rem < size) { size = p->rem;@@ -2178,15 +2802,15 @@ return p->matchFinder.GetPointerToCurrentPos(p->matchFinderObj) - p->additionalOffset;
} -SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, Bool reInit, +SRes LzmaEnc_CodeOneMemBlock(CLzmaEncHandle pp, BoolInt reInit, Byte *dest, size_t *destLen, UInt32 desiredPackSize, UInt32 *unpackSize) { CLzmaEnc *p = (CLzmaEnc *)pp; UInt64 nowPos64; SRes res; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False;@@ -2198,11 +2822,15 @@
if (reInit) LzmaEnc_Init(p); LzmaEnc_InitPrices(p); + nowPos64 = p->nowPos64; RangeEnc_Init(&p->rc); - p->rc.outStream = &outStream.funcTable; + p->rc.outStream = &outStream.vt; - res = LzmaEnc_CodeOneBlock(p, True, desiredPackSize, *unpackSize); + if (desiredPackSize == 0) + return SZ_ERROR_OUTPUT_EOF; + + res = LzmaEnc_CodeOneBlock(p, desiredPackSize, *unpackSize); *unpackSize = (UInt32)(p->nowPos64 - nowPos64); *destLen -= outStream.rem;@@ -2225,12 +2853,12 @@ #endif
for (;;) { - res = LzmaEnc_CodeOneBlock(p, False, 0, 0); + res = LzmaEnc_CodeOneBlock(p, 0, 0); if (res != SZ_OK || p->finished) break; if (progress) { - res = progress->Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); + res = ICompressProgress_Progress(progress, p->nowPos64, RangeEnc_GetProcessed(&p->rc)); if (res != SZ_OK) { res = SZ_ERROR_PROGRESS;@@ -2242,7 +2870,7 @@
LzmaEnc_Finish(p); /* - if (res == S_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) + if (res == SZ_OK && !Inline_MatchFinder_IsFinishedOK(&p->matchFinderBase)) res = SZ_ERROR_FAIL; } */@@ -2252,7 +2880,7 @@ }
SRes LzmaEnc_Encode(CLzmaEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress, - ISzAlloc *alloc, ISzAlloc *allocBig) + ISzAllocPtr alloc, ISzAllocPtr allocBig) { RINOK(LzmaEnc_Prepare(pp, outStream, inStream, alloc, allocBig)); return LzmaEnc_Encode2((CLzmaEnc *)pp, progress);@@ -2284,24 +2912,30 @@
for (i = 0; i < 4; i++) props[1 + i] = (Byte)(dictSize >> (8 * i)); return SZ_OK; +} + + +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle pp) +{ + return ((CLzmaEnc *)pp)->writeEndMark; } SRes LzmaEnc_MemEncode(CLzmaEncHandle pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { SRes res; CLzmaEnc *p = (CLzmaEnc *)pp; - CSeqOutStreamBuf outStream; + CLzmaEnc_SeqOutStreamBuf outStream; - outStream.funcTable.Write = MyWrite; + outStream.vt.Write = SeqOutStreamBuf_Write; outStream.data = dest; outStream.rem = *destLen; outStream.overflow = False; p->writeEndMark = writeEndMark; - p->rc.outStream = &outStream.funcTable; + p->rc.outStream = &outStream.vt; res = LzmaEnc_MemPrepare(pp, src, srcLen, 0, alloc, allocBig);@@ -2321,7 +2955,7 @@
SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig) + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig) { CLzmaEnc *p = (CLzmaEnc *)LzmaEnc_Create(alloc); SRes res;
M
src/third-party/lzma/LzmaEnc.h
→
src/third-party/lzma/LzmaEnc.h
@@ -1,5 +1,5 @@
/* LzmaEnc.h -- LZMA Encoder -2013-01-18 : Igor Pavlov : Public domain */ +2017-07-27 : Igor Pavlov : Public domain */ #ifndef __LZMA_ENC_H #define __LZMA_ENC_H@@ -12,12 +12,10 @@ #define LZMA_PROPS_SIZE 5
typedef struct _CLzmaEncProps { - int level; /* 0 <= level <= 9 */ + int level; /* 0 <= level <= 9 */ UInt32 dictSize; /* (1 << 12) <= dictSize <= (1 << 27) for 32-bit version - (1 << 12) <= dictSize <= (1 << 30) for 64-bit version - default = (1 << 24) */ - UInt64 reduceSize; /* estimated size of data that will be compressed. default = 0xFFFFFFFF. - Encoder uses this value to reduce dictionary size */ + (1 << 12) <= dictSize <= (3 << 29) for 64-bit version + default = (1 << 24) */ int lc; /* 0 <= lc <= 8, default = 3 */ int lp; /* 0 <= lp <= 4, default = 0 */ int pb; /* 0 <= pb <= 4, default = 2 */@@ -25,9 +23,12 @@ int algo; /* 0 - fast, 1 - normal, default = 1 */
int fb; /* 5 <= fb <= 273, default = 32 */ int btMode; /* 0 - hashChain Mode, 1 - binTree mode - normal, default = 1 */ int numHashBytes; /* 2, 3 or 4, default = 4 */ - UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ + UInt32 mc; /* 1 <= mc <= (1 << 30), default = 32 */ unsigned writeEndMark; /* 0 - do not write EOPM, 1 - write EOPM, default = 0 */ int numThreads; /* 1 or 2, default = 2 */ + + UInt64 reduceSize; /* estimated size of data that will be compressed. default = (UInt64)(Int64)-1. + Encoder uses this value to reduce dictionary size */ } CLzmaEncProps; void LzmaEncProps_Init(CLzmaEncProps *p);@@ -37,41 +38,38 @@
/* ---------- CLzmaEncHandle Interface ---------- */ -/* LzmaEnc_* functions can return the following exit codes: -Returns: +/* LzmaEnc* functions can return the following exit codes: +SRes: SZ_OK - OK SZ_ERROR_MEM - Memory allocation error SZ_ERROR_PARAM - Incorrect paramater in props - SZ_ERROR_WRITE - Write callback error. + SZ_ERROR_WRITE - ISeqOutStream write callback error + SZ_ERROR_OUTPUT_EOF - output buffer overflow - version with (Byte *) output SZ_ERROR_PROGRESS - some break from progress callback - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) + SZ_ERROR_THREAD - error in multithreading functions (only for Mt version) */ typedef void * CLzmaEncHandle; -CLzmaEncHandle LzmaEnc_Create(ISzAlloc *alloc); -void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAlloc *alloc, ISzAlloc *allocBig); +CLzmaEncHandle LzmaEnc_Create(ISzAllocPtr alloc); +void LzmaEnc_Destroy(CLzmaEncHandle p, ISzAllocPtr alloc, ISzAllocPtr allocBig); + SRes LzmaEnc_SetProps(CLzmaEncHandle p, const CLzmaEncProps *props); +void LzmaEnc_SetDataSize(CLzmaEncHandle p, UInt64 expectedDataSiize); SRes LzmaEnc_WriteProperties(CLzmaEncHandle p, Byte *properties, SizeT *size); +unsigned LzmaEnc_IsWriteEndMark(CLzmaEncHandle p); + SRes LzmaEnc_Encode(CLzmaEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); SRes LzmaEnc_MemEncode(CLzmaEncHandle p, Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, - int writeEndMark, ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + int writeEndMark, ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); -/* ---------- One Call Interface ---------- */ -/* LzmaEncode -Return code: - SZ_OK - OK - SZ_ERROR_MEM - Memory allocation error - SZ_ERROR_PARAM - Incorrect paramater - SZ_ERROR_OUTPUT_EOF - output buffer overflow - SZ_ERROR_THREAD - errors in multithreading functions (only for Mt version) -*/ +/* ---------- One Call Interface ---------- */ SRes LzmaEncode(Byte *dest, SizeT *destLen, const Byte *src, SizeT srcLen, const CLzmaEncProps *props, Byte *propsEncoded, SizeT *propsSize, int writeEndMark, - ICompressProgress *progress, ISzAlloc *alloc, ISzAlloc *allocBig); + ICompressProgress *progress, ISzAllocPtr alloc, ISzAllocPtr allocBig); EXTERN_C_END
M
src/third-party/lzma/MtCoder.c
→
src/third-party/lzma/MtCoder.c
@@ -1,168 +1,89 @@
/* MtCoder.c -- Multi-thread Coder -2015-10-13 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h" #include "MtCoder.h" -void LoopThread_Construct(CLoopThread *p) -{ - Thread_Construct(&p->thread); - Event_Construct(&p->startEvent); - Event_Construct(&p->finishedEvent); -} +#ifndef _7ZIP_ST -void LoopThread_Close(CLoopThread *p) +SRes MtProgressThunk_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) { - Thread_Close(&p->thread); - Event_Close(&p->startEvent); - Event_Close(&p->finishedEvent); -} - -static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE LoopThreadFunc(void *pp) -{ - CLoopThread *p = (CLoopThread *)pp; - for (;;) + CMtProgressThunk *thunk = CONTAINER_FROM_VTBL(pp, CMtProgressThunk, vt); + UInt64 inSize2 = 0; + UInt64 outSize2 = 0; + if (inSize != (UInt64)(Int64)-1) + { + inSize2 = inSize - thunk->inSize; + thunk->inSize = inSize; + } + if (outSize != (UInt64)(Int64)-1) { - if (Event_Wait(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - if (p->stop) - return 0; - p->res = p->func(p->param); - if (Event_Set(&p->finishedEvent) != 0) - return SZ_ERROR_THREAD; + outSize2 = outSize - thunk->outSize; + thunk->outSize = outSize; } + return MtProgress_ProgressAdd(thunk->mtProgress, inSize2, outSize2); } -WRes LoopThread_Create(CLoopThread *p) -{ - p->stop = 0; - RINOK(AutoResetEvent_CreateNotSignaled(&p->startEvent)); - RINOK(AutoResetEvent_CreateNotSignaled(&p->finishedEvent)); - return Thread_Create(&p->thread, LoopThreadFunc, p); -} -WRes LoopThread_StopAndWait(CLoopThread *p) +void MtProgressThunk_CreateVTable(CMtProgressThunk *p) { - p->stop = 1; - if (Event_Set(&p->startEvent) != 0) - return SZ_ERROR_THREAD; - return Thread_Wait(&p->thread); + p->vt.Progress = MtProgressThunk_Progress; } -WRes LoopThread_StartSubThread(CLoopThread *p) { return Event_Set(&p->startEvent); } -WRes LoopThread_WaitSubThread(CLoopThread *p) { return Event_Wait(&p->finishedEvent); } -static SRes Progress(ICompressProgress *p, UInt64 inSize, UInt64 outSize) -{ - return (p && p->Progress(p, inSize, outSize) != SZ_OK) ? SZ_ERROR_PROGRESS : SZ_OK; -} -static void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) -{ - unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - p->inSizes[i] = p->outSizes[i] = 0; - p->totalInSize = p->totalOutSize = 0; - p->progress = progress; - p->res = SZ_OK; -} +#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -static void MtProgress_Reinit(CMtProgress *p, unsigned index) -{ - p->inSizes[index] = 0; - p->outSizes[index] = 0; -} -#define UPDATE_PROGRESS(size, prev, total) \ - if (size != (UInt64)(Int64)-1) { total += size - prev; prev = size; } - -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize) +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) { - SRes res; - CriticalSection_Enter(&p->cs); - UPDATE_PROGRESS(inSize, p->inSizes[index], p->totalInSize) - UPDATE_PROGRESS(outSize, p->outSizes[index], p->totalOutSize) - if (p->res == SZ_OK) - p->res = Progress(p->progress, p->totalInSize, p->totalOutSize); - res = p->res; - CriticalSection_Leave(&p->cs); - return res; + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); } -static void MtProgress_SetError(CMtProgress *p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} -static void MtCoder_SetError(CMtCoder* p, SRes res) -{ - CriticalSection_Enter(&p->cs); - if (p->res == SZ_OK) - p->res = res; - CriticalSection_Leave(&p->cs); -} +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); -/* ---------- MtThread ---------- */ -void CMtThread_Construct(CMtThread *p, CMtCoder *mtCoder) +static SRes MtCoderThread_CreateAndStart(CMtCoderThread *t) { - p->mtCoder = mtCoder; - p->outBuf = 0; - p->inBuf = 0; - Event_Construct(&p->canRead); - Event_Construct(&p->canWrite); - LoopThread_Construct(&p->thread); + WRes wres = ArEvent_OptCreate_And_Reset(&t->startEvent); + if (wres == 0) + { + t->stop = False; + if (!Thread_WasCreated(&t->thread)) + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + wres = Event_Set(&t->startEvent); + } + if (wres == 0) + return SZ_OK; + return MY_SRes_HRESULT_FROM_WRes(wres); } -#define RINOK_THREAD(x) { if ((x) != 0) return SZ_ERROR_THREAD; } -static void CMtThread_CloseEvents(CMtThread *p) +static void MtCoderThread_Destruct(CMtCoderThread *t) { - Event_Close(&p->canRead); - Event_Close(&p->canWrite); -} + if (Thread_WasCreated(&t->thread)) + { + t->stop = 1; + Event_Set(&t->startEvent); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); + } -static void CMtThread_Destruct(CMtThread *p) -{ - CMtThread_CloseEvents(p); + Event_Close(&t->startEvent); - if (Thread_WasCreated(&p->thread.thread)) + if (t->inBuf) { - LoopThread_StopAndWait(&p->thread); - LoopThread_Close(&p->thread); + ISzAlloc_Free(t->mtCoder->allocBig, t->inBuf); + t->inBuf = NULL; } - - if (p->mtCoder->alloc) - IAlloc_Free(p->mtCoder->alloc, p->outBuf); - p->outBuf = 0; - - if (p->mtCoder->alloc) - IAlloc_Free(p->mtCoder->alloc, p->inBuf); - p->inBuf = 0; } -#define MY_BUF_ALLOC(buf, size, newSize) \ - if (buf == 0 || size != newSize) \ - { IAlloc_Free(p->mtCoder->alloc, buf); \ - size = newSize; buf = (Byte *)IAlloc_Alloc(p->mtCoder->alloc, size); \ - if (buf == 0) return SZ_ERROR_MEM; } -static SRes CMtThread_Prepare(CMtThread *p) -{ - MY_BUF_ALLOC(p->inBuf, p->inBufSize, p->mtCoder->blockSize) - MY_BUF_ALLOC(p->outBuf, p->outBufSize, p->mtCoder->destBlockSize) - - p->stopReading = False; - p->stopWriting = False; - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canRead)); - RINOK_THREAD(AutoResetEvent_CreateNotSignaled(&p->canWrite)); - - return SZ_OK; -} static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) {@@ -170,158 +91,511 @@ size_t size = *processedSize;
*processedSize = 0; while (size != 0) { - size_t curSize = size; - SRes res = stream->Read(stream, data, &curSize); - *processedSize += curSize; - data += curSize; - size -= curSize; + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; RINOK(res); - if (curSize == 0) + if (cur == 0) return SZ_OK; } return SZ_OK; } -#define GET_NEXT_THREAD(p) &p->mtCoder->threads[p->index == p->mtCoder->numThreads - 1 ? 0 : p->index + 1] + +/* + ThreadFunc2() returns: + SZ_OK - in all normal cases (even for stream error or memory allocation error) + SZ_ERROR_THREAD - in case of failure in system synch function +*/ -static SRes MtThread_Process(CMtThread *p, Bool *stop) +static SRes ThreadFunc2(CMtCoderThread *t) { - CMtThread *next; - *stop = True; - if (Event_Wait(&p->canRead) != 0) - return SZ_ERROR_THREAD; - - next = GET_NEXT_THREAD(p); - - if (p->stopReading) + CMtCoder *mtc = t->mtCoder; + + for (;;) { - next->stopReading = True; - return Event_Set(&next->canRead) == 0 ? SZ_OK : SZ_ERROR_THREAD; - } + unsigned bi; + SRes res; + SRes res2; + BoolInt finished; + unsigned bufIndex; + size_t size; + const Byte *inData; + UInt64 readProcessed = 0; + + RINOK_THREAD(Event_Wait(&mtc->readEvent)) + + /* after Event_Wait(&mtc->readEvent) we must call Event_Set(&mtc->readEvent) in any case to unlock another threads */ + + if (mtc->stopReading) + { + return Event_Set(&mtc->readEvent) == 0 ? SZ_OK : SZ_ERROR_THREAD; + } + + res = MtProgress_GetError(&mtc->mtProgress); + + size = 0; + inData = NULL; + finished = True; + + if (res == SZ_OK) + { + size = mtc->blockSize; + if (mtc->inStream) + { + if (!t->inBuf) + { + t->inBuf = (Byte *)ISzAlloc_Alloc(mtc->allocBig, mtc->blockSize); + if (!t->inBuf) + res = SZ_ERROR_MEM; + } + if (res == SZ_OK) + { + res = FullRead(mtc->inStream, t->inBuf, &size); + readProcessed = mtc->readProcessed + size; + mtc->readProcessed = readProcessed; + } + if (res != SZ_OK) + { + mtc->readRes = res; + /* after reading error - we can stop encoding of previous blocks */ + MtProgress_SetError(&mtc->mtProgress, res); + } + else + finished = (size != mtc->blockSize); + } + else + { + size_t rem; + readProcessed = mtc->readProcessed; + rem = mtc->inDataSize - (size_t)readProcessed; + if (size > rem) + size = rem; + inData = mtc->inData + (size_t)readProcessed; + readProcessed += size; + mtc->readProcessed = readProcessed; + finished = (mtc->inDataSize == (size_t)readProcessed); + } + } - { - size_t size = p->mtCoder->blockSize; - size_t destSize = p->outBufSize; + /* we must get some block from blocksSemaphore before Event_Set(&mtc->readEvent) */ - RINOK(FullRead(p->mtCoder->inStream, p->inBuf, &size)); - next->stopReading = *stop = (size != p->mtCoder->blockSize); - if (Event_Set(&next->canRead) != 0) - return SZ_ERROR_THREAD; + res2 = SZ_OK; - RINOK(p->mtCoder->mtCallback->Code(p->mtCoder->mtCallback, p->index, - p->outBuf, &destSize, p->inBuf, size, *stop)); + if (Semaphore_Wait(&mtc->blocksSemaphore) != 0) + { + res2 = SZ_ERROR_THREAD; + if (res == SZ_OK) + { + res = res2; + // MtProgress_SetError(&mtc->mtProgress, res); + } + } - MtProgress_Reinit(&p->mtCoder->mtProgress, p->index); + bi = mtc->blockIndex; - if (Event_Wait(&p->canWrite) != 0) - return SZ_ERROR_THREAD; - if (p->stopWriting) - return SZ_ERROR_FAIL; - if (p->mtCoder->outStream->Write(p->mtCoder->outStream, p->outBuf, destSize) != destSize) - return SZ_ERROR_WRITE; - return Event_Set(&next->canWrite) == 0 ? SZ_OK : SZ_ERROR_THREAD; + if (++mtc->blockIndex >= mtc->numBlocksMax) + mtc->blockIndex = 0; + + bufIndex = (unsigned)(int)-1; + + if (res == SZ_OK) + res = MtProgress_GetError(&mtc->mtProgress); + + if (res != SZ_OK) + finished = True; + + if (!finished) + { + if (mtc->numStartedThreads < mtc->numStartedThreadsLimit + && mtc->expectedDataSize != readProcessed) + { + res = MtCoderThread_CreateAndStart(&mtc->threads[mtc->numStartedThreads]); + if (res == SZ_OK) + mtc->numStartedThreads++; + else + { + MtProgress_SetError(&mtc->mtProgress, res); + finished = True; + } + } + } + + if (finished) + mtc->stopReading = True; + + RINOK_THREAD(Event_Set(&mtc->readEvent)) + + if (res2 != SZ_OK) + return res2; + + if (res == SZ_OK) + { + CriticalSection_Enter(&mtc->cs); + bufIndex = mtc->freeBlockHead; + mtc->freeBlockHead = mtc->freeBlockList[bufIndex]; + CriticalSection_Leave(&mtc->cs); + + res = mtc->mtCallback->Code(mtc->mtCallbackObject, t->index, bufIndex, + mtc->inStream ? t->inBuf : inData, size, finished); + + // MtProgress_Reinit(&mtc->mtProgress, t->index); + + if (res != SZ_OK) + MtProgress_SetError(&mtc->mtProgress, res); + } + + { + CMtCoderBlock *block = &mtc->blocks[bi]; + block->res = res; + block->bufIndex = bufIndex; + block->finished = finished; + } + + #ifdef MTCODER__USE_WRITE_THREAD + RINOK_THREAD(Event_Set(&mtc->writeEvents[bi])) + #else + { + unsigned wi; + { + CriticalSection_Enter(&mtc->cs); + wi = mtc->writeIndex; + if (wi == bi) + mtc->writeIndex = (unsigned)(int)-1; + else + mtc->ReadyBlocks[bi] = True; + CriticalSection_Leave(&mtc->cs); + } + + if (wi != bi) + { + if (res != SZ_OK || finished) + return 0; + continue; + } + + if (mtc->writeRes != SZ_OK) + res = mtc->writeRes; + + for (;;) + { + if (res == SZ_OK && bufIndex != (unsigned)(int)-1) + { + res = mtc->mtCallback->Write(mtc->mtCallbackObject, bufIndex); + if (res != SZ_OK) + { + mtc->writeRes = res; + MtProgress_SetError(&mtc->mtProgress, res); + } + } + + if (++wi >= mtc->numBlocksMax) + wi = 0; + { + BoolInt isReady; + + CriticalSection_Enter(&mtc->cs); + + if (bufIndex != (unsigned)(int)-1) + { + mtc->freeBlockList[bufIndex] = mtc->freeBlockHead; + mtc->freeBlockHead = bufIndex; + } + + isReady = mtc->ReadyBlocks[wi]; + + if (isReady) + mtc->ReadyBlocks[wi] = False; + else + mtc->writeIndex = wi; + + CriticalSection_Leave(&mtc->cs); + + RINOK_THREAD(Semaphore_Release1(&mtc->blocksSemaphore)) + + if (!isReady) + break; + } + + { + CMtCoderBlock *block = &mtc->blocks[wi]; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + bufIndex = block->bufIndex; + finished = block->finished; + } + } + } + #endif + + if (finished || res != SZ_OK) + return 0; } } + static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) { - CMtThread *p = (CMtThread *)pp; + CMtCoderThread *t = (CMtCoderThread *)pp; for (;;) { - Bool stop; - CMtThread *next = GET_NEXT_THREAD(p); - SRes res = MtThread_Process(p, &stop); - if (res != SZ_OK) + if (Event_Wait(&t->startEvent) != 0) + return SZ_ERROR_THREAD; + if (t->stop) + return 0; { - MtCoder_SetError(p->mtCoder, res); - MtProgress_SetError(&p->mtCoder->mtProgress, res); - next->stopReading = True; - next->stopWriting = True; - Event_Set(&next->canRead); - Event_Set(&next->canWrite); - return res; + SRes res = ThreadFunc2(t); + CMtCoder *mtc = t->mtCoder; + if (res != SZ_OK) + { + MtProgress_SetError(&mtc->mtProgress, res); + } + + #ifndef MTCODER__USE_WRITE_THREAD + { + unsigned numFinished = (unsigned)InterlockedIncrement(&mtc->numFinishedThreads); + if (numFinished == mtc->numStartedThreads) + if (Event_Set(&mtc->finishedEvent) != 0) + return SZ_ERROR_THREAD; + } + #endif } - if (stop) - return 0; } } -void MtCoder_Construct(CMtCoder* p) + + +void MtCoder_Construct(CMtCoder *p) { unsigned i; - p->alloc = 0; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) + + p->blockSize = 0; + p->numThreadsMax = 0; + p->expectedDataSize = (UInt64)(Int64)-1; + + p->inStream = NULL; + p->inData = NULL; + p->inDataSize = 0; + + p->progress = NULL; + p->allocBig = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + Event_Construct(&p->readEvent); + Semaphore_Construct(&p->blocksSemaphore); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) { - CMtThread *t = &p->threads[i]; + CMtCoderThread *t = &p->threads[i]; + t->mtCoder = p; t->index = i; - CMtThread_Construct(t, p); + t->inBuf = NULL; + t->stop = False; + Event_Construct(&t->startEvent); + Thread_Construct(&t->thread); } + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Construct(&p->writeEvents[i]); + #else + Event_Construct(&p->finishedEvent); + #endif + CriticalSection_Init(&p->cs); CriticalSection_Init(&p->mtProgress.cs); } -void MtCoder_Destruct(CMtCoder* p) + + + +static void MtCoder_Free(CMtCoder *p) { unsigned i; - for (i = 0; i < NUM_MT_CODER_THREADS_MAX; i++) - CMtThread_Destruct(&p->threads[i]); + + /* + p->stopReading = True; + if (Event_IsCreated(&p->readEvent)) + Event_Set(&p->readEvent); + */ + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + MtCoderThread_Destruct(&p->threads[i]); + + Event_Close(&p->readEvent); + Semaphore_Close(&p->blocksSemaphore); + + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + Event_Close(&p->writeEvents[i]); + #else + Event_Close(&p->finishedEvent); + #endif +} + + +void MtCoder_Destruct(CMtCoder *p) +{ + MtCoder_Free(p); + CriticalSection_Delete(&p->cs); CriticalSection_Delete(&p->mtProgress.cs); } + SRes MtCoder_Code(CMtCoder *p) { - unsigned i, numThreads = p->numThreads; + unsigned numThreads = p->numThreadsMax; + unsigned numBlocksMax; + unsigned i; SRes res = SZ_OK; - p->res = SZ_OK; + + if (numThreads > MTCODER__THREADS_MAX) + numThreads = MTCODER__THREADS_MAX; + numBlocksMax = MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads); + + if (p->blockSize < ((UInt32)1 << 26)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 24)) numBlocksMax++; + if (p->blockSize < ((UInt32)1 << 22)) numBlocksMax++; + + if (numBlocksMax > MTCODER__BLOCKS_MAX) + numBlocksMax = MTCODER__BLOCKS_MAX; + + if (p->blockSize != p->allocatedBufsSize) + { + for (i = 0; i < MTCODER__THREADS_MAX; i++) + { + CMtCoderThread *t = &p->threads[i]; + if (t->inBuf) + { + ISzAlloc_Free(p->allocBig, t->inBuf); + t->inBuf = NULL; + } + } + p->allocatedBufsSize = p->blockSize; + } + + p->readRes = SZ_OK; MtProgress_Init(&p->mtProgress, p->progress); - for (i = 0; i < numThreads; i++) + #ifdef MTCODER__USE_WRITE_THREAD + for (i = 0; i < numBlocksMax; i++) + { + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->writeEvents[i])); + } + #else + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + #endif + + { + RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->readEvent)); + + if (Semaphore_IsCreated(&p->blocksSemaphore)) + { + RINOK_THREAD(Semaphore_Close(&p->blocksSemaphore)); + } + RINOK_THREAD(Semaphore_Create(&p->blocksSemaphore, numBlocksMax, numBlocksMax)); + } + + for (i = 0; i < MTCODER__BLOCKS_MAX - 1; i++) + p->freeBlockList[i] = i + 1; + p->freeBlockList[MTCODER__BLOCKS_MAX - 1] = (unsigned)(int)-1; + p->freeBlockHead = 0; + + p->readProcessed = 0; + p->blockIndex = 0; + p->numBlocksMax = numBlocksMax; + p->stopReading = False; + + #ifndef MTCODER__USE_WRITE_THREAD + p->writeIndex = 0; + p->writeRes = SZ_OK; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->ReadyBlocks[i] = False; + p->numFinishedThreads = 0; + #endif + + p->numStartedThreadsLimit = numThreads; + p->numStartedThreads = 0; + + // for (i = 0; i < numThreads; i++) { - RINOK(CMtThread_Prepare(&p->threads[i])); + CMtCoderThread *nextThread = &p->threads[p->numStartedThreads++]; + RINOK(MtCoderThread_CreateAndStart(nextThread)); } - for (i = 0; i < numThreads; i++) + RINOK_THREAD(Event_Set(&p->readEvent)) + + #ifdef MTCODER__USE_WRITE_THREAD { - CMtThread *t = &p->threads[i]; - CLoopThread *lt = &t->thread; + unsigned bi = 0; - if (!Thread_WasCreated(<->thread)) + for (;; bi++) { - lt->func = ThreadFunc; - lt->param = t; + if (bi >= numBlocksMax) + bi = 0; + + RINOK_THREAD(Event_Wait(&p->writeEvents[bi])) - if (LoopThread_Create(lt) != SZ_OK) { - res = SZ_ERROR_THREAD; - break; + const CMtCoderBlock *block = &p->blocks[bi]; + unsigned bufIndex = block->bufIndex; + BoolInt finished = block->finished; + if (res == SZ_OK && block->res != SZ_OK) + res = block->res; + + if (bufIndex != (unsigned)(int)-1) + { + if (res == SZ_OK) + { + res = p->mtCallback->Write(p->mtCallbackObject, bufIndex); + if (res != SZ_OK) + MtProgress_SetError(&p->mtProgress, res); + } + + CriticalSection_Enter(&p->cs); + { + p->freeBlockList[bufIndex] = p->freeBlockHead; + p->freeBlockHead = bufIndex; + } + CriticalSection_Leave(&p->cs); + } + + RINOK_THREAD(Semaphore_Release1(&p->blocksSemaphore)) + + if (finished) + break; } } } + #else + { + WRes wres = Event_Wait(&p->finishedEvent); + res = MY_SRes_HRESULT_FROM_WRes(wres); + } + #endif if (res == SZ_OK) - { - unsigned j; - for (i = 0; i < numThreads; i++) - { - CMtThread *t = &p->threads[i]; - if (LoopThread_StartSubThread(&t->thread) != SZ_OK) - { - res = SZ_ERROR_THREAD; - p->threads[0].stopReading = True; - break; - } - } + res = p->readRes; - Event_Set(&p->threads[0].canWrite); - Event_Set(&p->threads[0].canRead); + if (res == SZ_OK) + res = p->mtProgress.res; - for (j = 0; j < i; j++) - LoopThread_WaitSubThread(&p->threads[j].thread); - } + #ifndef MTCODER__USE_WRITE_THREAD + if (res == SZ_OK) + res = p->writeRes; + #endif - for (i = 0; i < numThreads; i++) - CMtThread_CloseEvents(&p->threads[i]); - return (res == SZ_OK) ? p->res : res; + if (res != SZ_OK) + MtCoder_Free(p); + return res; } + +#endif
M
src/third-party/lzma/MtCoder.h
→
src/third-party/lzma/MtCoder.h
@@ -1,97 +1,140 @@
/* MtCoder.h -- Multi-thread Coder -2009-11-19 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __MT_CODER_H #define __MT_CODER_H -#include "Threads.h" +#include "MtDec.h" EXTERN_C_BEGIN -typedef struct -{ - CThread thread; - CAutoResetEvent startEvent; - CAutoResetEvent finishedEvent; - int stop; - - THREAD_FUNC_TYPE func; - LPVOID param; - THREAD_FUNC_RET_TYPE res; -} CLoopThread; - -void LoopThread_Construct(CLoopThread *p); -void LoopThread_Close(CLoopThread *p); -WRes LoopThread_Create(CLoopThread *p); -WRes LoopThread_StopAndWait(CLoopThread *p); -WRes LoopThread_StartSubThread(CLoopThread *p); -WRes LoopThread_WaitSubThread(CLoopThread *p); +/* + if ( defined MTCODER__USE_WRITE_THREAD) : main thread writes all data blocks to output stream + if (not defined MTCODER__USE_WRITE_THREAD) : any coder thread can write data blocks to output stream +*/ +/* #define MTCODER__USE_WRITE_THREAD */ #ifndef _7ZIP_ST -#define NUM_MT_CODER_THREADS_MAX 32 + #define MTCODER__GET_NUM_BLOCKS_FROM_THREADS(numThreads) ((numThreads) + (numThreads) / 8 + 1) + #define MTCODER__THREADS_MAX 64 + #define MTCODER__BLOCKS_MAX (MTCODER__GET_NUM_BLOCKS_FROM_THREADS(MTCODER__THREADS_MAX) + 3) #else -#define NUM_MT_CODER_THREADS_MAX 1 + #define MTCODER__THREADS_MAX 1 + #define MTCODER__BLOCKS_MAX 1 #endif + + +#ifndef _7ZIP_ST + typedef struct { - UInt64 totalInSize; - UInt64 totalOutSize; - ICompressProgress *progress; - SRes res; - CCriticalSection cs; - UInt64 inSizes[NUM_MT_CODER_THREADS_MAX]; - UInt64 outSizes[NUM_MT_CODER_THREADS_MAX]; -} CMtProgress; + ICompressProgress vt; + CMtProgress *mtProgress; + UInt64 inSize; + UInt64 outSize; +} CMtProgressThunk; + +void MtProgressThunk_CreateVTable(CMtProgressThunk *p); + +#define MtProgressThunk_Init(p) { (p)->inSize = 0; (p)->outSize = 0; } -SRes MtProgress_Set(CMtProgress *p, unsigned index, UInt64 inSize, UInt64 outSize); struct _CMtCoder; + typedef struct { struct _CMtCoder *mtCoder; - Byte *outBuf; - size_t outBufSize; + unsigned index; + int stop; Byte *inBuf; - size_t inBufSize; - unsigned index; - CLoopThread thread; - Bool stopReading; - Bool stopWriting; - CAutoResetEvent canRead; - CAutoResetEvent canWrite; -} CMtThread; + CAutoResetEvent startEvent; + CThread thread; +} CMtCoderThread; + typedef struct { - SRes (*Code)(void *p, unsigned index, Byte *dest, size_t *destSize, + SRes (*Code)(void *p, unsigned coderIndex, unsigned outBufIndex, const Byte *src, size_t srcSize, int finished); -} IMtCoderCallback; + SRes (*Write)(void *p, unsigned outBufIndex); +} IMtCoderCallback2; + + +typedef struct +{ + SRes res; + unsigned bufIndex; + BoolInt finished; +} CMtCoderBlock; + typedef struct _CMtCoder { - size_t blockSize; - size_t destBlockSize; - unsigned numThreads; + /* input variables */ + size_t blockSize; /* size of input block */ + unsigned numThreadsMax; + UInt64 expectedDataSize; + ISeqInStream *inStream; - ISeqOutStream *outStream; + const Byte *inData; + size_t inDataSize; + ICompressProgress *progress; - ISzAlloc *alloc; + ISzAllocPtr allocBig; - IMtCoderCallback *mtCallback; + IMtCoderCallback2 *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + CAutoResetEvent readEvent; + CSemaphore blocksSemaphore; + + BoolInt stopReading; + SRes readRes; + + #ifdef MTCODER__USE_WRITE_THREAD + CAutoResetEvent writeEvents[MTCODER__BLOCKS_MAX]; + #else + CAutoResetEvent finishedEvent; + SRes writeRes; + unsigned writeIndex; + Byte ReadyBlocks[MTCODER__BLOCKS_MAX]; + LONG numFinishedThreads; + #endif + + unsigned numStartedThreadsLimit; + unsigned numStartedThreads; + + unsigned numBlocksMax; + unsigned blockIndex; + UInt64 readProcessed; + CCriticalSection cs; - SRes res; + + unsigned freeBlockHead; + unsigned freeBlockList[MTCODER__BLOCKS_MAX]; CMtProgress mtProgress; - CMtThread threads[NUM_MT_CODER_THREADS_MAX]; + CMtCoderBlock blocks[MTCODER__BLOCKS_MAX]; + CMtCoderThread threads[MTCODER__THREADS_MAX]; } CMtCoder; -void MtCoder_Construct(CMtCoder* p); -void MtCoder_Destruct(CMtCoder* p); + +void MtCoder_Construct(CMtCoder *p); +void MtCoder_Destruct(CMtCoder *p); SRes MtCoder_Code(CMtCoder *p); + + +#endif + EXTERN_C_END
A
src/third-party/lzma/MtDec.c
@@ -0,0 +1,1138 @@
+/* MtDec.c -- Multi-thread Decoder +2019-02-02 : Igor Pavlov : Public domain */ + +#include "Precomp.h" + +// #define SHOW_DEBUG_INFO + +// #include <stdio.h> + +#ifdef SHOW_DEBUG_INFO +#include <stdio.h> +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + +#include "MtDec.h" + +#ifndef _7ZIP_ST + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress) +{ + p->progress = progress; + p->res = SZ_OK; + p->totalInSize = 0; + p->totalOutSize = 0; +} + + +SRes MtProgress_Progress_ST(CMtProgress *p) +{ + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + return p->res; +} + + +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize) +{ + SRes res; + CriticalSection_Enter(&p->cs); + + p->totalInSize += inSize; + p->totalOutSize += outSize; + if (p->res == SZ_OK && p->progress) + if (ICompressProgress_Progress(p->progress, p->totalInSize, p->totalOutSize) != SZ_OK) + p->res = SZ_ERROR_PROGRESS; + res = p->res; + + CriticalSection_Leave(&p->cs); + return res; +} + + +SRes MtProgress_GetError(CMtProgress *p) +{ + SRes res; + CriticalSection_Enter(&p->cs); + res = p->res; + CriticalSection_Leave(&p->cs); + return res; +} + + +void MtProgress_SetError(CMtProgress *p, SRes res) +{ + CriticalSection_Enter(&p->cs); + if (p->res == SZ_OK) + p->res = res; + CriticalSection_Leave(&p->cs); +} + + +#define RINOK_THREAD(x) RINOK(x) + + +static WRes ArEvent_OptCreate_And_Reset(CEvent *p) +{ + if (Event_IsCreated(p)) + return Event_Reset(p); + return AutoResetEvent_CreateNotSignaled(p); +} + + +struct __CMtDecBufLink +{ + struct __CMtDecBufLink *next; + void *pad[3]; +}; + +typedef struct __CMtDecBufLink CMtDecBufLink; + +#define MTDEC__LINK_DATA_OFFSET sizeof(CMtDecBufLink) +#define MTDEC__DATA_PTR_FROM_LINK(link) ((Byte *)(link) + MTDEC__LINK_DATA_OFFSET) + + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp); + + +static WRes MtDecThread_CreateEvents(CMtDecThread *t) +{ + WRes wres = ArEvent_OptCreate_And_Reset(&t->canWrite); + if (wres == 0) + { + wres = ArEvent_OptCreate_And_Reset(&t->canRead); + if (wres == 0) + return SZ_OK; + } + return wres; +} + + +static SRes MtDecThread_CreateAndStart(CMtDecThread *t) +{ + WRes wres = MtDecThread_CreateEvents(t); + // wres = 17; // for test + if (wres == 0) + { + if (Thread_WasCreated(&t->thread)) + return SZ_OK; + wres = Thread_Create(&t->thread, ThreadFunc, t); + if (wres == 0) + return SZ_OK; + } + return MY_SRes_HRESULT_FROM_WRes(wres); +} + + +void MtDecThread_FreeInBufs(CMtDecThread *t) +{ + if (t->inBuf) + { + void *link = t->inBuf; + t->inBuf = NULL; + do + { + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(t->mtDec->alloc, link); + link = next; + } + while (link); + } +} + + +static void MtDecThread_CloseThread(CMtDecThread *t) +{ + if (Thread_WasCreated(&t->thread)) + { + Event_Set(&t->canWrite); /* we can disable it. There are no threads waiting canWrite in normal cases */ + Event_Set(&t->canRead); + Thread_Wait(&t->thread); + Thread_Close(&t->thread); + } + + Event_Close(&t->canRead); + Event_Close(&t->canWrite); +} + +static void MtDec_CloseThreads(CMtDec *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_CloseThread(&p->threads[i]); +} + +static void MtDecThread_Destruct(CMtDecThread *t) +{ + MtDecThread_CloseThread(t); + MtDecThread_FreeInBufs(t); +} + + + +static SRes FullRead(ISeqInStream *stream, Byte *data, size_t *processedSize) +{ + size_t size = *processedSize; + *processedSize = 0; + while (size != 0) + { + size_t cur = size; + SRes res = ISeqInStream_Read(stream, data, &cur); + *processedSize += cur; + data += cur; + size -= cur; + RINOK(res); + if (cur == 0) + return SZ_OK; + } + return SZ_OK; +} + + +static SRes MtDec_GetError_Spec(CMtDec *p, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + CriticalSection_Leave(&p->mtProgress.cs); + return res; +} + +static SRes MtDec_Progress_GetError_Spec(CMtDec *p, UInt64 inSize, UInt64 outSize, UInt64 interruptIndex, BoolInt *wasInterrupted) +{ + SRes res; + CriticalSection_Enter(&p->mtProgress.cs); + + p->mtProgress.totalInSize += inSize; + p->mtProgress.totalOutSize += outSize; + if (p->mtProgress.res == SZ_OK && p->mtProgress.progress) + if (ICompressProgress_Progress(p->mtProgress.progress, p->mtProgress.totalInSize, p->mtProgress.totalOutSize) != SZ_OK) + p->mtProgress.res = SZ_ERROR_PROGRESS; + + *wasInterrupted = (p->needInterrupt && interruptIndex > p->interruptIndex); + res = p->mtProgress.res; + + CriticalSection_Leave(&p->mtProgress.cs); + + return res; +} + +static void MtDec_Interrupt(CMtDec *p, UInt64 interruptIndex) +{ + CriticalSection_Enter(&p->mtProgress.cs); + if (!p->needInterrupt || interruptIndex < p->interruptIndex) + { + p->interruptIndex = interruptIndex; + p->needInterrupt = True; + } + CriticalSection_Leave(&p->mtProgress.cs); +} + +Byte *MtDec_GetCrossBuff(CMtDec *p) +{ + Byte *cr = p->crossBlock; + if (!cr) + { + cr = (Byte *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!cr) + return NULL; + p->crossBlock = cr; + } + return MTDEC__DATA_PTR_FROM_LINK(cr); +} + + +/* + ThreadFunc2() returns: + 0 - in all normal cases (even for stream error or memory allocation error) + (!= 0) - WRes error return by system threading function +*/ + +// #define MTDEC_ProgessStep (1 << 22) +#define MTDEC_ProgessStep (1 << 0) + +static WRes ThreadFunc2(CMtDecThread *t) +{ + CMtDec *p = t->mtDec; + + PRF_STR_INT("ThreadFunc2", t->index); + + // SetThreadAffinityMask(GetCurrentThread(), 1 << t->index); + + for (;;) + { + SRes res, codeRes; + BoolInt wasInterrupted, isAllocError, overflow, finish; + SRes threadingErrorSRes; + BoolInt needCode, needWrite, needContinue; + + size_t inDataSize_Start; + UInt64 inDataSize; + // UInt64 inDataSize_Full; + + UInt64 blockIndex; + + UInt64 inPrev = 0; + UInt64 outPrev = 0; + UInt64 inCodePos; + UInt64 outCodePos; + + Byte *afterEndData = NULL; + size_t afterEndData_Size = 0; + + BoolInt canCreateNewThread = False; + // CMtDecCallbackInfo parse; + CMtDecThread *nextThread; + + PRF_STR_INT("Event_Wait(&t->canRead)", t->index); + + RINOK_THREAD(Event_Wait(&t->canRead)); + if (p->exitThread) + return 0; + + PRF_STR_INT("after Event_Wait(&t->canRead)", t->index); + + // if (t->index == 3) return 19; // for test + + blockIndex = p->blockIndex++; + + // PRF(printf("\ncanRead\n")) + + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + + finish = p->readWasFinished; + needCode = False; + needWrite = False; + isAllocError = False; + overflow = False; + + inDataSize_Start = 0; + inDataSize = 0; + // inDataSize_Full = 0; + + if (res == SZ_OK && !wasInterrupted) + { + // if (p->inStream) + { + CMtDecBufLink *prev = NULL; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + size_t crossSize = p->crossEnd - p->crossStart; + + PRF(printf("\ncrossSize = %d\n", crossSize)); + + for (;;) + { + if (!link) + { + link = (CMtDecBufLink *)ISzAlloc_Alloc(p->alloc, MTDEC__LINK_DATA_OFFSET + p->inBufSize); + if (!link) + { + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + link->next = NULL; + if (prev) + { + // static unsigned g_num = 0; + // printf("\n%6d : %x", ++g_num, (unsigned)(size_t)((Byte *)link - (Byte *)prev)); + prev->next = link; + } + else + t->inBuf = (void *)link; + } + + { + Byte *data = MTDEC__DATA_PTR_FROM_LINK(link); + Byte *parseData = data; + size_t size; + + if (crossSize != 0) + { + inDataSize = crossSize; + // inDataSize_Full = inDataSize; + inDataSize_Start = crossSize; + size = crossSize; + parseData = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + PRF(printf("\ncross : crossStart = %7d crossEnd = %7d finish = %1d", + (int)p->crossStart, (int)p->crossEnd, (int)finish)); + } + else + { + size = p->inBufSize; + + res = FullRead(p->inStream, data, &size); + + // size = 10; // test + + inDataSize += size; + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = size; + + p->readProcessed += size; + finish = (size != p->inBufSize); + if (finish) + p->readWasFinished = True; + + // res = E_INVALIDARG; // test + + if (res != SZ_OK) + { + // PRF(printf("\nRead error = %d\n", res)) + // we want to decode all data before error + p->readRes = res; + // p->readError_BlockIndex = blockIndex; + p->readWasFinished = True; + finish = True; + res = SZ_OK; + // break; + } + + if (inDataSize - inPrev >= MTDEC_ProgessStep) + { + res = MtDec_Progress_GetError_Spec(p, 0, 0, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inDataSize; + } + } + + { + CMtDecCallbackInfo parse; + + parse.startCall = (prev == NULL); + parse.src = parseData; + parse.srcSize = size; + parse.srcFinished = finish; + parse.canCreateNewThread = True; + + // PRF(printf("\nParse size = %d\n", (unsigned)size)) + + p->mtCallback->Parse(p->mtCallbackObject, t->index, &parse); + + needWrite = True; + canCreateNewThread = parse.canCreateNewThread; + + // printf("\n\n%12I64u %12I64u", (UInt64)p->mtProgress.totalInSize, (UInt64)p->mtProgress.totalOutSize); + + if ( + // parseRes != SZ_OK || + // inDataSize - (size - parse.srcSize) > p->inBlockMax + // || + parse.state == MTDEC_PARSE_OVERFLOW + // || wasInterrupted + ) + { + // Overflow or Parse error - switch from MT decoding to ST decoding + finish = True; + overflow = True; + + { + PRF(printf("\n Overflow")); + // PRF(printf("\nisBlockFinished = %d", (unsigned)parse.blockWasFinished)); + PRF(printf("\n inDataSize = %d", (unsigned)inDataSize)); + } + + if (crossSize != 0) + memcpy(data, parseData, size); + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (crossSize != 0) + { + memcpy(data, parseData, parse.srcSize); + p->crossStart += parse.srcSize; + } + + if (parse.state != MTDEC_PARSE_CONTINUE || finish) + { + // we don't need to parse in current thread anymore + + if (parse.state == MTDEC_PARSE_END) + finish = True; + + needCode = True; + // p->crossFinished = finish; + + if (parse.srcSize == size) + { + // full parsed - no cross transfer + p->crossStart = 0; + p->crossEnd = 0; + break; + } + + if (parse.state == MTDEC_PARSE_END) + { + p->crossStart = 0; + p->crossEnd = 0; + + if (crossSize != 0) + memcpy(data + parse.srcSize, parseData + parse.srcSize, size - parse.srcSize); // we need all data + afterEndData_Size = size - parse.srcSize; + afterEndData = parseData + parse.srcSize; + + // we reduce data size to required bytes (parsed only) + inDataSize -= (size - parse.srcSize); + if (!prev) + inDataSize_Start = parse.srcSize; + break; + } + + { + // partial parsed - need cross transfer + if (crossSize != 0) + inDataSize = parse.srcSize; // it's only parsed now + else + { + // partial parsed - is not in initial cross block - we need to copy new data to cross block + Byte *cr = MtDec_GetCrossBuff(p); + if (!cr) + { + { + PRF(printf("\ncross alloc error error\n")); + // res = SZ_ERROR_MEM; + finish = True; + // p->allocError_for_Read_BlockIndex = blockIndex; + isAllocError = True; + break; + } + } + + { + size_t crSize = size - parse.srcSize; + inDataSize -= crSize; + p->crossEnd = crSize; + p->crossStart = 0; + memcpy(cr, parseData + parse.srcSize, crSize); + } + } + + // inDataSize_Full = inDataSize; + if (!prev) + inDataSize_Start = parse.srcSize; // it's partial size (parsed only) + + finish = False; + break; + } + } + + if (parse.srcSize != size) + { + res = SZ_ERROR_FAIL; + PRF(printf("\nfinished error SZ_ERROR_FAIL = %d\n", res)); + break; + } + } + } + + prev = link; + link = link->next; + + if (crossSize != 0) + { + crossSize = 0; + p->crossStart = 0; + p->crossEnd = 0; + } + } + } + + if (res == SZ_OK) + res = MtDec_GetError_Spec(p, blockIndex, &wasInterrupted); + } + + codeRes = SZ_OK; + + if (res == SZ_OK && needCode && !wasInterrupted) + { + codeRes = p->mtCallback->PreCode(p->mtCallbackObject, t->index); + if (codeRes != SZ_OK) + { + needCode = False; + finish = True; + // SZ_ERROR_MEM is expected error here. + // if (codeRes == SZ_ERROR_MEM) - we will try single-thread decoding later. + // if (codeRes != SZ_ERROR_MEM) - we can stop decoding or try single-thread decoding. + } + } + + if (res != SZ_OK || wasInterrupted) + finish = True; + + nextThread = NULL; + threadingErrorSRes = SZ_OK; + + if (!finish) + { + if (p->numStartedThreads < p->numStartedThreads_Limit && canCreateNewThread) + { + SRes res2 = MtDecThread_CreateAndStart(&p->threads[p->numStartedThreads]); + if (res2 == SZ_OK) + { + // if (p->numStartedThreads % 1000 == 0) PRF(printf("\n numStartedThreads=%d\n", p->numStartedThreads)); + p->numStartedThreads++; + } + else + { + PRF(printf("\nERROR: numStartedThreads=%d\n", p->numStartedThreads)); + if (p->numStartedThreads == 1) + { + // if only one thread is possible, we leave muti-threading code + finish = True; + needCode = False; + threadingErrorSRes = res2; + } + else + p->numStartedThreads_Limit = p->numStartedThreads; + } + } + + if (!finish) + { + unsigned nextIndex = t->index + 1; + nextThread = &p->threads[nextIndex >= p->numStartedThreads ? 0 : nextIndex]; + RINOK_THREAD(Event_Set(&nextThread->canRead)) + // We have started executing for new iteration (with next thread) + // And that next thread now is responsible for possible exit from decoding (threading_code) + } + } + + // each call of Event_Set(&nextThread->canRead) must be followed by call of Event_Set(&nextThread->canWrite) + // if ( !finish ) we must call Event_Set(&nextThread->canWrite) in any case + // if ( finish ) we switch to single-thread mode and there are 2 ways at the end of current iteration (current block): + // - if (needContinue) after Write(&needContinue), we restore decoding with new iteration + // - otherwise we stop decoding and exit from ThreadFunc2() + + // Don't change (finish) variable in the further code + + + // ---------- CODE ---------- + + inPrev = 0; + outPrev = 0; + inCodePos = 0; + outCodePos = 0; + + if (res == SZ_OK && needCode && codeRes == SZ_OK) + { + BoolInt isStartBlock = True; + CMtDecBufLink *link = (CMtDecBufLink *)t->inBuf; + + for (;;) + { + size_t inSize; + int stop; + + if (isStartBlock) + inSize = inDataSize_Start; + else + { + UInt64 rem = inDataSize - inCodePos; + inSize = p->inBufSize; + if (inSize > rem) + inSize = (size_t)rem; + } + + inCodePos += inSize; + stop = True; + + codeRes = p->mtCallback->Code(p->mtCallbackObject, t->index, + (const Byte *)MTDEC__DATA_PTR_FROM_LINK(link), inSize, + (inCodePos == inDataSize), // srcFinished + &inCodePos, &outCodePos, &stop); + + if (codeRes != SZ_OK) + { + PRF(printf("\nCode Interrupt error = %x\n", codeRes)); + // we interrupt only later blocks + MtDec_Interrupt(p, blockIndex); + break; + } + + if (stop || inCodePos == inDataSize) + break; + + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + if (inDelta >= MTDEC_ProgessStep || outDelta >= MTDEC_ProgessStep) + { + // Sleep(1); + res = MtDec_Progress_GetError_Spec(p, inDelta, outDelta, blockIndex, &wasInterrupted); + if (res != SZ_OK || wasInterrupted) + break; + inPrev = inCodePos; + outPrev = outCodePos; + } + } + + link = link->next; + isStartBlock = False; + } + } + + + // ---------- WRITE ---------- + + RINOK_THREAD(Event_Wait(&t->canWrite)); + + { + BoolInt isErrorMode = False; + BoolInt canRecode = True; + BoolInt needWriteToStream = needWrite; + + if (p->exitThread) return 0; // it's never executed in normal cases + + if (p->wasInterrupted) + wasInterrupted = True; + else + { + if (codeRes != SZ_OK) // || !needCode // check it !!! + { + p->wasInterrupted = True; + p->codeRes = codeRes; + if (codeRes == SZ_ERROR_MEM) + isAllocError = True; + } + + if (threadingErrorSRes) + { + p->wasInterrupted = True; + p->threadingErrorSRes = threadingErrorSRes; + needWriteToStream = False; + } + if (isAllocError) + { + p->wasInterrupted = True; + p->isAllocError = True; + needWriteToStream = False; + } + if (overflow) + { + p->wasInterrupted = True; + p->overflow = True; + needWriteToStream = False; + } + } + + if (needCode) + { + if (wasInterrupted) + { + inCodePos = 0; + outCodePos = 0; + } + { + const UInt64 inDelta = inCodePos - inPrev; + const UInt64 outDelta = outCodePos - outPrev; + // if (inDelta != 0 || outDelta != 0) + res = MtProgress_ProgressAdd(&p->mtProgress, inDelta, outDelta); + } + } + + needContinue = (!finish); + + // if (res == SZ_OK && needWrite && !wasInterrupted) + if (needWrite) + { + // p->inProcessed += inCodePos; + + res = p->mtCallback->Write(p->mtCallbackObject, t->index, + res == SZ_OK && needWriteToStream && !wasInterrupted, // needWrite + afterEndData, afterEndData_Size, + &needContinue, + &canRecode); + + // res= E_INVALIDARG; // for test + + PRF(printf("\nAfter Write needContinue = %d\n", (unsigned)needContinue)); + PRF(printf("\nprocessed = %d\n", (unsigned)p->inProcessed)); + + if (res != SZ_OK) + { + PRF(printf("\nWrite error = %d\n", res)); + isErrorMode = True; + p->wasInterrupted = True; + } + if (res != SZ_OK + || (!needContinue && !finish)) + { + PRF(printf("\nWrite Interrupt error = %x\n", res)); + MtDec_Interrupt(p, blockIndex); + } + } + + if (canRecode) + if (!needCode + || res != SZ_OK + || p->wasInterrupted + || codeRes != SZ_OK + || wasInterrupted + || p->numFilledThreads != 0 + || isErrorMode) + { + if (p->numFilledThreads == 0) + p->filledThreadStart = t->index; + if (inDataSize != 0 || !finish) + { + t->inDataSize_Start = inDataSize_Start; + t->inDataSize = inDataSize; + p->numFilledThreads++; + } + PRF(printf("\np->numFilledThreads = %d\n", p->numFilledThreads)); + PRF(printf("p->filledThreadStart = %d\n", p->filledThreadStart)); + } + + if (!finish) + { + RINOK_THREAD(Event_Set(&nextThread->canWrite)); + } + else + { + if (needContinue) + { + // we restore decoding with new iteration + RINOK_THREAD(Event_Set(&p->threads[0].canWrite)); + } + else + { + // we exit from decoding + if (t->index == 0) + return SZ_OK; + p->exitThread = True; + } + RINOK_THREAD(Event_Set(&p->threads[0].canRead)); + } + } + } +} + +#ifdef _WIN32 +#define USE_ALLOCA +#endif + +#ifdef USE_ALLOCA +#ifdef _WIN32 +#include <malloc.h> +#else +#include <stdlib.h> +#endif +#endif + + +static THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc1(void *pp) +{ + WRes res; + + CMtDecThread *t = (CMtDecThread *)pp; + CMtDec *p; + + // fprintf(stdout, "\n%d = %p\n", t->index, &t); + + res = ThreadFunc2(t); + p = t->mtDec; + if (res == 0) + return p->exitThreadWRes; + { + // it's unexpected situation for some threading function error + if (p->exitThreadWRes == 0) + p->exitThreadWRes = res; + PRF(printf("\nthread exit error = %d\n", res)); + p->exitThread = True; + Event_Set(&p->threads[0].canRead); + Event_Set(&p->threads[0].canWrite); + MtProgress_SetError(&p->mtProgress, MY_SRes_HRESULT_FROM_WRes(res)); + } + return res; +} + +static MY_NO_INLINE THREAD_FUNC_RET_TYPE THREAD_FUNC_CALL_TYPE ThreadFunc(void *pp) +{ + CMtDecThread *t = (CMtDecThread *)pp; + + // fprintf(stderr, "\n%d = %p - before", t->index, &t); + #ifdef USE_ALLOCA + t->allocaPtr = alloca(t->index * 128); + #endif + return ThreadFunc1(pp); +} + + +int MtDec_PrepareRead(CMtDec *p) +{ + if (p->crossBlock && p->crossStart == p->crossEnd) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + if (i > p->numStartedThreads + || p->numFilledThreads <= + (i >= p->filledThreadStart ? + i - p->filledThreadStart : + i + p->numStartedThreads - p->filledThreadStart)) + MtDecThread_FreeInBufs(&p->threads[i]); + } + + return (p->numFilledThreads != 0) || (p->crossStart != p->crossEnd); +} + + +const Byte *MtDec_Read(CMtDec *p, size_t *inLim) +{ + while (p->numFilledThreads != 0) + { + CMtDecThread *t = &p->threads[p->filledThreadStart]; + + if (*inLim != 0) + { + { + void *link = t->inBuf; + void *next = ((CMtDecBufLink *)link)->next; + ISzAlloc_Free(p->alloc, link); + t->inBuf = next; + } + + if (t->inDataSize == 0) + { + MtDecThread_FreeInBufs(t); + if (--p->numFilledThreads == 0) + break; + if (++p->filledThreadStart == p->numStartedThreads) + p->filledThreadStart = 0; + t = &p->threads[p->filledThreadStart]; + } + } + + { + size_t lim = t->inDataSize_Start; + if (lim != 0) + t->inDataSize_Start = 0; + else + { + UInt64 rem = t->inDataSize; + lim = p->inBufSize; + if (lim > rem) + lim = (size_t)rem; + } + t->inDataSize -= lim; + *inLim = lim; + return (const Byte *)MTDEC__DATA_PTR_FROM_LINK(t->inBuf); + } + } + + { + size_t crossSize = p->crossEnd - p->crossStart; + if (crossSize != 0) + { + const Byte *data = MTDEC__DATA_PTR_FROM_LINK(p->crossBlock) + p->crossStart; + *inLim = crossSize; + p->crossStart = 0; + p->crossEnd = 0; + return data; + } + *inLim = 0; + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + return NULL; + } +} + + +void MtDec_Construct(CMtDec *p) +{ + unsigned i; + + p->inBufSize = (size_t)1 << 18; + + p->numThreadsMax = 0; + + p->inStream = NULL; + + // p->inData = NULL; + // p->inDataSize = 0; + + p->crossBlock = NULL; + p->crossStart = 0; + p->crossEnd = 0; + + p->numFilledThreads = 0; + + p->progress = NULL; + p->alloc = NULL; + + p->mtCallback = NULL; + p->mtCallbackObject = NULL; + + p->allocatedBufsSize = 0; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + t->mtDec = p; + t->index = i; + t->inBuf = NULL; + Event_Construct(&t->canRead); + Event_Construct(&t->canWrite); + Thread_Construct(&t->thread); + } + + // Event_Construct(&p->finishedEvent); + + CriticalSection_Init(&p->mtProgress.cs); +} + + +static void MtDec_Free(CMtDec *p) +{ + unsigned i; + + p->exitThread = True; + + for (i = 0; i < MTDEC__THREADS_MAX; i++) + MtDecThread_Destruct(&p->threads[i]); + + // Event_Close(&p->finishedEvent); + + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } +} + + +void MtDec_Destruct(CMtDec *p) +{ + MtDec_Free(p); + + CriticalSection_Delete(&p->mtProgress.cs); +} + + +SRes MtDec_Code(CMtDec *p) +{ + unsigned i; + + p->inProcessed = 0; + + p->blockIndex = 1; // it must be larger than not_defined index (0) + p->isAllocError = False; + p->overflow = False; + p->threadingErrorSRes = SZ_OK; + + p->needContinue = True; + + p->readWasFinished = False; + p->needInterrupt = False; + p->interruptIndex = (UInt64)(Int64)-1; + + p->readProcessed = 0; + p->readRes = SZ_OK; + p->codeRes = SZ_OK; + p->wasInterrupted = False; + + p->crossStart = 0; + p->crossEnd = 0; + + p->filledThreadStart = 0; + p->numFilledThreads = 0; + + { + unsigned numThreads = p->numThreadsMax; + if (numThreads > MTDEC__THREADS_MAX) + numThreads = MTDEC__THREADS_MAX; + p->numStartedThreads_Limit = numThreads; + p->numStartedThreads = 0; + } + + if (p->inBufSize != p->allocatedBufsSize) + { + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CMtDecThread *t = &p->threads[i]; + if (t->inBuf) + MtDecThread_FreeInBufs(t); + } + if (p->crossBlock) + { + ISzAlloc_Free(p->alloc, p->crossBlock); + p->crossBlock = NULL; + } + + p->allocatedBufsSize = p->inBufSize; + } + + MtProgress_Init(&p->mtProgress, p->progress); + + // RINOK_THREAD(ArEvent_OptCreate_And_Reset(&p->finishedEvent)); + p->exitThread = False; + p->exitThreadWRes = 0; + + { + WRes wres; + WRes sres; + CMtDecThread *nextThread = &p->threads[p->numStartedThreads++]; + // wres = MtDecThread_CreateAndStart(nextThread); + wres = MtDecThread_CreateEvents(nextThread); + if (wres == 0) { wres = Event_Set(&nextThread->canWrite); + if (wres == 0) { wres = Event_Set(&nextThread->canRead); + if (wres == 0) { wres = ThreadFunc(nextThread); + if (wres != 0) + { + p->needContinue = False; + MtDec_CloseThreads(p); + }}}} + + // wres = 17; // for test + // wres = Event_Wait(&p->finishedEvent); + + sres = MY_SRes_HRESULT_FROM_WRes(wres); + + if (sres != 0) + p->threadingErrorSRes = sres; + + if ( + // wres == 0 + // wres != 0 + // || p->mtc.codeRes == SZ_ERROR_MEM + p->isAllocError + || p->threadingErrorSRes != SZ_OK + || p->overflow) + { + // p->needContinue = True; + } + else + p->needContinue = False; + + if (p->needContinue) + return SZ_OK; + + // if (sres != SZ_OK) + return sres; + // return E_FAIL; + } +} + +#endif
A
src/third-party/lzma/MtDec.h
@@ -0,0 +1,201 @@
+/* MtDec.h -- Multi-thread Decoder +2018-07-04 : Igor Pavlov : Public domain */ + +#ifndef __MT_DEC_H +#define __MT_DEC_H + +#include "7zTypes.h" + +#ifndef _7ZIP_ST +#include "Threads.h" +#endif + +EXTERN_C_BEGIN + +#ifndef _7ZIP_ST + +#ifndef _7ZIP_ST + #define MTDEC__THREADS_MAX 32 +#else + #define MTDEC__THREADS_MAX 1 +#endif + + +typedef struct +{ + ICompressProgress *progress; + SRes res; + UInt64 totalInSize; + UInt64 totalOutSize; + CCriticalSection cs; +} CMtProgress; + +void MtProgress_Init(CMtProgress *p, ICompressProgress *progress); +SRes MtProgress_Progress_ST(CMtProgress *p); +SRes MtProgress_ProgressAdd(CMtProgress *p, UInt64 inSize, UInt64 outSize); +SRes MtProgress_GetError(CMtProgress *p); +void MtProgress_SetError(CMtProgress *p, SRes res); + +struct _CMtDec; + +typedef struct +{ + struct _CMtDec *mtDec; + unsigned index; + void *inBuf; + + size_t inDataSize_Start; // size of input data in start block + UInt64 inDataSize; // total size of input data in all blocks + + CThread thread; + CAutoResetEvent canRead; + CAutoResetEvent canWrite; + void *allocaPtr; +} CMtDecThread; + +void MtDecThread_FreeInBufs(CMtDecThread *t); + + +typedef enum +{ + MTDEC_PARSE_CONTINUE, // continue this block with more input data + MTDEC_PARSE_OVERFLOW, // MT buffers overflow, need switch to single-thread + MTDEC_PARSE_NEW, // new block + MTDEC_PARSE_END // end of block threading. But we still can return to threading after Write(&needContinue) +} EMtDecParseState; + +typedef struct +{ + // in + int startCall; + const Byte *src; + size_t srcSize; + // in : (srcSize == 0) is allowed + // out : it's allowed to return less that actually was used ? + int srcFinished; + + // out + EMtDecParseState state; + BoolInt canCreateNewThread; + UInt64 outPos; // check it (size_t) +} CMtDecCallbackInfo; + + +typedef struct +{ + void (*Parse)(void *p, unsigned coderIndex, CMtDecCallbackInfo *ci); + + // PreCode() and Code(): + // (SRes_return_result != SZ_OK) means stop decoding, no need another blocks + SRes (*PreCode)(void *p, unsigned coderIndex); + SRes (*Code)(void *p, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop); + // stop - means stop another Code calls + + + /* Write() must be called, if Parse() was called + set (needWrite) if + { + && (was not interrupted by progress) + && (was not interrupted in previous block) + } + + out: + if (*needContinue), decoder still need to continue decoding with new iteration, + even after MTDEC_PARSE_END + if (*canRecode), we didn't flush current block data, so we still can decode current block later. + */ + SRes (*Write)(void *p, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode); +} IMtDecCallback; + + + +typedef struct _CMtDec +{ + /* input variables */ + + size_t inBufSize; /* size of input block */ + unsigned numThreadsMax; + // size_t inBlockMax; + unsigned numThreadsMax_2; + + ISeqInStream *inStream; + // const Byte *inData; + // size_t inDataSize; + + ICompressProgress *progress; + ISzAllocPtr alloc; + + IMtDecCallback *mtCallback; + void *mtCallbackObject; + + + /* internal variables */ + + size_t allocatedBufsSize; + + BoolInt exitThread; + WRes exitThreadWRes; + + UInt64 blockIndex; + BoolInt isAllocError; + BoolInt overflow; + SRes threadingErrorSRes; + + BoolInt needContinue; + + // CAutoResetEvent finishedEvent; + + SRes readRes; + SRes codeRes; + + BoolInt wasInterrupted; + + unsigned numStartedThreads_Limit; + unsigned numStartedThreads; + + Byte *crossBlock; + size_t crossStart; + size_t crossEnd; + UInt64 readProcessed; + BoolInt readWasFinished; + UInt64 inProcessed; + + unsigned filledThreadStart; + unsigned numFilledThreads; + + #ifndef _7ZIP_ST + BoolInt needInterrupt; + UInt64 interruptIndex; + CMtProgress mtProgress; + CMtDecThread threads[MTDEC__THREADS_MAX]; + #endif +} CMtDec; + + +void MtDec_Construct(CMtDec *p); +void MtDec_Destruct(CMtDec *p); + +/* +MtDec_Code() returns: + SZ_OK - in most cases + MY_SRes_HRESULT_FROM_WRes(WRes_error) - in case of unexpected error in threading function +*/ + +SRes MtDec_Code(CMtDec *p); +Byte *MtDec_GetCrossBuff(CMtDec *p); + +int MtDec_PrepareRead(CMtDec *p); +const Byte *MtDec_Read(CMtDec *p, size_t *inLim); + +#endif + +EXTERN_C_END + +#endif
M
src/third-party/lzma/Ppmd.h
→
src/third-party/lzma/Ppmd.h
@@ -1,5 +1,5 @@
/* Ppmd.h -- PPMD codec common code -2013-01-18 : Igor Pavlov : Public domain +2017-04-03 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #ifndef __PPMD_H@@ -77,8 +77,8 @@ #endif
CPpmd_Byte_Ref; #define PPMD_SetAllBitsIn256Bytes(p) \ - { unsigned i; for (i = 0; i < 256 / sizeof(p[0]); i += 8) { \ - p[i+7] = p[i+6] = p[i+5] = p[i+4] = p[i+3] = p[i+2] = p[i+1] = p[i+0] = ~(size_t)0; }} + { size_t z; for (z = 0; z < 256 / sizeof(p[0]); z += 8) { \ + p[z+7] = p[z+6] = p[z+5] = p[z+4] = p[z+3] = p[z+2] = p[z+1] = p[z+0] = ~(size_t)0; }} EXTERN_C_END
M
src/third-party/lzma/Ppmd7.c
→
src/third-party/lzma/Ppmd7.c
@@ -1,5 +1,5 @@
/* Ppmd7.c -- PPMdH codec -2015-09-28 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h"@@ -15,7 +15,7 @@ #define MAX_FREQ 124
#define UNIT_SIZE 12 #define U2B(nu) ((UInt32)(nu) * UNIT_SIZE) -#define U2I(nu) (p->Units2Indx[(nu) - 1]) +#define U2I(nu) (p->Units2Indx[(size_t)(nu) - 1]) #define I2U(indx) (p->Indx2Units[indx]) #ifdef PPMD_32BIT@@ -88,29 +88,31 @@ memset(p->HB2Flag, 0, 0x40);
memset(p->HB2Flag + 0x40, 8, 0x100 - 0x40); } -void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc) +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->Base); + ISzAlloc_Free(alloc, p->Base); p->Size = 0; p->Base = 0; } -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc) +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc) { - if (p->Base == 0 || p->Size != size) + if (!p->Base || p->Size != size) { + size_t size2; Ppmd7_Free(p, alloc); + size2 = 0 + #ifndef PPMD_32BIT + + UNIT_SIZE + #endif + ; p->AlignOffset = #ifdef PPMD_32BIT (4 - size) & 3; #else 4 - (size & 3); #endif - if ((p->Base = (Byte *)alloc->Alloc(alloc, p->AlignOffset + size - #ifndef PPMD_32BIT - + UNIT_SIZE - #endif - )) == 0) + if ((p->Base = (Byte *)ISzAlloc_Alloc(alloc, p->AlignOffset + size + size2)) == 0) return False; p->Size = size; }@@ -340,7 +342,7 @@ p->DummySee.Summ = 0; /* unused */
p->DummySee.Count = 64; /* unused */ } -static CTX_PTR CreateSuccessors(CPpmd7 *p, Bool skip) +static CTX_PTR CreateSuccessors(CPpmd7 *p, BoolInt skip) { CPpmd_State upState; CTX_PTR c = p->MinContext;@@ -513,7 +515,7 @@ {
/* Expand for one UNIT */ unsigned oldNU = ns1 >> 1; unsigned i = U2I(oldNU); - if (i != U2I(oldNU + 1)) + if (i != U2I((size_t)oldNU + 1)) { void *ptr = AllocUnits(p, i + 1); void *oldPtr;@@ -639,10 +641,10 @@ CPpmd_See *see;
unsigned nonMasked = p->MinContext->NumStats - numMasked; if (p->MinContext->NumStats != 256) { - see = p->See[p->NS2Indx[nonMasked - 1]] + + see = p->See[(unsigned)p->NS2Indx[(size_t)nonMasked - 1]] + (nonMasked < (unsigned)SUFFIX(p->MinContext)->NumStats - p->MinContext->NumStats) + - 2 * (p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + - 4 * (numMasked > nonMasked) + + 2 * (unsigned)(p->MinContext->SummFreq < 11 * p->MinContext->NumStats) + + 4 * (unsigned)(numMasked > nonMasked) + p->HiBitsFlag; { unsigned r = (see->Summ >> see->Shift);
M
src/third-party/lzma/Ppmd7.h
→
src/third-party/lzma/Ppmd7.h
@@ -1,5 +1,5 @@
/* Ppmd7.h -- PPMdH compression codec -2010-03-12 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ /* This code supports virtual RangeDecoder and includes the implementation@@ -60,8 +60,8 @@ UInt16 BinSumm[128][64];
} CPpmd7; void Ppmd7_Construct(CPpmd7 *p); -Bool Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAlloc *alloc); -void Ppmd7_Free(CPpmd7 *p, ISzAlloc *alloc); +BoolInt Ppmd7_Alloc(CPpmd7 *p, UInt32 size, ISzAllocPtr alloc); +void Ppmd7_Free(CPpmd7 *p, ISzAllocPtr alloc); void Ppmd7_Init(CPpmd7 *p, unsigned maxOrder); #define Ppmd7_WasAllocated(p) ((p)->Base != NULL)@@ -86,10 +86,10 @@ void Ppmd7_Update2(CPpmd7 *p);
void Ppmd7_UpdateBin(CPpmd7 *p); #define Ppmd7_GetBinSumm(p) \ - &p->BinSumm[Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ - p->NS2BSIndx[Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ + &p->BinSumm[(size_t)(unsigned)Ppmd7Context_OneState(p->MinContext)->Freq - 1][p->PrevSuccess + \ + p->NS2BSIndx[(size_t)Ppmd7_GetContext(p, p->MinContext->Suffix)->NumStats - 1] + \ (p->HiBitsFlag = p->HB2Flag[p->FoundState->Symbol]) + \ - 2 * p->HB2Flag[Ppmd7Context_OneState(p->MinContext)->Symbol] + \ + 2 * p->HB2Flag[(unsigned)Ppmd7Context_OneState(p->MinContext)->Symbol] + \ ((p->RunLength >> 26) & 0x20)] CPpmd_See *Ppmd7_MakeEscFreq(CPpmd7 *p, unsigned numMasked, UInt32 *scale);@@ -97,26 +97,28 @@
/* ---------- Decode ---------- */ -typedef struct +typedef struct IPpmd7_RangeDec IPpmd7_RangeDec; + +struct IPpmd7_RangeDec { - UInt32 (*GetThreshold)(void *p, UInt32 total); - void (*Decode)(void *p, UInt32 start, UInt32 size); - UInt32 (*DecodeBit)(void *p, UInt32 size0); -} IPpmd7_RangeDec; + UInt32 (*GetThreshold)(const IPpmd7_RangeDec *p, UInt32 total); + void (*Decode)(const IPpmd7_RangeDec *p, UInt32 start, UInt32 size); + UInt32 (*DecodeBit)(const IPpmd7_RangeDec *p, UInt32 size0); +}; typedef struct { - IPpmd7_RangeDec p; + IPpmd7_RangeDec vt; UInt32 Range; UInt32 Code; IByteIn *Stream; } CPpmd7z_RangeDec; void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p); -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p); #define Ppmd7z_RangeDec_IsFinishedOK(p) ((p)->Code == 0) -int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc); +int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc); /* ---------- Encode ---------- */
M
src/third-party/lzma/Ppmd7Dec.c
→
src/third-party/lzma/Ppmd7Dec.c
@@ -1,5 +1,5 @@
/* Ppmd7Dec.c -- PPMdH Decoder -2010-03-12 : Igor Pavlov : Public domain +2018-07-04 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h"@@ -8,49 +8,51 @@ #include "Ppmd7.h"
#define kTopValue (1 << 24) -Bool Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) +BoolInt Ppmd7z_RangeDec_Init(CPpmd7z_RangeDec *p) { unsigned i; p->Code = 0; p->Range = 0xFFFFFFFF; - if (p->Stream->Read((void *)p->Stream) != 0) + if (IByteIn_Read(p->Stream) != 0) return False; for (i = 0; i < 4; i++) - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); return (p->Code < 0xFFFFFFFF); } -static UInt32 Range_GetThreshold(void *pp, UInt32 total) +#define GET_Ppmd7z_RangeDec CPpmd7z_RangeDec *p = CONTAINER_FROM_VTBL(pp, CPpmd7z_RangeDec, vt); + +static UInt32 Range_GetThreshold(const IPpmd7_RangeDec *pp, UInt32 total) { - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; - return (p->Code) / (p->Range /= total); + GET_Ppmd7z_RangeDec + return p->Code / (p->Range /= total); } static void Range_Normalize(CPpmd7z_RangeDec *p) { if (p->Range < kTopValue) { - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); p->Range <<= 8; if (p->Range < kTopValue) { - p->Code = (p->Code << 8) | p->Stream->Read((void *)p->Stream); + p->Code = (p->Code << 8) | IByteIn_Read(p->Stream); p->Range <<= 8; } } } -static void Range_Decode(void *pp, UInt32 start, UInt32 size) +static void Range_Decode(const IPpmd7_RangeDec *pp, UInt32 start, UInt32 size) { - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; + GET_Ppmd7z_RangeDec p->Code -= start * p->Range; p->Range *= size; Range_Normalize(p); } -static UInt32 Range_DecodeBit(void *pp, UInt32 size0) +static UInt32 Range_DecodeBit(const IPpmd7_RangeDec *pp, UInt32 size0) { - CPpmd7z_RangeDec *p = (CPpmd7z_RangeDec *)pp; + GET_Ppmd7z_RangeDec UInt32 newBound = (p->Range >> 14) * size0; UInt32 symbol; if (p->Code < newBound)@@ -70,15 +72,15 @@ }
void Ppmd7z_RangeDec_CreateVTable(CPpmd7z_RangeDec *p) { - p->p.GetThreshold = Range_GetThreshold; - p->p.Decode = Range_Decode; - p->p.DecodeBit = Range_DecodeBit; + p->vt.GetThreshold = Range_GetThreshold; + p->vt.Decode = Range_Decode; + p->vt.DecodeBit = Range_DecodeBit; } #define MASK(sym) ((signed char *)charMask)[sym] -int Ppmd7_DecodeSymbol(CPpmd7 *p, IPpmd7_RangeDec *rc) +int Ppmd7_DecodeSymbol(CPpmd7 *p, const IPpmd7_RangeDec *rc) { size_t charMask[256 / sizeof(size_t)]; if (p->MinContext->NumStats != 1)
M
src/third-party/lzma/Ppmd7Enc.c
→
src/third-party/lzma/Ppmd7Enc.c
@@ -1,5 +1,5 @@
/* Ppmd7Enc.c -- PPMdH Encoder -2015-09-28 : Igor Pavlov : Public domain +2017-04-03 : Igor Pavlov : Public domain This code is based on PPMd var.H (2001): Dmitry Shkarin : Public domain */ #include "Precomp.h"@@ -23,7 +23,7 @@ {
Byte temp = p->Cache; do { - p->Stream->Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); + IByteOut_Write(p->Stream, (Byte)(temp + (Byte)(p->Low >> 32))); temp = 0xFF; } while (--p->CacheSize != 0);
M
src/third-party/lzma/Sha256.c
→
src/third-party/lzma/Sha256.c
@@ -1,5 +1,5 @@
/* Crypto/Sha256.c -- SHA-256 Hash -2015-11-14 : Igor Pavlov : Public domain +2017-04-03 : Igor Pavlov : Public domain This code is based on public domain code from Wei Dai's Crypto++ library. */ #include "Precomp.h"@@ -45,7 +45,7 @@
#ifdef _SHA256_UNROLL2 #define R(a,b,c,d,e,f,g,h, i) \ - h += S1(e) + Ch(e,f,g) + K[(i)+(j)] + (j ? blk2(i) : blk0(i)); \ + h += S1(e) + Ch(e,f,g) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ d += h; \ h += S0(a) + Maj(a, b, c)@@ -73,7 +73,7 @@ #define g(i) T[(6-(i))&7]
#define h(i) T[(7-(i))&7] #define R(i) \ - h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(j)] + (j ? blk2(i) : blk0(i)); \ + h(i) += S1(e(i)) + Ch(e(i),f(i),g(i)) + K[(i)+(size_t)(j)] + (j ? blk2(i) : blk0(i)); \ d(i) += h(i); \ h(i) += S0(a(i)) + Maj(a(i), b(i), c(i)) \
M
src/third-party/lzma/Threads.c
→
src/third-party/lzma/Threads.c
@@ -1,5 +1,5 @@
/* Threads.c -- multithreading library -2014-09-21 : Igor Pavlov : Public domain */ +2017-06-26 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -12,18 +12,20 @@
static WRes GetError() { DWORD res = GetLastError(); - return (res) ? (WRes)(res) : 1; + return res ? (WRes)res : 1; } -WRes HandleToWRes(HANDLE h) { return (h != 0) ? 0 : GetError(); } -WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } +static WRes HandleToWRes(HANDLE h) { return (h != NULL) ? 0 : GetError(); } +static WRes BOOLToWRes(BOOL v) { return v ? 0 : GetError(); } WRes HandlePtr_Close(HANDLE *p) { if (*p != NULL) + { if (!CloseHandle(*p)) return GetError(); - *p = NULL; + *p = NULL; + } return 0; }@@ -49,7 +51,7 @@ /* maybe we must use errno here, but probably GetLastError() is also OK. */
return HandleToWRes(*p); } -WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) +static WRes Event_Create(CEvent *p, BOOL manualReset, int signaled) { *p = CreateEvent(NULL, manualReset, (signaled ? TRUE : FALSE), NULL); return HandleToWRes(*p);
M
src/third-party/lzma/Threads.h
→
src/third-party/lzma/Threads.h
@@ -1,5 +1,5 @@
/* Threads.h -- multithreading library -2013-11-12 : Igor Pavlov : Public domain */ +2017-06-18 : Igor Pavlov : Public domain */ #ifndef __7Z_THREADS_H #define __7Z_THREADS_H@@ -49,7 +49,8 @@ WRes AutoResetEvent_Create(CAutoResetEvent *p, int signaled);
WRes AutoResetEvent_CreateNotSignaled(CAutoResetEvent *p); typedef HANDLE CSemaphore; -#define Semaphore_Construct(p) (*p) = NULL +#define Semaphore_Construct(p) *(p) = NULL +#define Semaphore_IsCreated(p) (*(p) != NULL) #define Semaphore_Close(p) HandlePtr_Close(p) #define Semaphore_Wait(p) Handle_WaitObject(*(p)) WRes Semaphore_Create(CSemaphore *p, UInt32 initCount, UInt32 maxCount);
M
src/third-party/lzma/Util/7z/7zMain.c
→
src/third-party/lzma/Util/7z/7zMain.c
@@ -1,10 +1,12 @@
/* 7zMain.c - Test application for 7z Decoder -2015-08-02 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" #include <stdio.h> #include <string.h> + +#include "../../CpuArch.h" #include "../../7z.h" #include "../../7zAlloc.h"@@ -23,7 +25,17 @@ #include <errno.h>
#endif #endif -static ISzAlloc g_Alloc = { SzAlloc, SzFree }; + +#define kInputBufSize ((size_t)1 << 18) + +static const ISzAlloc g_Alloc = { SzAlloc, SzFree }; + + +static void Print(const char *s) +{ + fputs(s, stdout); +} + static int Buf_EnsureSize(CBuf *dest, size_t size) {@@ -164,7 +176,7 @@ {
char defaultChar = '_'; BOOL defUsed; unsigned numChars = 0; - numChars = WideCharToMultiByte(codePage, 0, s, len, (char *)buf->data, size, &defaultChar, &defUsed); + numChars = WideCharToMultiByte(codePage, 0, (LPCWSTR)s, len, (char *)buf->data, size, &defaultChar, &defUsed); if (numChars == 0 || numChars >= size) return SZ_ERROR_FAIL; buf->data[numChars] = 0;@@ -190,7 +202,7 @@ static WRes MyCreateDir(const UInt16 *name)
{ #ifdef USE_WINDOWS_FILE - return CreateDirectoryW(name, NULL) ? 0 : GetLastError(); + return CreateDirectoryW((LPCWSTR)name, NULL) ? 0 : GetLastError(); #else@@ -215,7 +227,7 @@
static WRes OutFile_OpenUtf16(CSzFile *p, const UInt16 *name) { #ifdef USE_WINDOWS_FILE - return OutFile_OpenW(p, name); + return OutFile_OpenW(p, (LPCWSTR)name); #else CBuf buf; WRes res;@@ -227,6 +239,7 @@ return res;
#endif } + static SRes PrintString(const UInt16 *s) { CBuf buf;@@ -238,12 +251,12 @@ , CP_OEMCP
#endif ); if (res == SZ_OK) - fputs((const char *)buf.data, stdout); + Print((const char *)buf.data); Buf_Free(&buf, &g_Alloc); return res; } -static void UInt64ToStr(UInt64 value, char *s) +static void UInt64ToStr(UInt64 value, char *s, int numDigits) { char temp[32]; int pos = 0;@@ -253,6 +266,10 @@ temp[pos++] = (char)('0' + (unsigned)(value % 10));
value /= 10; } while (value != 0); + + for (numDigits -= pos; numDigits > 0; numDigits--) + *s++ = ' '; + do *s++ = temp[--pos]; while (pos);@@ -266,8 +283,10 @@ int pos = 0;
do temp[pos++] = (char)('0' + (value % 10)); while (value /= 10); + for (numDigits -= pos; numDigits > 0; numDigits--) *s++ = '0'; + do *s++ = temp[--pos]; while (pos);@@ -310,10 +329,10 @@ if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0))
ms[1] = 29; for (mon = 0;; mon++) { - unsigned s = ms[mon]; - if (v < s) + unsigned d = ms[mon]; + if (v < d) break; - v -= s; + v -= d; } s = UIntToStr(s, year, 4); *s++ = '-'; UIntToStr_2(s, mon + 1); s[2] = '-'; s += 3;@@ -323,12 +342,19 @@ UIntToStr_2(s, min); s[2] = ':'; s += 3;
UIntToStr_2(s, sec); s[2] = 0; } -void PrintError(char *sz) +static void PrintLF() +{ + Print("\n"); +} + +static void PrintError(char *s) { - printf("\nERROR: %s\n", sz); + Print("\nERROR: "); + Print(s); + PrintLF(); } -static void GetAttribString(UInt32 wa, Bool isDir, char *s) +static void GetAttribString(UInt32 wa, BoolInt isDir, char *s) { #ifdef USE_WINDOWS_FILE s[0] = (char)(((wa & FILE_ATTRIBUTE_DIRECTORY) != 0 || isDir) ? 'D' : '.');@@ -343,25 +369,27 @@ s[1] = 0;
#endif } + // #define NUM_PARENTS_MAX 128 int MY_CDECL main(int numargs, char *args[]) { + ISzAlloc allocImp; + ISzAlloc allocTempImp; + CFileInStream archiveStream; - CLookToRead lookStream; + CLookToRead2 lookStream; CSzArEx db; SRes res; - ISzAlloc allocImp; - ISzAlloc allocTempImp; UInt16 *temp = NULL; size_t tempSize = 0; // UInt32 parents[NUM_PARENTS_MAX]; - printf("\n7z ANSI-C Decoder " MY_VERSION_COPYRIGHT_DATE "\n\n"); + Print("\n7z Decoder " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n"); if (numargs == 1) { - printf( + Print( "Usage: 7zDec <command> <archive_name>\n\n" "<Commands>\n" " e: Extract files from archive (without using directory names)\n"@@ -370,7 +398,7 @@ " t: Test integrity of archive\n"
" x: eXtract files with full paths\n"); return 0; } - + if (numargs < 3) { PrintError("incorrect command");@@ -381,11 +409,9 @@ #if defined(_WIN32) && !defined(USE_WINDOWS_FILE) && !defined(UNDER_CE)
g_FileCodePage = AreFileApisANSI() ? CP_ACP : CP_OEMCP; #endif - allocImp.Alloc = SzAlloc; - allocImp.Free = SzFree; - allocTempImp.Alloc = SzAllocTemp; - allocTempImp.Free = SzFreeTemp; + allocImp = g_Alloc; + allocTempImp = g_Alloc; #ifdef UNDER_CE if (InFile_OpenW(&archiveStream.file, L"\test.7z"))@@ -398,16 +424,31 @@ return 1;
} FileInStream_CreateVTable(&archiveStream); - LookToRead_CreateVTable(&lookStream, False); - - lookStream.realStream = &archiveStream.s; - LookToRead_Init(&lookStream); + LookToRead2_CreateVTable(&lookStream, False); + lookStream.buf = NULL; - CrcGenerateTable(); + res = SZ_OK; + { + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); + if (!lookStream.buf) + res = SZ_ERROR_MEM; + else + { + lookStream.bufSize = kInputBufSize; + lookStream.realStream = &archiveStream.vt; + LookToRead2_Init(&lookStream); + } + } + + CrcGenerateTable(); + SzArEx_Init(&db); - - res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); + + if (res == SZ_OK) + { + res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); + } if (res == SZ_OK) {@@ -477,7 +518,7 @@
GetAttribString(SzBitWithVals_Check(&db.Attribs, i) ? db.Attribs.Vals[i] : 0, isDir, attr); fileSize = SzArEx_GetFileSize(&db, i); - UInt64ToStr(fileSize, s); + UInt64ToStr(fileSize, s, 10); if (SzBitWithVals_Check(&db.MTime, i)) ConvertFileTimeToString(&db.MTime.Vals[i], t);@@ -489,29 +530,33 @@ t[j] = ' ';
t[j] = '\0'; } - printf("%s %s %10s ", t, attr, s); + Print(t); + Print(" "); + Print(attr); + Print(" "); + Print(s); + Print(" "); res = PrintString(temp); if (res != SZ_OK) break; if (isDir) - printf("/"); - printf("\n"); + Print("/"); + PrintLF(); continue; } - fputs(testCommand ? + Print(testCommand ? "Testing ": - "Extracting ", - stdout); + "Extracting "); res = PrintString(temp); if (res != SZ_OK) break; if (isDir) - printf("/"); + Print("/"); else { - res = SzArEx_Extract(&db, &lookStream.s, i, + res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, &offset, &outSizeProcessed, &allocImp, &allocTempImp);@@ -543,7 +588,7 @@
if (isDir) { MyCreateDir(destPath); - printf("\n"); + PrintLF(); continue; } else if (OutFile_OpenUtf16(&outFile, destPath))@@ -561,6 +606,31 @@ PrintError("can not write output file");
res = SZ_ERROR_FAIL; break; } + + #ifdef USE_WINDOWS_FILE + { + FILETIME mtime, ctime; + FILETIME *mtimePtr = NULL; + FILETIME *ctimePtr = NULL; + + if (SzBitWithVals_Check(&db.MTime, i)) + { + const CNtfsFileTime *t = &db.MTime.Vals[i]; + mtime.dwLowDateTime = (DWORD)(t->Low); + mtime.dwHighDateTime = (DWORD)(t->High); + mtimePtr = &mtime; + } + if (SzBitWithVals_Check(&db.CTime, i)) + { + const CNtfsFileTime *t = &db.CTime.Vals[i]; + ctime.dwLowDateTime = (DWORD)(t->Low); + ctime.dwHighDateTime = (DWORD)(t->High); + ctimePtr = &ctime; + } + if (mtimePtr || ctimePtr) + SetFileTime(outFile.handle, ctimePtr, NULL, mtimePtr); + } + #endif if (File_Close(&outFile)) {@@ -571,23 +641,31 @@ }
#ifdef USE_WINDOWS_FILE if (SzBitWithVals_Check(&db.Attribs, i)) - SetFileAttributesW(destPath, db.Attribs.Vals[i]); + { + UInt32 attrib = db.Attribs.Vals[i]; + /* p7zip stores posix attributes in high 16 bits and adds 0x8000 as marker. + We remove posix bits, if we detect posix mode field */ + if ((attrib & 0xF0000000) != 0) + attrib &= 0x7FFF; + SetFileAttributesW((LPCWSTR)destPath, attrib); + } #endif } - printf("\n"); + PrintLF(); } - IAlloc_Free(&allocImp, outBuffer); + ISzAlloc_Free(&allocImp, outBuffer); } } + SzFree(NULL, temp); SzArEx_Free(&db, &allocImp); - SzFree(NULL, temp); + ISzAlloc_Free(&allocImp, lookStream.buf); File_Close(&archiveStream.file); if (res == SZ_OK) { - printf("\nEverything is Ok\n"); + Print("\nEverything is Ok\n"); return 0; }@@ -598,7 +676,11 @@ PrintError("can not allocate memory");
else if (res == SZ_ERROR_CRC) PrintError("CRC error"); else - printf("\nERROR #%d\n", res); + { + char s[32]; + UInt64ToStr(res, s, 0); + PrintError(s); + } return 1; }
M
src/third-party/lzma/Util/Lzma/LzmaUtil.c
→
src/third-party/lzma/Util/Lzma/LzmaUtil.c
@@ -1,5 +1,5 @@
/* LzmaUtil.c -- Test application for LZMA compression -2015-11-08 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "../../Precomp.h"@@ -7,26 +7,29 @@ #include <stdio.h>
#include <stdlib.h> #include <string.h> +#include "../../CpuArch.h" + #include "../../Alloc.h" #include "../../7zFile.h" #include "../../7zVersion.h" #include "../../LzmaDec.h" #include "../../LzmaEnc.h" -const char *kCantReadMessage = "Can not read input file"; -const char *kCantWriteMessage = "Can not write output file"; -const char *kCantAllocateMessage = "Can not allocate memory"; -const char *kDataErrorMessage = "Data error"; +static const char * const kCantReadMessage = "Can not read input file"; +static const char * const kCantWriteMessage = "Can not write output file"; +static const char * const kCantAllocateMessage = "Can not allocate memory"; +static const char * const kDataErrorMessage = "Data error"; -void PrintHelp(char *buffer) +static void PrintHelp(char *buffer) { - strcat(buffer, "\nLZMA Utility " MY_VERSION_COPYRIGHT_DATE "\n" - "\nUsage: lzma <e|d> inputFile outputFile\n" - " e: encode file\n" - " d: decode file\n"); + strcat(buffer, + "\nLZMA-C " MY_VERSION_CPU " : " MY_COPYRIGHT_DATE "\n\n" + "Usage: lzma <e|d> inputFile outputFile\n" + " e: encode file\n" + " d: decode file\n"); } -int PrintError(char *buffer, const char *message) +static int PrintError(char *buffer, const char *message) { strcat(buffer, "\nError: "); strcat(buffer, message);@@ -34,19 +37,21 @@ strcat(buffer, "\n");
return 1; } -int PrintErrorNumber(char *buffer, SRes val) +static int PrintErrorNumber(char *buffer, SRes val) { sprintf(buffer + strlen(buffer), "\nError code: %x\n", (unsigned)val); return 1; } -int PrintUserError(char *buffer) +static int PrintUserError(char *buffer) { return PrintError(buffer, "Incorrect command"); } + #define IN_BUF_SIZE (1 << 16) #define OUT_BUF_SIZE (1 << 16) + static SRes Decode2(CLzmaDec *state, ISeqOutStream *outStream, ISeqInStream *inStream, UInt64 unpackSize)@@ -100,6 +105,7 @@ }
} } } + static SRes Decode(ISeqOutStream *outStream, ISeqInStream *inStream) {@@ -163,14 +169,15 @@ LzmaEnc_Destroy(enc, &g_Alloc, &g_Alloc);
return res; } -int main2(int numArgs, const char *args[], char *rs) + +static int main2(int numArgs, const char *args[], char *rs) { CFileSeqInStream inStream; CFileOutStream outStream; char c; int res; int encodeMode; - Bool useOutFile = False; + BoolInt useOutFile = False; FileSeqInStream_CreateVTable(&inStream); File_Construct(&inStream.file);@@ -215,11 +222,11 @@ if (encodeMode)
{ UInt64 fileSize; File_GetLength(&inStream.file, &fileSize); - res = Encode(&outStream.s, &inStream.s, fileSize, rs); + res = Encode(&outStream.vt, &inStream.vt, fileSize, rs); } else { - res = Decode(&outStream.s, useOutFile ? &inStream.s : NULL); + res = Decode(&outStream.vt, useOutFile ? &inStream.vt : NULL); } if (useOutFile)@@ -240,6 +247,7 @@ return PrintErrorNumber(rs, res);
} return 0; } + int MY_CDECL main(int numArgs, const char *args[]) {
M
src/third-party/lzma/Util/Lzma/LzmaUtil.dsp
→
src/third-party/lzma/Util/Lzma/LzmaUtil.dsp
@@ -98,6 +98,10 @@ SOURCE=..\..\7zStream.c
# End Source File # Begin Source File +SOURCE=..\..\7zTypes.h +# End Source File +# Begin Source File + SOURCE=..\..\7zVersion.h # End Source File # Begin Source File@@ -159,10 +163,6 @@ # End Source File
# Begin Source File SOURCE=..\..\Threads.h -# End Source File -# Begin Source File - -SOURCE=..\..\Types.h # End Source File # End Target # End Project
M
src/third-party/lzma/Util/SfxSetup/SfxSetup.c
→
src/third-party/lzma/Util/SfxSetup/SfxSetup.c
@@ -1,5 +1,5 @@
/* SfxSetup.c - 7z SFX Setup -2015-11-08 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -20,8 +20,11 @@ #include "../../7zAlloc.h"
#include "../../7zCrc.h" #include "../../7zFile.h" #include "../../CpuArch.h" +#include "../../DllSecur.h" #define k_EXE_ExtIndex 2 + +#define kInputBufSize ((size_t)1 << 18) static const char * const kExts[] = {@@ -124,7 +127,7 @@ #endif
#define kSignatureSearchLimit (1 << 22) -static Bool FindSignature(CSzFile *stream, UInt64 *resPos) +static BoolInt FindSignature(CSzFile *stream, UInt64 *resPos) { Byte buf[kBufferSize]; size_t numPrevBytes = 0;@@ -160,7 +163,7 @@ memmove(buf, buf + processed, k7zStartHeaderSize);
} } -static Bool DoesFileOrDirExist(const WCHAR *path) +static BoolInt DoesFileOrDirExist(const WCHAR *path) { WIN32_FIND_DATAW fd; HANDLE handle;@@ -237,7 +240,7 @@ lpCmdLine, int nCmdShow)
#endif { CFileInStream archiveStream; - CLookToRead lookStream; + CLookToRead2 lookStream; CSzArEx db; SRes res = SZ_OK; ISzAlloc allocImp;@@ -251,9 +254,11 @@ size_t pathLen;
DWORD winRes; const wchar_t *cmdLineParams; const char *errorMessage = NULL; - Bool useShellExecute = True; + BoolInt useShellExecute = True; DWORD exitCode = 0; + LoadSecurityDlls(); + #ifdef _CONSOLE SetConsoleCtrlHandler(HandlerRoutine, TRUE); #else@@ -272,7 +277,8 @@ allocTempImp.Alloc = SzAllocTemp;
allocTempImp.Free = SzFreeTemp; FileInStream_CreateVTable(&archiveStream); - LookToRead_CreateVTable(&lookStream, False); + LookToRead2_CreateVTable(&lookStream, False); + lookStream.buf = NULL; winRes = GetModuleFileNameW(NULL, sfxPath, MAX_PATH); if (winRes == 0 || winRes > MAX_PATH)@@ -281,7 +287,7 @@ {
cmdLineParams = GetCommandLineW(); #ifndef UNDER_CE { - Bool quoteMode = False; + BoolInt quoteMode = False; for (;; cmdLineParams++) { wchar_t c = *cmdLineParams;@@ -373,14 +379,22 @@ }
if (res == SZ_OK) { - lookStream.realStream = &archiveStream.s; - LookToRead_Init(&lookStream); + lookStream.buf = (Byte *)ISzAlloc_Alloc(&allocImp, kInputBufSize); + if (!lookStream.buf) + res = SZ_ERROR_MEM; + else + { + lookStream.bufSize = kInputBufSize; + lookStream.realStream = &archiveStream.vt; + LookToRead2_Init(&lookStream); + } } SzArEx_Init(&db); + if (res == SZ_OK) { - res = SzArEx_Open(&db, &lookStream.s, &allocImp, &allocTempImp); + res = SzArEx_Open(&db, &lookStream.vt, &allocImp, &allocTempImp); } if (res == SZ_OK)@@ -396,11 +410,9 @@ for (i = 0; i < db.NumFiles; i++)
{ size_t offset = 0; size_t outSizeProcessed = 0; - size_t len; WCHAR *temp; - len = SzArEx_GetFileNameUtf16(&db, i, NULL); - - if (len >= MAX_PATH) + + if (SzArEx_GetFileNameUtf16(&db, i, NULL) >= MAX_PATH) { res = SZ_ERROR_FAIL; break;@@ -408,9 +420,9 @@ }
temp = path + pathLen; - SzArEx_GetFileNameUtf16(&db, i, temp); + SzArEx_GetFileNameUtf16(&db, i, (UInt16 *)temp); { - res = SzArEx_Extract(&db, &lookStream.s, i, + res = SzArEx_Extract(&db, &lookStream.vt, i, &blockIndex, &outBuffer, &outBufferSize, &offset, &outSizeProcessed, &allocImp, &allocTempImp);@@ -515,15 +527,18 @@ else
{ WCHAR *temp = path + pathLen; UInt32 j; - SzArEx_GetFileNameUtf16(&db, executeFileIndex, temp); + SzArEx_GetFileNameUtf16(&db, executeFileIndex, (UInt16 *)temp); for (j = 0; temp[j] != 0; j++) if (temp[j] == '/') temp[j] = CHAR_PATH_SEPARATOR; } } - IAlloc_Free(&allocImp, outBuffer); + ISzAlloc_Free(&allocImp, outBuffer); } + SzArEx_Free(&db, &allocImp); + + ISzAlloc_Free(&allocImp, lookStream.buf); File_Close(&archiveStream.file);
M
src/third-party/lzma/Util/SfxSetup/SfxSetup.dsp
→
src/third-party/lzma/Util/SfxSetup/SfxSetup.dsp
@@ -187,6 +187,14 @@ SOURCE=..\..\Delta.h
# End Source File # Begin Source File +SOURCE=..\..\DllSecur.c +# End Source File +# Begin Source File + +SOURCE=..\..\DllSecur.h +# End Source File +# Begin Source File + SOURCE=..\..\Lzma2Dec.c # End Source File # Begin Source File
M
src/third-party/lzma/Util/SfxSetup/makefile
→
src/third-party/lzma/Util/SfxSetup/makefile
@@ -1,4 +1,5 @@
PROG = 7zS2.sfx +MY_FIXED = 1 C_OBJS = \ $O\7zAlloc.obj \@@ -16,6 +17,7 @@ $O\Bra86.obj \
$O\BraIA64.obj \ $O\CpuArch.obj \ $O\Delta.obj \ + $O\DllSecur.obj \ $O\Lzma2Dec.obj \ $O\LzmaDec.obj \
M
src/third-party/lzma/Util/SfxSetup/makefile_con
→
src/third-party/lzma/Util/SfxSetup/makefile_con
@@ -1,4 +1,5 @@
PROG = 7zS2con.sfx +MY_FIXED = 1 CFLAGS = $(CFLAGS) -D_CONSOLE C_OBJS = \@@ -17,6 +18,7 @@ $O\Bra86.obj \
$O\BraIA64.obj \ $O\CpuArch.obj \ $O\Delta.obj \ + $O\DllSecur.obj \ $O\Lzma2Dec.obj \ $O\LzmaDec.obj \
M
src/third-party/lzma/Xz.c
→
src/third-party/lzma/Xz.c
@@ -1,5 +1,5 @@
/* Xz.c - Xz -2015-05-01 : Igor Pavlov : Public domain */ +2017-05-12 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -9,7 +9,7 @@ #include "Xz.h"
#include "XzCrc64.h" const Byte XZ_SIG[XZ_SIG_SIZE] = { 0xFD, '7', 'z', 'X', 'Z', 0 }; -const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; +/* const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE] = { 'Y', 'Z' }; */ unsigned Xz_WriteVarInt(Byte *buf, UInt64 v) {@@ -20,22 +20,22 @@ buf[i++] = (Byte)((v & 0x7F) | 0x80);
v >>= 7; } while (v != 0); - buf[i - 1] &= 0x7F; + buf[(size_t)i - 1] &= 0x7F; return i; } void Xz_Construct(CXzStream *p) { - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + p->numBlocks = 0; + p->blocks = NULL; p->flags = 0; } -void Xz_Free(CXzStream *p, ISzAlloc *alloc) +void Xz_Free(CXzStream *p, ISzAllocPtr alloc) { - alloc->Free(alloc, p->blocks); - p->numBlocks = p->numBlocksAllocated = 0; - p->blocks = 0; + ISzAlloc_Free(alloc, p->blocks); + p->numBlocks = 0; + p->blocks = NULL; } unsigned XzFlags_GetCheckSize(CXzStreamFlags f)
M
src/third-party/lzma/Xz.h
→
src/third-party/lzma/Xz.h
@@ -1,5 +1,5 @@
/* Xz.h - Xz interface -2015-05-01 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #ifndef __XZ_H #define __XZ_H@@ -50,9 +50,10 @@
#define XzBlock_GetNumFilters(p) (((p)->flags & XZ_BF_NUM_FILTERS_MASK) + 1) #define XzBlock_HasPackSize(p) (((p)->flags & XZ_BF_PACK_SIZE) != 0) #define XzBlock_HasUnpackSize(p) (((p)->flags & XZ_BF_UNPACK_SIZE) != 0) +#define XzBlock_HasUnsupportedFlags(p) (((p)->flags & ~(XZ_BF_NUM_FILTERS_MASK | XZ_BF_PACK_SIZE | XZ_BF_UNPACK_SIZE)) != 0) SRes XzBlock_Parse(CXzBlock *p, const Byte *header); -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes); +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes); /* ---------- xz stream ---------- */@@ -60,7 +61,13 @@ #define XZ_SIG_SIZE 6
#define XZ_FOOTER_SIG_SIZE 2 extern const Byte XZ_SIG[XZ_SIG_SIZE]; + +/* extern const Byte XZ_FOOTER_SIG[XZ_FOOTER_SIG_SIZE]; +*/ + +#define XZ_FOOTER_SIG_0 'Y' +#define XZ_FOOTER_SIG_1 'Z' #define XZ_STREAM_FLAGS_SIZE 2 #define XZ_STREAM_CRC_SIZE 4@@ -106,13 +113,12 @@ typedef struct
{ CXzStreamFlags flags; size_t numBlocks; - size_t numBlocksAllocated; CXzBlockSizes *blocks; UInt64 startOffset; } CXzStream; void Xz_Construct(CXzStream *p); -void Xz_Free(CXzStream *p, ISzAlloc *alloc); +void Xz_Free(CXzStream *p, ISzAllocPtr alloc); #define XZ_SIZE_OVERFLOW ((UInt64)(Int64)-1)@@ -127,12 +133,15 @@ CXzStream *streams;
} CXzs; void Xzs_Construct(CXzs *p); -void Xzs_Free(CXzs *p, ISzAlloc *alloc); -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAlloc *alloc); +void Xzs_Free(CXzs *p, ISzAllocPtr alloc); +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *inStream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc); UInt64 Xzs_GetNumBlocks(const CXzs *p); UInt64 Xzs_GetUnpackSize(const CXzs *p); + +// ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + typedef enum { CODER_STATUS_NOT_SPECIFIED, /* use main error code instead */@@ -141,43 +150,55 @@ CODER_STATUS_NOT_FINISHED, /* stream was not finished */
CODER_STATUS_NEEDS_MORE_INPUT /* you must provide more input bytes */ } ECoderStatus; + +// ECoderFinishMode values are identical to ELzmaFinishMode + typedef enum { CODER_FINISH_ANY, /* finish at any point */ CODER_FINISH_END /* block must be finished at the end */ } ECoderFinishMode; + typedef struct _IStateCoder { void *p; - void (*Free)(void *p, ISzAlloc *alloc); - SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAlloc *alloc); + void (*Free)(void *p, ISzAllocPtr alloc); + SRes (*SetProps)(void *p, const Byte *props, size_t propSize, ISzAllocPtr alloc); void (*Init)(void *p); - SRes (*Code)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished); + SRes (*Code2)(void *p, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status); + SizeT (*Filter)(void *p, Byte *data, SizeT size); } IStateCoder; + + #define MIXCODER_NUM_FILTERS_MAX 4 typedef struct { - ISzAlloc *alloc; + ISzAllocPtr alloc; Byte *buf; unsigned numCoders; + + Byte *outBuf; + size_t outBufSize; + size_t outWritten; // is equal to lzmaDecoder.dicPos (in outBuf mode) + BoolInt wasFinished; + SRes res; + ECoderStatus status; + // BoolInt SingleBufMode; + int finished[MIXCODER_NUM_FILTERS_MAX - 1]; size_t pos[MIXCODER_NUM_FILTERS_MAX - 1]; size_t size[MIXCODER_NUM_FILTERS_MAX - 1]; UInt64 ids[MIXCODER_NUM_FILTERS_MAX]; + SRes results[MIXCODER_NUM_FILTERS_MAX]; IStateCoder coders[MIXCODER_NUM_FILTERS_MAX]; } CMixCoder; -void MixCoder_Construct(CMixCoder *p, ISzAlloc *alloc); -void MixCoder_Free(CMixCoder *p); -void MixCoder_Init(CMixCoder *p); -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId); -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status); typedef enum {@@ -191,6 +212,7 @@ XZ_STATE_BLOCK,
XZ_STATE_BLOCK_FOOTER } EXzState; + typedef struct { EXzState state;@@ -204,7 +226,7 @@ UInt32 blockHeaderSize;
UInt64 packSize; UInt64 unpackSize; - UInt64 numBlocks; + UInt64 numBlocks; // number of finished blocks in current stream UInt64 indexSize; UInt64 indexPos; UInt64 padSize;@@ -218,15 +240,66 @@ CMixCoder decoder;
CXzBlock block; CXzCheck check; CSha256 sha; + + BoolInt parseMode; + BoolInt headerParsedOk; + BoolInt decodeToStreamSignature; + unsigned decodeOnlyOneBlock; + + Byte *outBuf; + size_t outBufSize; + size_t outDataWritten; // the size of data in (outBuf) that were fully unpacked + Byte shaDigest[SHA256_DIGEST_SIZE]; Byte buf[XZ_BLOCK_HEADER_SIZE_MAX]; } CXzUnpacker; -void XzUnpacker_Construct(CXzUnpacker *p, ISzAlloc *alloc); +/* alloc : aligned for cache line allocation is better */ +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc); void XzUnpacker_Init(CXzUnpacker *p); +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize); void XzUnpacker_Free(CXzUnpacker *p); /* + XzUnpacker + The sequence for decoding functions: + { + XzUnpacker_Construct() + [Decoding_Calls] + XzUnpacker_Free() + } + + [Decoding_Calls] + + There are 3 types of interfaces for [Decoding_Calls] calls: + + Interface-1 : Partial output buffers: + { + XzUnpacker_Init() + for() + XzUnpacker_Code(); + } + + Interface-2 : Direct output buffer: + Use it, if you know exact size of decoded data, and you need + whole xz unpacked data in one output buffer. + xz unpacker doesn't allocate additional buffer for lzma2 dictionary in that mode. + { + XzUnpacker_Init() + XzUnpacker_SetOutBufMode(); // to set output buffer and size + for() + XzUnpacker_Code(); // (dest = NULL) in XzUnpacker_Code() + } + + Interface-3 : Direct output buffer : One call full decoding + It unpacks whole input buffer to output buffer in one call. + It uses Interface-2 internally. + { + XzUnpacker_CodeFull() + } +*/ + +/* finishMode: It has meaning only if the decoding reaches output limit (*destLen). CODER_FINISH_ANY - use smallest number of input bytes@@ -255,20 +328,132 @@ */
SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, - ECoderStatus *status); + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status); + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status); -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p); +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p); /* -Call XzUnpacker_GetExtraSize after XzUnpacker_Code function to detect real size of -xz stream in two cases: -XzUnpacker_Code() returns: +XzUnpacker_GetExtraSize() returns then number of uncofirmed bytes, + if it's in (XZ_STATE_STREAM_HEADER) state or in (XZ_STATE_STREAM_PADDING) state. +These bytes can be some bytes after xz archive, or +it can be start of new xz stream. + +Call XzUnpacker_GetExtraSize() after XzUnpacker_Code() function to detect real size of +xz stream in two cases, if XzUnpacker_Code() returns: res == SZ_OK && status == CODER_STATUS_NEEDS_MORE_INPUT res == SZ_ERROR_NO_ARCHIVE */ -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p); +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p); + + +/* + for random block decoding: + XzUnpacker_Init(); + set CXzUnpacker::streamFlags + XzUnpacker_PrepareToRandomBlockDecoding() + loop + { + XzUnpacker_Code() + XzUnpacker_IsBlockFinished() + } +*/ + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p); +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p); + +#define XzUnpacker_GetPackSizeForIndex(p) ((p)->packSize + (p)->blockHeaderSize + XzFlags_GetCheckSize((p)->streamFlags)) + + + +/* ---------- Multi Threading Decoding ---------- */ + + +typedef struct +{ + size_t inBufSize_ST; + size_t outStep_ST; + BoolInt ignoreErrors; + + #ifndef _7ZIP_ST + unsigned numThreads; + size_t inBufSize_MT; + size_t memUseMax; + #endif +} CXzDecMtProps; + +void XzDecMtProps_Init(CXzDecMtProps *p); + + +typedef void * CXzDecMtHandle; + +/* + alloc : XzDecMt uses CAlignOffsetAlloc for addresses allocated by (alloc). + allocMid : for big allocations, aligned allocation is better +*/ + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid); +void XzDecMt_Destroy(CXzDecMtHandle p); + + +typedef struct +{ + Byte UnpackSize_Defined; + Byte NumStreams_Defined; + Byte NumBlocks_Defined; + + Byte DataAfterEnd; + Byte DecodingTruncated; // Decoding was Truncated, we need only partial output data + + UInt64 InSize; // pack size processed + UInt64 OutSize; + + UInt64 NumStreams; + UInt64 NumBlocks; + + SRes DecodeRes; + SRes ReadRes; + SRes ProgressRes; + SRes CombinedRes; + SRes CombinedRes_Type; + +} CXzStatInfo; + +void XzStatInfo_Clear(CXzStatInfo *p); + +/* +XzDecMt_Decode() +SRes: + SZ_OK - OK + SZ_ERROR_MEM - Memory allocation error + SZ_ERROR_NO_ARCHIVE - is not xz archive + SZ_ERROR_ARCHIVE - Headers error + SZ_ERROR_DATA - Data Error + SZ_ERROR_CRC - CRC Error + SZ_ERROR_INPUT_EOF - it needs more input data + SZ_ERROR_WRITE - ISeqOutStream error + (SZ_ERROR_READ) - ISeqInStream errors + (SZ_ERROR_PROGRESS) - ICompressProgress errors + // SZ_ERROR_THREAD - error in multi-threading functions + MY_SRes_HRESULT_FROM_WRes(WRes_error) - error in multi-threading function +*/ + +SRes XzDecMt_Decode(CXzDecMtHandle p, + const CXzDecMtProps *props, + const UInt64 *outDataSize, // NULL means undefined + int finishMode, // 0 - partial unpacking is allowed, 1 - xz stream(s) must be finished + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, // 0 means that ST (Single-Thread) version was used + ICompressProgress *progress); EXTERN_C_END
M
src/third-party/lzma/XzCrc64.c
→
src/third-party/lzma/XzCrc64.c
@@ -1,5 +1,5 @@
/* XzCrc64.c -- CRC64 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2017-05-23 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -9,9 +9,9 @@
#define kCrc64Poly UINT64_CONST(0xC96C5795D7870F42) #ifdef MY_CPU_LE - #define CRC_NUM_TABLES 4 + #define CRC64_NUM_TABLES 4 #else - #define CRC_NUM_TABLES 5 + #define CRC64_NUM_TABLES 5 #define CRC_UINT64_SWAP(v) \ ((v >> 56) \ | ((v >> 40) & ((UInt64)0xFF << 8)) \@@ -29,10 +29,10 @@ #ifndef MY_CPU_BE
UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table); #endif -typedef UInt64 (MY_FAST_CALL *CRC_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); +typedef UInt64 (MY_FAST_CALL *CRC64_FUNC)(UInt64 v, const void *data, size_t size, const UInt64 *table); -static CRC_FUNC g_Crc64Update; -UInt64 g_Crc64Table[256 * CRC_NUM_TABLES]; +static CRC64_FUNC g_Crc64Update; +UInt64 g_Crc64Table[256 * CRC64_NUM_TABLES]; UInt64 MY_FAST_CALL Crc64Update(UInt64 v, const void *data, size_t size) {@@ -52,12 +52,12 @@ {
UInt64 r = i; unsigned j; for (j = 0; j < 8; j++) - r = (r >> 1) ^ (kCrc64Poly & ~((r & 1) - 1)); + r = (r >> 1) ^ (kCrc64Poly & ((UInt64)0 - (r & 1))); g_Crc64Table[i] = r; } - for (; i < 256 * CRC_NUM_TABLES; i++) + for (i = 256; i < 256 * CRC64_NUM_TABLES; i++) { - UInt64 r = g_Crc64Table[i - 256]; + UInt64 r = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = g_Crc64Table[r & 0xFF] ^ (r >> 8); }@@ -74,9 +74,9 @@ g_Crc64Update = XzCrc64UpdateT4;
else #endif { - for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--) + for (i = 256 * CRC64_NUM_TABLES - 1; i >= 256; i--) { - UInt64 x = g_Crc64Table[i - 256]; + UInt64 x = g_Crc64Table[(size_t)i - 256]; g_Crc64Table[i] = CRC_UINT64_SWAP(x); } g_Crc64Update = XzCrc64UpdateT1_BeT4;
M
src/third-party/lzma/XzCrc64Opt.c
→
src/third-party/lzma/XzCrc64Opt.c
@@ -1,5 +1,5 @@
/* XzCrc64Opt.c -- CRC64 calculation -2015-03-01 : Igor Pavlov : Public domain */ +2017-06-30 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -7,24 +7,24 @@ #include "CpuArch.h"
#ifndef MY_CPU_BE -#define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) +#define CRC64_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8)) UInt64 MY_FAST_CALL XzCrc64UpdateT4(UInt64 v, const void *data, size_t size, const UInt64 *table) { const Byte *p = (const Byte *)data; for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); for (; size >= 4; size -= 4, p += 4) { UInt32 d = (UInt32)v ^ *(const UInt32 *)p; v = (v >> 32) - ^ table[0x300 + ((d ) & 0xFF)] - ^ table[0x200 + ((d >> 8) & 0xFF)] - ^ table[0x100 + ((d >> 16) & 0xFF)] - ^ table[0x000 + ((d >> 24))]; + ^ (table + 0x300)[((d ) & 0xFF)] + ^ (table + 0x200)[((d >> 8) & 0xFF)] + ^ (table + 0x100)[((d >> 16) & 0xFF)] + ^ (table + 0x000)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2(v, *p); + v = CRC64_UPDATE_BYTE_2(v, *p); return v; }@@ -43,7 +43,7 @@ | ((v << 24) & ((UInt64)0xFF << 40)) \
| ((v << 40) & ((UInt64)0xFF << 48)) \ | ((v << 56))) -#define CRC_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) +#define CRC64_UPDATE_BYTE_2_BE(crc, b) (table[(Byte)((crc) >> 56) ^ (b)] ^ ((crc) << 8)) UInt64 MY_FAST_CALL XzCrc64UpdateT1_BeT4(UInt64 v, const void *data, size_t size, const UInt64 *table) {@@ -51,18 +51,18 @@ const Byte *p = (const Byte *)data;
table += 0x100; v = CRC_UINT64_SWAP(v); for (; size > 0 && ((unsigned)(ptrdiff_t)p & 3) != 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); for (; size >= 4; size -= 4, p += 4) { UInt32 d = (UInt32)(v >> 32) ^ *(const UInt32 *)p; v = (v << 32) - ^ table[0x000 + ((d ) & 0xFF)] - ^ table[0x100 + ((d >> 8) & 0xFF)] - ^ table[0x200 + ((d >> 16) & 0xFF)] - ^ table[0x300 + ((d >> 24))]; + ^ (table + 0x000)[((d ) & 0xFF)] + ^ (table + 0x100)[((d >> 8) & 0xFF)] + ^ (table + 0x200)[((d >> 16) & 0xFF)] + ^ (table + 0x300)[((d >> 24))]; } for (; size > 0; size--, p++) - v = CRC_UPDATE_BYTE_2_BE(v, *p); + v = CRC64_UPDATE_BYTE_2_BE(v, *p); return CRC_UINT64_SWAP(v); }
M
src/third-party/lzma/XzDec.c
→
src/third-party/lzma/XzDec.c
@@ -1,14 +1,33 @@
/* XzDec.c -- Xz Decode -2015-11-09 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h" +// #include <stdio.h> + +// #define XZ_DUMP + /* #define XZ_DUMP */ #ifdef XZ_DUMP #include <stdio.h> #endif +// #define SHOW_DEBUG_INFO + +#ifdef SHOW_DEBUG_INFO +#include <stdio.h> +#endif + +#ifdef SHOW_DEBUG_INFO +#define PRF(x) x +#else +#define PRF(x) +#endif + +#define PRF_STR(s) PRF(printf("\n" s "\n")) +#define PRF_STR_INT(s, d) PRF(printf("\n" s " %d\n", (unsigned)d)) + #include <stdlib.h> #include <string.h>@@ -18,17 +37,19 @@ #include "Bra.h"
#include "CpuArch.h" #include "Delta.h" #include "Lzma2Dec.h" + +// #define USE_SUBBLOCK #ifdef USE_SUBBLOCK #include "Bcj3Dec.c" -#include "SbDec.c" +#include "SbDec.h" #endif #include "Xz.h" #define XZ_CHECK_SIZE_MAX 64 -#define CODER_BUF_SIZE (1 << 17) +#define CODER_BUF_SIZE ((size_t)1 << 17) unsigned Xz_ReadVarInt(const Byte *p, size_t maxSize, UInt64 *value) {@@ -56,8 +77,9 @@ size_t bufPos;
size_t bufConv; size_t bufTotal; + int encodeMode; + UInt32 methodId; - int encodeMode; UInt32 delta; UInt32 ip; UInt32 x86State;@@ -66,12 +88,12 @@
Byte buf[BRA_BUF_SIZE]; } CBraState; -static void BraState_Free(void *pp, ISzAlloc *alloc) +static void BraState_Free(void *pp, ISzAllocPtr alloc) { - alloc->Free(alloc, pp); + ISzAlloc_Free(alloc, pp); } -static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes BraState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { CBraState *p = ((CBraState *)pp); UNUSED_VAR(alloc);@@ -121,69 +143,84 @@ if (p->methodId == XZ_ID_Delta)
Delta_Init(p->deltaState); } -#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: p->bufConv = isa ## _Convert(p->buf, p->bufTotal, p->ip, p->encodeMode); break; + +#define CASE_BRA_CONV(isa) case XZ_ID_ ## isa: size = isa ## _Convert(data, size, p->ip, p->encodeMode); break; -static SRes BraState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +static SizeT BraState_Filter(void *pp, Byte *data, SizeT size) { CBraState *p = ((CBraState *)pp); - SizeT destLenOrig = *destLen; - SizeT srcLenOrig = *srcLen; + switch (p->methodId) + { + case XZ_ID_Delta: + if (p->encodeMode) + Delta_Encode(p->deltaState, p->delta, data, size); + else + Delta_Decode(p->deltaState, p->delta, data, size); + break; + case XZ_ID_X86: + size = x86_Convert(data, size, p->ip, &p->x86State, p->encodeMode); + break; + CASE_BRA_CONV(PPC) + CASE_BRA_CONV(IA64) + CASE_BRA_CONV(ARM) + CASE_BRA_CONV(ARMT) + CASE_BRA_CONV(SPARC) + } + p->ip += (UInt32)size; + return size; +} + + +static SRes BraState_Code2(void *pp, + Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, int srcWasFinished, + ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) +{ + CBraState *p = ((CBraState *)pp); + SizeT destRem = *destLen; + SizeT srcRem = *srcLen; UNUSED_VAR(finishMode); + *destLen = 0; *srcLen = 0; - *wasFinished = 0; - while (destLenOrig > 0) + // *wasFinished = False; + *status = CODER_STATUS_NOT_FINISHED; + + while (destRem > 0) { if (p->bufPos != p->bufConv) { - size_t curSize = p->bufConv - p->bufPos; - if (curSize > destLenOrig) - curSize = destLenOrig; - memcpy(dest, p->buf + p->bufPos, curSize); - p->bufPos += curSize; - *destLen += curSize; - dest += curSize; - destLenOrig -= curSize; + size_t size = p->bufConv - p->bufPos; + if (size > destRem) + size = destRem; + memcpy(dest, p->buf + p->bufPos, size); + p->bufPos += size; + *destLen += size; + dest += size; + destRem -= size; continue; } + p->bufTotal -= p->bufPos; memmove(p->buf, p->buf + p->bufPos, p->bufTotal); p->bufPos = 0; p->bufConv = 0; { - size_t curSize = BRA_BUF_SIZE - p->bufTotal; - if (curSize > srcLenOrig) - curSize = srcLenOrig; - memcpy(p->buf + p->bufTotal, src, curSize); - *srcLen += curSize; - src += curSize; - srcLenOrig -= curSize; - p->bufTotal += curSize; + size_t size = BRA_BUF_SIZE - p->bufTotal; + if (size > srcRem) + size = srcRem; + memcpy(p->buf + p->bufTotal, src, size); + *srcLen += size; + src += size; + srcRem -= size; + p->bufTotal += size; } if (p->bufTotal == 0) break; - switch (p->methodId) - { - case XZ_ID_Delta: - if (p->encodeMode) - Delta_Encode(p->deltaState, p->delta, p->buf, p->bufTotal); - else - Delta_Decode(p->deltaState, p->delta, p->buf, p->bufTotal); - p->bufConv = p->bufTotal; - break; - case XZ_ID_X86: - p->bufConv = x86_Convert(p->buf, p->bufTotal, p->ip, &p->x86State, p->encodeMode); - break; - CASE_BRA_CONV(PPC) - CASE_BRA_CONV(IA64) - CASE_BRA_CONV(ARM) - CASE_BRA_CONV(ARMT) - CASE_BRA_CONV(SPARC) - default: - return SZ_ERROR_UNSUPPORTED; - } - p->ip += (UInt32)p->bufConv; + + p->bufConv = BraState_Filter(pp, p->buf, p->bufTotal); if (p->bufConv == 0) {@@ -192,48 +229,54 @@ break;
p->bufConv = p->bufTotal; } } - if (p->bufTotal == p->bufPos && srcLenOrig == 0 && srcWasFinished) - *wasFinished = 1; + + if (p->bufTotal == p->bufPos && srcRem == 0 && srcWasFinished) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + // *wasFinished = 1; + } + return SZ_OK; } -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc) + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc) { CBraState *decoder; - if (id != XZ_ID_Delta && - id != XZ_ID_X86 && - id != XZ_ID_PPC && - id != XZ_ID_IA64 && - id != XZ_ID_ARM && - id != XZ_ID_ARMT && - id != XZ_ID_SPARC) + if (id < XZ_ID_Delta || id > XZ_ID_SPARC) return SZ_ERROR_UNSUPPORTED; - p->p = 0; - decoder = (CBraState *)alloc->Alloc(alloc, sizeof(CBraState)); - if (decoder == 0) - return SZ_ERROR_MEM; + decoder = (CBraState *)p->p; + if (!decoder) + { + decoder = (CBraState *)ISzAlloc_Alloc(alloc, sizeof(CBraState)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = BraState_Free; + p->SetProps = BraState_SetProps; + p->Init = BraState_Init; + p->Code2 = BraState_Code2; + p->Filter = BraState_Filter; + } decoder->methodId = (UInt32)id; decoder->encodeMode = encodeMode; - p->p = decoder; - p->Free = BraState_Free; - p->SetProps = BraState_SetProps; - p->Init = BraState_Init; - p->Code = BraState_Code; return SZ_OK; } + + /* ---------- SbState ---------- */ #ifdef USE_SUBBLOCK -static void SbState_Free(void *pp, ISzAlloc *alloc) +static void SbState_Free(void *pp, ISzAllocPtr alloc) { CSbDec *p = (CSbDec *)pp; SbDec_Free(p); - alloc->Free(alloc, pp); + ISzAlloc_Free(alloc, pp); } -static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes SbState_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { UNUSED_VAR(pp); UNUSED_VAR(props);@@ -246,8 +289,10 @@ {
SbDec_Init((CSbDec *)pp); } -static SRes SbState_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) +static SRes SbState_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished + ECoderStatus *status) { CSbDec *p = (CSbDec *)pp; SRes res;@@ -260,102 +305,185 @@ p->finish = finishMode; /* change it */
res = SbDec_Decode((CSbDec *)pp); *destLen -= p->destLen; *srcLen -= p->srcLen; - *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + // *wasFinished = (*destLen == 0 && *srcLen == 0); /* change it */ + *status = (*destLen == 0 && *srcLen == 0) ? + CODER_STATUS_FINISHED_WITH_MARK : + CODER_STATUS_NOT_FINISHED; return res; } -SRes SbState_SetFromMethod(IStateCoder *p, ISzAlloc *alloc) +static SRes SbState_SetFromMethod(IStateCoder *p, ISzAllocPtr alloc) { - CSbDec *decoder; - p->p = 0; - decoder = alloc->Alloc(alloc, sizeof(CSbDec)); - if (decoder == 0) - return SZ_ERROR_MEM; - p->p = decoder; - p->Free = SbState_Free; - p->SetProps = SbState_SetProps; - p->Init = SbState_Init; - p->Code = SbState_Code; + CSbDec *decoder = (CSbDec *)p->p; + if (!decoder) + { + decoder = (CSbDec *)ISzAlloc_Alloc(alloc, sizeof(CSbDec)); + if (!decoder) + return SZ_ERROR_MEM; + p->p = decoder; + p->Free = SbState_Free; + p->SetProps = SbState_SetProps; + p->Init = SbState_Init; + p->Code2 = SbState_Code2; + p->Filter = NULL; + } SbDec_Construct(decoder); SbDec_SetAlloc(decoder, alloc); return SZ_OK; } + #endif -/* ---------- Lzma2State ---------- */ + -static void Lzma2State_Free(void *pp, ISzAlloc *alloc) +/* ---------- Lzma2 ---------- */ + +typedef struct { - Lzma2Dec_Free((CLzma2Dec *)pp, alloc); - alloc->Free(alloc, pp); + CLzma2Dec decoder; + BoolInt outBufMode; +} CLzma2Dec_Spec; + + +static void Lzma2State_Free(void *pp, ISzAllocPtr alloc) +{ + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + Lzma2Dec_FreeProbs(&p->decoder, alloc); + else + Lzma2Dec_Free(&p->decoder, alloc); + ISzAlloc_Free(alloc, pp); } -static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAlloc *alloc) +static SRes Lzma2State_SetProps(void *pp, const Byte *props, size_t propSize, ISzAllocPtr alloc) { if (propSize != 1) return SZ_ERROR_UNSUPPORTED; - return Lzma2Dec_Allocate((CLzma2Dec *)pp, props[0], alloc); + { + CLzma2Dec_Spec *p = (CLzma2Dec_Spec *)pp; + if (p->outBufMode) + return Lzma2Dec_AllocateProbs(&p->decoder, props[0], alloc); + else + return Lzma2Dec_Allocate(&p->decoder, props[0], alloc); + } } static void Lzma2State_Init(void *pp) { - Lzma2Dec_Init((CLzma2Dec *)pp); + Lzma2Dec_Init(&((CLzma2Dec_Spec *)pp)->decoder); } -static SRes Lzma2State_Code(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, - int srcWasFinished, ECoderFinishMode finishMode, int *wasFinished) + +/* + if (outBufMode), then (dest) is not used. Use NULL. + Data is unpacked to (spec->decoder.decoder.dic) output buffer. +*/ + +static SRes Lzma2State_Code2(void *pp, Byte *dest, SizeT *destLen, const Byte *src, SizeT *srcLen, + int srcWasFinished, ECoderFinishMode finishMode, + // int *wasFinished, + ECoderStatus *status) { - ELzmaStatus status; + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)pp; + ELzmaStatus status2; /* ELzmaFinishMode fm = (finishMode == LZMA_FINISH_ANY) ? LZMA_FINISH_ANY : LZMA_FINISH_END; */ - SRes res = Lzma2Dec_DecodeToBuf((CLzma2Dec *)pp, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status); + SRes res; UNUSED_VAR(srcWasFinished); - *wasFinished = (status == LZMA_STATUS_FINISHED_WITH_MARK); + if (spec->outBufMode) + { + SizeT dicPos = spec->decoder.decoder.dicPos; + SizeT dicLimit = dicPos + *destLen; + res = Lzma2Dec_DecodeToDic(&spec->decoder, dicLimit, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + *destLen = spec->decoder.decoder.dicPos - dicPos; + } + else + res = Lzma2Dec_DecodeToBuf(&spec->decoder, dest, destLen, src, srcLen, (ELzmaFinishMode)finishMode, &status2); + // *wasFinished = (status2 == LZMA_STATUS_FINISHED_WITH_MARK); + // ECoderStatus values are identical to ELzmaStatus values of LZMA2 decoder + *status = (ECoderStatus)status2; return res; } -static SRes Lzma2State_SetFromMethod(IStateCoder *p, ISzAlloc *alloc) + +static SRes Lzma2State_SetFromMethod(IStateCoder *p, Byte *outBuf, size_t outBufSize, ISzAllocPtr alloc) { - CLzma2Dec *decoder = (CLzma2Dec *)alloc->Alloc(alloc, sizeof(CLzma2Dec)); - p->p = decoder; - if (decoder == 0) - return SZ_ERROR_MEM; - p->Free = Lzma2State_Free; - p->SetProps = Lzma2State_SetProps; - p->Init = Lzma2State_Init; - p->Code = Lzma2State_Code; - Lzma2Dec_Construct(decoder); + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if (!spec) + { + spec = (CLzma2Dec_Spec *)ISzAlloc_Alloc(alloc, sizeof(CLzma2Dec_Spec)); + if (!spec) + return SZ_ERROR_MEM; + p->p = spec; + p->Free = Lzma2State_Free; + p->SetProps = Lzma2State_SetProps; + p->Init = Lzma2State_Init; + p->Code2 = Lzma2State_Code2; + p->Filter = NULL; + Lzma2Dec_Construct(&spec->decoder); + } + spec->outBufMode = False; + if (outBuf) + { + spec->outBufMode = True; + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } + return SZ_OK; +} + + +static SRes Lzma2State_ResetOutBuf(IStateCoder *p, Byte *outBuf, size_t outBufSize) +{ + CLzma2Dec_Spec *spec = (CLzma2Dec_Spec *)p->p; + if ((spec->outBufMode && !outBuf) || (!spec->outBufMode && outBuf)) + return SZ_ERROR_FAIL; + if (outBuf) + { + spec->decoder.decoder.dic = outBuf; + spec->decoder.decoder.dicBufSize = outBufSize; + } return SZ_OK; } -void MixCoder_Construct(CMixCoder *p, ISzAlloc *alloc) + +static void MixCoder_Construct(CMixCoder *p, ISzAllocPtr alloc) { unsigned i; p->alloc = alloc; p->buf = NULL; p->numCoders = 0; + + p->outBufSize = 0; + p->outBuf = NULL; + // p->SingleBufMode = False; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) p->coders[i].p = NULL; } -void MixCoder_Free(CMixCoder *p) + +static void MixCoder_Free(CMixCoder *p) { unsigned i; - for (i = 0; i < p->numCoders; i++) + p->numCoders = 0; + for (i = 0; i < MIXCODER_NUM_FILTERS_MAX; i++) { IStateCoder *sc = &p->coders[i]; - if (p->alloc && sc->p) + if (sc->p) + { sc->Free(sc->p, p->alloc); + sc->p = NULL; + } } - p->numCoders = 0; if (p->buf) { - p->alloc->Free(p->alloc, p->buf); + ISzAlloc_Free(p->alloc, p->buf); p->buf = NULL; /* 9.31: the BUG was fixed */ } } -void MixCoder_Init(CMixCoder *p) +static void MixCoder_Init(CMixCoder *p) { unsigned i; for (i = 0; i < MIXCODER_NUM_FILTERS_MAX - 1; i++)@@ -368,16 +496,22 @@ for (i = 0; i < p->numCoders; i++)
{ IStateCoder *coder = &p->coders[i]; coder->Init(coder->p); + p->results[i] = SZ_OK; } + p->outWritten = 0; + p->wasFinished = False; + p->res = SZ_OK; + p->status = CODER_STATUS_NOT_SPECIFIED; } -SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId) + +static SRes MixCoder_SetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) { IStateCoder *sc = &p->coders[coderIndex]; p->ids[coderIndex] = methodId; switch (methodId) { - case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, p->alloc); + case XZ_ID_LZMA2: return Lzma2State_SetFromMethod(sc, outBuf, outBufSize, p->alloc); #ifdef USE_SUBBLOCK case XZ_ID_Subblock: return SbState_SetFromMethod(sc, p->alloc); #endif@@ -387,31 +521,128 @@ return SZ_ERROR_UNSUPPORTED;
return BraState_SetFromMethod(sc, methodId, 0, p->alloc); } -SRes MixCoder_Code(CMixCoder *p, Byte *dest, SizeT *destLen, + +static SRes MixCoder_ResetFromMethod(CMixCoder *p, unsigned coderIndex, UInt64 methodId, Byte *outBuf, size_t outBufSize) +{ + IStateCoder *sc = &p->coders[coderIndex]; + switch (methodId) + { + case XZ_ID_LZMA2: return Lzma2State_ResetOutBuf(sc, outBuf, outBufSize); + } + return SZ_ERROR_UNSUPPORTED; +} + + + +/* + if (destFinish) - then unpack data block is finished at (*destLen) position, + and we can return data that were not processed by filter + +output (status) can be : + CODER_STATUS_NOT_FINISHED + CODER_STATUS_FINISHED_WITH_MARK + CODER_STATUS_NEEDS_MORE_INPUT - not implemented still +*/ + +static SRes MixCoder_Code(CMixCoder *p, + Byte *dest, SizeT *destLen, int destFinish, const Byte *src, SizeT *srcLen, int srcWasFinished, - ECoderFinishMode finishMode, ECoderStatus *status) + ECoderFinishMode finishMode) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; - Bool allFinished = True; + *destLen = 0; *srcLen = 0; - *status = CODER_STATUS_NOT_FINISHED; + + if (p->wasFinished) + return p->res; + + p->status = CODER_STATUS_NOT_FINISHED; - if (!p->buf) + // if (p->SingleBufMode) + if (p->outBuf) { - p->buf = (Byte *)p->alloc->Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); - if (!p->buf) - return SZ_ERROR_MEM; + SRes res; + SizeT destLen2, srcLen2; + int wasFinished; + + PRF_STR("------- MixCoder Single ----------"); + + srcLen2 = srcLenOrig; + destLen2 = destLenOrig; + + { + IStateCoder *coder = &p->coders[0]; + res = coder->Code2(coder->p, NULL, &destLen2, src, &srcLen2, srcWasFinished, finishMode, + // &wasFinished, + &p->status); + wasFinished = (p->status == CODER_STATUS_FINISHED_WITH_MARK); + } + + p->res = res; + + /* + if (wasFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + else + { + if (res == SZ_OK) + if (destLen2 != destLenOrig) + p->status = CODER_STATUS_NEEDS_MORE_INPUT; + } + */ + + + *srcLen = srcLen2; + src += srcLen2; + p->outWritten += destLen2; + + if (res != SZ_OK || srcWasFinished || wasFinished) + p->wasFinished = True; + + if (p->numCoders == 1) + *destLen = destLen2; + else if (p->wasFinished) + { + unsigned i; + size_t processed = p->outWritten; + + for (i = 1; i < p->numCoders; i++) + { + IStateCoder *coder = &p->coders[i]; + processed = coder->Filter(coder->p, p->outBuf, processed); + if (wasFinished || (destFinish && p->outWritten == destLenOrig)) + processed = p->outWritten; + PRF_STR_INT("filter", i); + } + *destLen = processed; + } + return res; } + PRF_STR("standard mix"); + if (p->numCoders != 1) + { + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(p->alloc, CODER_BUF_SIZE * (MIXCODER_NUM_FILTERS_MAX - 1)); + if (!p->buf) + return SZ_ERROR_MEM; + } + finishMode = CODER_FINISH_ANY; + } for (;;) { - Bool processed = False; + BoolInt processed = False; + BoolInt allFinished = True; + SRes resMain = SZ_OK; unsigned i; + + p->status = CODER_STATUS_NOT_FINISHED; /* if (p->numCoders == 1 && *destLen == destLenOrig && finishMode == LZMA_FINISH_ANY) break;@@ -421,78 +652,105 @@ for (i = 0; i < p->numCoders; i++)
{ SRes res; IStateCoder *coder = &p->coders[i]; - Byte *destCur; - SizeT destLenCur, srcLenCur; - const Byte *srcCur; - int srcFinishedCur; + Byte *dest2; + SizeT destLen2, srcLen2; // destLen2_Orig; + const Byte *src2; + int srcFinished2; int encodingWasFinished; + ECoderStatus status2; if (i == 0) { - srcCur = src; - srcLenCur = srcLenOrig - *srcLen; - srcFinishedCur = srcWasFinished; + src2 = src; + srcLen2 = srcLenOrig - *srcLen; + srcFinished2 = srcWasFinished; } else { - srcCur = p->buf + (CODER_BUF_SIZE * (i - 1)) + p->pos[i - 1]; - srcLenCur = p->size[i - 1] - p->pos[i - 1]; - srcFinishedCur = p->finished[i - 1]; + size_t k = i - 1; + src2 = p->buf + (CODER_BUF_SIZE * k) + p->pos[k]; + srcLen2 = p->size[k] - p->pos[k]; + srcFinished2 = p->finished[k]; } if (i == p->numCoders - 1) { - destCur = dest; - destLenCur = destLenOrig - *destLen; + dest2 = dest; + destLen2 = destLenOrig - *destLen; } else { if (p->pos[i] != p->size[i]) continue; - destCur = p->buf + (CODER_BUF_SIZE * i); - destLenCur = CODER_BUF_SIZE; + dest2 = p->buf + (CODER_BUF_SIZE * i); + destLen2 = CODER_BUF_SIZE; } - res = coder->Code(coder->p, destCur, &destLenCur, srcCur, &srcLenCur, srcFinishedCur, finishMode, &encodingWasFinished); + // destLen2_Orig = destLen2; + + if (p->results[i] != SZ_OK) + { + if (resMain == SZ_OK) + resMain = p->results[i]; + continue; + } + + res = coder->Code2(coder->p, + dest2, &destLen2, + src2, &srcLen2, srcFinished2, + finishMode, + // &encodingWasFinished, + &status2); + if (res != SZ_OK) + { + p->results[i] = res; + if (resMain == SZ_OK) + resMain = res; + } + + encodingWasFinished = (status2 == CODER_STATUS_FINISHED_WITH_MARK); + if (!encodingWasFinished) + { allFinished = False; + if (p->numCoders == 1 && res == SZ_OK) + p->status = status2; + } if (i == 0) { - *srcLen += srcLenCur; - src += srcLenCur; + *srcLen += srcLen2; + src += srcLen2; } else - { - p->pos[i - 1] += srcLenCur; - } + p->pos[(size_t)i - 1] += srcLen2; if (i == p->numCoders - 1) { - *destLen += destLenCur; - dest += destLenCur; + *destLen += destLen2; + dest += destLen2; } else { - p->size[i] = destLenCur; + p->size[i] = destLen2; p->pos[i] = 0; p->finished[i] = encodingWasFinished; } - if (res != SZ_OK) - return res; - - if (destLenCur != 0 || srcLenCur != 0) + if (destLen2 != 0 || srcLen2 != 0) processed = True; } + if (!processed) - break; + { + if (allFinished) + p->status = CODER_STATUS_FINISHED_WITH_MARK; + return resMain; + } } - if (allFinished) - *status = CODER_STATUS_FINISHED_WITH_MARK; - return SZ_OK; } + SRes Xz_ParseHeader(CXzStreamFlags *p, const Byte *buf) {@@ -503,13 +761,13 @@ return SZ_ERROR_NO_ARCHIVE;
return XzFlags_IsSupported(*p) ? SZ_OK : SZ_ERROR_UNSUPPORTED; } -static Bool Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) +static BoolInt Xz_CheckFooter(CXzStreamFlags flags, UInt64 indexSize, const Byte *buf) { - return - indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) && - (GetUi32(buf) == CrcCalc(buf + 4, 6) && - flags == GetBe16(buf + 8) && - memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0); + return indexSize == (((UInt64)GetUi32(buf + 4) + 1) << 2) + && GetUi32(buf) == CrcCalc(buf + 4, 6) + && flags == GetBe16(buf + 8) + && buf[10] == XZ_FOOTER_SIG_0 + && buf[11] == XZ_FOOTER_SIG_1; } #define READ_VARINT_AND_CHECK(buf, pos, size, res) \@@ -517,20 +775,48 @@ { unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \
if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } +static BoolInt XzBlock_AreSupportedFilters(const CXzBlock *p) +{ + unsigned numFilters = XzBlock_GetNumFilters(p) - 1; + unsigned i; + { + const CXzFilter *f = &p->filters[numFilters]; + if (f->id != XZ_ID_LZMA2 || f->propsSize != 1 || f->props[0] > 40) + return False; + } + + for (i = 0; i < numFilters; i++) + { + const CXzFilter *f = &p->filters[i]; + if (f->id == XZ_ID_Delta) + { + if (f->propsSize != 1) + return False; + } + else if (f->id < XZ_ID_Delta + || f->id > XZ_ID_SPARC + || (f->propsSize != 0 && f->propsSize != 4)) + return False; + } + return True; +} + + SRes XzBlock_Parse(CXzBlock *p, const Byte *header) { unsigned pos; unsigned numFilters, i; unsigned headerSize = (unsigned)header[0] << 2; + /* (headerSize != 0) : another code checks */ + if (CrcCalc(header, headerSize) != GetUi32(header + headerSize)) return SZ_ERROR_ARCHIVE; pos = 1; - if (pos == headerSize) - return SZ_ERROR_ARCHIVE; p->flags = header[pos++]; + p->packSize = (UInt64)(Int64)-1; if (XzBlock_HasPackSize(p)) { READ_VARINT_AND_CHECK(header, pos, headerSize, &p->packSize);@@ -538,6 +824,7 @@ if (p->packSize == 0 || p->packSize + headerSize >= (UInt64)1 << 63)
return SZ_ERROR_ARCHIVE; } + p->unpackSize = (UInt64)(Int64)-1; if (XzBlock_HasUnpackSize(p)) READ_VARINT_AND_CHECK(header, pos, headerSize, &p->unpackSize);@@ -564,37 +851,56 @@ }
#endif } + if (XzBlock_HasUnsupportedFlags(p)) + return SZ_ERROR_UNSUPPORTED; + while (pos < headerSize) if (header[pos++] != 0) return SZ_ERROR_ARCHIVE; return SZ_OK; } -SRes XzDec_Init(CMixCoder *p, const CXzBlock *block) + + + +static SRes XzDecMix_Init(CMixCoder *p, const CXzBlock *block, Byte *outBuf, size_t outBufSize) { unsigned i; - Bool needReInit = True; + BoolInt needReInit = True; unsigned numFilters = XzBlock_GetNumFilters(block); - - if (numFilters == p->numCoders) + + if (numFilters == p->numCoders && ((p->outBuf && outBuf) || (!p->outBuf && !outBuf))) { + needReInit = False; for (i = 0; i < numFilters; i++) if (p->ids[i] != block->filters[numFilters - 1 - i].id) + { + needReInit = True; break; - needReInit = (i != numFilters); + } } + + // p->SingleBufMode = (outBuf != NULL); + p->outBuf = outBuf; + p->outBufSize = outBufSize; + + // p->SingleBufMode = False; + // outBuf = NULL; if (needReInit) { MixCoder_Free(p); - p->numCoders = numFilters; for (i = 0; i < numFilters; i++) { - const CXzFilter *f = &block->filters[numFilters - 1 - i]; - RINOK(MixCoder_SetFromMethod(p, i, f->id)); + RINOK(MixCoder_SetFromMethod(p, i, block->filters[numFilters - 1 - i].id, outBuf, outBufSize)); } + p->numCoders = numFilters; } - + else + { + RINOK(MixCoder_ResetFromMethod(p, 0, block->filters[numFilters - 1].id, outBuf, outBufSize)); + } + for (i = 0; i < numFilters; i++) { const CXzFilter *f = &block->filters[numFilters - 1 - i];@@ -606,6 +912,8 @@ MixCoder_Init(p);
return SZ_OK; } + + void XzUnpacker_Init(CXzUnpacker *p) { p->state = XZ_STATE_STREAM_HEADER;@@ -614,75 +922,171 @@ p->numStartedStreams = 0;
p->numFinishedStreams = 0; p->numTotalBlocks = 0; p->padSize = 0; + p->decodeOnlyOneBlock = 0; + + p->parseMode = False; + p->decodeToStreamSignature = False; + + // p->outBuf = NULL; + // p->outBufSize = 0; + p->outDataWritten = 0; } -void XzUnpacker_Construct(CXzUnpacker *p, ISzAlloc *alloc) + +void XzUnpacker_SetOutBuf(CXzUnpacker *p, Byte *outBuf, size_t outBufSize) +{ + p->outBuf = outBuf; + p->outBufSize = outBufSize; +} + + +void XzUnpacker_Construct(CXzUnpacker *p, ISzAllocPtr alloc) { MixCoder_Construct(&p->decoder, alloc); + p->outBuf = NULL; + p->outBufSize = 0; XzUnpacker_Init(p); } + void XzUnpacker_Free(CXzUnpacker *p) { MixCoder_Free(&p->decoder); } + +void XzUnpacker_PrepareToRandomBlockDecoding(CXzUnpacker *p) +{ + p->indexSize = 0; + p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; + p->pos = 0; + p->decodeOnlyOneBlock = 1; +} + + +static void XzUnpacker_UpdateIndex(CXzUnpacker *p, UInt64 packSize, UInt64 unpackSize) +{ + Byte temp[32]; + unsigned num = Xz_WriteVarInt(temp, packSize); + num += Xz_WriteVarInt(temp + num, unpackSize); + Sha256_Update(&p->sha, temp, num); + p->indexSize += num; + p->numBlocks++; +} + + + SRes XzUnpacker_Code(CXzUnpacker *p, Byte *dest, SizeT *destLen, - const Byte *src, SizeT *srcLen, ECoderFinishMode finishMode, ECoderStatus *status) + const Byte *src, SizeT *srcLen, int srcFinished, + ECoderFinishMode finishMode, ECoderStatus *status) { SizeT destLenOrig = *destLen; SizeT srcLenOrig = *srcLen; *destLen = 0; *srcLen = 0; *status = CODER_STATUS_NOT_SPECIFIED; + for (;;) { - SizeT srcRem = srcLenOrig - *srcLen; + SizeT srcRem; if (p->state == XZ_STATE_BLOCK) { SizeT destLen2 = destLenOrig - *destLen; SizeT srcLen2 = srcLenOrig - *srcLen; SRes res; + + ECoderFinishMode finishMode2 = finishMode; + BoolInt srcFinished2 = srcFinished; + BoolInt destFinish = False; + + if (p->block.packSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.packSize - p->packSize; + if (srcLen2 >= rem) + { + srcFinished2 = True; + srcLen2 = (SizeT)rem; + } + if (rem == 0 && p->block.unpackSize == p->unpackSize) + return SZ_ERROR_DATA; + } + + if (p->block.unpackSize != (UInt64)(Int64)-1) + { + UInt64 rem = p->block.unpackSize - p->unpackSize; + if (destLen2 >= rem) + { + destFinish = True; + finishMode2 = CODER_FINISH_END; + destLen2 = (SizeT)rem; + } + } + + /* if (srcLen2 == 0 && destLen2 == 0) { *status = CODER_STATUS_NOT_FINISHED; return SZ_OK; } + */ - res = MixCoder_Code(&p->decoder, dest, &destLen2, src, &srcLen2, False, finishMode, status); - XzCheck_Update(&p->check, dest, destLen2); + { + res = MixCoder_Code(&p->decoder, + (p->outBuf ? NULL : dest), &destLen2, destFinish, + src, &srcLen2, srcFinished2, + finishMode2); + + *status = p->decoder.status; + XzCheck_Update(&p->check, (p->outBuf ? p->outBuf + p->outDataWritten : dest), destLen2); + if (!p->outBuf) + dest += destLen2; + p->outDataWritten += destLen2; + } (*srcLen) += srcLen2; src += srcLen2; p->packSize += srcLen2; - (*destLen) += destLen2; - dest += destLen2; p->unpackSize += destLen2; - + RINOK(res); - - if (*status == CODER_STATUS_FINISHED_WITH_MARK) + + if (*status != CODER_STATUS_FINISHED_WITH_MARK) { - Byte temp[32]; - unsigned num = Xz_WriteVarInt(temp, p->packSize + p->blockHeaderSize + XzFlags_GetCheckSize(p->streamFlags)); - num += Xz_WriteVarInt(temp + num, p->unpackSize); - Sha256_Update(&p->sha, temp, num); - p->indexSize += num; - p->numBlocks++; + if (p->block.packSize == p->packSize + && *status == CODER_STATUS_NEEDS_MORE_INPUT) + { + PRF_STR("CODER_STATUS_NEEDS_MORE_INPUT"); + *status = CODER_STATUS_NOT_SPECIFIED; + return SZ_ERROR_DATA; + } + return SZ_OK; + } + { + XzUnpacker_UpdateIndex(p, XzUnpacker_GetPackSizeForIndex(p), p->unpackSize); p->state = XZ_STATE_BLOCK_FOOTER; p->pos = 0; p->alignPos = 0; + *status = CODER_STATUS_NOT_SPECIFIED; + + if ((p->block.packSize != (UInt64)(Int64)-1 && p->block.packSize != p->packSize) + || (p->block.unpackSize != (UInt64)(Int64)-1 && p->block.unpackSize != p->unpackSize)) + { + PRF_STR("ERROR: block.size mismatch"); + return SZ_ERROR_DATA; + } } - else if (srcLen2 == 0 && destLen2 == 0) - return SZ_OK; - - continue; + // continue; } - if (srcRem == 0) + srcRem = srcLenOrig - *srcLen; + + // XZ_STATE_BLOCK_FOOTER can transit to XZ_STATE_BLOCK_HEADER without input bytes + if (srcRem == 0 && p->state != XZ_STATE_BLOCK_FOOTER) { *status = CODER_STATUS_NEEDS_MORE_INPUT; return SZ_OK;@@ -696,6 +1100,8 @@ if (p->pos < XZ_STREAM_HEADER_SIZE)
{ if (p->pos < XZ_SIG_SIZE && *src != XZ_SIG[p->pos]) return SZ_ERROR_NO_ARCHIVE; + if (p->decodeToStreamSignature) + return SZ_OK; p->buf[p->pos++] = *src++; (*srcLen)++; }@@ -703,10 +1109,10 @@ else
{ RINOK(Xz_ParseHeader(&p->streamFlags, p->buf)); p->numStartedStreams++; - p->state = XZ_STATE_BLOCK_HEADER; - Sha256_Init(&p->sha); p->indexSize = 0; p->numBlocks = 0; + Sha256_Init(&p->sha); + p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; } break;@@ -720,6 +1126,8 @@ p->buf[p->pos++] = *src++;
(*srcLen)++; if (p->buf[0] == 0) { + if (p->decodeOnlyOneBlock) + return SZ_ERROR_DATA; p->indexPreSize = 1 + Xz_WriteVarInt(p->buf + 1, p->numBlocks); p->indexPos = p->indexPreSize; p->indexSize += p->indexPreSize;@@ -727,10 +1135,13 @@ Sha256_Final(&p->sha, p->shaDigest);
Sha256_Init(&p->sha); p->crc = CrcUpdate(CRC_INIT_VAL, p->buf, p->indexPreSize); p->state = XZ_STATE_STREAM_INDEX; + break; } p->blockHeaderSize = ((UInt32)p->buf[0] << 2) + 4; + break; } - else if (p->pos != p->blockHeaderSize) + + if (p->pos != p->blockHeaderSize) { UInt32 cur = p->blockHeaderSize - p->pos; if (cur > srcRem)@@ -743,20 +1154,32 @@ }
else { RINOK(XzBlock_Parse(&p->block, p->buf)); + if (!XzBlock_AreSupportedFilters(&p->block)) + return SZ_ERROR_UNSUPPORTED; p->numTotalBlocks++; p->state = XZ_STATE_BLOCK; p->packSize = 0; p->unpackSize = 0; XzCheck_Init(&p->check, XzFlags_GetCheckType(p->streamFlags)); - RINOK(XzDec_Init(&p->decoder, &p->block)); + if (p->parseMode) + { + p->headerParsedOk = True; + return SZ_OK; + } + RINOK(XzDecMix_Init(&p->decoder, &p->block, p->outBuf, p->outBufSize)); } break; } case XZ_STATE_BLOCK_FOOTER: { - if (((p->packSize + p->alignPos) & 3) != 0) + if ((((unsigned)p->packSize + p->alignPos) & 3) != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } (*srcLen)++; p->alignPos++; if (*src++ != 0)@@ -768,20 +1191,31 @@ UInt32 checkSize = XzFlags_GetCheckSize(p->streamFlags);
UInt32 cur = checkSize - p->pos; if (cur != 0) { + if (srcRem == 0) + { + *status = CODER_STATUS_NEEDS_MORE_INPUT; + return SZ_OK; + } if (cur > srcRem) cur = (UInt32)srcRem; memcpy(p->buf + p->pos, src, cur); p->pos += cur; (*srcLen) += cur; src += cur; + if (checkSize != p->pos) + break; } - else { Byte digest[XZ_CHECK_SIZE_MAX]; p->state = XZ_STATE_BLOCK_HEADER; p->pos = 0; if (XzCheck_Final(&p->check, digest) && memcmp(digest, p->buf, checkSize) != 0) return SZ_ERROR_CRC; + if (p->decodeOnlyOneBlock) + { + *status = CODER_STATUS_FINISHED_WITH_MARK; + return SZ_OK; + } } } break;@@ -897,17 +1331,1436 @@ return SZ_OK;
*/ } -Bool XzUnpacker_IsStreamWasFinished(CXzUnpacker *p) + +SRes XzUnpacker_CodeFull(CXzUnpacker *p, Byte *dest, SizeT *destLen, + const Byte *src, SizeT *srcLen, + ECoderFinishMode finishMode, ECoderStatus *status) +{ + XzUnpacker_Init(p); + XzUnpacker_SetOutBuf(p, dest, *destLen); + + return XzUnpacker_Code(p, + NULL, destLen, + src, srcLen, True, + finishMode, status); +} + + +BoolInt XzUnpacker_IsBlockFinished(const CXzUnpacker *p) +{ + return (p->state == XZ_STATE_BLOCK_HEADER) && (p->pos == 0); +} + +BoolInt XzUnpacker_IsStreamWasFinished(const CXzUnpacker *p) { return (p->state == XZ_STATE_STREAM_PADDING) && (((UInt32)p->padSize & 3) == 0); } -UInt64 XzUnpacker_GetExtraSize(CXzUnpacker *p) +UInt64 XzUnpacker_GetExtraSize(const CXzUnpacker *p) { UInt64 num = 0; if (p->state == XZ_STATE_STREAM_PADDING) - num += p->padSize; + num = p->padSize; else if (p->state == XZ_STATE_STREAM_HEADER) - num += p->padSize + p->pos; + num = p->padSize + p->pos; return num; } + + + + + + + + + + + + + + + + + + + + + +#ifndef _7ZIP_ST +#include "MtDec.h" +#endif + + +void XzDecMtProps_Init(CXzDecMtProps *p) +{ + p->inBufSize_ST = 1 << 18; + p->outStep_ST = 1 << 20; + p->ignoreErrors = False; + + #ifndef _7ZIP_ST + p->numThreads = 1; + p->inBufSize_MT = 1 << 18; + p->memUseMax = sizeof(size_t) << 28; + #endif +} + + + +#ifndef _7ZIP_ST + +/* ---------- CXzDecMtThread ---------- */ + +typedef struct +{ + Byte *outBuf; + size_t outBufSize; + size_t outPreSize; + size_t inPreSize; + size_t inPreHeaderSize; + size_t blockPackSize_for_Index; // including block header and checksum. + size_t blockPackTotal; // including stream header, block header and checksum. + size_t inCodeSize; + size_t outCodeSize; + ECoderStatus status; + SRes codeRes; + BoolInt skipMode; + // BoolInt finishedWithMark; + EMtDecParseState parseState; + BoolInt parsing_Truncated; + BoolInt atBlockHeader; + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + BoolInt dec_created; + CXzUnpacker dec; + + Byte mtPad[1 << 7]; +} CXzDecMtThread; + +#endif + + +/* ---------- CXzDecMt ---------- */ + +typedef struct +{ + CAlignOffsetAlloc alignOffsetAlloc; + ISzAllocPtr allocMid; + + CXzDecMtProps props; + size_t unpackBlockMaxSize; + + ISeqInStream *inStream; + ISeqOutStream *outStream; + ICompressProgress *progress; + // CXzStatInfo *stat; + + BoolInt finishMode; + BoolInt outSize_Defined; + UInt64 outSize; + + UInt64 outProcessed; + UInt64 inProcessed; + UInt64 readProcessed; + BoolInt readWasFinished; + SRes readRes; + SRes writeRes; + + Byte *outBuf; + size_t outBufSize; + Byte *inBuf; + size_t inBufSize; + + CXzUnpacker dec; + + ECoderStatus status; + SRes codeRes; + + #ifndef _7ZIP_ST + BoolInt mainDecoderWasCalled; + // int statErrorDefined; + int finishedDecoderIndex; + + // global values that are used in Parse stage + CXzStreamFlags streamFlags; + // UInt64 numFinishedStreams + UInt64 numStreams; + UInt64 numTotalBlocks; + UInt64 numBlocks; + + // UInt64 numBadBlocks; + SRes mainErrorCode; + + BoolInt isBlockHeaderState_Parse; + BoolInt isBlockHeaderState_Write; + UInt64 outProcessed_Parse; + BoolInt parsing_Truncated; + + BoolInt mtc_WasConstructed; + CMtDec mtc; + CXzDecMtThread coders[MTDEC__THREADS_MAX]; + #endif + +} CXzDecMt; + + + +CXzDecMtHandle XzDecMt_Create(ISzAllocPtr alloc, ISzAllocPtr allocMid) +{ + CXzDecMt *p = (CXzDecMt *)ISzAlloc_Alloc(alloc, sizeof(CXzDecMt)); + if (!p) + return NULL; + + AlignOffsetAlloc_CreateVTable(&p->alignOffsetAlloc); + p->alignOffsetAlloc.baseAlloc = alloc; + p->alignOffsetAlloc.numAlignBits = 7; + p->alignOffsetAlloc.offset = 0; + + p->allocMid = allocMid; + + p->outBuf = NULL; + p->outBufSize = 0; + p->inBuf = NULL; + p->inBufSize = 0; + + XzUnpacker_Construct(&p->dec, &p->alignOffsetAlloc.vt); + + p->unpackBlockMaxSize = 0; + + XzDecMtProps_Init(&p->props); + + #ifndef _7ZIP_ST + p->mtc_WasConstructed = False; + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + coder->dec_created = False; + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + #endif + + return p; +} + + +#ifndef _7ZIP_ST + +static void XzDecMt_FreeOutBufs(CXzDecMt *p) +{ + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *coder = &p->coders[i]; + if (coder->outBuf) + { + ISzAlloc_Free(p->allocMid, coder->outBuf); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + } + p->unpackBlockMaxSize = 0; +} + +#endif + + + +static void XzDecMt_FreeSt(CXzDecMt *p) +{ + XzUnpacker_Free(&p->dec); + + if (p->outBuf) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBuf = NULL; + } + p->outBufSize = 0; + + if (p->inBuf) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBuf = NULL; + } + p->inBufSize = 0; +} + + +void XzDecMt_Destroy(CXzDecMtHandle pp) +{ + CXzDecMt *p = (CXzDecMt *)pp; + + XzDecMt_FreeSt(p); + + #ifndef _7ZIP_ST + + if (p->mtc_WasConstructed) + { + MtDec_Destruct(&p->mtc); + p->mtc_WasConstructed = False; + } + { + unsigned i; + for (i = 0; i < MTDEC__THREADS_MAX; i++) + { + CXzDecMtThread *t = &p->coders[i]; + if (t->dec_created) + { + // we don't need to free dict here + XzUnpacker_Free(&t->dec); + t->dec_created = False; + } + } + } + XzDecMt_FreeOutBufs(p); + + #endif + + ISzAlloc_Free(p->alignOffsetAlloc.baseAlloc, pp); +} + + + +#ifndef _7ZIP_ST + +static void XzDecMt_Callback_Parse(void *obj, unsigned coderIndex, CMtDecCallbackInfo *cc) +{ + CXzDecMt *me = (CXzDecMt *)obj; + CXzDecMtThread *coder = &me->coders[coderIndex]; + size_t srcSize = cc->srcSize; + + cc->srcSize = 0; + cc->outPos = 0; + cc->state = MTDEC_PARSE_CONTINUE; + + cc->canCreateNewThread = True; + + if (cc->startCall) + { + coder->outPreSize = 0; + coder->inPreSize = 0; + coder->inPreHeaderSize = 0; + coder->parseState = MTDEC_PARSE_CONTINUE; + coder->parsing_Truncated = False; + coder->skipMode = False; + coder->codeRes = SZ_OK; + coder->status = CODER_STATUS_NOT_SPECIFIED; + coder->inCodeSize = 0; + coder->outCodeSize = 0; + + coder->numStreams = me->numStreams; + coder->numTotalBlocks = me->numTotalBlocks; + coder->numBlocks = me->numBlocks; + + if (!coder->dec_created) + { + XzUnpacker_Construct(&coder->dec, &me->alignOffsetAlloc.vt); + coder->dec_created = True; + } + + XzUnpacker_Init(&coder->dec); + + if (me->isBlockHeaderState_Parse) + { + coder->dec.streamFlags = me->streamFlags; + coder->atBlockHeader = True; + XzUnpacker_PrepareToRandomBlockDecoding(&coder->dec); + } + else + { + coder->atBlockHeader = False; + me->isBlockHeaderState_Parse = True; + } + + coder->dec.numStartedStreams = me->numStreams; + coder->dec.numTotalBlocks = me->numTotalBlocks; + coder->dec.numBlocks = me->numBlocks; + } + + while (!coder->skipMode) + { + ECoderStatus status; + SRes res; + size_t srcSize2 = srcSize; + size_t destSize = (size_t)0 - 1; + + coder->dec.parseMode = True; + coder->dec.headerParsedOk = False; + + PRF_STR_INT("Parse", srcSize2); + + res = XzUnpacker_Code(&coder->dec, + NULL, &destSize, + cc->src, &srcSize2, cc->srcFinished, + CODER_FINISH_END, &status); + + // PRF(printf(" res = %d, srcSize2 = %d", res, (unsigned)srcSize2)); + + coder->codeRes = res; + coder->status = status; + cc->srcSize += srcSize2; + srcSize -= srcSize2; + coder->inPreHeaderSize += srcSize2; + coder->inPreSize = coder->inPreHeaderSize; + + if (res != SZ_OK) + { + cc->state = + coder->parseState = MTDEC_PARSE_END; + /* + if (res == SZ_ERROR_MEM) + return res; + return SZ_OK; + */ + return; // res; + } + + if (coder->dec.headerParsedOk) + { + const CXzBlock *block = &coder->dec.block; + if (XzBlock_HasUnpackSize(block) + // && block->unpackSize <= me->props.outBlockMax + && XzBlock_HasPackSize(block)) + { + { + if (block->unpackSize * 2 * me->mtc.numStartedThreads > me->props.memUseMax) + { + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + } + { + UInt64 packSize = block->packSize; + UInt64 packSizeAligned = packSize + ((0 - (unsigned)packSize) & 3); + UInt32 checkSize = XzFlags_GetCheckSize(coder->dec.streamFlags); + UInt64 blockPackSum = coder->inPreSize + packSizeAligned + checkSize; + // if (blockPackSum <= me->props.inBlockMax) + // unpackBlockMaxSize + { + coder->blockPackSize_for_Index = (size_t)(coder->dec.blockHeaderSize + packSize + checkSize); + coder->blockPackTotal = (size_t)blockPackSum; + coder->outPreSize = (size_t)block->unpackSize; + coder->streamFlags = coder->dec.streamFlags; + me->streamFlags = coder->dec.streamFlags; + coder->skipMode = True; + break; + } + } + } + } + else + // if (coder->inPreSize <= me->props.inBlockMax) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = + coder->parseState = MTDEC_PARSE_END; + return; // SZ_OK; + } + cc->state = MTDEC_PARSE_OVERFLOW; + return; // SZ_OK; + } + + // ---------- skipMode ---------- + { + UInt64 rem = coder->blockPackTotal - coder->inPreSize; + size_t cur = srcSize; + if (cur > rem) + cur = (size_t)rem; + cc->srcSize += cur; + coder->inPreSize += cur; + srcSize -= cur; + + if (coder->inPreSize == coder->blockPackTotal) + { + if (srcSize == 0) + { + if (!cc->srcFinished) + return; // SZ_OK; + cc->state = MTDEC_PARSE_END; + } + else if ((cc->src)[cc->srcSize] == 0) // we check control byte of next block + cc->state = MTDEC_PARSE_END; + else + { + cc->state = MTDEC_PARSE_NEW; + + { + size_t blockMax = me->unpackBlockMaxSize; + if (blockMax < coder->outPreSize) + blockMax = coder->outPreSize; + { + UInt64 required = (UInt64)blockMax * (me->mtc.numStartedThreads + 1) * 2; + if (me->props.memUseMax < required) + cc->canCreateNewThread = False; + } + } + + if (me->outSize_Defined) + { + // next block can be zero size + const UInt64 rem2 = me->outSize - me->outProcessed_Parse; + if (rem2 < coder->outPreSize) + { + coder->parsing_Truncated = True; + cc->state = MTDEC_PARSE_END; + } + me->outProcessed_Parse += coder->outPreSize; + } + } + } + else if (cc->srcFinished) + cc->state = MTDEC_PARSE_END; + else + return; // SZ_OK; + + coder->parseState = cc->state; + cc->outPos = coder->outPreSize; + + me->numStreams = coder->dec.numStartedStreams; + me->numTotalBlocks = coder->dec.numTotalBlocks; + me->numBlocks = coder->dec.numBlocks + 1; + return; // SZ_OK; + } +} + + +static SRes XzDecMt_Callback_PreCode(void *pp, unsigned coderIndex) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + Byte *dest; + + if (!coder->dec.headerParsedOk) + return SZ_OK; + + dest = coder->outBuf; + + if (!dest || coder->outBufSize < coder->outPreSize) + { + if (dest) + { + ISzAlloc_Free(me->allocMid, dest); + coder->outBuf = NULL; + coder->outBufSize = 0; + } + { + size_t outPreSize = coder->outPreSize; + if (outPreSize == 0) + outPreSize = 1; + dest = (Byte *)ISzAlloc_Alloc(me->allocMid, outPreSize); + } + if (!dest) + return SZ_ERROR_MEM; + coder->outBuf = dest; + coder->outBufSize = coder->outPreSize; + + if (coder->outBufSize > me->unpackBlockMaxSize) + me->unpackBlockMaxSize = coder->outBufSize; + } + + // return SZ_ERROR_MEM; + + XzUnpacker_SetOutBuf(&coder->dec, coder->outBuf, coder->outBufSize); + + { + SRes res = XzDecMix_Init(&coder->dec.decoder, &coder->dec.block, coder->outBuf, coder->outBufSize); + // res = SZ_ERROR_UNSUPPORTED; // to test + coder->codeRes = res; + if (res != SZ_OK) + { + // if (res == SZ_ERROR_MEM) return res; + if (me->props.ignoreErrors && res != SZ_ERROR_MEM) + return S_OK; + return res; + } + } + + return SZ_OK; +} + + +static SRes XzDecMt_Callback_Code(void *pp, unsigned coderIndex, + const Byte *src, size_t srcSize, int srcFinished, + // int finished, int blockFinished, + UInt64 *inCodePos, UInt64 *outCodePos, int *stop) +{ + CXzDecMt *me = (CXzDecMt *)pp; + CXzDecMtThread *coder = &me->coders[coderIndex]; + + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + *stop = True; + + if (coder->inCodeSize < coder->inPreHeaderSize) + { + UInt64 rem = coder->inPreHeaderSize - coder->inCodeSize; + size_t step = srcSize; + if (step > rem) + step = (size_t)rem; + src += step; + srcSize -= step; + coder->inCodeSize += step; + if (coder->inCodeSize < coder->inPreHeaderSize) + { + *stop = False; + return SZ_OK; + } + } + + if (!coder->dec.headerParsedOk) + return SZ_OK; + if (!coder->outBuf) + return SZ_OK; + + if (coder->codeRes == SZ_OK) + { + ECoderStatus status; + SRes res; + size_t srcProcessed = srcSize; + size_t outSizeCur = coder->outPreSize - coder->dec.outDataWritten; + + // PRF(printf("\nCallback_Code: Code %d %d\n", (unsigned)srcSize, (unsigned)outSizeCur)); + + res = XzUnpacker_Code(&coder->dec, + NULL, &outSizeCur, + src, &srcProcessed, srcFinished, + // coder->finishedWithMark ? CODER_FINISH_END : CODER_FINISH_ANY, + CODER_FINISH_END, + &status); + + // PRF(printf(" res = %d, srcSize2 = %d, outSizeCur = %d", res, (unsigned)srcProcessed, (unsigned)outSizeCur)); + + coder->codeRes = res; + coder->status = status; + coder->inCodeSize += srcProcessed; + coder->outCodeSize = coder->dec.outDataWritten; + *inCodePos = coder->inCodeSize; + *outCodePos = coder->outCodeSize; + + if (res == SZ_OK) + { + if (srcProcessed == srcSize) + *stop = False; + return SZ_OK; + } + } + + if (me->props.ignoreErrors && coder->codeRes != SZ_ERROR_MEM) + { + *inCodePos = coder->inPreSize; + *outCodePos = coder->outPreSize; + return S_OK; + } + return coder->codeRes; +} + + +#define XZDECMT_STREAM_WRITE_STEP (1 << 24) + +static SRes XzDecMt_Callback_Write(void *pp, unsigned coderIndex, + BoolInt needWriteToStream, + const Byte *src, size_t srcSize, + // int srcFinished, + BoolInt *needContinue, + BoolInt *canRecode) +{ + CXzDecMt *me = (CXzDecMt *)pp; + const CXzDecMtThread *coder = &me->coders[coderIndex]; + + // PRF(printf("\nWrite processed = %d srcSize = %d\n", (unsigned)me->mtc.inProcessed, (unsigned)srcSize)); + + *needContinue = False; + *canRecode = True; + + if (!needWriteToStream) + return SZ_OK; + + if (!coder->dec.headerParsedOk || !coder->outBuf) + { + if (me->finishedDecoderIndex < 0) + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (me->finishedDecoderIndex >= 0) + return SZ_OK; + + me->mtc.inProcessed += coder->inCodeSize; + + *canRecode = False; + + { + SRes res; + size_t size = coder->outCodeSize; + Byte *data = coder->outBuf; + + // we use in me->dec: sha, numBlocks, indexSize + + if (!me->isBlockHeaderState_Write) + { + XzUnpacker_PrepareToRandomBlockDecoding(&me->dec); + me->dec.decodeOnlyOneBlock = False; + me->dec.numStartedStreams = coder->dec.numStartedStreams; + me->dec.streamFlags = coder->streamFlags; + + me->isBlockHeaderState_Write = True; + } + + me->dec.numTotalBlocks = coder->dec.numTotalBlocks; + XzUnpacker_UpdateIndex(&me->dec, coder->blockPackSize_for_Index, coder->outPreSize); + + if (coder->outPreSize != size) + { + if (me->props.ignoreErrors) + { + memset(data + size, 0, coder->outPreSize - size); + size = coder->outPreSize; + } + // me->numBadBlocks++; + if (me->mainErrorCode == SZ_OK) + { + if ((int)coder->status == LZMA_STATUS_NEEDS_MORE_INPUT) + me->mainErrorCode = SZ_ERROR_INPUT_EOF; + else + me->mainErrorCode = SZ_ERROR_DATA; + } + } + + if (me->writeRes != SZ_OK) + return me->writeRes; + + res = SZ_OK; + { + if (me->outSize_Defined) + { + const UInt64 rem = me->outSize - me->outProcessed; + if (size > rem) + size = (SizeT)rem; + } + + for (;;) + { + size_t cur = size; + size_t written; + if (cur > XZDECMT_STREAM_WRITE_STEP) + cur = XZDECMT_STREAM_WRITE_STEP; + + written = ISeqOutStream_Write(me->outStream, data, cur); + + // PRF(printf("\nWritten ask = %d written = %d\n", (unsigned)cur, (unsigned)written)); + + me->outProcessed += written; + if (written != cur) + { + me->writeRes = SZ_ERROR_WRITE; + res = me->writeRes; + break; + } + data += cur; + size -= cur; + // PRF_STR_INT("Written size =", size); + if (size == 0) + break; + res = MtProgress_ProgressAdd(&me->mtc.mtProgress, 0, 0); + if (res != SZ_OK) + break; + } + } + + if (coder->codeRes != SZ_OK) + if (!me->props.ignoreErrors) + { + me->finishedDecoderIndex = coderIndex; + return res; + } + + RINOK(res); + + if (coder->inPreSize != coder->inCodeSize + || coder->blockPackTotal != coder->inCodeSize) + { + me->finishedDecoderIndex = coderIndex; + return SZ_OK; + } + + if (coder->parseState != MTDEC_PARSE_END) + { + *needContinue = True; + return SZ_OK; + } + } + + // (coder->state == MTDEC_PARSE_END) means that there are no other working threads + // so we can use mtc variables without lock + + PRF_STR_INT("Write MTDEC_PARSE_END", me->mtc.inProcessed); + + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + { + CXzUnpacker *dec = &me->dec; + + PRF_STR_INT("PostSingle", srcSize); + + { + size_t srcProcessed = srcSize; + ECoderStatus status; + size_t outSizeCur = 0; + SRes res; + + // dec->decodeOnlyOneBlock = False; + dec->decodeToStreamSignature = True; + + me->mainDecoderWasCalled = True; + + if (coder->parsing_Truncated) + { + me->parsing_Truncated = True; + return SZ_OK; + } + + res = XzUnpacker_Code(dec, + NULL, &outSizeCur, + src, &srcProcessed, + me->mtc.readWasFinished, // srcFinished + CODER_FINISH_END, // CODER_FINISH_ANY, + &status); + + me->status = status; + me->codeRes = res; + + me->mtc.inProcessed += srcProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + { + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + memcpy(crossBuf, src + srcProcessed, srcSize - srcProcessed); + } + me->mtc.crossStart = 0; + me->mtc.crossEnd = srcSize - srcProcessed; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + { + return E_FAIL; + } + + if (me->mtc.readWasFinished) + { + return SZ_OK; + } + } + + { + size_t inPos; + size_t inLim; + const Byte *inData; + UInt64 inProgressPrev = me->mtc.inProcessed; + + // XzDecMt_Prepare_InBuf_ST(p); + Byte *crossBuf = MtDec_GetCrossBuff(&me->mtc); + if (!crossBuf) + return SZ_ERROR_MEM; + + inPos = 0; + inLim = 0; + // outProcessed = 0; + + inData = crossBuf; + + for (;;) + { + SizeT inProcessed; + SizeT outProcessed; + ECoderStatus status; + SRes res; + + if (inPos == inLim) + { + if (!me->mtc.readWasFinished) + { + inPos = 0; + inLim = me->mtc.inBufSize; + me->mtc.readRes = ISeqInStream_Read(me->inStream, (void *)inData, &inLim); + me->mtc.readProcessed += inLim; + if (inLim == 0 || me->mtc.readRes != SZ_OK) + me->mtc.readWasFinished = True; + } + } + + inProcessed = inLim - inPos; + outProcessed = 0; + + res = XzUnpacker_Code(dec, + NULL, &outProcessed, + inData + inPos, &inProcessed, + (inProcessed == 0), // srcFinished + CODER_FINISH_END, &status); + + me->codeRes = res; + me->status = status; + inPos += inProcessed; + me->mtc.inProcessed += inProcessed; + me->mtc.mtProgress.totalInSize = me->mtc.inProcessed; + + if (res != SZ_OK) + { + return S_OK; + // return res; + } + + if (dec->state == XZ_STATE_STREAM_HEADER) + { + *needContinue = True; + me->mtc.crossStart = inPos; + me->mtc.crossEnd = inLim; + me->isBlockHeaderState_Parse = False; + me->isBlockHeaderState_Write = False; + return SZ_OK; + } + + if (status != CODER_STATUS_NEEDS_MORE_INPUT) + return E_FAIL; + + if (me->mtc.progress) + { + UInt64 inDelta = me->mtc.inProcessed - inProgressPrev; + if (inDelta >= (1 << 22)) + { + RINOK(MtProgress_Progress_ST(&me->mtc.mtProgress)); + inProgressPrev = me->mtc.inProcessed; + } + } + if (me->mtc.readWasFinished) + return SZ_OK; + } + } + } +} + + +#endif + + + +void XzStatInfo_Clear(CXzStatInfo *p) +{ + p->InSize = 0; + p->OutSize = 0; + + p->NumStreams = 0; + p->NumBlocks = 0; + + p->UnpackSize_Defined = False; + + p->NumStreams_Defined = False; + p->NumBlocks_Defined = False; + + // p->IsArc = False; + // p->UnexpectedEnd = False; + // p->Unsupported = False; + // p->HeadersError = False; + // p->DataError = False; + // p->CrcError = False; + + p->DataAfterEnd = False; + p->DecodingTruncated = False; + + p->DecodeRes = SZ_OK; + p->ReadRes = SZ_OK; + p->ProgressRes = SZ_OK; + + p->CombinedRes = SZ_OK; + p->CombinedRes_Type = SZ_OK; +} + + + + +static SRes XzDecMt_Decode_ST(CXzDecMt *p + #ifndef _7ZIP_ST + , BoolInt tMode + #endif + , CXzStatInfo *stat) +{ + size_t outPos; + size_t inPos, inLim; + const Byte *inData; + UInt64 inPrev, outPrev; + + CXzUnpacker *dec; + + #ifndef _7ZIP_ST + if (tMode) + { + XzDecMt_FreeOutBufs(p); + tMode = MtDec_PrepareRead(&p->mtc); + } + #endif + + if (!p->outBuf || p->outBufSize != p->props.outStep_ST) + { + ISzAlloc_Free(p->allocMid, p->outBuf); + p->outBufSize = 0; + p->outBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.outStep_ST); + if (!p->outBuf) + return SZ_ERROR_MEM; + p->outBufSize = p->props.outStep_ST; + } + + if (!p->inBuf || p->inBufSize != p->props.inBufSize_ST) + { + ISzAlloc_Free(p->allocMid, p->inBuf); + p->inBufSize = 0; + p->inBuf = (Byte *)ISzAlloc_Alloc(p->allocMid, p->props.inBufSize_ST); + if (!p->inBuf) + return SZ_ERROR_MEM; + p->inBufSize = p->props.inBufSize_ST; + } + + dec = &p->dec; + dec->decodeToStreamSignature = False; + // dec->decodeOnlyOneBlock = False; + + XzUnpacker_SetOutBuf(dec, NULL, 0); + + inPrev = p->inProcessed; + outPrev = p->outProcessed; + + inPos = 0; + inLim = 0; + inData = NULL; + outPos = 0; + + for (;;) + { + SizeT outSize; + BoolInt finished; + ECoderFinishMode finishMode; + SizeT inProcessed; + ECoderStatus status; + SRes res; + + SizeT outProcessed; + + + + if (inPos == inLim) + { + #ifndef _7ZIP_ST + if (tMode) + { + inData = MtDec_Read(&p->mtc, &inLim); + inPos = 0; + if (inData) + continue; + tMode = False; + inLim = 0; + } + #endif + + if (!p->readWasFinished) + { + inPos = 0; + inLim = p->inBufSize; + inData = p->inBuf; + p->readRes = ISeqInStream_Read(p->inStream, (void *)inData, &inLim); + p->readProcessed += inLim; + if (inLim == 0 || p->readRes != SZ_OK) + p->readWasFinished = True; + } + } + + outSize = p->props.outStep_ST - outPos; + + finishMode = CODER_FINISH_ANY; + if (p->outSize_Defined) + { + const UInt64 rem = p->outSize - p->outProcessed; + if (outSize >= rem) + { + outSize = (SizeT)rem; + if (p->finishMode) + finishMode = CODER_FINISH_END; + } + } + + inProcessed = inLim - inPos; + outProcessed = outSize; + + res = XzUnpacker_Code(dec, p->outBuf + outPos, &outProcessed, + inData + inPos, &inProcessed, + (inPos == inLim), // srcFinished + finishMode, &status); + + p->codeRes = res; + p->status = status; + + inPos += inProcessed; + outPos += outProcessed; + p->inProcessed += inProcessed; + p->outProcessed += outProcessed; + + finished = ((inProcessed == 0 && outProcessed == 0) || res != SZ_OK); + + if (finished || outProcessed >= outSize) + if (outPos != 0) + { + size_t written = ISeqOutStream_Write(p->outStream, p->outBuf, outPos); + p->outProcessed += written; + if (written != outPos) + { + stat->CombinedRes_Type = SZ_ERROR_WRITE; + return SZ_ERROR_WRITE; + } + outPos = 0; + } + + if (p->progress && res == SZ_OK) + { + UInt64 inDelta = p->inProcessed - inPrev; + UInt64 outDelta = p->outProcessed - outPrev; + if (inDelta >= (1 << 22) || outDelta >= (1 << 22)) + { + res = ICompressProgress_Progress(p->progress, p->inProcessed, p->outProcessed); + if (res != SZ_OK) + { + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + stat->ProgressRes = res; + return res; + } + inPrev = p->inProcessed; + outPrev = p->outProcessed; + } + } + + if (finished) + return res; + } +} + +static SRes XzStatInfo_SetStat(const CXzUnpacker *dec, + int finishMode, + UInt64 readProcessed, UInt64 inProcessed, + SRes res, ECoderStatus status, + BoolInt decodingTruncated, + CXzStatInfo *stat) +{ + UInt64 extraSize; + + stat->DecodingTruncated = (Byte)(decodingTruncated ? 1 : 0); + stat->InSize = inProcessed; + stat->NumStreams = dec->numStartedStreams; + stat->NumBlocks = dec->numTotalBlocks; + + stat->UnpackSize_Defined = True; + stat->NumStreams_Defined = True; + stat->NumBlocks_Defined = True; + + extraSize = XzUnpacker_GetExtraSize(dec); + + if (res == SZ_OK) + { + if (status == CODER_STATUS_NEEDS_MORE_INPUT) + { + // CODER_STATUS_NEEDS_MORE_INPUT is expected status for correct xz streams + extraSize = 0; + if (!XzUnpacker_IsStreamWasFinished(dec)) + res = SZ_ERROR_INPUT_EOF; + } + else if (!decodingTruncated || finishMode) // (status == CODER_STATUS_NOT_FINISHED) + res = SZ_ERROR_DATA; + } + else if (res == SZ_ERROR_NO_ARCHIVE) + { + /* + SZ_ERROR_NO_ARCHIVE is possible for 2 states: + XZ_STATE_STREAM_HEADER - if bad signature or bad CRC + XZ_STATE_STREAM_PADDING - if non-zero padding data + extraSize / inProcessed don't include "bad" byte + */ + if (inProcessed != extraSize) // if good streams before error + if (extraSize != 0 || readProcessed != inProcessed) + { + stat->DataAfterEnd = True; + // there is some good xz stream before. So we set SZ_OK + res = SZ_OK; + } + } + + stat->DecodeRes = res; + + stat->InSize -= extraSize; + return res; +} + + +SRes XzDecMt_Decode(CXzDecMtHandle pp, + const CXzDecMtProps *props, + const UInt64 *outDataSize, int finishMode, + ISeqOutStream *outStream, + // Byte *outBuf, size_t *outBufSize, + ISeqInStream *inStream, + // const Byte *inData, size_t inDataSize, + CXzStatInfo *stat, + int *isMT, + ICompressProgress *progress) +{ + CXzDecMt *p = (CXzDecMt *)pp; + #ifndef _7ZIP_ST + BoolInt tMode; + #endif + + XzStatInfo_Clear(stat); + + p->props = *props; + + p->inStream = inStream; + p->outStream = outStream; + p->progress = progress; + // p->stat = stat; + + p->outSize = 0; + p->outSize_Defined = False; + if (outDataSize) + { + p->outSize_Defined = True; + p->outSize = *outDataSize; + } + + p->finishMode = finishMode; + + // p->outSize = 457; p->outSize_Defined = True; p->finishMode = False; // for test + + p->writeRes = SZ_OK; + p->outProcessed = 0; + p->inProcessed = 0; + p->readProcessed = 0; + p->readWasFinished = False; + + p->codeRes = 0; + p->status = CODER_STATUS_NOT_SPECIFIED; + + XzUnpacker_Init(&p->dec); + + *isMT = False; + + /* + p->outBuf = NULL; + p->outBufSize = 0; + if (!outStream) + { + p->outBuf = outBuf; + p->outBufSize = *outBufSize; + *outBufSize = 0; + } + */ + + + #ifndef _7ZIP_ST + + p->isBlockHeaderState_Parse = False; + p->isBlockHeaderState_Write = False; + // p->numBadBlocks = 0; + p->mainErrorCode = SZ_OK; + p->mainDecoderWasCalled = False; + + tMode = False; + + if (p->props.numThreads > 1) + { + IMtDecCallback vt; + + // we just free ST buffers here + // but we still keep state variables, that was set in XzUnpacker_Init() + XzDecMt_FreeSt(p); + + p->outProcessed_Parse = 0; + p->parsing_Truncated = False; + + p->numStreams = 0; + p->numTotalBlocks = 0; + p->numBlocks = 0; + p->finishedDecoderIndex = -1; + + if (!p->mtc_WasConstructed) + { + p->mtc_WasConstructed = True; + MtDec_Construct(&p->mtc); + } + + p->mtc.mtCallback = &vt; + p->mtc.mtCallbackObject = p; + + p->mtc.progress = progress; + p->mtc.inStream = inStream; + p->mtc.alloc = &p->alignOffsetAlloc.vt; + // p->mtc.inData = inData; + // p->mtc.inDataSize = inDataSize; + p->mtc.inBufSize = p->props.inBufSize_MT; + // p->mtc.inBlockMax = p->props.inBlockMax; + p->mtc.numThreadsMax = p->props.numThreads; + + *isMT = True; + + vt.Parse = XzDecMt_Callback_Parse; + vt.PreCode = XzDecMt_Callback_PreCode; + vt.Code = XzDecMt_Callback_Code; + vt.Write = XzDecMt_Callback_Write; + + { + BoolInt needContinue; + + SRes res = MtDec_Code(&p->mtc); + + stat->InSize = p->mtc.inProcessed; + + p->inProcessed = p->mtc.inProcessed; + p->readRes = p->mtc.readRes; + p->readWasFinished = p->mtc.readWasFinished; + p->readProcessed = p->mtc.readProcessed; + + tMode = True; + needContinue = False; + + if (res == SZ_OK) + { + if (p->mtc.mtProgress.res != SZ_OK) + { + res = p->mtc.mtProgress.res; + stat->ProgressRes = res; + stat->CombinedRes_Type = SZ_ERROR_PROGRESS; + } + else + needContinue = p->mtc.needContinue; + } + + if (!needContinue) + { + SRes codeRes; + BoolInt truncated = False; + ECoderStatus status; + CXzUnpacker *dec; + + stat->OutSize = p->outProcessed; + + if (p->finishedDecoderIndex >= 0) + { + CXzDecMtThread *coder = &p->coders[(unsigned)p->finishedDecoderIndex]; + codeRes = coder->codeRes; + dec = &coder->dec; + status = coder->status; + } + else if (p->mainDecoderWasCalled) + { + codeRes = p->codeRes; + dec = &p->dec; + status = p->status; + truncated = p->parsing_Truncated; + } + else + return E_FAIL; + + XzStatInfo_SetStat(dec, p->finishMode, + p->mtc.readProcessed, p->mtc.inProcessed, + codeRes, status, + truncated, + stat); + + if (res == SZ_OK) + { + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else if (p->mtc.readRes != SZ_OK && p->mtc.inProcessed == p->mtc.readProcessed) + { + res = p->mtc.readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + else if (p->mainErrorCode != SZ_OK) + { + res = p->mainErrorCode; + } + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } + + PRF_STR("----- decoding ST -----"); + } + } + + #endif + + + *isMT = False; + + { + SRes res = XzDecMt_Decode_ST(p + #ifndef _7ZIP_ST + , tMode + #endif + , stat + ); + + XzStatInfo_SetStat(&p->dec, + p->finishMode, + p->readProcessed, p->inProcessed, + p->codeRes, p->status, + False, // truncated + stat); + + if (res == SZ_OK) + { + /* + if (p->writeRes != SZ_OK) + { + res = p->writeRes; + stat->CombinedRes_Type = SZ_ERROR_WRITE; + } + else + */ + if (p->readRes != SZ_OK && p->inProcessed == p->readProcessed) + { + res = p->readRes; + stat->ReadRes = res; + stat->CombinedRes_Type = SZ_ERROR_READ; + } + #ifndef _7ZIP_ST + else if (p->mainErrorCode != SZ_OK) + res = p->mainErrorCode; + #endif + } + + stat->CombinedRes = res; + if (stat->CombinedRes_Type == SZ_OK) + stat->CombinedRes_Type = res; + return res; + } +}
M
src/third-party/lzma/XzEnc.c
→
src/third-party/lzma/XzEnc.c
@@ -1,5 +1,5 @@
/* XzEnc.c -- Xz Encode -2015-09-16 : Igor Pavlov : Public domain */ +2019-02-02 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -7,7 +7,6 @@ #include <stdlib.h>
#include <string.h> #include "7zCrc.h" -#include "Alloc.h" #include "Bra.h" #include "CpuArch.h"@@ -18,22 +17,41 @@ #include "SbEnc.c"
#endif #include "XzEnc.h" + +// #define _7ZIP_ST + +#ifndef _7ZIP_ST +#include "MtCoder.h" +#else +#define MTCODER__THREADS_MAX 1 +#define MTCODER__BLOCKS_MAX 1 +#endif + +#define XZ_GET_PAD_SIZE(dataSize) ((4 - ((unsigned)(dataSize) & 3)) & 3) + +/* max pack size for LZMA2 block + check-64bytrs: */ +#define XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize) ((unpackSize) + ((unpackSize) >> 10) + 16 + 64) + +#define XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(unpackSize) (XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(unpackSize)) + #define XzBlock_ClearFlags(p) (p)->flags = 0; #define XzBlock_SetNumFilters(p, n) (p)->flags |= ((n) - 1); #define XzBlock_SetHasPackSize(p) (p)->flags |= XZ_BF_PACK_SIZE; #define XzBlock_SetHasUnpackSize(p) (p)->flags |= XZ_BF_UNPACK_SIZE; -static SRes WriteBytes(ISeqOutStream *s, const void *buf, UInt32 size) + +static SRes WriteBytes(ISeqOutStream *s, const void *buf, size_t size) { - return (s->Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; + return (ISeqOutStream_Write(s, buf, size) == size) ? SZ_OK : SZ_ERROR_WRITE; } -static SRes WriteBytesAndCrc(ISeqOutStream *s, const void *buf, UInt32 size, UInt32 *crc) +static SRes WriteBytesUpdateCrc(ISeqOutStream *s, const void *buf, size_t size, UInt32 *crc) { *crc = CrcUpdate(*crc, buf, size); return WriteBytes(s, buf, size); } + static SRes Xz_WriteHeader(CXzStreamFlags f, ISeqOutStream *s) {@@ -78,98 +96,154 @@ return WriteBytes(s, header, pos + 4);
} -static SRes Xz_WriteFooter(CXzStream *p, ISeqOutStream *s) + + +typedef struct { - Byte buf[32]; - UInt64 globalPos; - { - UInt32 crc = CRC_INIT_VAL; - unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); - size_t i; + size_t numBlocks; + size_t size; + size_t allocated; + Byte *blocks; +} CXzEncIndex; - globalPos = pos; - buf[0] = 0; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - for (i = 0; i < p->numBlocks; i++) - { - const CXzBlockSizes *block = &p->blocks[i]; - pos = Xz_WriteVarInt(buf, block->totalSize); - pos += Xz_WriteVarInt(buf + pos, block->unpackSize); - globalPos += pos; - RINOK(WriteBytesAndCrc(s, buf, pos, &crc)); - } - - pos = ((unsigned)globalPos & 3); - - if (pos != 0) - { - buf[0] = buf[1] = buf[2] = 0; - RINOK(WriteBytesAndCrc(s, buf, 4 - pos, &crc)); - globalPos += 4 - pos; - } - { - SetUi32(buf, CRC_GET_DIGEST(crc)); - RINOK(WriteBytes(s, buf, 4)); - globalPos += 4; - } - } +static void XzEncIndex_Construct(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; + p->blocks = NULL; +} +static void XzEncIndex_Init(CXzEncIndex *p) +{ + p->numBlocks = 0; + p->size = 0; +} + +static void XzEncIndex_Free(CXzEncIndex *p, ISzAllocPtr alloc) +{ + if (p->blocks) { - UInt32 indexSize = (UInt32)((globalPos >> 2) - 1); - SetUi32(buf + 4, indexSize); - buf[8] = (Byte)(p->flags >> 8); - buf[9] = (Byte)(p->flags & 0xFF); - SetUi32(buf, CrcCalc(buf + 4, 6)); - memcpy(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE); - return WriteBytes(s, buf, 12); + ISzAlloc_Free(alloc, p->blocks); + p->blocks = NULL; } + p->numBlocks = 0; + p->size = 0; + p->allocated = 0; } -static SRes Xz_AddIndexRecord(CXzStream *p, UInt64 unpackSize, UInt64 totalSize, ISzAlloc *alloc) +static SRes XzEncIndex_ReAlloc(CXzEncIndex *p, size_t newSize, ISzAllocPtr alloc) +{ + Byte *blocks = (Byte *)ISzAlloc_Alloc(alloc, newSize); + if (!blocks) + return SZ_ERROR_MEM; + if (p->size != 0) + memcpy(blocks, p->blocks, p->size); + if (p->blocks) + ISzAlloc_Free(alloc, p->blocks); + p->blocks = blocks; + p->allocated = newSize; + return SZ_OK; +} + + +static SRes XzEncIndex_PreAlloc(CXzEncIndex *p, UInt64 numBlocks, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) { - if (!p->blocks || p->numBlocksAllocated == p->numBlocks) + UInt64 pos; + { + Byte buf[32]; + unsigned pos2 = Xz_WriteVarInt(buf, totalSize); + pos2 += Xz_WriteVarInt(buf + pos2, unpackSize); + pos = numBlocks * pos2; + } + + if (pos <= p->allocated - p->size) + return SZ_OK; { - size_t num = p->numBlocks * 2 + 1; - size_t newSize = sizeof(CXzBlockSizes) * num; - CXzBlockSizes *blocks; - if (newSize / sizeof(CXzBlockSizes) != num) - return SZ_ERROR_MEM; - blocks = (CXzBlockSizes *)alloc->Alloc(alloc, newSize); - if (!blocks) + UInt64 newSize64 = p->size + pos; + size_t newSize = (size_t)newSize64; + if (newSize != newSize64) return SZ_ERROR_MEM; - if (p->numBlocks != 0) - { - memcpy(blocks, p->blocks, p->numBlocks * sizeof(CXzBlockSizes)); - alloc->Free(alloc, p->blocks); - } - p->blocks = blocks; - p->numBlocksAllocated = num; + return XzEncIndex_ReAlloc(p, newSize, alloc); } +} + + +static SRes XzEncIndex_AddIndexRecord(CXzEncIndex *p, UInt64 unpackSize, UInt64 totalSize, ISzAllocPtr alloc) +{ + Byte buf[32]; + unsigned pos = Xz_WriteVarInt(buf, totalSize); + pos += Xz_WriteVarInt(buf + pos, unpackSize); + + if (pos > p->allocated - p->size) { - CXzBlockSizes *block = &p->blocks[p->numBlocks++]; - block->unpackSize = unpackSize; - block->totalSize = totalSize; + size_t newSize = p->allocated * 2 + 16 * 2; + if (newSize < p->size + pos) + return SZ_ERROR_MEM; + RINOK(XzEncIndex_ReAlloc(p, newSize, alloc)); } + memcpy(p->blocks + p->size, buf, pos); + p->size += pos; + p->numBlocks++; return SZ_OK; } +static SRes XzEncIndex_WriteFooter(const CXzEncIndex *p, CXzStreamFlags flags, ISeqOutStream *s) +{ + Byte buf[32]; + UInt64 globalPos; + UInt32 crc = CRC_INIT_VAL; + unsigned pos = 1 + Xz_WriteVarInt(buf + 1, p->numBlocks); + + globalPos = pos; + buf[0] = 0; + RINOK(WriteBytesUpdateCrc(s, buf, pos, &crc)); + RINOK(WriteBytesUpdateCrc(s, p->blocks, p->size, &crc)); + globalPos += p->size; + + pos = XZ_GET_PAD_SIZE(globalPos); + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + globalPos += pos; + + crc = CrcUpdate(crc, buf + 4 - pos, pos); + SetUi32(buf + 4, CRC_GET_DIGEST(crc)); + + SetUi32(buf + 8 + 4, (UInt32)(globalPos >> 2)); + buf[8 + 8] = (Byte)(flags >> 8); + buf[8 + 9] = (Byte)(flags & 0xFF); + SetUi32(buf + 8, CrcCalc(buf + 8 + 4, 6)); + buf[8 + 10] = XZ_FOOTER_SIG_0; + buf[8 + 11] = XZ_FOOTER_SIG_1; + + return WriteBytes(s, buf + 4 - pos, pos + 4 + 12); +} + + + /* ---------- CSeqCheckInStream ---------- */ typedef struct { - ISeqInStream p; + ISeqInStream vt; ISeqInStream *realStream; + const Byte *data; + UInt64 limit; UInt64 processed; + int realStreamFinished; CXzCheck check; } CSeqCheckInStream; -static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned mode) +static void SeqCheckInStream_Init(CSeqCheckInStream *p, unsigned checkMode) { + p->limit = (UInt64)(Int64)-1; p->processed = 0; - XzCheck_Init(&p->check, mode); + p->realStreamFinished = 0; + XzCheck_Init(&p->check, checkMode); } static void SeqCheckInStream_GetDigest(CSeqCheckInStream *p, Byte *digest)@@ -177,12 +251,31 @@ {
XzCheck_Final(&p->check, digest); } -static SRes SeqCheckInStream_Read(void *pp, void *data, size_t *size) +static SRes SeqCheckInStream_Read(const ISeqInStream *pp, void *data, size_t *size) { - CSeqCheckInStream *p = (CSeqCheckInStream *)pp; - SRes res = p->realStream->Read(p->realStream, data, size); - XzCheck_Update(&p->check, data, *size); - p->processed += *size; + CSeqCheckInStream *p = CONTAINER_FROM_VTBL(pp, CSeqCheckInStream, vt); + size_t size2 = *size; + SRes res = SZ_OK; + + if (p->limit != (UInt64)(Int64)-1) + { + UInt64 rem = p->limit - p->processed; + if (size2 > rem) + size2 = (size_t)rem; + } + if (size2 != 0) + { + if (p->realStream) + { + res = ISeqInStream_Read(p->realStream, data, &size2); + p->realStreamFinished = (size2 == 0) ? 1 : 0; + } + else + memcpy(data, p->data + (size_t)p->processed, size2); + XzCheck_Update(&p->check, data, size2); + p->processed += size2; + } + *size = size2; return res; }@@ -191,15 +284,24 @@ /* ---------- CSeqSizeOutStream ---------- */
typedef struct { - ISeqOutStream p; + ISeqOutStream vt; ISeqOutStream *realStream; + Byte *outBuf; + size_t outBufLimit; UInt64 processed; } CSeqSizeOutStream; -static size_t MyWrite(void *pp, const void *data, size_t size) +static size_t SeqSizeOutStream_Write(const ISeqOutStream *pp, const void *data, size_t size) { - CSeqSizeOutStream *p = (CSeqSizeOutStream *)pp; - size = p->realStream->Write(p->realStream, data, size); + CSeqSizeOutStream *p = CONTAINER_FROM_VTBL(pp, CSeqSizeOutStream, vt); + if (p->realStream) + size = ISeqOutStream_Write(p->realStream, data, size); + else + { + if (size > p->outBufLimit - (size_t)p->processed) + return 0; + memcpy(p->outBuf + (size_t)p->processed, data, size); + } p->processed += size; return size; }@@ -220,9 +322,29 @@ size_t endPos;
int srcWasFinished; } CSeqInFilter; -static SRes SeqInFilter_Read(void *pp, void *data, size_t *size) + +SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAllocPtr alloc); + +static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props, ISzAllocPtr alloc) { - CSeqInFilter *p = (CSeqInFilter *)pp; + if (!p->buf) + { + p->buf = (Byte *)ISzAlloc_Alloc(alloc, FILTER_BUF_SIZE); + if (!p->buf) + return SZ_ERROR_MEM; + } + p->curPos = p->endPos = 0; + p->srcWasFinished = 0; + RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, alloc)); + RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, alloc)); + p->StateCoder.Init(p->StateCoder.p); + return SZ_OK; +} + + +static SRes SeqInFilter_Read(const ISeqInStream *pp, void *data, size_t *size) +{ + CSeqInFilter *p = CONTAINER_FROM_VTBL(pp, CSeqInFilter, p); size_t sizeOriginal = *size; if (sizeOriginal == 0) return SZ_OK;@@ -234,19 +356,22 @@ if (!p->srcWasFinished && p->curPos == p->endPos)
{ p->curPos = 0; p->endPos = FILTER_BUF_SIZE; - RINOK(p->realStream->Read(p->realStream, p->buf, &p->endPos)); + RINOK(ISeqInStream_Read(p->realStream, p->buf, &p->endPos)); if (p->endPos == 0) p->srcWasFinished = 1; } { SizeT srcLen = p->endPos - p->curPos; - int wasFinished; + ECoderStatus status; SRes res; *size = sizeOriginal; - res = p->StateCoder.Code(p->StateCoder.p, data, size, p->buf + p->curPos, &srcLen, - p->srcWasFinished, CODER_FINISH_ANY, &wasFinished); + res = p->StateCoder.Code2(p->StateCoder.p, + (Byte *)data, size, + p->buf + p->curPos, &srcLen, + p->srcWasFinished, CODER_FINISH_ANY, + &status); p->curPos += srcLen; - if (*size != 0 || srcLen == 0 || res != 0) + if (*size != 0 || srcLen == 0 || res != SZ_OK) return res; } }@@ -255,34 +380,22 @@
static void SeqInFilter_Construct(CSeqInFilter *p) { p->buf = NULL; + p->StateCoder.p = NULL; p->p.Read = SeqInFilter_Read; } -static void SeqInFilter_Free(CSeqInFilter *p) +static void SeqInFilter_Free(CSeqInFilter *p, ISzAllocPtr alloc) { - if (p->buf) + if (p->StateCoder.p) { - g_Alloc.Free(&g_Alloc, p->buf); - p->buf = NULL; + p->StateCoder.Free(p->StateCoder.p, alloc); + p->StateCoder.p = NULL; } -} - -SRes BraState_SetFromMethod(IStateCoder *p, UInt64 id, int encodeMode, ISzAlloc *alloc); - -static SRes SeqInFilter_Init(CSeqInFilter *p, const CXzFilter *props) -{ - if (!p->buf) + if (p->buf) { - p->buf = g_Alloc.Alloc(&g_Alloc, FILTER_BUF_SIZE); - if (!p->buf) - return SZ_ERROR_MEM; + ISzAlloc_Free(alloc, p->buf); + p->buf = NULL; } - p->curPos = p->endPos = 0; - p->srcWasFinished = 0; - RINOK(BraState_SetFromMethod(&p->StateCoder, props->id, 1, &g_Alloc)); - RINOK(p->StateCoder.SetProps(p->StateCoder.p, props->props, props->propsSize, &g_Alloc)); - p->StateCoder.Init(p->StateCoder.p); - return SZ_OK; }@@ -292,17 +405,17 @@ #ifdef USE_SUBBLOCK
typedef struct { - ISeqInStream p; + ISeqInStream vt; ISeqInStream *inStream; CSbEnc enc; } CSbEncInStream; -static SRes SbEncInStream_Read(void *pp, void *data, size_t *size) +static SRes SbEncInStream_Read(const ISeqInStream *pp, void *data, size_t *size) { - CSbEncInStream *p = (CSbEncInStream *)pp; + CSbEncInStream *p = CONTAINER_FROM_VTBL(pp, CSbEncInStream, vt); size_t sizeOriginal = *size; if (sizeOriginal == 0) - return S_OK; + return SZ_OK; for (;;) {@@ -322,14 +435,14 @@
*size = sizeOriginal; RINOK(SbEnc_Read(&p->enc, data, size)); if (*size != 0 || !p->enc.needRead) - return S_OK; + return SZ_OK; } } -void SbEncInStream_Construct(CSbEncInStream *p, ISzAlloc *alloc) +void SbEncInStream_Construct(CSbEncInStream *p, ISzAllocPtr alloc) { SbEnc_Construct(&p->enc, alloc); - p->p.Read = SbEncInStream_Read; + p->vt.Read = SbEncInStream_Read; } SRes SbEncInStream_Init(CSbEncInStream *p)@@ -345,43 +458,233 @@
#endif + +/* ---------- CXzProps ---------- */ + + +void XzFilterProps_Init(CXzFilterProps *p) +{ + p->id = 0; + p->delta = 0; + p->ip = 0; + p->ipDefined = False; +} + +void XzProps_Init(CXzProps *p) +{ + p->checkId = XZ_CHECK_CRC32; + p->blockSize = XZ_PROPS__BLOCK_SIZE__AUTO; + p->numBlockThreads_Reduced = -1; + p->numBlockThreads_Max = -1; + p->numTotalThreads = -1; + p->reduceSize = (UInt64)(Int64)-1; + p->forceWriteSizesInHeader = 0; + // p->forceWriteSizesInHeader = 1; + + XzFilterProps_Init(&p->filterProps); + Lzma2EncProps_Init(&p->lzma2Props); +} + + +static void XzEncProps_Normalize_Fixed(CXzProps *p) +{ + UInt64 fileSize; + int t1, t1n, t2, t2r, t3; + { + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + Lzma2EncProps_Normalize(&tp); + t1n = tp.numTotalThreads; + } + + t1 = p->lzma2Props.numTotalThreads; + t2 = p->numBlockThreads_Max; + t3 = p->numTotalThreads; + + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + + if (t3 <= 0) + { + if (t2 <= 0) + t2 = 1; + t3 = t1n * t2; + } + else if (t2 <= 0) + { + t2 = t3 / t1n; + if (t2 == 0) + { + t1 = 1; + t2 = t3; + } + if (t2 > MTCODER__THREADS_MAX) + t2 = MTCODER__THREADS_MAX; + } + else if (t1 <= 0) + { + t1 = t3 / t2; + if (t1 == 0) + t1 = 1; + } + else + t3 = t1n * t2; + + p->lzma2Props.numTotalThreads = t1; + + t2r = t2; + + fileSize = p->reduceSize; + + if ((p->blockSize < fileSize || fileSize == (UInt64)(Int64)-1)) + p->lzma2Props.lzmaProps.reduceSize = p->blockSize; + + Lzma2EncProps_Normalize(&p->lzma2Props); + + t1 = p->lzma2Props.numTotalThreads; + + { + if (t2 > 1 && fileSize != (UInt64)(Int64)-1) + { + UInt64 numBlocks = fileSize / p->blockSize; + if (numBlocks * p->blockSize != fileSize) + numBlocks++; + if (numBlocks < (unsigned)t2) + { + t2r = (unsigned)numBlocks; + if (t2r == 0) + t2r = 1; + t3 = t1 * t2r; + } + } + } + + p->numBlockThreads_Max = t2; + p->numBlockThreads_Reduced = t2r; + p->numTotalThreads = t3; +} + + +static void XzProps_Normalize(CXzProps *p) +{ + /* we normalize xzProps properties, but we normalize only some of CXzProps::lzma2Props properties. + Lzma2Enc_SetProps() will normalize lzma2Props later. */ + + if (p->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID) + { + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + return; + } + else + { + CLzma2EncProps *lzma2 = &p->lzma2Props; + if (p->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + { + // xz-auto + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + { + // if (xz-auto && lzma2-solid) - we use solid for both + p->blockSize = XZ_PROPS__BLOCK_SIZE__SOLID; + p->numBlockThreads_Reduced = 1; + p->numBlockThreads_Max = 1; + if (p->lzma2Props.numTotalThreads <= 0) + p->lzma2Props.numTotalThreads = p->numTotalThreads; + } + else + { + // if (xz-auto && (lzma2-auto || lzma2-fixed_) + // we calculate block size for lzma2 and use that block size for xz, lzma2 uses single-chunk per block + CLzma2EncProps tp = p->lzma2Props; + if (tp.numTotalThreads <= 0) + tp.numTotalThreads = p->numTotalThreads; + + Lzma2EncProps_Normalize(&tp); + + p->blockSize = tp.blockSize; // fixed or solid + p->numBlockThreads_Reduced = tp.numBlockThreads_Reduced; + p->numBlockThreads_Max = tp.numBlockThreads_Max; + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = tp.blockSize; // fixed or solid, LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + if (lzma2->lzmaProps.reduceSize > tp.blockSize && tp.blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->lzmaProps.reduceSize = tp.blockSize; + lzma2->numBlockThreads_Reduced = 1; + lzma2->numBlockThreads_Max = 1; + return; + } + } + else + { + // xz-fixed + // we can use xz::reduceSize or xz::blockSize as base for lzmaProps::reduceSize + + p->lzma2Props.lzmaProps.reduceSize = p->reduceSize; + { + UInt64 r = p->reduceSize; + if (r > p->blockSize || r == (UInt64)(Int64)-1) + r = p->blockSize; + lzma2->lzmaProps.reduceSize = r; + } + if (lzma2->blockSize == LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO) + lzma2->blockSize = LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID; + else if (lzma2->blockSize > p->blockSize && lzma2->blockSize != LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID) + lzma2->blockSize = p->blockSize; + + XzEncProps_Normalize_Fixed(p); + } + } +} + + +/* ---------- CLzma2WithFilters ---------- */ + typedef struct { CLzma2EncHandle lzma2; + CSeqInFilter filter; + #ifdef USE_SUBBLOCK CSbEncInStream sb; #endif - CSeqInFilter filter; - ISzAlloc *alloc; - ISzAlloc *bigAlloc; } CLzma2WithFilters; -static void Lzma2WithFilters_Construct(CLzma2WithFilters *p, ISzAlloc *alloc, ISzAlloc *bigAlloc) +static void Lzma2WithFilters_Construct(CLzma2WithFilters *p) { - p->alloc = alloc; - p->bigAlloc = bigAlloc; p->lzma2 = NULL; + SeqInFilter_Construct(&p->filter); + #ifdef USE_SUBBLOCK SbEncInStream_Construct(&p->sb, alloc); #endif - SeqInFilter_Construct(&p->filter); } -static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p) + +static SRes Lzma2WithFilters_Create(CLzma2WithFilters *p, ISzAllocPtr alloc, ISzAllocPtr bigAlloc) { - p->lzma2 = Lzma2Enc_Create(p->alloc, p->bigAlloc); if (!p->lzma2) - return SZ_ERROR_MEM; + { + p->lzma2 = Lzma2Enc_Create(alloc, bigAlloc); + if (!p->lzma2) + return SZ_ERROR_MEM; + } return SZ_OK; } -static void Lzma2WithFilters_Free(CLzma2WithFilters *p) + +static void Lzma2WithFilters_Free(CLzma2WithFilters *p, ISzAllocPtr alloc) { - SeqInFilter_Free(&p->filter); #ifdef USE_SUBBLOCK SbEncInStream_Free(&p->sb); #endif + + SeqInFilter_Free(&p->filter, alloc); if (p->lzma2) { Lzma2Enc_Destroy(p->lzma2);@@ -390,137 +693,625 @@ }
} -void XzProps_Init(CXzProps *p) +typedef struct +{ + UInt64 unpackSize; + UInt64 totalSize; + size_t headerSize; +} CXzEncBlockInfo; + + +static SRes Xz_CompressBlock( + CLzma2WithFilters *lzmaf, + + ISeqOutStream *outStream, + Byte *outBufHeader, + Byte *outBufData, size_t outBufDataLimit, + + ISeqInStream *inStream, + // UInt64 expectedSize, + const Byte *inBuf, // used if (!inStream) + size_t inBufSize, // used if (!inStream), it's block size, props->blockSize is ignored + + const CXzProps *props, + ICompressProgress *progress, + int *inStreamFinished, /* only for inStream version */ + CXzEncBlockInfo *blockSizes, + ISzAllocPtr alloc, + ISzAllocPtr allocBig) +{ + CSeqCheckInStream checkInStream; + CSeqSizeOutStream seqSizeOutStream; + CXzBlock block; + unsigned filterIndex = 0; + CXzFilter *filter = NULL; + const CXzFilterProps *fp = &props->filterProps; + if (fp->id == 0) + fp = NULL; + + *inStreamFinished = False; + + RINOK(Lzma2WithFilters_Create(lzmaf, alloc, allocBig)); + + RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, &props->lzma2Props)); + + XzBlock_ClearFlags(&block); + XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + + if (fp) + { + filter = &block.filters[filterIndex++]; + filter->id = fp->id; + filter->propsSize = 0; + + if (fp->id == XZ_ID_Delta) + { + filter->props[0] = (Byte)(fp->delta - 1); + filter->propsSize = 1; + } + else if (fp->ipDefined) + { + SetUi32(filter->props, fp->ip); + filter->propsSize = 4; + } + } + + { + CXzFilter *f = &block.filters[filterIndex++]; + f->id = XZ_ID_LZMA2; + f->propsSize = 1; + f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + } + + seqSizeOutStream.vt.Write = SeqSizeOutStream_Write; + seqSizeOutStream.realStream = outStream; + seqSizeOutStream.outBuf = outBufData; + seqSizeOutStream.outBufLimit = outBufDataLimit; + seqSizeOutStream.processed = 0; + + /* + if (expectedSize != (UInt64)(Int64)-1) + { + block.unpackSize = expectedSize; + if (props->blockSize != (UInt64)(Int64)-1) + if (expectedSize > props->blockSize) + block.unpackSize = props->blockSize; + XzBlock_SetHasUnpackSize(&block); + } + */ + + if (outStream) + { + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + } + + checkInStream.vt.Read = SeqCheckInStream_Read; + SeqCheckInStream_Init(&checkInStream, props->checkId); + + checkInStream.realStream = inStream; + checkInStream.data = inBuf; + checkInStream.limit = props->blockSize; + if (!inStream) + checkInStream.limit = inBufSize; + + if (fp) + { + #ifdef USE_SUBBLOCK + if (fp->id == XZ_ID_Subblock) + { + lzmaf->sb.inStream = &checkInStream.vt; + RINOK(SbEncInStream_Init(&lzmaf->sb)); + } + else + #endif + { + lzmaf->filter.realStream = &checkInStream.vt; + RINOK(SeqInFilter_Init(&lzmaf->filter, filter, alloc)); + } + } + + { + SRes res; + Byte *outBuf = NULL; + size_t outSize = 0; + BoolInt useStream = (fp || inStream); + // useStream = True; + + if (!useStream) + { + XzCheck_Update(&checkInStream.check, inBuf, inBufSize); + checkInStream.processed = inBufSize; + } + + if (!outStream) + { + outBuf = seqSizeOutStream.outBuf; // + (size_t)seqSizeOutStream.processed; + outSize = seqSizeOutStream.outBufLimit; // - (size_t)seqSizeOutStream.processed; + } + + res = Lzma2Enc_Encode2(lzmaf->lzma2, + outBuf ? NULL : &seqSizeOutStream.vt, + outBuf, + outBuf ? &outSize : NULL, + + useStream ? + (fp ? + ( + #ifdef USE_SUBBLOCK + (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.vt: + #endif + &lzmaf->filter.p) : + &checkInStream.vt) : NULL, + + useStream ? NULL : inBuf, + useStream ? 0 : inBufSize, + + progress); + + if (outBuf) + seqSizeOutStream.processed += outSize; + + RINOK(res); + blockSizes->unpackSize = checkInStream.processed; + } + { + Byte buf[4 + 64]; + unsigned padSize = XZ_GET_PAD_SIZE(seqSizeOutStream.processed); + UInt64 packSize = seqSizeOutStream.processed; + + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + + SeqCheckInStream_GetDigest(&checkInStream, buf + 4); + RINOK(WriteBytes(&seqSizeOutStream.vt, buf + (4 - padSize), padSize + XzFlags_GetCheckSize((CXzStreamFlags)props->checkId))); + + blockSizes->totalSize = seqSizeOutStream.processed - padSize; + + if (!outStream) + { + seqSizeOutStream.outBuf = outBufHeader; + seqSizeOutStream.outBufLimit = XZ_BLOCK_HEADER_SIZE_MAX; + seqSizeOutStream.processed = 0; + + block.unpackSize = blockSizes->unpackSize; + XzBlock_SetHasUnpackSize(&block); + + block.packSize = packSize; + XzBlock_SetHasPackSize(&block); + + RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.vt)); + + blockSizes->headerSize = (size_t)seqSizeOutStream.processed; + blockSizes->totalSize += seqSizeOutStream.processed; + } + } + + if (inStream) + *inStreamFinished = checkInStream.realStreamFinished; + else + { + *inStreamFinished = False; + if (checkInStream.processed != inBufSize) + return SZ_ERROR_FAIL; + } + + return SZ_OK; +} + + + +typedef struct +{ + ICompressProgress vt; + ICompressProgress *progress; + UInt64 inOffset; + UInt64 outOffset; +} CCompressProgress_XzEncOffset; + + +static SRes CompressProgress_XzEncOffset_Progress(const ICompressProgress *pp, UInt64 inSize, UInt64 outSize) +{ + const CCompressProgress_XzEncOffset *p = CONTAINER_FROM_VTBL(pp, CCompressProgress_XzEncOffset, vt); + inSize += p->inOffset; + outSize += p->outOffset; + return ICompressProgress_Progress(p->progress, inSize, outSize); +} + + + + +typedef struct +{ + ISzAllocPtr alloc; + ISzAllocPtr allocBig; + + CXzProps xzProps; + UInt64 expectedDataSize; + + CXzEncIndex xzIndex; + + CLzma2WithFilters lzmaf_Items[MTCODER__THREADS_MAX]; + + size_t outBufSize; /* size of allocated outBufs[i] */ + Byte *outBufs[MTCODER__BLOCKS_MAX]; + + #ifndef _7ZIP_ST + unsigned checkType; + ISeqOutStream *outStream; + BoolInt mtCoder_WasConstructed; + CMtCoder mtCoder; + CXzEncBlockInfo EncBlocks[MTCODER__BLOCKS_MAX]; + #endif + +} CXzEnc; + + +static void XzEnc_Construct(CXzEnc *p) +{ + unsigned i; + + XzEncIndex_Construct(&p->xzIndex); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Construct(&p->lzmaf_Items[i]); + + #ifndef _7ZIP_ST + p->mtCoder_WasConstructed = False; + { + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + p->outBufs[i] = NULL; + p->outBufSize = 0; + } + #endif +} + + +static void XzEnc_FreeOutBufs(CXzEnc *p) +{ + unsigned i; + for (i = 0; i < MTCODER__BLOCKS_MAX; i++) + if (p->outBufs[i]) + { + ISzAlloc_Free(p->alloc, p->outBufs[i]); + p->outBufs[i] = NULL; + } + p->outBufSize = 0; +} + + +static void XzEnc_Free(CXzEnc *p, ISzAllocPtr alloc) +{ + unsigned i; + + XzEncIndex_Free(&p->xzIndex, alloc); + + for (i = 0; i < MTCODER__THREADS_MAX; i++) + Lzma2WithFilters_Free(&p->lzmaf_Items[i], alloc); + + #ifndef _7ZIP_ST + if (p->mtCoder_WasConstructed) + { + MtCoder_Destruct(&p->mtCoder); + p->mtCoder_WasConstructed = False; + } + XzEnc_FreeOutBufs(p); + #endif +} + + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig) { - p->lzma2Props = NULL; - p->filterProps = NULL; - p->checkId = XZ_CHECK_CRC32; + CXzEnc *p = (CXzEnc *)ISzAlloc_Alloc(alloc, sizeof(CXzEnc)); + if (!p) + return NULL; + XzEnc_Construct(p); + XzProps_Init(&p->xzProps); + XzProps_Normalize(&p->xzProps); + p->expectedDataSize = (UInt64)(Int64)-1; + p->alloc = alloc; + p->allocBig = allocBig; + return p; +} + + +void XzEnc_Destroy(CXzEncHandle pp) +{ + CXzEnc *p = (CXzEnc *)pp; + XzEnc_Free(p, p->alloc); + ISzAlloc_Free(p->alloc, p); +} + + +SRes XzEnc_SetProps(CXzEncHandle pp, const CXzProps *props) +{ + CXzEnc *p = (CXzEnc *)pp; + p->xzProps = *props; + XzProps_Normalize(&p->xzProps); + return SZ_OK; +} + + +void XzEnc_SetDataSize(CXzEncHandle pp, UInt64 expectedDataSiize) +{ + CXzEnc *p = (CXzEnc *)pp; + p->expectedDataSize = expectedDataSiize; +} + + + + +#ifndef _7ZIP_ST + +static SRes XzEnc_MtCallback_Code(void *pp, unsigned coderIndex, unsigned outBufIndex, + const Byte *src, size_t srcSize, int finished) +{ + CXzEnc *me = (CXzEnc *)pp; + SRes res; + CMtProgressThunk progressThunk; + + Byte *dest = me->outBufs[outBufIndex]; + + UNUSED_VAR(finished) + + { + CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + bInfo->totalSize = 0; + bInfo->unpackSize = 0; + bInfo->headerSize = 0; + } + + if (!dest) + { + dest = (Byte *)ISzAlloc_Alloc(me->alloc, me->outBufSize); + if (!dest) + return SZ_ERROR_MEM; + me->outBufs[outBufIndex] = dest; + } + + MtProgressThunk_CreateVTable(&progressThunk); + progressThunk.mtProgress = &me->mtCoder.mtProgress; + MtProgressThunk_Init(&progressThunk); + + { + CXzEncBlockInfo blockSizes; + int inStreamFinished; + + res = Xz_CompressBlock( + &me->lzmaf_Items[coderIndex], + + NULL, + dest, + dest + XZ_BLOCK_HEADER_SIZE_MAX, me->outBufSize - XZ_BLOCK_HEADER_SIZE_MAX, + + NULL, + // srcSize, // expectedSize + src, srcSize, + + &me->xzProps, + &progressThunk.vt, + &inStreamFinished, + &blockSizes, + me->alloc, + me->allocBig); + + if (res == SZ_OK) + me->EncBlocks[outBufIndex] = blockSizes; + + return res; + } } -void XzFilterProps_Init(CXzFilterProps *p) + +static SRes XzEnc_MtCallback_Write(void *pp, unsigned outBufIndex) { - p->id = 0; - p->delta = 0; - p->ip = 0; - p->ipDefined = False; + CXzEnc *me = (CXzEnc *)pp; + + const CXzEncBlockInfo *bInfo = &me->EncBlocks[outBufIndex]; + const Byte *data = me->outBufs[outBufIndex]; + + RINOK(WriteBytes(me->outStream, data, bInfo->headerSize)); + + { + UInt64 totalPackFull = bInfo->totalSize + XZ_GET_PAD_SIZE(bInfo->totalSize); + RINOK(WriteBytes(me->outStream, data + XZ_BLOCK_HEADER_SIZE_MAX, (size_t)totalPackFull - bInfo->headerSize)); + } + + return XzEncIndex_AddIndexRecord(&me->xzIndex, bInfo->unpackSize, bInfo->totalSize, me->alloc); } +#endif -static SRes Xz_Compress(CXzStream *xz, CLzma2WithFilters *lzmaf, - ISeqOutStream *outStream, ISeqInStream *inStream, - const CXzProps *props, ICompressProgress *progress) + + +SRes XzEnc_Encode(CXzEncHandle pp, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress) { - xz->flags = (Byte)props->checkId; + CXzEnc *p = (CXzEnc *)pp; - RINOK(Lzma2Enc_SetProps(lzmaf->lzma2, props->lzma2Props)); - RINOK(Xz_WriteHeader(xz->flags, outStream)); + const CXzProps *props = &p->xzProps; + XzEncIndex_Init(&p->xzIndex); { - CSeqCheckInStream checkInStream; - CSeqSizeOutStream seqSizeOutStream; - CXzBlock block; - unsigned filterIndex = 0; - CXzFilter *filter = NULL; - const CXzFilterProps *fp = props->filterProps; + UInt64 numBlocks = 1; + UInt64 blockSize = props->blockSize; - XzBlock_ClearFlags(&block); - XzBlock_SetNumFilters(&block, 1 + (fp ? 1 : 0)); + if (blockSize != XZ_PROPS__BLOCK_SIZE__SOLID + && props->reduceSize != (UInt64)(Int64)-1) + { + numBlocks = props->reduceSize / blockSize; + if (numBlocks * blockSize != props->reduceSize) + numBlocks++; + } + else + blockSize = (UInt64)1 << 62; - if (fp) + RINOK(XzEncIndex_PreAlloc(&p->xzIndex, numBlocks, blockSize, XZ_GET_ESTIMATED_BLOCK_TOTAL_PACK_SIZE(blockSize), p->alloc)); + } + + RINOK(Xz_WriteHeader((CXzStreamFlags)props->checkId, outStream)); + + + #ifndef _7ZIP_ST + if (props->numBlockThreads_Reduced > 1) + { + IMtCoderCallback2 vt; + + if (!p->mtCoder_WasConstructed) { - filter = &block.filters[filterIndex++]; - filter->id = fp->id; - filter->propsSize = 0; - - if (fp->id == XZ_ID_Delta) - { - filter->props[0] = (Byte)(fp->delta - 1); - filter->propsSize = 1; - } - else if (fp->ipDefined) - { - SetUi32(filter->props, fp->ip); - filter->propsSize = 4; - } + p->mtCoder_WasConstructed = True; + MtCoder_Construct(&p->mtCoder); } + vt.Code = XzEnc_MtCallback_Code; + vt.Write = XzEnc_MtCallback_Write; + + p->checkType = props->checkId; + p->xzProps = *props; + + p->outStream = outStream; + + p->mtCoder.allocBig = p->allocBig; + p->mtCoder.progress = progress; + p->mtCoder.inStream = inStream; + p->mtCoder.inData = NULL; + p->mtCoder.inDataSize = 0; + p->mtCoder.mtCallback = &vt; + p->mtCoder.mtCallbackObject = p; + + if ( props->blockSize == XZ_PROPS__BLOCK_SIZE__SOLID + || props->blockSize == XZ_PROPS__BLOCK_SIZE__AUTO) + return SZ_ERROR_FAIL; + + p->mtCoder.blockSize = (size_t)props->blockSize; + if (p->mtCoder.blockSize != props->blockSize) + return SZ_ERROR_PARAM; /* SZ_ERROR_MEM */ + { - CXzFilter *f = &block.filters[filterIndex++]; - f->id = XZ_ID_LZMA2; - f->propsSize = 1; - f->props[0] = Lzma2Enc_WriteProperties(lzmaf->lzma2); + size_t destBlockSize = XZ_BLOCK_HEADER_SIZE_MAX + XZ_GET_MAX_BLOCK_PACK_SIZE(p->mtCoder.blockSize); + if (destBlockSize < p->mtCoder.blockSize) + return SZ_ERROR_PARAM; + if (p->outBufSize != destBlockSize) + XzEnc_FreeOutBufs(p); + p->outBufSize = destBlockSize; } - seqSizeOutStream.p.Write = MyWrite; - seqSizeOutStream.realStream = outStream; - seqSizeOutStream.processed = 0; + p->mtCoder.numThreadsMax = props->numBlockThreads_Max; + p->mtCoder.expectedDataSize = p->expectedDataSize; - RINOK(XzBlock_WriteHeader(&block, &seqSizeOutStream.p)); + RINOK(MtCoder_Code(&p->mtCoder)); + } + else + #endif + { + int writeStartSizes; + CCompressProgress_XzEncOffset progress2; + Byte *bufData = NULL; + size_t bufSize = 0; + + progress2.vt.Progress = CompressProgress_XzEncOffset_Progress; + progress2.inOffset = 0; + progress2.outOffset = 0; + progress2.progress = progress; - checkInStream.p.Read = SeqCheckInStream_Read; - checkInStream.realStream = inStream; - SeqCheckInStream_Init(&checkInStream, XzFlags_GetCheckType(xz->flags)); + writeStartSizes = 0; - if (fp) + if (props->blockSize != XZ_PROPS__BLOCK_SIZE__SOLID) { - #ifdef USE_SUBBLOCK - if (fp->id == XZ_ID_Subblock) + writeStartSizes = (props->forceWriteSizesInHeader > 0); + + if (writeStartSizes) { - lzmaf->sb.inStream = &checkInStream.p; - RINOK(SbEncInStream_Init(&lzmaf->sb)); - } - else - #endif - { - lzmaf->filter.realStream = &checkInStream.p; - RINOK(SeqInFilter_Init(&lzmaf->filter, filter)); + size_t t2; + size_t t = (size_t)props->blockSize; + if (t != props->blockSize) + return SZ_ERROR_PARAM; + t = XZ_GET_MAX_BLOCK_PACK_SIZE(t); + if (t < props->blockSize) + return SZ_ERROR_PARAM; + t2 = XZ_BLOCK_HEADER_SIZE_MAX + t; + if (!p->outBufs[0] || t2 != p->outBufSize) + { + XzEnc_FreeOutBufs(p); + p->outBufs[0] = (Byte *)ISzAlloc_Alloc(p->alloc, t2); + if (!p->outBufs[0]) + return SZ_ERROR_MEM; + p->outBufSize = t2; + } + bufData = p->outBufs[0] + XZ_BLOCK_HEADER_SIZE_MAX; + bufSize = t; } } - + + for (;;) { - UInt64 packPos = seqSizeOutStream.processed; + CXzEncBlockInfo blockSizes; + int inStreamFinished; - SRes res = Lzma2Enc_Encode(lzmaf->lzma2, &seqSizeOutStream.p, - fp ? - #ifdef USE_SUBBLOCK - (fp->id == XZ_ID_Subblock) ? &lzmaf->sb.p: - #endif - &lzmaf->filter.p: - &checkInStream.p, - progress); + /* + UInt64 rem = (UInt64)(Int64)-1; + if (props->reduceSize != (UInt64)(Int64)-1 + && props->reduceSize >= progress2.inOffset) + rem = props->reduceSize - progress2.inOffset; + */ + + blockSizes.headerSize = 0; // for GCC - RINOK(res); - block.unpackSize = checkInStream.processed; - block.packSize = seqSizeOutStream.processed - packPos; - } + RINOK(Xz_CompressBlock( + &p->lzmaf_Items[0], + + writeStartSizes ? NULL : outStream, + writeStartSizes ? p->outBufs[0] : NULL, + bufData, bufSize, + + inStream, + // rem, + NULL, 0, + + props, + progress ? &progress2.vt : NULL, + &inStreamFinished, + &blockSizes, + p->alloc, + p->allocBig)); - { - unsigned padSize = 0; - Byte buf[128]; - while ((((unsigned)block.packSize + padSize) & 3) != 0) - buf[padSize++] = 0; - SeqCheckInStream_GetDigest(&checkInStream, buf + padSize); - RINOK(WriteBytes(&seqSizeOutStream.p, buf, padSize + XzFlags_GetCheckSize(xz->flags))); - RINOK(Xz_AddIndexRecord(xz, block.unpackSize, seqSizeOutStream.processed - padSize, &g_Alloc)); + { + UInt64 totalPackFull = blockSizes.totalSize + XZ_GET_PAD_SIZE(blockSizes.totalSize); + + if (writeStartSizes) + { + RINOK(WriteBytes(outStream, p->outBufs[0], blockSizes.headerSize)); + RINOK(WriteBytes(outStream, bufData, (size_t)totalPackFull - blockSizes.headerSize)); + } + + RINOK(XzEncIndex_AddIndexRecord(&p->xzIndex, blockSizes.unpackSize, blockSizes.totalSize, p->alloc)); + + progress2.inOffset += blockSizes.unpackSize; + progress2.outOffset += totalPackFull; + } + + if (inStreamFinished) + break; } } - return Xz_WriteFooter(xz, outStream); + + return XzEncIndex_WriteFooter(&p->xzIndex, (CXzStreamFlags)props->checkId, outStream); } + +#include "Alloc.h" SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress) { SRes res; - CXzStream xz; - CLzma2WithFilters lzmaf; - Xz_Construct(&xz); - Lzma2WithFilters_Construct(&lzmaf, &g_Alloc, &g_BigAlloc); - res = Lzma2WithFilters_Create(&lzmaf); + CXzEncHandle xz = XzEnc_Create(&g_Alloc, &g_BigAlloc); + if (!xz) + return SZ_ERROR_MEM; + res = XzEnc_SetProps(xz, props); if (res == SZ_OK) - res = Xz_Compress(&xz, &lzmaf, outStream, inStream, props, progress); - Lzma2WithFilters_Free(&lzmaf); - Xz_Free(&xz, &g_Alloc); + res = XzEnc_Encode(xz, outStream, inStream, progress); + XzEnc_Destroy(xz); return res; }@@ -528,11 +1319,11 @@
SRes Xz_EncodeEmpty(ISeqOutStream *outStream) { SRes res; - CXzStream xz; - Xz_Construct(&xz); - res = Xz_WriteHeader(xz.flags, outStream); + CXzEncIndex xzIndex; + XzEncIndex_Construct(&xzIndex); + res = Xz_WriteHeader((CXzStreamFlags)0, outStream); if (res == SZ_OK) - res = Xz_WriteFooter(&xz, outStream); - Xz_Free(&xz, &g_Alloc); + res = XzEncIndex_WriteFooter(&xzIndex, (CXzStreamFlags)0, outStream); + XzEncIndex_Free(&xzIndex, NULL); // g_Alloc return res; }
M
src/third-party/lzma/XzEnc.h
→
src/third-party/lzma/XzEnc.h
@@ -1,5 +1,5 @@
/* XzEnc.h -- Xz Encode -2011-02-07 : Igor Pavlov : Public domain */ +2017-06-27 : Igor Pavlov : Public domain */ #ifndef __XZ_ENC_H #define __XZ_ENC_H@@ -9,6 +9,11 @@
#include "Xz.h" EXTERN_C_BEGIN + + +#define XZ_PROPS__BLOCK_SIZE__AUTO LZMA2_ENC_PROPS__BLOCK_SIZE__AUTO +#define XZ_PROPS__BLOCK_SIZE__SOLID LZMA2_ENC_PROPS__BLOCK_SIZE__SOLID + typedef struct {@@ -20,14 +25,30 @@ } CXzFilterProps;
void XzFilterProps_Init(CXzFilterProps *p); + typedef struct { - const CLzma2EncProps *lzma2Props; - const CXzFilterProps *filterProps; + CLzma2EncProps lzma2Props; + CXzFilterProps filterProps; unsigned checkId; + UInt64 blockSize; + int numBlockThreads_Reduced; + int numBlockThreads_Max; + int numTotalThreads; + int forceWriteSizesInHeader; + UInt64 reduceSize; } CXzProps; void XzProps_Init(CXzProps *p); + + +typedef void * CXzEncHandle; + +CXzEncHandle XzEnc_Create(ISzAllocPtr alloc, ISzAllocPtr allocBig); +void XzEnc_Destroy(CXzEncHandle p); +SRes XzEnc_SetProps(CXzEncHandle p, const CXzProps *props); +void XzEnc_SetDataSize(CXzEncHandle p, UInt64 expectedDataSiize); +SRes XzEnc_Encode(CXzEncHandle p, ISeqOutStream *outStream, ISeqInStream *inStream, ICompressProgress *progress); SRes Xz_Encode(ISeqOutStream *outStream, ISeqInStream *inStream, const CXzProps *props, ICompressProgress *progress);
M
src/third-party/lzma/XzIn.c
→
src/third-party/lzma/XzIn.c
@@ -1,5 +1,5 @@
/* XzIn.c - Xz input -2015-11-08 : Igor Pavlov : Public domain */ +2018-07-04 : Igor Pavlov : Public domain */ #include "Precomp.h"@@ -8,6 +8,12 @@
#include "7zCrc.h" #include "CpuArch.h" #include "Xz.h" + +/* +#define XZ_FOOTER_SIG_CHECK(p) (memcmp((p), XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) == 0) +*/ +#define XZ_FOOTER_SIG_CHECK(p) ((p)[0] == XZ_FOOTER_SIG_0 && (p)[1] == XZ_FOOTER_SIG_1) + SRes Xz_ReadHeader(CXzStreamFlags *p, ISeqInStream *inStream) {@@ -22,13 +28,13 @@ #define READ_VARINT_AND_CHECK(buf, pos, size, res) \
{ unsigned s = Xz_ReadVarInt(buf + pos, size - pos, res); \ if (s == 0) return SZ_ERROR_ARCHIVE; pos += s; } -SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, Bool *isIndex, UInt32 *headerSizeRes) +SRes XzBlock_ReadHeader(CXzBlock *p, ISeqInStream *inStream, BoolInt *isIndex, UInt32 *headerSizeRes) { Byte header[XZ_BLOCK_HEADER_SIZE_MAX]; unsigned headerSize; *headerSizeRes = 0; RINOK(SeqInStream_ReadByte(inStream, &header[0])); - headerSize = ((unsigned)header[0] << 2) + 4; + headerSize = (unsigned)header[0]; if (headerSize == 0) { *headerSizeRes = 1;@@ -37,12 +43,13 @@ return SZ_OK;
} *isIndex = False; + headerSize = (headerSize << 2) + 4; *headerSizeRes = headerSize; RINOK(SeqInStream_Read(inStream, header + 1, headerSize - 1)); return XzBlock_Parse(p, header); } -#define ADD_SIZE_CHECH(size, val) \ +#define ADD_SIZE_CHECK(size, val) \ { UInt64 newSize = size + (val); if (newSize < size) return XZ_SIZE_OVERFLOW; size = newSize; } UInt64 Xz_GetUnpackSize(const CXzStream *p)@@ -50,7 +57,7 @@ {
UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, p->blocks[i].unpackSize); + ADD_SIZE_CHECK(size, p->blocks[i].unpackSize); return size; }@@ -59,7 +66,7 @@ {
UInt64 size = 0; size_t i; for (i = 0; i < p->numBlocks; i++) - ADD_SIZE_CHECH(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); + ADD_SIZE_CHECK(size, (p->blocks[i].totalSize + 3) & ~(UInt64)3); return size; }@@ -70,7 +77,7 @@ return SeqInStream_Read(inStream, p->check, XzFlags_GetCheckSize(f));
} */ -static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAlloc *alloc) +static SRes Xz_ReadIndex2(CXzStream *p, const Byte *buf, size_t size, ISzAllocPtr alloc) { size_t numBlocks, pos = 1; UInt32 crc;@@ -96,9 +103,8 @@ if (numBlocks != 0)
{ size_t i; p->numBlocks = numBlocks; - p->numBlocksAllocated = numBlocks; - p->blocks = alloc->Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); - if (p->blocks == 0) + p->blocks = (CXzBlockSizes *)ISzAlloc_Alloc(alloc, sizeof(CXzBlockSizes) * numBlocks); + if (!p->blocks) return SZ_ERROR_MEM; for (i = 0; i < numBlocks; i++) {@@ -115,7 +121,7 @@ return SZ_ERROR_ARCHIVE;
return (pos == size) ? SZ_OK : SZ_ERROR_ARCHIVE; } -static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAlloc *alloc) +static SRes Xz_ReadIndex(CXzStream *p, ILookInStream *stream, UInt64 indexSize, ISzAllocPtr alloc) { SRes res; size_t size;@@ -125,13 +131,13 @@ return SZ_ERROR_UNSUPPORTED;
size = (size_t)indexSize; if (size != indexSize) return SZ_ERROR_UNSUPPORTED; - buf = alloc->Alloc(alloc, size); - if (buf == 0) + buf = (Byte *)ISzAlloc_Alloc(alloc, size); + if (!buf) return SZ_ERROR_MEM; res = LookInStream_Read2(stream, buf, size, SZ_ERROR_UNSUPPORTED); if (res == SZ_OK) res = Xz_ReadIndex2(p, buf, size, alloc); - alloc->Free(alloc, buf); + ISzAlloc_Free(alloc, buf); return res; }@@ -142,7 +148,7 @@ return LookInStream_Read(stream, buf, size);
/* return LookInStream_Read2(stream, buf, size, SZ_ERROR_NO_ARCHIVE); */ } -static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAlloc *alloc) +static SRes Xz_ReadBackward(CXzStream *p, ILookInStream *stream, Int64 *startOffset, ISzAllocPtr alloc) { UInt64 indexSize; Byte buf[XZ_STREAM_FOOTER_SIZE];@@ -154,7 +160,7 @@
pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) { UInt32 total = 0; pos += XZ_STREAM_FOOTER_SIZE;@@ -187,7 +193,7 @@ if (pos < XZ_STREAM_FOOTER_SIZE)
return SZ_ERROR_NO_ARCHIVE; pos -= XZ_STREAM_FOOTER_SIZE; RINOK(LookInStream_SeekRead_ForArc(stream, pos, buf, XZ_STREAM_FOOTER_SIZE)); - if (memcmp(buf + 10, XZ_FOOTER_SIG, XZ_FOOTER_SIG_SIZE) != 0) + if (!XZ_FOOTER_SIG_CHECK(buf + 10)) return SZ_ERROR_NO_ARCHIVE; }@@ -224,7 +230,7 @@ CSecToRead secToRead;
SecToRead_CreateVTable(&secToRead); secToRead.realStream = stream; - RINOK(Xz_ReadHeader(&headerFlags, &secToRead.s)); + RINOK(Xz_ReadHeader(&headerFlags, &secToRead.vt)); return (p->flags == headerFlags) ? SZ_OK : SZ_ERROR_ARCHIVE; } }@@ -238,12 +244,12 @@ p->num = p->numAllocated = 0;
p->streams = 0; } -void Xzs_Free(CXzs *p, ISzAlloc *alloc) +void Xzs_Free(CXzs *p, ISzAllocPtr alloc) { size_t i; for (i = 0; i < p->num; i++) Xz_Free(&p->streams[i], alloc); - alloc->Free(alloc, p->streams); + ISzAlloc_Free(alloc, p->streams); p->num = p->numAllocated = 0; p->streams = 0; }@@ -262,7 +268,7 @@ {
UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetUnpackSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetUnpackSize(&p->streams[i])); return size; }@@ -272,15 +278,15 @@ {
UInt64 size = 0; size_t i; for (i = 0; i < p->num; i++) - ADD_SIZE_CHECH(size, Xz_GetTotalSize(&p->streams[i])); + ADD_SIZE_CHECK(size, Xz_GetTotalSize(&p->streams[i])); return size; } */ -SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAlloc *alloc) +SRes Xzs_ReadBackward(CXzs *p, ILookInStream *stream, Int64 *startOffset, ICompressProgress *progress, ISzAllocPtr alloc) { Int64 endOffset = 0; - RINOK(stream->Seek(stream, &endOffset, SZ_SEEK_END)); + RINOK(ILookInStream_Seek(stream, &endOffset, SZ_SEEK_END)); *startOffset = endOffset; for (;;) {@@ -293,20 +299,20 @@ RINOK(res);
if (p->num == p->numAllocated) { size_t newNum = p->num + p->num / 4 + 1; - Byte *data = (Byte *)alloc->Alloc(alloc, newNum * sizeof(CXzStream)); - if (data == 0) + Byte *data = (Byte *)ISzAlloc_Alloc(alloc, newNum * sizeof(CXzStream)); + if (!data) return SZ_ERROR_MEM; p->numAllocated = newNum; if (p->num != 0) memcpy(data, p->streams, p->num * sizeof(CXzStream)); - alloc->Free(alloc, p->streams); + ISzAlloc_Free(alloc, p->streams); p->streams = (CXzStream *)data; } p->streams[p->num++] = st; if (*startOffset == 0) break; RINOK(LookInStream_SeekTo(stream, *startOffset)); - if (progress && progress->Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) + if (progress && ICompressProgress_Progress(progress, endOffset - *startOffset, (UInt64)(Int64)-1) != SZ_OK) return SZ_ERROR_PROGRESS; } return SZ_OK;
M
src/util/gui.c
→
src/util/gui.c
@@ -16,7 +16,8 @@
void GUIPollInput(struct GUIParams* params, uint32_t* newInputOut, uint32_t* heldInput) { uint32_t input = params->pollInput(¶ms->keyMap); uint32_t newInput = 0; - for (int i = 0; i < GUI_INPUT_MAX; ++i) { + int i; + for (i = 0; i < GUI_INPUT_MAX; ++i) { if (input & (1 << i)) { ++params->inputHistory[i]; } else {@@ -66,7 +67,8 @@ return params->cursorState;
} void GUIInvalidateKeys(struct GUIParams* params) { - for (int i = 0; i < GUI_INPUT_MAX; ++i) { + int i; + for (i = 0; i < GUI_INPUT_MAX; ++i) { params->inputHistory[i] = 0; } }
M
src/util/hash.c
→
src/util/hash.c
@@ -60,7 +60,8 @@ // body
const uint32_t * blocks = (const uint32_t *)(data + nblocks*4); - for(int i = -nblocks; i; i++) + int i; + for(i = -nblocks; i; i++) { uint32_t k1 = getblock32(blocks,i);@@ -82,10 +83,15 @@ uint32_t k1 = 0;
switch(len & 3) { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; - case 1: k1 ^= tail[0]; - k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; + case 3: + k1 ^= tail[2] << 16; + // Fall through + case 2: + k1 ^= tail[1] << 8; + // Fall through + case 1: + k1 ^= tail[0]; + k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1; }; //----------
M
src/util/string.c
→
src/util/string.c
@@ -234,8 +234,8 @@
extern const uint16_t gbkUnicodeTable[]; char* gbkToUtf8(const char* gbk, size_t length) { - char* utf8 = 0; - char* utf8Offset; + char* utf8 = NULL; + char* utf8Offset = NULL; size_t offset; uint8_t gbk1 = 0; char buffer[4];@@ -266,17 +266,17 @@ }
size_t bytes = toUtf8(unichar, buffer); utf8Length += bytes; - if (utf8Length < utf8TotalBytes) { - memcpy(utf8Offset, buffer, bytes); - utf8Offset += bytes; - } else if (!utf8) { + if (!utf8) { utf8 = malloc(length); if (!utf8) { - return 0; + return NULL; } utf8TotalBytes = length; memcpy(utf8, buffer, bytes); utf8Offset = utf8 + bytes; + } else if (utf8Length < utf8TotalBytes) { + memcpy(utf8Offset, buffer, bytes); + utf8Offset += bytes; } else if (utf8Length >= utf8TotalBytes) { ptrdiff_t o = utf8Offset - utf8; char* newUTF8 = realloc(utf8, utf8TotalBytes * 2);
M
src/util/vfs.c
→
src/util/vfs.c
@@ -199,7 +199,7 @@ size_t len = strlen(dotPoint);
if (PATH_MAX <= len) { len = PATH_MAX - 1; } - strncpy(extension, dotPoint, len); + strncpy(extension, dotPoint, PATH_MAX - 1); extension[len] = '\0'; } else { extension[0] = '\0';
M
src/util/vfs/vfs-lzma.c
→
src/util/vfs/vfs-lzma.c
@@ -16,6 +16,8 @@ #include "third-party/lzma/7zCrc.h"
#include "third-party/lzma/7zFile.h" #include "third-party/lzma/7zVersion.h" +#define BUFFER_SIZE 0x2000 + struct VDirEntry7z { struct VDirEntry d;@@ -30,7 +32,7 @@ struct VDirEntry7z dirent;
// What is all this garbage? CFileInStream archiveStream; - CLookToRead lookStream; + CLookToRead2 lookStream; CSzArEx db; ISzAlloc allocImp; ISzAlloc allocTempImp;@@ -88,15 +90,18 @@ vd->allocTempImp.Alloc = SzAllocTemp;
vd->allocTempImp.Free = SzFreeTemp; FileInStream_CreateVTable(&vd->archiveStream); - LookToRead_CreateVTable(&vd->lookStream, False); + LookToRead2_CreateVTable(&vd->lookStream, False); + + vd->lookStream.realStream = &vd->archiveStream.vt; + vd->lookStream.buf = malloc(BUFFER_SIZE); + vd->lookStream.bufSize = BUFFER_SIZE; - vd->lookStream.realStream = &vd->archiveStream.s; - LookToRead_Init(&vd->lookStream); + LookToRead2_Init(&vd->lookStream); CrcGenerateTable(); SzArEx_Init(&vd->db); - SRes res = SzArEx_Open(&vd->db, &vd->lookStream.s, &vd->allocImp, &vd->allocTempImp); + SRes res = SzArEx_Open(&vd->db, &vd->lookStream.vt, &vd->allocImp, &vd->allocTempImp); if (res != SZ_OK) { SzArEx_Free(&vd->db, &vd->allocImp); File_Close(&vd->archiveStream.file);@@ -123,6 +128,7 @@
bool _vf7zClose(struct VFile* vf) { struct VFile7z* vf7z = (struct VFile7z*) vf; IAlloc_Free(&vf7z->vd->allocImp, vf7z->outBuffer); + free(vf7z); return true; }@@ -211,6 +217,7 @@ struct VDir7z* vd7z = (struct VDir7z*) vd;
SzArEx_Free(&vd7z->db, &vd7z->allocImp); File_Close(&vd7z->archiveStream.file); + free(vd7z->lookStream.buf); free(vd7z->dirent.utf8); vd7z->dirent.utf8 = 0;@@ -281,7 +288,7 @@ size_t outBufferSize;
UInt32 blockIndex; vf->outBuffer = 0; - SRes res = SzArEx_Extract(&vd7z->db, &vd7z->lookStream.s, i, &blockIndex, + SRes res = SzArEx_Extract(&vd7z->db, &vd7z->lookStream.vt, i, &blockIndex, &vf->outBuffer, &outBufferSize, &vf->bufferOffset, &vf->size, &vd7z->allocImp, &vd7z->allocTempImp);
M
tools/perf.py
→
tools/perf.py
@@ -90,7 +90,7 @@ if self.command:
server_command = list(self.command) else: server_command = [os.path.join(os.getcwd(), PerfTest.EXECUTABLE)] - server_command.extend(['--', '-PD', '0']) + server_command.extend(['--', '-PD']) if hasattr(test, "frames"): server_command.extend(['-F', str(test.frames)]) if not test.renderer: