DS GX: Fix overflow causing a division crash
Vicki Pfau vi@endrift.com
Wed, 29 Jul 2020 18:03:04 -0700
2 files changed,
4 insertions(+),
3 deletions(-)
M
CHANGES
→
CHANGES
@@ -22,6 +22,7 @@ - DS GX: Mask off invalid bits of the palette base
- DS: Properly close save file on close - DS Video: Fix size of VRAM zeroes buffer - DS GX: Fix depth test scaling + - DS GX: Fix overflow causing a division crash Misc: - DS GX: Clean up and unify texture mapping - DS Core: Add symbol loading
M
src/ds/gx/software.c
→
src/ds/gx/software.c
@@ -500,7 +500,7 @@ struct DSGXVertex* v0 = &verts[poly->poly->vertIds[0]];
struct DSGXVertex* v1; int32_t v0w = v0->viewCoord[3]; - if (!v0w) { + if (!v0w || v0w == INT32_MIN) { v0w = 1; }@@ -518,7 +518,7 @@ for (v = 1; v < poly->poly->verts; ++v) {
v1 = &verts[poly->poly->vertIds[v]]; int32_t v1w = v1->viewCoord[3]; - if (!v1w) { + if (!v1w || v1w == INT32_MIN) { v1w = 1; } int32_t v1x = (v1->viewCoord[0] + v1w) * (int64_t) (v1->viewportWidth << 12) / (v1w * 2) + (v1->viewportX << 12);@@ -577,7 +577,7 @@
v1 = &verts[poly->poly->vertIds[0]]; int32_t v1w = v1->viewCoord[3]; - if (!v1w) { + if (!v1w || v1w == INT32_MIN) { v1w = 1; } int32_t v1x = (v1->viewCoord[0] + v1w) * (int64_t) (v1->viewportWidth << 12) / (v1w * 2) + (v1->viewportX << 12);