src/platform/test/cinema-main.c (view raw)
1/* Copyright (c) 2013-2020 Jeffrey Pfau
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6#include <mgba/core/config.h>
7#include <mgba/core/core.h>
8#include <mgba/core/log.h>
9#include <mgba/core/version.h>
10#include <mgba/feature/commandline.h>
11#include <mgba/feature/video-logger.h>
12
13#include <mgba-util/png-io.h>
14#include <mgba-util/string.h>
15#include <mgba-util/table.h>
16#include <mgba-util/vector.h>
17#include <mgba-util/vfs.h>
18
19#ifdef USE_FFMPEG
20#include "feature/ffmpeg/ffmpeg-decoder.h"
21#include "feature/ffmpeg/ffmpeg-encoder.h"
22#endif
23
24#ifdef _MSC_VER
25#include <mgba-util/platform/windows/getopt.h>
26#else
27#include <getopt.h>
28#endif
29
30#include <stdlib.h>
31#include <sys/stat.h>
32#include <sys/types.h>
33
34#define MAX_TEST 200
35
36static const struct option longOpts[] = {
37 { "base", required_argument, 0, 'b' },
38 { "diffs", no_argument, 0, 'd' },
39 { "help", no_argument, 0, 'h' },
40 { "dry-run", no_argument, 0, 'n' },
41 { "outdir", required_argument, 0, 'o' },
42 { "quiet", no_argument, 0, 'q' },
43 { "rebaseline", no_argument, 0, 'r' },
44 { "verbose", no_argument, 0, 'v' },
45 { "version", no_argument, 0, '\0' },
46 { 0, 0, 0, 0 }
47};
48
49static const char shortOpts[] = "b:dhno:qrv";
50
51enum CInemaStatus {
52 CI_PASS,
53 CI_FAIL,
54 CI_XPASS,
55 CI_XFAIL,
56 CI_ERROR,
57 CI_SKIP
58};
59
60struct CInemaTest {
61 char directory[MAX_TEST];
62 char filename[MAX_TEST];
63 char name[MAX_TEST];
64 enum CInemaStatus status;
65 unsigned failedFrames;
66 uint64_t failedPixels;
67 unsigned totalFrames;
68 uint64_t totalDistance;
69 uint64_t totalPixels;
70};
71
72struct CInemaImage {
73 void* data;
74 unsigned width;
75 unsigned height;
76 unsigned stride;
77};
78
79DECLARE_VECTOR(CInemaTestList, struct CInemaTest)
80DEFINE_VECTOR(CInemaTestList, struct CInemaTest)
81
82DECLARE_VECTOR(ImageList, void*)
83DEFINE_VECTOR(ImageList, void*)
84
85static bool showVersion = false;
86static bool showUsage = false;
87static char base[PATH_MAX] = {0};
88static char outdir[PATH_MAX] = {'.'};
89static bool dryRun = false;
90static bool diffs = false;
91static bool rebaseline = false;
92static int verbosity = 0;
93
94bool CInemaTestInit(struct CInemaTest*, const char* directory, const char* filename);
95void CInemaTestRun(struct CInemaTest*, struct Table* configTree);
96
97bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* value);
98void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core);
99
100static void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args);
101
102ATTRIBUTE_FORMAT(printf, 2, 3) void CIlog(int minlevel, const char* format, ...) {
103 if (verbosity < minlevel) {
104 return;
105 }
106 va_list args;
107 va_start(args, format);
108 vprintf(format, args);
109 va_end(args);
110}
111
112ATTRIBUTE_FORMAT(printf, 2, 3) void CIerr(int minlevel, const char* format, ...) {
113 if (verbosity < minlevel) {
114 return;
115 }
116 va_list args;
117 va_start(args, format);
118 vfprintf(stderr, format, args);
119 va_end(args);
120}
121
122static bool parseCInemaArgs(int argc, char* const* argv) {
123 int ch;
124 int index = 0;
125 while ((ch = getopt_long(argc, argv, shortOpts, longOpts, &index)) != -1) {
126 const struct option* opt = &longOpts[index];
127 switch (ch) {
128 case '\0':
129 if (strcmp(opt->name, "version") == 0) {
130 showVersion = true;
131 } else {
132 return false;
133 }
134 break;
135 case 'b':
136 strlcpy(base, optarg, sizeof(base));
137 // TODO: Verify path exists
138 break;
139 case 'd':
140 diffs = true;
141 break;
142 case 'h':
143 showUsage = true;
144 break;
145 case 'n':
146 dryRun = true;
147 break;
148 case 'o':
149 strlcpy(outdir, optarg, sizeof(outdir));
150 // TODO: Make directory
151 break;
152 case 'q':
153 --verbosity;
154 break;
155 case 'r':
156 rebaseline = true;
157 break;
158 case 'v':
159 ++verbosity;
160 break;
161 default:
162 return false;
163 }
164 }
165
166 return true;
167}
168
169static void usageCInema(const char* arg0) {
170 printf("usage: %s [-dhnqv] [-b BASE] [-o DIR] [--version] [test...]\n", arg0);
171 puts(" -b, --base [BASE] Path to the CInema base directory");
172 puts(" -d, --diffs Output image diffs from failures");
173 puts(" -h, --help Print this usage and exit");
174 puts(" -n, --dry-run List all collected tests instead of running them");
175 puts(" -o, --output [DIR] Path to output applicable results");
176 puts(" -q, --quiet Decrease log verbosity (can be repeated)");
177 puts(" -r, --rebaseline Rewrite the baseline for failing tests");
178 puts(" -v, --verbose Increase log verbosity (can be repeated)");
179 puts(" --version Print version and exit");
180}
181
182static bool determineBase(int argc, char* const* argv) {
183 // TODO: Better dynamic detection
184 separatePath(__FILE__, base, NULL, NULL);
185 strncat(base, PATH_SEP ".." PATH_SEP ".." PATH_SEP ".." PATH_SEP "cinema", sizeof(base) - strlen(base) - 1);
186 return true;
187}
188
189static bool collectTests(struct CInemaTestList* tests, const char* path) {
190 CIerr(2, "Considering path %s\n", path);
191 struct VDir* dir = VDirOpen(path);
192 if (!dir) {
193 return false;
194 }
195 struct VDirEntry* entry = dir->listNext(dir);
196 while (entry) {
197 char subpath[PATH_MAX];
198 snprintf(subpath, sizeof(subpath), "%s" PATH_SEP "%s", path, entry->name(entry));
199 if (entry->type(entry) == VFS_DIRECTORY && strncmp(entry->name(entry), ".", 2) != 0 && strncmp(entry->name(entry), "..", 3) != 0) {
200 if (!collectTests(tests, subpath)) {
201 dir->close(dir);
202 return false;
203 }
204 } else if (entry->type(entry) == VFS_FILE && strncmp(entry->name(entry), "test.", 5) == 0) {
205 CIerr(3, "Found potential test %s\n", subpath);
206 struct VFile* vf = dir->openFile(dir, entry->name(entry), O_RDONLY);
207 if (vf) {
208 if (mCoreIsCompatible(vf) != PLATFORM_NONE || mVideoLogIsCompatible(vf) != PLATFORM_NONE) {
209 struct CInemaTest* test = CInemaTestListAppend(tests);
210 if (!CInemaTestInit(test, path, entry->name(entry))) {
211 CIerr(3, "Failed to create test\n");
212 CInemaTestListResize(tests, -1);
213 } else {
214 CIerr(2, "Found test %s\n", test->name);
215 }
216 } else {
217 CIerr(3, "Not a compatible file\n");
218 }
219 vf->close(vf);
220 } else {
221 CIerr(3, "Failed to open file\n");
222 }
223 }
224 entry = dir->listNext(dir);
225 }
226 dir->close(dir);
227 return true;
228}
229
230static int _compareNames(const void* a, const void* b) {
231 const struct CInemaTest* ta = a;
232 const struct CInemaTest* tb = b;
233
234 return strncmp(ta->name, tb->name, sizeof(ta->name));
235}
236
237static void reduceTestList(struct CInemaTestList* tests) {
238 qsort(CInemaTestListGetPointer(tests, 0), CInemaTestListSize(tests), sizeof(struct CInemaTest), _compareNames);
239
240 size_t i;
241 for (i = 1; i < CInemaTestListSize(tests);) {
242 struct CInemaTest* cur = CInemaTestListGetPointer(tests, i);
243 struct CInemaTest* prev = CInemaTestListGetPointer(tests, i - 1);
244 if (strncmp(cur->name, prev->name, sizeof(cur->name)) != 0) {
245 ++i;
246 continue;
247 }
248 CInemaTestListShift(tests, i, 1);
249 }
250}
251
252static void testToPath(const char* testName, char* path) {
253 strlcpy(path, base, PATH_MAX);
254
255 bool dotSeen = true;
256 size_t i;
257 for (i = strlen(path); testName[0] && i < PATH_MAX; ++testName) {
258 if (testName[0] == '.') {
259 dotSeen = true;
260 } else {
261 if (dotSeen) {
262 strlcpy(&path[i], PATH_SEP, PATH_MAX - i);
263 i += strlen(PATH_SEP);
264 dotSeen = false;
265 if (!i) {
266 break;
267 }
268 }
269 path[i] = testName[0];
270 ++i;
271 }
272 }
273}
274
275static void _loadConfigTree(struct Table* configTree, const char* testName) {
276 char key[MAX_TEST];
277 strlcpy(key, testName, sizeof(key));
278
279 struct mCoreConfig* config;
280 while (!(config = HashTableLookup(configTree, key))) {
281 char path[PATH_MAX];
282 config = malloc(sizeof(*config));
283 mCoreConfigInit(config, "cinema");
284 testToPath(key, path);
285 strncat(path, PATH_SEP, sizeof(path) - 1);
286 strncat(path, "config.ini", sizeof(path) - 1);
287 mCoreConfigLoadPath(config, path);
288 HashTableInsert(configTree, key, config);
289 char* pos = strrchr(key, '.');
290 if (pos) {
291 pos[0] = '\0';
292 } else if (key[0]) {
293 key[0] = '\0';
294 } else {
295 break;
296 }
297 }
298}
299
300static void _unloadConfigTree(const char* key, void* value, void* user) {
301 UNUSED(key);
302 UNUSED(user);
303 mCoreConfigDeinit(value);
304}
305
306static const char* _lookupValue(struct Table* configTree, const char* testName, const char* key) {
307 _loadConfigTree(configTree, testName);
308
309 char testKey[MAX_TEST];
310 strlcpy(testKey, testName, sizeof(testKey));
311
312 struct mCoreConfig* config;
313 while (true) {
314 config = HashTableLookup(configTree, testKey);
315 if (!config) {
316 continue;
317 }
318 const char* str = ConfigurationGetValue(&config->configTable, "testinfo", key);
319 if (str) {
320 return str;
321 }
322 char* pos = strrchr(testKey, '.');
323 if (pos) {
324 pos[0] = '\0';
325 } else if (testKey[0]) {
326 testKey[0] = '\0';
327 } else {
328 break;
329 }
330 }
331 return NULL;
332}
333
334bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* out) {
335 const char* charValue = _lookupValue(configTree, testName, key);
336 if (!charValue) {
337 return false;
338 }
339 char* end;
340 unsigned long value = strtoul(charValue, &end, 10);
341 if (*end) {
342 return false;
343 }
344 *out = value;
345 return true;
346}
347
348void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core) {
349 _loadConfigTree(configTree, testName);
350
351 char testKey[MAX_TEST] = {0};
352 char* keyEnd = testKey;
353 const char* pos;
354 while (true) {
355 pos = strchr(testName, '.');
356 size_t maxlen = sizeof(testKey) - (keyEnd - testKey) - 1;
357 size_t len;
358 if (pos) {
359 len = pos - testName;
360 } else {
361 len = strlen(testName);
362 }
363 if (len > maxlen) {
364 len = maxlen;
365 }
366 strncpy(keyEnd, testName, len);
367 keyEnd += len;
368
369 struct mCoreConfig* config = HashTableLookup(configTree, testKey);
370 if (config) {
371 core->loadConfig(core, config);
372 }
373 if (!pos) {
374 break;
375 }
376 testName = pos + 1;
377 keyEnd[0] = '.';
378 ++keyEnd;
379 }
380}
381
382bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* filename) {
383 if (strncmp(base, directory, strlen(base)) != 0) {
384 return false;
385 }
386 memset(test, 0, sizeof(*test));
387 strlcpy(test->directory, directory, sizeof(test->directory));
388 strlcpy(test->filename, filename, sizeof(test->filename));
389 directory += strlen(base) + 1;
390 strlcpy(test->name, directory, sizeof(test->name));
391 char* str = strstr(test->name, PATH_SEP);
392 while (str) {
393 str[0] = '.';
394 str = strstr(str, PATH_SEP);
395 }
396 return true;
397}
398
399static bool _loadBaselinePNG(struct VDir* dir, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) {
400 char baselineName[32];
401 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
402 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_RDONLY);
403 if (!baselineVF) {
404 if (*status == CI_PASS) {
405 *status = CI_FAIL;
406 }
407 return false;
408 }
409
410 png_structp png = PNGReadOpen(baselineVF, 0);
411 png_infop info = png_create_info_struct(png);
412 png_infop end = png_create_info_struct(png);
413 if (!png || !info || !end || !PNGReadHeader(png, info)) {
414 PNGReadClose(png, info, end);
415 baselineVF->close(baselineVF);
416 CIerr(1, "Failed to load %s\n", baselineName);
417 *status = CI_ERROR;
418 return false;
419 }
420
421 unsigned pwidth = png_get_image_width(png, info);
422 unsigned pheight = png_get_image_height(png, info);
423 if (pheight != image->height || pwidth != image->width) {
424 PNGReadClose(png, info, end);
425 baselineVF->close(baselineVF);
426 CIlog(1, "Size mismatch for %s, expected %ux%u, got %ux%u\n", baselineName, pwidth, pheight, image->width, image->height);
427 if (*status == CI_PASS) {
428 *status = CI_FAIL;
429 }
430 return false;
431 }
432
433 image->data = malloc(pwidth * pheight * BYTES_PER_PIXEL);
434 if (!image->data) {
435 CIerr(1, "Failed to allocate baseline buffer\n");
436 *status = CI_ERROR;
437 PNGReadClose(png, info, end);
438 baselineVF->close(baselineVF);
439 return false;
440 }
441 if (!PNGReadPixels(png, info, image->data, pwidth, pheight, pwidth) || !PNGReadFooter(png, end)) {
442 CIerr(1, "Failed to read %s\n", baselineName);
443 *status = CI_ERROR;
444 free(image->data);
445 return false;
446 }
447 PNGReadClose(png, info, end);
448 baselineVF->close(baselineVF);
449 image->stride = pwidth;
450 return true;
451}
452
453#ifdef USE_FFMPEG
454struct CInemaStream {
455 struct mAVStream d;
456 struct CInemaImage* image;
457 enum CInemaStatus* status;
458};
459
460static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) {
461 struct CInemaStream* cistream = (struct CInemaStream*) stream;
462 if (height != cistream->image->height || width != cistream->image->width) {
463 CIlog(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height);
464 if (*cistream->status == CI_PASS) {
465 *cistream->status = CI_FAIL;
466 }
467 }
468}
469
470static void _cinemaVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) {
471 struct CInemaStream* cistream = (struct CInemaStream*) stream;
472 cistream->image->stride = stride;
473 size_t bufferSize = cistream->image->stride * cistream->image->height * BYTES_PER_PIXEL;
474 cistream->image->data = malloc(bufferSize);
475 memcpy(cistream->image->data, pixels, bufferSize);
476}
477#endif
478
479static struct VDir* _makeOutDir(const char* testName) {
480 char path[PATH_MAX] = {0};
481 strlcpy(path, outdir, sizeof(path));
482 char* pathEnd = path + strlen(path);
483 const char* pos;
484 while (true) {
485 pathEnd[0] = PATH_SEP[0];
486 ++pathEnd;
487 pos = strchr(testName, '.');
488 size_t maxlen = sizeof(path) - (pathEnd - path) - 1;
489 size_t len;
490 if (pos) {
491 len = pos - testName;
492 } else {
493 len = strlen(testName);
494 }
495 if (len > maxlen) {
496 len = maxlen;
497 }
498 strncpy(pathEnd, testName, len);
499 pathEnd += len;
500
501 mkdir(path, 0777);
502
503 if (!pos) {
504 break;
505 }
506 testName = pos + 1;
507 }
508 return VDirOpen(path);
509}
510
511static void _writeImage(struct VFile* vf, const struct CInemaImage* image) {
512 png_structp png = PNGWriteOpen(vf);
513 png_infop info = PNGWriteHeader(png, image->width, image->height);
514 if (!PNGWritePixels(png, image->width, image->height, image->stride, image->data)) {
515 CIerr(0, "Could not write output image\n");
516 }
517 PNGWriteClose(png, info);
518
519 vf->close(vf);
520}
521
522static void _writeDiff(const char* testName, const struct CInemaImage* image, size_t frame, const char* type) {
523 struct VDir* dir = _makeOutDir(testName);
524 if (!dir) {
525 CIerr(0, "Could not open directory for %s\n", testName);
526 return;
527 }
528 char name[32];
529 snprintf(name, sizeof(name), "%s_%04" PRIz "u.png", type, frame);
530 struct VFile* vf = dir->openFile(dir, name, O_CREAT | O_TRUNC | O_WRONLY);
531 if (!vf) {
532 CIerr(0, "Could not open output file %s\n", name);
533 dir->close(dir);
534 return;
535 }
536 _writeImage(vf, image);
537 dir->close(dir);
538}
539
540static void _writeBaseline(struct VDir* dir, const struct CInemaImage* image, size_t frame) {
541 char baselineName[32];
542 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
543 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_CREAT | O_TRUNC | O_WRONLY);
544 if (baselineVF) {
545 _writeImage(baselineVF, image);
546 } else {
547 CIerr(0, "Could not open output file %s\n", baselineName);
548 }
549}
550
551void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) {
552 unsigned ignore = 0;
553 CInemaConfigGetUInt(configTree, test->name, "ignore", &ignore);
554 if (ignore) {
555 test->status = CI_SKIP;
556 return;
557 }
558
559 struct VDir* dir = VDirOpen(test->directory);
560 if (!dir) {
561 CIerr(0, "Failed to open test directory\n");
562 test->status = CI_ERROR;
563 return;
564 }
565 struct VFile* rom = dir->openFile(dir, test->filename, O_RDONLY);
566 if (!rom) {
567 CIerr(0, "Failed to open test\n");
568 test->status = CI_ERROR;
569 return;
570 }
571 struct mCore* core = mCoreFindVF(rom);
572 if (!core) {
573 CIerr(0, "Failed to load test\n");
574 test->status = CI_ERROR;
575 rom->close(rom);
576 return;
577 }
578 if (!core->init(core)) {
579 CIerr(0, "Failed to init test\n");
580 test->status = CI_ERROR;
581 core->deinit(core);
582 return;
583 }
584 struct CInemaImage image;
585 core->desiredVideoDimensions(core, &image.width, &image.height);
586 ssize_t bufferSize = image.width * image.height * BYTES_PER_PIXEL;
587 image.data = malloc(bufferSize);
588 image.stride = image.width;
589 if (!image.data) {
590 CIerr(0, "Failed to allocate video buffer\n");
591 test->status = CI_ERROR;
592 core->deinit(core);
593 }
594 core->setVideoBuffer(core, image.data, image.stride);
595 mCoreConfigInit(&core->config, "cinema");
596
597 unsigned limit = 9999;
598 unsigned skip = 0;
599 unsigned fail = 0;
600 unsigned video = 0;
601
602 CInemaConfigGetUInt(configTree, test->name, "frames", &limit);
603 CInemaConfigGetUInt(configTree, test->name, "skip", &skip);
604 CInemaConfigGetUInt(configTree, test->name, "fail", &fail);
605 CInemaConfigGetUInt(configTree, test->name, "video", &video);
606 CInemaConfigLoad(configTree, test->name, core);
607
608 core->loadROM(core, rom);
609 core->rtc.override = RTC_FAKE_EPOCH;
610 core->rtc.value = 1200000000;
611 core->reset(core);
612
613 test->status = CI_PASS;
614
615 unsigned minFrame = core->frameCounter(core);
616 size_t frame;
617 for (frame = 0; frame < skip; ++frame) {
618 core->runFrame(core);
619 }
620 core->desiredVideoDimensions(core, &image.width, &image.height);
621
622#ifdef USE_FFMPEG
623 struct FFmpegDecoder decoder;
624 struct FFmpegEncoder encoder;
625 struct CInemaStream stream = {0};
626 if (video) {
627 char fname[PATH_MAX];
628 snprintf(fname, sizeof(fname), "%s" PATH_SEP "baseline.mkv", test->directory);
629 if (rebaseline) {
630 FFmpegEncoderInit(&encoder);
631 FFmpegEncoderSetAudio(&encoder, NULL, 0);
632 FFmpegEncoderSetVideo(&encoder, "png", 0, 0);
633 FFmpegEncoderSetContainer(&encoder, "mkv");
634 FFmpegEncoderSetDimensions(&encoder, image.width, image.height);
635 if (!FFmpegEncoderOpen(&encoder, fname)) {
636 CIerr(1, "Failed to save baseline video\n");
637 } else {
638 core->setAVStream(core, &encoder.d);
639 }
640 } else {
641 FFmpegDecoderInit(&decoder);
642 stream.d.postVideoFrame = _cinemaVideoFrame;
643 stream.d.videoDimensionsChanged = _cinemaDimensionsChanged;
644 stream.status = &test->status;
645 decoder.out = &stream.d;
646
647 if (!FFmpegDecoderOpen(&decoder, fname)) {
648 CIerr(1, "Failed to load baseline video\n");
649 }
650 }
651 }
652#else
653 if (video) {
654 CIerr(0, "Failed to run video test without ffmpeg linked in\n");
655 test->status = CI_ERROR;
656 }
657#endif
658
659 for (frame = 0; limit; ++frame, --limit) {
660 core->runFrame(core);
661 ++test->totalFrames;
662 unsigned frameCounter = core->frameCounter(core);
663 if (frameCounter <= minFrame) {
664 break;
665 }
666 CIlog(3, "Test frame: %u\n", frameCounter);
667 core->desiredVideoDimensions(core, &image.width, &image.height);
668 uint8_t* diff = NULL;
669 struct CInemaImage expected = {
670 .data = NULL,
671 .width = image.width,
672 .height = image.height,
673 .stride = image.width,
674 };
675 bool baselineFound;
676 if (video) {
677 baselineFound = false;
678#ifdef USE_FFMPEG
679 if (!rebaseline && FFmpegDecoderIsOpen(&decoder)) {
680 stream.image = &expected;
681 while (!expected.data) {
682 if (!FFmpegDecoderRead(&decoder)) {
683 CIlog(1, "Failed to read more frames. EOF?\n");
684 test->status = CI_FAIL;
685 break;
686 }
687 }
688 baselineFound = expected.data;
689 }
690#endif
691 } else {
692 baselineFound = _loadBaselinePNG(dir, &expected, frame, &test->status);
693 }
694 if (baselineFound) {
695 uint8_t* testPixels = image.data;
696 uint8_t* expectPixels = expected.data;
697 size_t x;
698 size_t y;
699 int max = 0;
700 bool failed = false;
701 for (y = 0; y < image.height; ++y) {
702 for (x = 0; x < image.width; ++x) {
703 size_t pix = expected.stride * y + x;
704 size_t tpix = image.stride * y + x;
705 int testR = testPixels[tpix * 4 + 0];
706 int testG = testPixels[tpix * 4 + 1];
707 int testB = testPixels[tpix * 4 + 2];
708 int expectR = expectPixels[pix * 4 + 0];
709 int expectG = expectPixels[pix * 4 + 1];
710 int expectB = expectPixels[pix * 4 + 2];
711 int r = expectR - testR;
712 int g = expectG - testG;
713 int b = expectB - testB;
714 if (r | g | b) {
715 failed = true;
716 if (diffs && !diff) {
717 diff = calloc(expected.stride * expected.height, BYTES_PER_PIXEL);
718 }
719 CIlog(3, "Frame %u failed at pixel %" PRIz "ux%" PRIz "u with diff %i,%i,%i (expected %02x%02x%02x, got %02x%02x%02x)\n",
720 frameCounter, x, y, r, g, b,
721 expectR, expectG, expectB,
722 testR, testG, testB);
723 test->status = CI_FAIL;
724 if (r < 0) {
725 r = -r;
726 }
727 if (g < 0) {
728 g = -g;
729 }
730 if (b < 0) {
731 b = -b;
732 }
733
734 if (diff) {
735 if (r > max) {
736 max = r;
737 }
738 if (g > max) {
739 max = g;
740 }
741 if (b > max) {
742 max = b;
743 }
744 diff[pix * 4 + 0] = r;
745 diff[pix * 4 + 1] = g;
746 diff[pix * 4 + 2] = b;
747 }
748
749 test->totalDistance += r + g + b;
750 ++test->failedPixels;
751 }
752 }
753 }
754 if (failed) {
755 ++test->failedFrames;
756 }
757 test->totalPixels += image.height * image.width;
758 if (rebaseline && failed) {
759 _writeBaseline(dir, &image, frame);
760 }
761 if (diff) {
762 if (failed) {
763 struct CInemaImage outdiff = {
764 .data = diff,
765 .width = expected.width,
766 .height = expected.height,
767 .stride = expected.stride,
768 };
769
770 _writeDiff(test->name, &image, frame, "result");
771 _writeDiff(test->name, &expected, frame, "expected");
772 _writeDiff(test->name, &outdiff, frame, "diff");
773
774 for (y = 0; y < outdiff.height; ++y) {
775 for (x = 0; x < outdiff.width; ++x) {
776 size_t pix = outdiff.stride * y + x;
777 diff[pix * 4 + 0] = diff[pix * 4 + 0] * 255 / max;
778 diff[pix * 4 + 1] = diff[pix * 4 + 1] * 255 / max;
779 diff[pix * 4 + 2] = diff[pix * 4 + 2] * 255 / max;
780 }
781 }
782 _writeDiff(test->name, &outdiff, frame, "normalized");
783 }
784 free(diff);
785 }
786 free(expected.data);
787 } else if (test->status == CI_ERROR) {
788 break;
789 } else if (rebaseline && !video) {
790 _writeBaseline(dir, &image, frame);
791 } else if (!rebaseline) {
792 test->status = CI_FAIL;
793 }
794 }
795
796 if (fail) {
797 if (test->status == CI_FAIL) {
798 test->status = CI_XFAIL;
799 } else if (test->status == CI_PASS) {
800 test->status = CI_XPASS;
801 }
802 }
803
804#ifdef USE_FFMPEG
805 if (video) {
806 if (rebaseline) {
807 FFmpegEncoderClose(&encoder);
808 } else {
809 FFmpegDecoderClose(&decoder);
810 }
811 }
812#endif
813
814 free(image.data);
815 mCoreConfigDeinit(&core->config);
816 core->deinit(core);
817 dir->close(dir);
818}
819
820void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args) {
821 UNUSED(log);
822 if (verbosity < 0) {
823 return;
824 }
825 int mask = mLOG_FATAL;
826 if (verbosity >= 1) {
827 mask |= mLOG_ERROR;
828 }
829 if (verbosity >= 2) {
830 mask |= mLOG_WARN;
831 }
832 if (verbosity >= 4) {
833 mask |= mLOG_INFO;
834 }
835 if (verbosity >= 5) {
836 mask |= mLOG_ALL;
837 }
838 if (!(mask & level)) {
839 return;
840 }
841
842 char buffer[256];
843 vsnprintf(buffer, sizeof(buffer), format, args);
844 CIerr(0, "[%s] %s\n", mLogCategoryName(category), buffer);
845}
846
847int main(int argc, char** argv) {
848 int status = 0;
849 if (!parseCInemaArgs(argc, argv)) {
850 status = 1;
851 goto cleanup;
852 }
853
854 if (showVersion) {
855 version(argv[0]);
856 goto cleanup;
857 }
858
859 if (showUsage) {
860 usageCInema(argv[0]);
861 goto cleanup;
862 }
863
864 argc -= optind;
865 argv += optind;
866
867 if (!base[0] && !determineBase(argc, argv)) {
868 CIlog(0, "Could not determine CInema test base. Please specify manually.");
869 status = 1;
870 goto cleanup;
871 }
872#ifndef _WIN32
873 char* rbase = realpath(base, NULL);
874 if (rbase) {
875 strlcpy(base, rbase, sizeof(base));
876 free(rbase);
877 }
878#endif
879
880 struct CInemaTestList tests;
881 CInemaTestListInit(&tests, 0);
882
883 struct mLogger logger = { .log = _log };
884 mLogSetDefaultLogger(&logger);
885#ifdef USE_FFMPEG
886 if (verbosity < 2) {
887 av_log_set_level(AV_LOG_ERROR);
888 }
889#endif
890
891 if (argc > 0) {
892 size_t i;
893 for (i = 0; i < (size_t) argc; ++i) {
894 char path[PATH_MAX + 1] = {0};
895 testToPath(argv[i], path);
896
897 if (!collectTests(&tests, path)) {
898 status = 1;
899 break;
900 }
901 }
902 } else if (!collectTests(&tests, base)) {
903 status = 1;
904 }
905
906 if (CInemaTestListSize(&tests) == 0) {
907 CIlog(1, "No tests found.");
908 status = 1;
909 } else {
910 reduceTestList(&tests);
911 }
912
913 struct Table configTree;
914 HashTableInit(&configTree, 0, free);
915
916 size_t i;
917 for (i = 0; i < CInemaTestListSize(&tests); ++i) {
918 struct CInemaTest* test = CInemaTestListGetPointer(&tests, i);
919 if (dryRun) {
920 CIlog(-1, "%s\n", test->name);
921 } else {
922 CIlog(1, "%s: ", test->name);
923 fflush(stdout);
924 CInemaTestRun(test, &configTree);
925 switch (test->status) {
926 case CI_PASS:
927 CIlog(1, "pass\n");
928 break;
929 case CI_FAIL:
930 status = 1;
931 CIlog(1, "fail\n");
932 break;
933 case CI_XPASS:
934 CIlog(1, "xpass\n");
935 break;
936 case CI_XFAIL:
937 CIlog(1, "xfail\n");
938 break;
939 case CI_SKIP:
940 CIlog(1, "skip\n");
941 break;
942 case CI_ERROR:
943 status = 1;
944 CIlog(1, "error\n");
945 break;
946 }
947 if (test->failedFrames) {
948 CIlog(2, "\tfailed frames: %u/%u (%1.3g%%)\n", test->failedFrames, test->totalFrames, test->failedFrames / (test->totalFrames * 0.01));
949 CIlog(2, "\tfailed pixels: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->failedPixels, test->totalPixels, test->failedPixels / (test->totalPixels * 0.01));
950 CIlog(2, "\tdistance: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->totalDistance, test->totalPixels * 765, test->totalDistance / (test->totalPixels * 7.65));
951 }
952 }
953 }
954
955 HashTableEnumerate(&configTree, _unloadConfigTree, NULL);
956 HashTableDeinit(&configTree);
957 CInemaTestListDeinit(&tests);
958
959cleanup:
960 return status;
961}