src/platform/test/cinema-main.c (view raw)
1/* Copyright (c) 2013-2020 Jeffrey Pfau
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6#include <mgba/core/config.h>
7#include <mgba/core/core.h>
8#include <mgba/core/log.h>
9#include <mgba/core/version.h>
10#include <mgba/feature/commandline.h>
11#include <mgba/feature/video-logger.h>
12
13#include <mgba-util/png-io.h>
14#include <mgba-util/string.h>
15#include <mgba-util/table.h>
16#include <mgba-util/vector.h>
17#include <mgba-util/vfs.h>
18
19#ifdef USE_FFMPEG
20#include "feature/ffmpeg/ffmpeg-decoder.h"
21#include "feature/ffmpeg/ffmpeg-encoder.h"
22#endif
23
24#ifdef _MSC_VER
25#include <mgba-util/platform/windows/getopt.h>
26#else
27#include <getopt.h>
28#endif
29
30#include <stdlib.h>
31#include <sys/stat.h>
32#include <sys/types.h>
33
34#define MAX_TEST 200
35
36static const struct option longOpts[] = {
37 { "base", required_argument, 0, 'b' },
38 { "diffs", no_argument, 0, 'd' },
39 { "help", no_argument, 0, 'h' },
40 { "dry-run", no_argument, 0, 'n' },
41 { "outdir", required_argument, 0, 'o' },
42 { "quiet", no_argument, 0, 'q' },
43 { "rebaseline", no_argument, 0, 'r' },
44 { "verbose", no_argument, 0, 'v' },
45 { "version", no_argument, 0, '\0' },
46 { 0, 0, 0, 0 }
47};
48
49static const char shortOpts[] = "b:dhno:qrv";
50
51enum CInemaStatus {
52 CI_PASS,
53 CI_FAIL,
54 CI_XPASS,
55 CI_XFAIL,
56 CI_ERROR,
57 CI_SKIP
58};
59
60struct CInemaTest {
61 char directory[MAX_TEST];
62 char filename[MAX_TEST];
63 char name[MAX_TEST];
64 enum CInemaStatus status;
65 unsigned failedFrames;
66 uint64_t failedPixels;
67 unsigned totalFrames;
68 uint64_t totalDistance;
69 uint64_t totalPixels;
70};
71
72struct CInemaImage {
73 void* data;
74 unsigned width;
75 unsigned height;
76 unsigned stride;
77};
78
79DECLARE_VECTOR(CInemaTestList, struct CInemaTest)
80DEFINE_VECTOR(CInemaTestList, struct CInemaTest)
81
82DECLARE_VECTOR(ImageList, void*)
83DEFINE_VECTOR(ImageList, void*)
84
85static bool showVersion = false;
86static bool showUsage = false;
87static char base[PATH_MAX] = {0};
88static char outdir[PATH_MAX] = {'.'};
89static bool dryRun = false;
90static bool diffs = false;
91static bool rebaseline = false;
92static int verbosity = 0;
93
94bool CInemaTestInit(struct CInemaTest*, const char* directory, const char* filename);
95void CInemaTestRun(struct CInemaTest*, struct Table* configTree);
96
97bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* value);
98void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core);
99
100static void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args);
101
102ATTRIBUTE_FORMAT(printf, 2, 3) void CIlog(int minlevel, const char* format, ...) {
103 if (verbosity < minlevel) {
104 return;
105 }
106 va_list args;
107 va_start(args, format);
108 vprintf(format, args);
109 va_end(args);
110}
111
112ATTRIBUTE_FORMAT(printf, 2, 3) void CIerr(int minlevel, const char* format, ...) {
113 if (verbosity < minlevel) {
114 return;
115 }
116 va_list args;
117 va_start(args, format);
118 vfprintf(stderr, format, args);
119 va_end(args);
120}
121
122static bool parseCInemaArgs(int argc, char* const* argv) {
123 int ch;
124 int index = 0;
125 while ((ch = getopt_long(argc, argv, shortOpts, longOpts, &index)) != -1) {
126 const struct option* opt = &longOpts[index];
127 switch (ch) {
128 case '\0':
129 if (strcmp(opt->name, "version") == 0) {
130 showVersion = true;
131 } else {
132 return false;
133 }
134 break;
135 case 'b':
136 strlcpy(base, optarg, sizeof(base));
137 // TODO: Verify path exists
138 break;
139 case 'd':
140 diffs = true;
141 break;
142 case 'h':
143 showUsage = true;
144 break;
145 case 'n':
146 dryRun = true;
147 break;
148 case 'o':
149 strlcpy(outdir, optarg, sizeof(outdir));
150 // TODO: Make directory
151 break;
152 case 'q':
153 --verbosity;
154 break;
155 case 'r':
156 rebaseline = true;
157 break;
158 case 'v':
159 ++verbosity;
160 break;
161 default:
162 return false;
163 }
164 }
165
166 return true;
167}
168
169static void usageCInema(const char* arg0) {
170 printf("usage: %s [-dhnqv] [-b BASE] [-o DIR] [--version] [test...]\n", arg0);
171 puts(" -b, --base [BASE] Path to the CInema base directory");
172 puts(" -d, --diffs Output image diffs from failures");
173 puts(" -h, --help Print this usage and exit");
174 puts(" -n, --dry-run List all collected tests instead of running them");
175 puts(" -o, --output [DIR] Path to output applicable results");
176 puts(" -q, --quiet Decrease log verbosity (can be repeated)");
177 puts(" -r, --rebaseline Rewrite the baseline for failing tests");
178 puts(" -v, --verbose Increase log verbosity (can be repeated)");
179 puts(" --version Print version and exit");
180}
181
182static bool determineBase(int argc, char* const* argv) {
183 // TODO: Better dynamic detection
184 separatePath(__FILE__, base, NULL, NULL);
185 strncat(base, PATH_SEP ".." PATH_SEP ".." PATH_SEP ".." PATH_SEP "cinema", sizeof(base) - strlen(base) - 1);
186 return true;
187}
188
189static bool collectTests(struct CInemaTestList* tests, const char* path) {
190 CIerr(2, "Considering path %s\n", path);
191 struct VDir* dir = VDirOpen(path);
192 if (!dir) {
193 return false;
194 }
195 struct VDirEntry* entry = dir->listNext(dir);
196 while (entry) {
197 char subpath[PATH_MAX];
198 snprintf(subpath, sizeof(subpath), "%s" PATH_SEP "%s", path, entry->name(entry));
199 if (entry->type(entry) == VFS_DIRECTORY && strncmp(entry->name(entry), ".", 2) != 0 && strncmp(entry->name(entry), "..", 3) != 0) {
200 if (!collectTests(tests, subpath)) {
201 dir->close(dir);
202 return false;
203 }
204 } else if (entry->type(entry) == VFS_FILE && strncmp(entry->name(entry), "test.", 5) == 0) {
205 CIerr(3, "Found potential test %s\n", subpath);
206 struct VFile* vf = dir->openFile(dir, entry->name(entry), O_RDONLY);
207 if (vf) {
208 if (mCoreIsCompatible(vf) != PLATFORM_NONE || mVideoLogIsCompatible(vf) != PLATFORM_NONE) {
209 struct CInemaTest* test = CInemaTestListAppend(tests);
210 if (!CInemaTestInit(test, path, entry->name(entry))) {
211 CIerr(3, "Failed to create test\n");
212 CInemaTestListResize(tests, -1);
213 } else {
214 CIerr(2, "Found test %s\n", test->name);
215 }
216 } else {
217 CIerr(3, "Not a compatible file\n");
218 }
219 vf->close(vf);
220 } else {
221 CIerr(3, "Failed to open file\n");
222 }
223 }
224 entry = dir->listNext(dir);
225 }
226 dir->close(dir);
227 return true;
228}
229
230static int _compareNames(const void* a, const void* b) {
231 const struct CInemaTest* ta = a;
232 const struct CInemaTest* tb = b;
233
234 return strncmp(ta->name, tb->name, sizeof(ta->name));
235}
236
237static void reduceTestList(struct CInemaTestList* tests) {
238 qsort(CInemaTestListGetPointer(tests, 0), CInemaTestListSize(tests), sizeof(struct CInemaTest), _compareNames);
239
240 size_t i;
241 for (i = 1; i < CInemaTestListSize(tests);) {
242 struct CInemaTest* cur = CInemaTestListGetPointer(tests, i);
243 struct CInemaTest* prev = CInemaTestListGetPointer(tests, i - 1);
244 if (strncmp(cur->name, prev->name, sizeof(cur->name)) != 0) {
245 ++i;
246 continue;
247 }
248 CInemaTestListShift(tests, i, 1);
249 }
250}
251
252static void testToPath(const char* testName, char* path) {
253 strlcpy(path, base, PATH_MAX);
254
255 bool dotSeen = true;
256 size_t i;
257 for (i = strlen(path); testName[0] && i < PATH_MAX; ++testName) {
258 if (testName[0] == '.') {
259 dotSeen = true;
260 } else {
261 if (dotSeen) {
262 strlcpy(&path[i], PATH_SEP, PATH_MAX - i);
263 i += strlen(PATH_SEP);
264 dotSeen = false;
265 if (!i) {
266 break;
267 }
268 }
269 path[i] = testName[0];
270 ++i;
271 }
272 }
273 if (i == PATH_MAX) {
274 --i;
275 }
276 path[i] = '\0';
277}
278
279static void _loadConfigTree(struct Table* configTree, const char* testName) {
280 char key[MAX_TEST];
281 strlcpy(key, testName, sizeof(key));
282
283 struct mCoreConfig* config;
284 while (!(config = HashTableLookup(configTree, key))) {
285 char path[PATH_MAX];
286 config = malloc(sizeof(*config));
287 mCoreConfigInit(config, "cinema");
288 testToPath(key, path);
289 strncat(path, PATH_SEP, sizeof(path) - 1);
290 strncat(path, "config.ini", sizeof(path) - 1);
291 mCoreConfigLoadPath(config, path);
292 HashTableInsert(configTree, key, config);
293 char* pos = strrchr(key, '.');
294 if (pos) {
295 pos[0] = '\0';
296 } else if (key[0]) {
297 key[0] = '\0';
298 } else {
299 break;
300 }
301 }
302}
303
304static void _unloadConfigTree(const char* key, void* value, void* user) {
305 UNUSED(key);
306 UNUSED(user);
307 mCoreConfigDeinit(value);
308}
309
310static const char* _lookupValue(struct Table* configTree, const char* testName, const char* key) {
311 _loadConfigTree(configTree, testName);
312
313 char testKey[MAX_TEST];
314 strlcpy(testKey, testName, sizeof(testKey));
315
316 struct mCoreConfig* config;
317 while (true) {
318 config = HashTableLookup(configTree, testKey);
319 if (!config) {
320 continue;
321 }
322 const char* str = ConfigurationGetValue(&config->configTable, "testinfo", key);
323 if (str) {
324 return str;
325 }
326 char* pos = strrchr(testKey, '.');
327 if (pos) {
328 pos[0] = '\0';
329 } else if (testKey[0]) {
330 testKey[0] = '\0';
331 } else {
332 break;
333 }
334 }
335 return NULL;
336}
337
338bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* out) {
339 const char* charValue = _lookupValue(configTree, testName, key);
340 if (!charValue) {
341 return false;
342 }
343 char* end;
344 unsigned long value = strtoul(charValue, &end, 10);
345 if (*end) {
346 return false;
347 }
348 *out = value;
349 return true;
350}
351
352void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core) {
353 _loadConfigTree(configTree, testName);
354
355 char testKey[MAX_TEST] = {0};
356 char* keyEnd = testKey;
357 const char* pos;
358 while (true) {
359 pos = strchr(testName, '.');
360 size_t maxlen = sizeof(testKey) - (keyEnd - testKey) - 1;
361 size_t len;
362 if (pos) {
363 len = pos - testName;
364 } else {
365 len = strlen(testName);
366 }
367 if (len > maxlen) {
368 len = maxlen;
369 }
370 strncpy(keyEnd, testName, len);
371 keyEnd += len;
372
373 struct mCoreConfig* config = HashTableLookup(configTree, testKey);
374 if (config) {
375 core->loadConfig(core, config);
376 }
377 if (!pos) {
378 break;
379 }
380 testName = pos + 1;
381 keyEnd[0] = '.';
382 ++keyEnd;
383 }
384}
385
386bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* filename) {
387 if (strncmp(base, directory, strlen(base)) != 0) {
388 return false;
389 }
390 memset(test, 0, sizeof(*test));
391 strlcpy(test->directory, directory, sizeof(test->directory));
392 strlcpy(test->filename, filename, sizeof(test->filename));
393 directory += strlen(base) + 1;
394 strlcpy(test->name, directory, sizeof(test->name));
395 char* str = strstr(test->name, PATH_SEP);
396 while (str) {
397 str[0] = '.';
398 str = strstr(str, PATH_SEP);
399 }
400 return true;
401}
402
403static bool _loadBaselinePNG(struct VDir* dir, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) {
404 char baselineName[32];
405 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
406 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_RDONLY);
407 if (!baselineVF) {
408 if (*status == CI_PASS) {
409 *status = CI_FAIL;
410 }
411 return false;
412 }
413
414 png_structp png = PNGReadOpen(baselineVF, 0);
415 png_infop info = png_create_info_struct(png);
416 png_infop end = png_create_info_struct(png);
417 if (!png || !info || !end || !PNGReadHeader(png, info)) {
418 PNGReadClose(png, info, end);
419 baselineVF->close(baselineVF);
420 CIerr(1, "Failed to load %s\n", baselineName);
421 *status = CI_ERROR;
422 return false;
423 }
424
425 unsigned pwidth = png_get_image_width(png, info);
426 unsigned pheight = png_get_image_height(png, info);
427 if (pheight != image->height || pwidth != image->width) {
428 PNGReadClose(png, info, end);
429 baselineVF->close(baselineVF);
430 CIlog(1, "Size mismatch for %s, expected %ux%u, got %ux%u\n", baselineName, pwidth, pheight, image->width, image->height);
431 if (*status == CI_PASS) {
432 *status = CI_FAIL;
433 }
434 return false;
435 }
436
437 image->data = malloc(pwidth * pheight * BYTES_PER_PIXEL);
438 if (!image->data) {
439 CIerr(1, "Failed to allocate baseline buffer\n");
440 *status = CI_ERROR;
441 PNGReadClose(png, info, end);
442 baselineVF->close(baselineVF);
443 return false;
444 }
445 if (!PNGReadPixels(png, info, image->data, pwidth, pheight, pwidth) || !PNGReadFooter(png, end)) {
446 CIerr(1, "Failed to read %s\n", baselineName);
447 *status = CI_ERROR;
448 free(image->data);
449 return false;
450 }
451 PNGReadClose(png, info, end);
452 baselineVF->close(baselineVF);
453 image->stride = pwidth;
454 return true;
455}
456
457#ifdef USE_FFMPEG
458struct CInemaStream {
459 struct mAVStream d;
460 struct CInemaImage* image;
461 enum CInemaStatus* status;
462};
463
464static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) {
465 struct CInemaStream* cistream = (struct CInemaStream*) stream;
466 if (height != cistream->image->height || width != cistream->image->width) {
467 CIlog(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height);
468 if (*cistream->status == CI_PASS) {
469 *cistream->status = CI_FAIL;
470 }
471 }
472}
473
474static void _cinemaVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) {
475 struct CInemaStream* cistream = (struct CInemaStream*) stream;
476 cistream->image->stride = stride;
477 size_t bufferSize = cistream->image->stride * cistream->image->height * BYTES_PER_PIXEL;
478 cistream->image->data = malloc(bufferSize);
479 memcpy(cistream->image->data, pixels, bufferSize);
480}
481#endif
482
483static struct VDir* _makeOutDir(const char* testName) {
484 char path[PATH_MAX] = {0};
485 strlcpy(path, outdir, sizeof(path));
486 char* pathEnd = path + strlen(path);
487 const char* pos;
488 while (true) {
489 pathEnd[0] = PATH_SEP[0];
490 ++pathEnd;
491 pos = strchr(testName, '.');
492 size_t maxlen = sizeof(path) - (pathEnd - path) - 1;
493 size_t len;
494 if (pos) {
495 len = pos - testName;
496 } else {
497 len = strlen(testName);
498 }
499 if (len > maxlen) {
500 len = maxlen;
501 }
502 strncpy(pathEnd, testName, len);
503 pathEnd += len;
504
505 mkdir(path, 0777);
506
507 if (!pos) {
508 break;
509 }
510 testName = pos + 1;
511 }
512 return VDirOpen(path);
513}
514
515static void _writeImage(struct VFile* vf, const struct CInemaImage* image) {
516 png_structp png = PNGWriteOpen(vf);
517 png_infop info = PNGWriteHeader(png, image->width, image->height);
518 if (!PNGWritePixels(png, image->width, image->height, image->stride, image->data)) {
519 CIerr(0, "Could not write output image\n");
520 }
521 PNGWriteClose(png, info);
522
523 vf->close(vf);
524}
525
526static void _writeDiff(const char* testName, const struct CInemaImage* image, size_t frame, const char* type) {
527 struct VDir* dir = _makeOutDir(testName);
528 if (!dir) {
529 CIerr(0, "Could not open directory for %s\n", testName);
530 return;
531 }
532 char name[32];
533 snprintf(name, sizeof(name), "%s_%04" PRIz "u.png", type, frame);
534 struct VFile* vf = dir->openFile(dir, name, O_CREAT | O_TRUNC | O_WRONLY);
535 if (!vf) {
536 CIerr(0, "Could not open output file %s\n", name);
537 dir->close(dir);
538 return;
539 }
540 _writeImage(vf, image);
541 dir->close(dir);
542}
543
544static void _writeBaseline(struct VDir* dir, const struct CInemaImage* image, size_t frame) {
545 char baselineName[32];
546 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
547 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_CREAT | O_TRUNC | O_WRONLY);
548 if (baselineVF) {
549 _writeImage(baselineVF, image);
550 } else {
551 CIerr(0, "Could not open output file %s\n", baselineName);
552 }
553}
554
555void CInemaTestRun(struct CInemaTest* test, struct Table* configTree) {
556 unsigned ignore = 0;
557 CInemaConfigGetUInt(configTree, test->name, "ignore", &ignore);
558 if (ignore) {
559 test->status = CI_SKIP;
560 return;
561 }
562
563 struct VDir* dir = VDirOpen(test->directory);
564 if (!dir) {
565 CIerr(0, "Failed to open test directory\n");
566 test->status = CI_ERROR;
567 return;
568 }
569 struct VFile* rom = dir->openFile(dir, test->filename, O_RDONLY);
570 if (!rom) {
571 CIerr(0, "Failed to open test\n");
572 test->status = CI_ERROR;
573 return;
574 }
575 struct mCore* core = mCoreFindVF(rom);
576 if (!core) {
577 CIerr(0, "Failed to load test\n");
578 test->status = CI_ERROR;
579 rom->close(rom);
580 return;
581 }
582 if (!core->init(core)) {
583 CIerr(0, "Failed to init test\n");
584 test->status = CI_ERROR;
585 core->deinit(core);
586 return;
587 }
588 struct CInemaImage image;
589 core->desiredVideoDimensions(core, &image.width, &image.height);
590 ssize_t bufferSize = image.width * image.height * BYTES_PER_PIXEL;
591 image.data = malloc(bufferSize);
592 image.stride = image.width;
593 if (!image.data) {
594 CIerr(0, "Failed to allocate video buffer\n");
595 test->status = CI_ERROR;
596 core->deinit(core);
597 }
598 core->setVideoBuffer(core, image.data, image.stride);
599 mCoreConfigInit(&core->config, "cinema");
600
601 unsigned limit = 9999;
602 unsigned skip = 0;
603 unsigned fail = 0;
604 unsigned video = 0;
605
606 CInemaConfigGetUInt(configTree, test->name, "frames", &limit);
607 CInemaConfigGetUInt(configTree, test->name, "skip", &skip);
608 CInemaConfigGetUInt(configTree, test->name, "fail", &fail);
609 CInemaConfigGetUInt(configTree, test->name, "video", &video);
610 CInemaConfigLoad(configTree, test->name, core);
611
612 struct VFile* save = VFileMemChunk(NULL, 0);
613 core->loadROM(core, rom);
614 if (!core->loadSave(core, save)) {
615 save->close(save);
616 }
617 core->rtc.override = RTC_FAKE_EPOCH;
618 core->rtc.value = 1200000000;
619 core->reset(core);
620
621 test->status = CI_PASS;
622
623 unsigned minFrame = core->frameCounter(core);
624 size_t frame;
625 for (frame = 0; frame < skip; ++frame) {
626 core->runFrame(core);
627 }
628 core->desiredVideoDimensions(core, &image.width, &image.height);
629
630#ifdef USE_FFMPEG
631 struct FFmpegDecoder decoder;
632 struct FFmpegEncoder encoder;
633 struct CInemaStream stream = {0};
634 if (video) {
635 char fname[PATH_MAX];
636 snprintf(fname, sizeof(fname), "%s" PATH_SEP "baseline.mkv", test->directory);
637 if (rebaseline) {
638 FFmpegEncoderInit(&encoder);
639 FFmpegEncoderSetAudio(&encoder, NULL, 0);
640 FFmpegEncoderSetVideo(&encoder, "png", 0, 0);
641 FFmpegEncoderSetContainer(&encoder, "mkv");
642 FFmpegEncoderSetDimensions(&encoder, image.width, image.height);
643 if (!FFmpegEncoderOpen(&encoder, fname)) {
644 CIerr(1, "Failed to save baseline video\n");
645 } else {
646 core->setAVStream(core, &encoder.d);
647 }
648 } else {
649 FFmpegDecoderInit(&decoder);
650 stream.d.postVideoFrame = _cinemaVideoFrame;
651 stream.d.videoDimensionsChanged = _cinemaDimensionsChanged;
652 stream.status = &test->status;
653 decoder.out = &stream.d;
654
655 if (!FFmpegDecoderOpen(&decoder, fname)) {
656 CIerr(1, "Failed to load baseline video\n");
657 }
658 }
659 }
660#else
661 if (video) {
662 CIerr(0, "Failed to run video test without ffmpeg linked in\n");
663 test->status = CI_ERROR;
664 }
665#endif
666
667 for (frame = 0; limit; ++frame, --limit) {
668 core->runFrame(core);
669 ++test->totalFrames;
670 unsigned frameCounter = core->frameCounter(core);
671 if (frameCounter <= minFrame) {
672 break;
673 }
674 CIlog(3, "Test frame: %u\n", frameCounter);
675 core->desiredVideoDimensions(core, &image.width, &image.height);
676 uint8_t* diff = NULL;
677 struct CInemaImage expected = {
678 .data = NULL,
679 .width = image.width,
680 .height = image.height,
681 .stride = image.width,
682 };
683 bool baselineFound;
684 if (video) {
685 baselineFound = false;
686#ifdef USE_FFMPEG
687 if (!rebaseline && FFmpegDecoderIsOpen(&decoder)) {
688 stream.image = &expected;
689 while (!expected.data) {
690 if (!FFmpegDecoderRead(&decoder)) {
691 CIlog(1, "Failed to read more frames. EOF?\n");
692 test->status = CI_FAIL;
693 break;
694 }
695 }
696 baselineFound = expected.data;
697 }
698#endif
699 } else {
700 baselineFound = _loadBaselinePNG(dir, &expected, frame, &test->status);
701 }
702 if (baselineFound) {
703 uint8_t* testPixels = image.data;
704 uint8_t* expectPixels = expected.data;
705 size_t x;
706 size_t y;
707 int max = 0;
708 bool failed = false;
709 for (y = 0; y < image.height; ++y) {
710 for (x = 0; x < image.width; ++x) {
711 size_t pix = expected.stride * y + x;
712 size_t tpix = image.stride * y + x;
713 int testR = testPixels[tpix * 4 + 0];
714 int testG = testPixels[tpix * 4 + 1];
715 int testB = testPixels[tpix * 4 + 2];
716 int expectR = expectPixels[pix * 4 + 0];
717 int expectG = expectPixels[pix * 4 + 1];
718 int expectB = expectPixels[pix * 4 + 2];
719 int r = expectR - testR;
720 int g = expectG - testG;
721 int b = expectB - testB;
722 if (r | g | b) {
723 failed = true;
724 if (diffs && !diff) {
725 diff = calloc(expected.stride * expected.height, BYTES_PER_PIXEL);
726 }
727 CIlog(3, "Frame %u failed at pixel %" PRIz "ux%" PRIz "u with diff %i,%i,%i (expected %02x%02x%02x, got %02x%02x%02x)\n",
728 frameCounter, x, y, r, g, b,
729 expectR, expectG, expectB,
730 testR, testG, testB);
731 test->status = CI_FAIL;
732 if (r < 0) {
733 r = -r;
734 }
735 if (g < 0) {
736 g = -g;
737 }
738 if (b < 0) {
739 b = -b;
740 }
741
742 if (diff) {
743 if (r > max) {
744 max = r;
745 }
746 if (g > max) {
747 max = g;
748 }
749 if (b > max) {
750 max = b;
751 }
752 diff[pix * 4 + 0] = r;
753 diff[pix * 4 + 1] = g;
754 diff[pix * 4 + 2] = b;
755 }
756
757 test->totalDistance += r + g + b;
758 ++test->failedPixels;
759 }
760 }
761 }
762 if (failed) {
763 ++test->failedFrames;
764 }
765 test->totalPixels += image.height * image.width;
766 if (rebaseline && failed) {
767 _writeBaseline(dir, &image, frame);
768 }
769 if (diff) {
770 if (failed) {
771 struct CInemaImage outdiff = {
772 .data = diff,
773 .width = expected.width,
774 .height = expected.height,
775 .stride = expected.stride,
776 };
777
778 _writeDiff(test->name, &image, frame, "result");
779 _writeDiff(test->name, &expected, frame, "expected");
780 _writeDiff(test->name, &outdiff, frame, "diff");
781
782 for (y = 0; y < outdiff.height; ++y) {
783 for (x = 0; x < outdiff.width; ++x) {
784 size_t pix = outdiff.stride * y + x;
785 diff[pix * 4 + 0] = diff[pix * 4 + 0] * 255 / max;
786 diff[pix * 4 + 1] = diff[pix * 4 + 1] * 255 / max;
787 diff[pix * 4 + 2] = diff[pix * 4 + 2] * 255 / max;
788 }
789 }
790 _writeDiff(test->name, &outdiff, frame, "normalized");
791 }
792 free(diff);
793 }
794 free(expected.data);
795 } else if (test->status == CI_ERROR) {
796 break;
797 } else if (rebaseline && !video) {
798 _writeBaseline(dir, &image, frame);
799 } else if (!rebaseline) {
800 test->status = CI_FAIL;
801 }
802 }
803
804 if (fail) {
805 if (test->status == CI_FAIL) {
806 test->status = CI_XFAIL;
807 } else if (test->status == CI_PASS) {
808 test->status = CI_XPASS;
809 }
810 }
811
812#ifdef USE_FFMPEG
813 if (video) {
814 if (rebaseline) {
815 FFmpegEncoderClose(&encoder);
816 } else {
817 FFmpegDecoderClose(&decoder);
818 }
819 }
820#endif
821
822 free(image.data);
823 mCoreConfigDeinit(&core->config);
824 core->deinit(core);
825 dir->close(dir);
826}
827
828void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args) {
829 UNUSED(log);
830 if (verbosity < 0) {
831 return;
832 }
833 int mask = mLOG_FATAL;
834 if (verbosity >= 1) {
835 mask |= mLOG_ERROR;
836 }
837 if (verbosity >= 2) {
838 mask |= mLOG_WARN;
839 }
840 if (verbosity >= 4) {
841 mask |= mLOG_INFO;
842 }
843 if (verbosity >= 5) {
844 mask |= mLOG_ALL;
845 }
846 if (!(mask & level)) {
847 return;
848 }
849
850 char buffer[256];
851 vsnprintf(buffer, sizeof(buffer), format, args);
852 CIerr(0, "[%s] %s\n", mLogCategoryName(category), buffer);
853}
854
855int main(int argc, char** argv) {
856 int status = 0;
857 if (!parseCInemaArgs(argc, argv)) {
858 status = 1;
859 goto cleanup;
860 }
861
862 if (showVersion) {
863 version(argv[0]);
864 goto cleanup;
865 }
866
867 if (showUsage) {
868 usageCInema(argv[0]);
869 goto cleanup;
870 }
871
872 argc -= optind;
873 argv += optind;
874
875 if (!base[0] && !determineBase(argc, argv)) {
876 CIlog(0, "Could not determine CInema test base. Please specify manually.");
877 status = 1;
878 goto cleanup;
879 }
880#ifndef _WIN32
881 char* rbase = realpath(base, NULL);
882 if (rbase) {
883 strlcpy(base, rbase, sizeof(base));
884 free(rbase);
885 }
886#endif
887
888 struct CInemaTestList tests;
889 CInemaTestListInit(&tests, 0);
890
891 struct mLogger logger = { .log = _log };
892 mLogSetDefaultLogger(&logger);
893#ifdef USE_FFMPEG
894 if (verbosity < 2) {
895 av_log_set_level(AV_LOG_ERROR);
896 }
897#endif
898
899 if (argc > 0) {
900 size_t i;
901 for (i = 0; i < (size_t) argc; ++i) {
902 char path[PATH_MAX + 1] = {0};
903 testToPath(argv[i], path);
904
905 if (!collectTests(&tests, path)) {
906 status = 1;
907 break;
908 }
909 }
910 } else if (!collectTests(&tests, base)) {
911 status = 1;
912 }
913
914 if (CInemaTestListSize(&tests) == 0) {
915 CIlog(1, "No tests found.");
916 status = 1;
917 } else {
918 reduceTestList(&tests);
919 }
920
921 struct Table configTree;
922 HashTableInit(&configTree, 0, free);
923
924 size_t i;
925 for (i = 0; i < CInemaTestListSize(&tests); ++i) {
926 struct CInemaTest* test = CInemaTestListGetPointer(&tests, i);
927 if (dryRun) {
928 CIlog(-1, "%s\n", test->name);
929 } else {
930 CIlog(1, "%s: ", test->name);
931 fflush(stdout);
932 CInemaTestRun(test, &configTree);
933 switch (test->status) {
934 case CI_PASS:
935 CIlog(1, "pass\n");
936 break;
937 case CI_FAIL:
938 status = 1;
939 CIlog(1, "fail\n");
940 break;
941 case CI_XPASS:
942 CIlog(1, "xpass\n");
943 break;
944 case CI_XFAIL:
945 CIlog(1, "xfail\n");
946 break;
947 case CI_SKIP:
948 CIlog(1, "skip\n");
949 break;
950 case CI_ERROR:
951 status = 1;
952 CIlog(1, "error\n");
953 break;
954 }
955 if (test->failedFrames) {
956 CIlog(2, "\tfailed frames: %u/%u (%1.3g%%)\n", test->failedFrames, test->totalFrames, test->failedFrames / (test->totalFrames * 0.01));
957 CIlog(2, "\tfailed pixels: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->failedPixels, test->totalPixels, test->failedPixels / (test->totalPixels * 0.01));
958 CIlog(2, "\tdistance: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->totalDistance, test->totalPixels * 765, test->totalDistance / (test->totalPixels * 7.65));
959 }
960 }
961 }
962
963 HashTableEnumerate(&configTree, _unloadConfigTree, NULL);
964 HashTableDeinit(&configTree);
965 CInemaTestListDeinit(&tests);
966
967cleanup:
968 return status;
969}