src/platform/test/cinema-main.c (view raw)
1/* Copyright (c) 2013-2020 Jeffrey Pfau
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6#include <mgba/core/config.h>
7#include <mgba/core/core.h>
8#include <mgba/core/log.h>
9#include <mgba/core/version.h>
10#include <mgba/feature/commandline.h>
11#include <mgba/feature/video-logger.h>
12
13#include <mgba-util/png-io.h>
14#include <mgba-util/string.h>
15#include <mgba-util/table.h>
16#include <mgba-util/threading.h>
17#include <mgba-util/vector.h>
18#include <mgba-util/vfs.h>
19
20#ifdef USE_FFMPEG
21#include "feature/ffmpeg/ffmpeg-decoder.h"
22#include "feature/ffmpeg/ffmpeg-encoder.h"
23#endif
24
25#ifdef _MSC_VER
26#include <mgba-util/platform/windows/getopt.h>
27#else
28#include <getopt.h>
29#endif
30
31#include <stdlib.h>
32#include <sys/stat.h>
33#include <sys/types.h>
34
35#define MAX_TEST 200
36#define MAX_JOBS 128
37
38static const struct option longOpts[] = {
39 { "base", required_argument, 0, 'b' },
40 { "diffs", no_argument, 0, 'd' },
41 { "help", no_argument, 0, 'h' },
42 { "jobs", required_argument, 0, 'j' },
43 { "dry-run", no_argument, 0, 'n' },
44 { "outdir", required_argument, 0, 'o' },
45 { "quiet", no_argument, 0, 'q' },
46 { "rebaseline", no_argument, 0, 'r' },
47 { "verbose", no_argument, 0, 'v' },
48 { "version", no_argument, 0, '\0' },
49 { 0, 0, 0, 0 }
50};
51
52static const char shortOpts[] = "b:dhj:no:qrv";
53
54enum CInemaStatus {
55 CI_PASS,
56 CI_FAIL,
57 CI_XPASS,
58 CI_XFAIL,
59 CI_ERROR,
60 CI_SKIP
61};
62
63struct CInemaTest {
64 char directory[MAX_TEST];
65 char filename[MAX_TEST];
66 char name[MAX_TEST];
67 enum CInemaStatus status;
68 unsigned failedFrames;
69 uint64_t failedPixels;
70 unsigned totalFrames;
71 uint64_t totalDistance;
72 uint64_t totalPixels;
73};
74
75struct CInemaImage {
76 void* data;
77 unsigned width;
78 unsigned height;
79 unsigned stride;
80};
81
82DECLARE_VECTOR(CInemaTestList, struct CInemaTest)
83DEFINE_VECTOR(CInemaTestList, struct CInemaTest)
84
85DECLARE_VECTOR(ImageList, void*)
86DEFINE_VECTOR(ImageList, void*)
87
88static bool showVersion = false;
89static bool showUsage = false;
90static char base[PATH_MAX] = {0};
91static char outdir[PATH_MAX] = {'.'};
92static bool dryRun = false;
93static bool diffs = false;
94static bool rebaseline = false;
95static int verbosity = 0;
96
97static struct Table configTree;
98static Mutex configMutex;
99
100static int jobs = 1;
101static size_t jobIndex = 0;
102static Mutex jobMutex;
103static Thread jobThreads[MAX_JOBS];
104static int jobStatus;
105
106bool CInemaTestInit(struct CInemaTest*, const char* directory, const char* filename);
107void CInemaTestRun(struct CInemaTest*);
108
109bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* value);
110void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core);
111
112static void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args);
113
114ATTRIBUTE_FORMAT(printf, 2, 3) void CIlog(int minlevel, const char* format, ...) {
115 if (verbosity < minlevel) {
116 return;
117 }
118 va_list args;
119 va_start(args, format);
120 vprintf(format, args);
121 va_end(args);
122}
123
124ATTRIBUTE_FORMAT(printf, 2, 3) void CIerr(int minlevel, const char* format, ...) {
125 if (verbosity < minlevel) {
126 return;
127 }
128 va_list args;
129 va_start(args, format);
130 vfprintf(stderr, format, args);
131 va_end(args);
132}
133
134static bool parseCInemaArgs(int argc, char* const* argv) {
135 int ch;
136 int index = 0;
137 while ((ch = getopt_long(argc, argv, shortOpts, longOpts, &index)) != -1) {
138 const struct option* opt = &longOpts[index];
139 switch (ch) {
140 case '\0':
141 if (strcmp(opt->name, "version") == 0) {
142 showVersion = true;
143 } else {
144 return false;
145 }
146 break;
147 case 'b':
148 strlcpy(base, optarg, sizeof(base));
149 // TODO: Verify path exists
150 break;
151 case 'd':
152 diffs = true;
153 break;
154 case 'h':
155 showUsage = true;
156 break;
157 case 'j':
158 jobs = atoi(optarg);
159 break;
160 case 'n':
161 dryRun = true;
162 break;
163 case 'o':
164 strlcpy(outdir, optarg, sizeof(outdir));
165 // TODO: Make directory
166 break;
167 case 'q':
168 --verbosity;
169 break;
170 case 'r':
171 rebaseline = true;
172 break;
173 case 'v':
174 ++verbosity;
175 break;
176 default:
177 return false;
178 }
179 }
180
181 return true;
182}
183
184static void usageCInema(const char* arg0) {
185 printf("usage: %s [-dhnqv] [-b BASE] [-o DIR] [--version] [test...]\n", arg0);
186 puts(" -b, --base [BASE] Path to the CInema base directory");
187 puts(" -d, --diffs Output image diffs from failures");
188 puts(" -h, --help Print this usage and exit");
189 puts(" -n, --dry-run List all collected tests instead of running them");
190 puts(" -o, --output [DIR] Path to output applicable results");
191 puts(" -q, --quiet Decrease log verbosity (can be repeated)");
192 puts(" -r, --rebaseline Rewrite the baseline for failing tests");
193 puts(" -v, --verbose Increase log verbosity (can be repeated)");
194 puts(" --version Print version and exit");
195}
196
197static bool determineBase(int argc, char* const* argv) {
198 // TODO: Better dynamic detection
199 separatePath(__FILE__, base, NULL, NULL);
200 strncat(base, PATH_SEP ".." PATH_SEP ".." PATH_SEP ".." PATH_SEP "cinema", sizeof(base) - strlen(base) - 1);
201 return true;
202}
203
204static bool collectTests(struct CInemaTestList* tests, const char* path) {
205 CIerr(2, "Considering path %s\n", path);
206 struct VDir* dir = VDirOpen(path);
207 if (!dir) {
208 return false;
209 }
210 struct VDirEntry* entry = dir->listNext(dir);
211 while (entry) {
212 char subpath[PATH_MAX];
213 snprintf(subpath, sizeof(subpath), "%s" PATH_SEP "%s", path, entry->name(entry));
214 if (entry->type(entry) == VFS_DIRECTORY && strncmp(entry->name(entry), ".", 2) != 0 && strncmp(entry->name(entry), "..", 3) != 0) {
215 if (!collectTests(tests, subpath)) {
216 dir->close(dir);
217 return false;
218 }
219 } else if (entry->type(entry) == VFS_FILE && strncmp(entry->name(entry), "test.", 5) == 0) {
220 CIerr(3, "Found potential test %s\n", subpath);
221 struct VFile* vf = dir->openFile(dir, entry->name(entry), O_RDONLY);
222 if (vf) {
223 if (mCoreIsCompatible(vf) != PLATFORM_NONE || mVideoLogIsCompatible(vf) != PLATFORM_NONE) {
224 struct CInemaTest* test = CInemaTestListAppend(tests);
225 if (!CInemaTestInit(test, path, entry->name(entry))) {
226 CIerr(3, "Failed to create test\n");
227 CInemaTestListResize(tests, -1);
228 } else {
229 CIerr(2, "Found test %s\n", test->name);
230 }
231 } else {
232 CIerr(3, "Not a compatible file\n");
233 }
234 vf->close(vf);
235 } else {
236 CIerr(3, "Failed to open file\n");
237 }
238 }
239 entry = dir->listNext(dir);
240 }
241 dir->close(dir);
242 return true;
243}
244
245static int _compareNames(const void* a, const void* b) {
246 const struct CInemaTest* ta = a;
247 const struct CInemaTest* tb = b;
248
249 return strncmp(ta->name, tb->name, sizeof(ta->name));
250}
251
252static void reduceTestList(struct CInemaTestList* tests) {
253 qsort(CInemaTestListGetPointer(tests, 0), CInemaTestListSize(tests), sizeof(struct CInemaTest), _compareNames);
254
255 size_t i;
256 for (i = 1; i < CInemaTestListSize(tests);) {
257 struct CInemaTest* cur = CInemaTestListGetPointer(tests, i);
258 struct CInemaTest* prev = CInemaTestListGetPointer(tests, i - 1);
259 if (strncmp(cur->name, prev->name, sizeof(cur->name)) != 0) {
260 ++i;
261 continue;
262 }
263 CInemaTestListShift(tests, i, 1);
264 }
265}
266
267static void testToPath(const char* testName, char* path) {
268 strlcpy(path, base, PATH_MAX);
269
270 bool dotSeen = true;
271 size_t i;
272 for (i = strlen(path); testName[0] && i < PATH_MAX; ++testName) {
273 if (testName[0] == '.') {
274 dotSeen = true;
275 } else {
276 if (dotSeen) {
277 strlcpy(&path[i], PATH_SEP, PATH_MAX - i);
278 i += strlen(PATH_SEP);
279 dotSeen = false;
280 if (!i) {
281 break;
282 }
283 }
284 path[i] = testName[0];
285 ++i;
286 }
287 }
288 if (i == PATH_MAX) {
289 --i;
290 }
291 path[i] = '\0';
292}
293
294static void _loadConfigTree(struct Table* configTree, const char* testName) {
295 char key[MAX_TEST];
296 strlcpy(key, testName, sizeof(key));
297
298 struct mCoreConfig* config;
299 while (!(config = HashTableLookup(configTree, key))) {
300 char path[PATH_MAX];
301 config = malloc(sizeof(*config));
302 mCoreConfigInit(config, "cinema");
303 testToPath(key, path);
304 strncat(path, PATH_SEP, sizeof(path) - 1);
305 strncat(path, "config.ini", sizeof(path) - 1);
306 mCoreConfigLoadPath(config, path);
307 HashTableInsert(configTree, key, config);
308 char* pos = strrchr(key, '.');
309 if (pos) {
310 pos[0] = '\0';
311 } else if (key[0]) {
312 key[0] = '\0';
313 } else {
314 break;
315 }
316 }
317}
318
319static void _unloadConfigTree(const char* key, void* value, void* user) {
320 UNUSED(key);
321 UNUSED(user);
322 mCoreConfigDeinit(value);
323}
324
325static const char* _lookupValue(struct Table* configTree, const char* testName, const char* key) {
326 _loadConfigTree(configTree, testName);
327
328 char testKey[MAX_TEST];
329 strlcpy(testKey, testName, sizeof(testKey));
330
331 struct mCoreConfig* config;
332 while (true) {
333 config = HashTableLookup(configTree, testKey);
334 if (!config) {
335 continue;
336 }
337 const char* str = ConfigurationGetValue(&config->configTable, "testinfo", key);
338 if (str) {
339 return str;
340 }
341 char* pos = strrchr(testKey, '.');
342 if (pos) {
343 pos[0] = '\0';
344 } else if (testKey[0]) {
345 testKey[0] = '\0';
346 } else {
347 break;
348 }
349 }
350 return NULL;
351}
352
353bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* out) {
354 const char* charValue = _lookupValue(configTree, testName, key);
355 if (!charValue) {
356 return false;
357 }
358 char* end;
359 unsigned long value = strtoul(charValue, &end, 10);
360 if (*end) {
361 return false;
362 }
363 *out = value;
364 return true;
365}
366
367void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core) {
368 _loadConfigTree(configTree, testName);
369
370 char testKey[MAX_TEST] = {0};
371 char* keyEnd = testKey;
372 const char* pos;
373 while (true) {
374 pos = strchr(testName, '.');
375 size_t maxlen = sizeof(testKey) - (keyEnd - testKey) - 1;
376 size_t len;
377 if (pos) {
378 len = pos - testName;
379 } else {
380 len = strlen(testName);
381 }
382 if (len > maxlen) {
383 len = maxlen;
384 }
385 strncpy(keyEnd, testName, len);
386 keyEnd += len;
387
388 struct mCoreConfig* config = HashTableLookup(configTree, testKey);
389 if (config) {
390 core->loadConfig(core, config);
391 }
392 if (!pos) {
393 break;
394 }
395 testName = pos + 1;
396 keyEnd[0] = '.';
397 ++keyEnd;
398 }
399}
400
401bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* filename) {
402 if (strncmp(base, directory, strlen(base)) != 0) {
403 return false;
404 }
405 memset(test, 0, sizeof(*test));
406 strlcpy(test->directory, directory, sizeof(test->directory));
407 strlcpy(test->filename, filename, sizeof(test->filename));
408 directory += strlen(base) + 1;
409 strlcpy(test->name, directory, sizeof(test->name));
410 char* str = strstr(test->name, PATH_SEP);
411 while (str) {
412 str[0] = '.';
413 str = strstr(str, PATH_SEP);
414 }
415 return true;
416}
417
418static bool _loadBaselinePNG(struct VDir* dir, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) {
419 char baselineName[32];
420 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
421 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_RDONLY);
422 if (!baselineVF) {
423 if (*status == CI_PASS) {
424 *status = CI_FAIL;
425 }
426 return false;
427 }
428
429 png_structp png = PNGReadOpen(baselineVF, 0);
430 png_infop info = png_create_info_struct(png);
431 png_infop end = png_create_info_struct(png);
432 if (!png || !info || !end || !PNGReadHeader(png, info)) {
433 PNGReadClose(png, info, end);
434 baselineVF->close(baselineVF);
435 CIerr(1, "Failed to load %s\n", baselineName);
436 *status = CI_ERROR;
437 return false;
438 }
439
440 unsigned pwidth = png_get_image_width(png, info);
441 unsigned pheight = png_get_image_height(png, info);
442 if (pheight != image->height || pwidth != image->width) {
443 PNGReadClose(png, info, end);
444 baselineVF->close(baselineVF);
445 CIlog(1, "Size mismatch for %s, expected %ux%u, got %ux%u\n", baselineName, pwidth, pheight, image->width, image->height);
446 if (*status == CI_PASS) {
447 *status = CI_FAIL;
448 }
449 return false;
450 }
451
452 image->data = malloc(pwidth * pheight * BYTES_PER_PIXEL);
453 if (!image->data) {
454 CIerr(1, "Failed to allocate baseline buffer\n");
455 *status = CI_ERROR;
456 PNGReadClose(png, info, end);
457 baselineVF->close(baselineVF);
458 return false;
459 }
460 if (!PNGReadPixels(png, info, image->data, pwidth, pheight, pwidth) || !PNGReadFooter(png, end)) {
461 CIerr(1, "Failed to read %s\n", baselineName);
462 *status = CI_ERROR;
463 free(image->data);
464 return false;
465 }
466 PNGReadClose(png, info, end);
467 baselineVF->close(baselineVF);
468 image->stride = pwidth;
469 return true;
470}
471
472#ifdef USE_FFMPEG
473struct CInemaStream {
474 struct mAVStream d;
475 struct CInemaImage* image;
476 enum CInemaStatus* status;
477};
478
479static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) {
480 struct CInemaStream* cistream = (struct CInemaStream*) stream;
481 if (height != cistream->image->height || width != cistream->image->width) {
482 CIlog(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height);
483 if (*cistream->status == CI_PASS) {
484 *cistream->status = CI_FAIL;
485 }
486 }
487}
488
489static void _cinemaVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) {
490 struct CInemaStream* cistream = (struct CInemaStream*) stream;
491 cistream->image->stride = stride;
492 size_t bufferSize = cistream->image->stride * cistream->image->height * BYTES_PER_PIXEL;
493 cistream->image->data = malloc(bufferSize);
494 memcpy(cistream->image->data, pixels, bufferSize);
495}
496#endif
497
498static struct VDir* _makeOutDir(const char* testName) {
499 char path[PATH_MAX] = {0};
500 strlcpy(path, outdir, sizeof(path));
501 char* pathEnd = path + strlen(path);
502 const char* pos;
503 while (true) {
504 pathEnd[0] = PATH_SEP[0];
505 ++pathEnd;
506 pos = strchr(testName, '.');
507 size_t maxlen = sizeof(path) - (pathEnd - path) - 1;
508 size_t len;
509 if (pos) {
510 len = pos - testName;
511 } else {
512 len = strlen(testName);
513 }
514 if (len > maxlen) {
515 len = maxlen;
516 }
517 strncpy(pathEnd, testName, len);
518 pathEnd += len;
519
520 mkdir(path, 0777);
521
522 if (!pos) {
523 break;
524 }
525 testName = pos + 1;
526 }
527 return VDirOpen(path);
528}
529
530static void _writeImage(struct VFile* vf, const struct CInemaImage* image) {
531 png_structp png = PNGWriteOpen(vf);
532 png_infop info = PNGWriteHeader(png, image->width, image->height);
533 if (!PNGWritePixels(png, image->width, image->height, image->stride, image->data)) {
534 CIerr(0, "Could not write output image\n");
535 }
536 PNGWriteClose(png, info);
537
538 vf->close(vf);
539}
540
541static void _writeDiff(const char* testName, const struct CInemaImage* image, size_t frame, const char* type) {
542 struct VDir* dir = _makeOutDir(testName);
543 if (!dir) {
544 CIerr(0, "Could not open directory for %s\n", testName);
545 return;
546 }
547 char name[32];
548 snprintf(name, sizeof(name), "%s_%04" PRIz "u.png", type, frame);
549 struct VFile* vf = dir->openFile(dir, name, O_CREAT | O_TRUNC | O_WRONLY);
550 if (!vf) {
551 CIerr(0, "Could not open output file %s\n", name);
552 dir->close(dir);
553 return;
554 }
555 _writeImage(vf, image);
556 dir->close(dir);
557}
558
559static void _writeBaseline(struct VDir* dir, const struct CInemaImage* image, size_t frame) {
560 char baselineName[32];
561 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
562 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_CREAT | O_TRUNC | O_WRONLY);
563 if (baselineVF) {
564 _writeImage(baselineVF, image);
565 } else {
566 CIerr(0, "Could not open output file %s\n", baselineName);
567 }
568}
569
570void CInemaTestRun(struct CInemaTest* test) {
571 unsigned ignore = 0;
572 MutexLock(&configMutex);
573 CInemaConfigGetUInt(&configTree, test->name, "ignore", &ignore);
574 MutexUnlock(&configMutex);
575 if (ignore) {
576 test->status = CI_SKIP;
577 return;
578 }
579
580 struct VDir* dir = VDirOpen(test->directory);
581 if (!dir) {
582 CIerr(0, "Failed to open test directory\n");
583 test->status = CI_ERROR;
584 return;
585 }
586 struct VFile* rom = dir->openFile(dir, test->filename, O_RDONLY);
587 if (!rom) {
588 CIerr(0, "Failed to open test\n");
589 test->status = CI_ERROR;
590 return;
591 }
592 struct mCore* core = mCoreFindVF(rom);
593 if (!core) {
594 CIerr(0, "Failed to load test\n");
595 test->status = CI_ERROR;
596 rom->close(rom);
597 return;
598 }
599 if (!core->init(core)) {
600 CIerr(0, "Failed to init test\n");
601 test->status = CI_ERROR;
602 core->deinit(core);
603 return;
604 }
605 struct CInemaImage image;
606 core->desiredVideoDimensions(core, &image.width, &image.height);
607 ssize_t bufferSize = image.width * image.height * BYTES_PER_PIXEL;
608 image.data = malloc(bufferSize);
609 image.stride = image.width;
610 if (!image.data) {
611 CIerr(0, "Failed to allocate video buffer\n");
612 test->status = CI_ERROR;
613 core->deinit(core);
614 }
615 core->setVideoBuffer(core, image.data, image.stride);
616 mCoreConfigInit(&core->config, "cinema");
617
618 unsigned limit = 9999;
619 unsigned skip = 0;
620 unsigned fail = 0;
621 unsigned video = 0;
622
623 MutexLock(&configMutex);
624 CInemaConfigGetUInt(&configTree, test->name, "frames", &limit);
625 CInemaConfigGetUInt(&configTree, test->name, "skip", &skip);
626 CInemaConfigGetUInt(&configTree, test->name, "fail", &fail);
627 CInemaConfigGetUInt(&configTree, test->name, "video", &video);
628 CInemaConfigLoad(&configTree, test->name, core);
629 MutexUnlock(&configMutex);
630
631 struct VFile* save = VFileMemChunk(NULL, 0);
632 core->loadROM(core, rom);
633 if (!core->loadSave(core, save)) {
634 save->close(save);
635 }
636 core->rtc.override = RTC_FAKE_EPOCH;
637 core->rtc.value = 1200000000;
638 core->reset(core);
639
640 test->status = CI_PASS;
641
642 unsigned minFrame = core->frameCounter(core);
643 size_t frame;
644 for (frame = 0; frame < skip; ++frame) {
645 core->runFrame(core);
646 }
647 core->desiredVideoDimensions(core, &image.width, &image.height);
648
649#ifdef USE_FFMPEG
650 struct FFmpegDecoder decoder;
651 struct FFmpegEncoder encoder;
652 struct CInemaStream stream = {0};
653 if (video) {
654 char fname[PATH_MAX];
655 snprintf(fname, sizeof(fname), "%s" PATH_SEP "baseline.mkv", test->directory);
656 if (rebaseline) {
657 FFmpegEncoderInit(&encoder);
658 FFmpegEncoderSetAudio(&encoder, NULL, 0);
659 FFmpegEncoderSetVideo(&encoder, "png", 0, 0);
660 FFmpegEncoderSetContainer(&encoder, "mkv");
661 FFmpegEncoderSetDimensions(&encoder, image.width, image.height);
662 if (!FFmpegEncoderOpen(&encoder, fname)) {
663 CIerr(1, "Failed to save baseline video\n");
664 } else {
665 core->setAVStream(core, &encoder.d);
666 }
667 } else {
668 FFmpegDecoderInit(&decoder);
669 stream.d.postVideoFrame = _cinemaVideoFrame;
670 stream.d.videoDimensionsChanged = _cinemaDimensionsChanged;
671 stream.status = &test->status;
672 decoder.out = &stream.d;
673
674 if (!FFmpegDecoderOpen(&decoder, fname)) {
675 CIerr(1, "Failed to load baseline video\n");
676 }
677 }
678 }
679#else
680 if (video) {
681 CIerr(0, "Failed to run video test without ffmpeg linked in\n");
682 test->status = CI_ERROR;
683 }
684#endif
685
686 for (frame = 0; limit; ++frame, --limit) {
687 core->runFrame(core);
688 ++test->totalFrames;
689 unsigned frameCounter = core->frameCounter(core);
690 if (frameCounter <= minFrame) {
691 break;
692 }
693 CIlog(3, "Test frame: %u\n", frameCounter);
694 core->desiredVideoDimensions(core, &image.width, &image.height);
695 uint8_t* diff = NULL;
696 struct CInemaImage expected = {
697 .data = NULL,
698 .width = image.width,
699 .height = image.height,
700 .stride = image.width,
701 };
702 bool baselineFound;
703 if (video) {
704 baselineFound = false;
705#ifdef USE_FFMPEG
706 if (!rebaseline && FFmpegDecoderIsOpen(&decoder)) {
707 stream.image = &expected;
708 while (!expected.data) {
709 if (!FFmpegDecoderRead(&decoder)) {
710 CIlog(1, "Failed to read more frames. EOF?\n");
711 test->status = CI_FAIL;
712 break;
713 }
714 }
715 baselineFound = expected.data;
716 }
717#endif
718 } else {
719 baselineFound = _loadBaselinePNG(dir, &expected, frame, &test->status);
720 }
721 if (baselineFound) {
722 uint8_t* testPixels = image.data;
723 uint8_t* expectPixels = expected.data;
724 size_t x;
725 size_t y;
726 int max = 0;
727 bool failed = false;
728 for (y = 0; y < image.height; ++y) {
729 for (x = 0; x < image.width; ++x) {
730 size_t pix = expected.stride * y + x;
731 size_t tpix = image.stride * y + x;
732 int testR = testPixels[tpix * 4 + 0];
733 int testG = testPixels[tpix * 4 + 1];
734 int testB = testPixels[tpix * 4 + 2];
735 int expectR = expectPixels[pix * 4 + 0];
736 int expectG = expectPixels[pix * 4 + 1];
737 int expectB = expectPixels[pix * 4 + 2];
738 int r = expectR - testR;
739 int g = expectG - testG;
740 int b = expectB - testB;
741 if (r | g | b) {
742 failed = true;
743 if (diffs && !diff) {
744 diff = calloc(expected.stride * expected.height, BYTES_PER_PIXEL);
745 }
746 CIlog(3, "Frame %u failed at pixel %" PRIz "ux%" PRIz "u with diff %i,%i,%i (expected %02x%02x%02x, got %02x%02x%02x)\n",
747 frameCounter, x, y, r, g, b,
748 expectR, expectG, expectB,
749 testR, testG, testB);
750 test->status = CI_FAIL;
751 if (r < 0) {
752 r = -r;
753 }
754 if (g < 0) {
755 g = -g;
756 }
757 if (b < 0) {
758 b = -b;
759 }
760
761 if (diff) {
762 if (r > max) {
763 max = r;
764 }
765 if (g > max) {
766 max = g;
767 }
768 if (b > max) {
769 max = b;
770 }
771 diff[pix * 4 + 0] = r;
772 diff[pix * 4 + 1] = g;
773 diff[pix * 4 + 2] = b;
774 }
775
776 test->totalDistance += r + g + b;
777 ++test->failedPixels;
778 }
779 }
780 }
781 if (failed) {
782 ++test->failedFrames;
783 }
784 test->totalPixels += image.height * image.width;
785 if (rebaseline && failed) {
786 _writeBaseline(dir, &image, frame);
787 }
788 if (diff) {
789 if (failed) {
790 struct CInemaImage outdiff = {
791 .data = diff,
792 .width = expected.width,
793 .height = expected.height,
794 .stride = expected.stride,
795 };
796
797 _writeDiff(test->name, &image, frame, "result");
798 _writeDiff(test->name, &expected, frame, "expected");
799 _writeDiff(test->name, &outdiff, frame, "diff");
800
801 for (y = 0; y < outdiff.height; ++y) {
802 for (x = 0; x < outdiff.width; ++x) {
803 size_t pix = outdiff.stride * y + x;
804 diff[pix * 4 + 0] = diff[pix * 4 + 0] * 255 / max;
805 diff[pix * 4 + 1] = diff[pix * 4 + 1] * 255 / max;
806 diff[pix * 4 + 2] = diff[pix * 4 + 2] * 255 / max;
807 }
808 }
809 _writeDiff(test->name, &outdiff, frame, "normalized");
810 }
811 free(diff);
812 }
813 free(expected.data);
814 } else if (test->status == CI_ERROR) {
815 break;
816 } else if (rebaseline && !video) {
817 _writeBaseline(dir, &image, frame);
818 } else if (!rebaseline) {
819 test->status = CI_FAIL;
820 }
821 }
822
823 if (fail) {
824 if (test->status == CI_FAIL) {
825 test->status = CI_XFAIL;
826 } else if (test->status == CI_PASS) {
827 test->status = CI_XPASS;
828 }
829 }
830
831#ifdef USE_FFMPEG
832 if (video) {
833 if (rebaseline) {
834 FFmpegEncoderClose(&encoder);
835 } else {
836 FFmpegDecoderClose(&decoder);
837 }
838 }
839#endif
840
841 free(image.data);
842 mCoreConfigDeinit(&core->config);
843 core->deinit(core);
844 dir->close(dir);
845}
846
847static bool CInemaTask(struct CInemaTestList* tests, size_t i) {
848 bool success = true;
849 struct CInemaTest* test = CInemaTestListGetPointer(tests, i);
850 if (dryRun) {
851 CIlog(-1, "%s\n", test->name);
852 } else {
853 CIlog(1, "%s: ", test->name);
854 fflush(stdout);
855 CInemaTestRun(test);
856 switch (test->status) {
857 case CI_PASS:
858 CIlog(1, "pass\n");
859 break;
860 case CI_FAIL:
861 success = false;
862 CIlog(1, "fail\n");
863 break;
864 case CI_XPASS:
865 CIlog(1, "xpass\n");
866 break;
867 case CI_XFAIL:
868 CIlog(1, "xfail\n");
869 break;
870 case CI_SKIP:
871 CIlog(1, "skip\n");
872 break;
873 case CI_ERROR:
874 success = false;
875 CIlog(1, "error\n");
876 break;
877 }
878 if (test->failedFrames) {
879 CIlog(2, "\tfailed frames: %u/%u (%1.3g%%)\n", test->failedFrames, test->totalFrames, test->failedFrames / (test->totalFrames * 0.01));
880 CIlog(2, "\tfailed pixels: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->failedPixels, test->totalPixels, test->failedPixels / (test->totalPixels * 0.01));
881 CIlog(2, "\tdistance: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->totalDistance, test->totalPixels * 765, test->totalDistance / (test->totalPixels * 7.65));
882 }
883 }
884 return success;
885}
886
887static THREAD_ENTRY CInemaJob(void* context) {
888 struct CInemaTestList* tests = context;
889 bool success = true;
890 while (true) {
891 size_t i;
892 MutexLock(&jobMutex);
893 i = jobIndex;
894 ++jobIndex;
895 MutexUnlock(&jobMutex);
896 if (i >= CInemaTestListSize(tests)) {
897 break;
898 }
899 if (!CInemaTask(tests, i)) {
900 success = false;
901 }
902 }
903 MutexLock(&jobMutex);
904 if (!success) {
905 jobStatus = 1;
906 }
907 MutexUnlock(&jobMutex);
908}
909
910void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args) {
911 UNUSED(log);
912 if (verbosity < 0) {
913 return;
914 }
915 int mask = mLOG_FATAL;
916 if (verbosity >= 1) {
917 mask |= mLOG_ERROR;
918 }
919 if (verbosity >= 2) {
920 mask |= mLOG_WARN;
921 }
922 if (verbosity >= 4) {
923 mask |= mLOG_INFO;
924 }
925 if (verbosity >= 5) {
926 mask |= mLOG_ALL;
927 }
928 if (!(mask & level)) {
929 return;
930 }
931
932 char buffer[256];
933 vsnprintf(buffer, sizeof(buffer), format, args);
934 CIerr(0, "[%s] %s\n", mLogCategoryName(category), buffer);
935}
936
937int main(int argc, char** argv) {
938 int status = 0;
939 if (!parseCInemaArgs(argc, argv)) {
940 status = 1;
941 goto cleanup;
942 }
943
944 if (showVersion) {
945 version(argv[0]);
946 goto cleanup;
947 }
948
949 if (showUsage) {
950 usageCInema(argv[0]);
951 goto cleanup;
952 }
953
954 argc -= optind;
955 argv += optind;
956
957 if (!base[0] && !determineBase(argc, argv)) {
958 CIlog(0, "Could not determine CInema test base. Please specify manually.");
959 status = 1;
960 goto cleanup;
961 }
962#ifndef _WIN32
963 char* rbase = realpath(base, NULL);
964 if (rbase) {
965 strlcpy(base, rbase, sizeof(base));
966 free(rbase);
967 }
968#endif
969
970 struct CInemaTestList tests;
971 CInemaTestListInit(&tests, 0);
972
973 struct mLogger logger = { .log = _log };
974 mLogSetDefaultLogger(&logger);
975#ifdef USE_FFMPEG
976 if (verbosity < 2) {
977 av_log_set_level(AV_LOG_ERROR);
978 }
979#endif
980
981 if (argc > 0) {
982 size_t i;
983 for (i = 0; i < (size_t) argc; ++i) {
984 char path[PATH_MAX + 1] = {0};
985 testToPath(argv[i], path);
986
987 if (!collectTests(&tests, path)) {
988 status = 1;
989 break;
990 }
991 }
992 } else if (!collectTests(&tests, base)) {
993 status = 1;
994 }
995
996 if (CInemaTestListSize(&tests) == 0) {
997 CIlog(1, "No tests found.");
998 status = 1;
999 } else {
1000 reduceTestList(&tests);
1001 }
1002
1003 HashTableInit(&configTree, 0, free);
1004 MutexInit(&configMutex);
1005
1006 if (jobs == 1) {
1007 size_t i;
1008 for (i = 0; i < CInemaTestListSize(&tests); ++i) {
1009 bool success = CInemaTask(&tests, i);
1010 if (!success) {
1011 status = 1;
1012 }
1013 }
1014 } else {
1015 MutexInit(&jobMutex);
1016 int i;
1017 for (i = 0; i < jobs; ++i) {
1018 ThreadCreate(&jobThreads[i], CInemaJob, &tests);
1019 }
1020 for (i = 0; i < jobs; ++i) {
1021 ThreadJoin(&jobThreads[i]);
1022 }
1023 MutexDeinit(&jobMutex);
1024 status = jobStatus;
1025 }
1026
1027 MutexDeinit(&configMutex);
1028 HashTableEnumerate(&configTree, _unloadConfigTree, NULL);
1029 HashTableDeinit(&configTree);
1030 CInemaTestListDeinit(&tests);
1031
1032cleanup:
1033 return status;
1034}