src/platform/test/cinema-main.c (view raw)
1/* Copyright (c) 2013-2020 Jeffrey Pfau
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6#include <mgba/core/config.h>
7#include <mgba/core/core.h>
8#include <mgba/core/log.h>
9#include <mgba/core/version.h>
10#include <mgba/feature/commandline.h>
11#include <mgba/feature/video-logger.h>
12
13#include <mgba-util/png-io.h>
14#include <mgba-util/string.h>
15#include <mgba-util/table.h>
16#include <mgba-util/threading.h>
17#include <mgba-util/vector.h>
18#include <mgba-util/vfs.h>
19
20#ifdef USE_FFMPEG
21#include "feature/ffmpeg/ffmpeg-decoder.h"
22#include "feature/ffmpeg/ffmpeg-encoder.h"
23#endif
24
25#ifdef _MSC_VER
26#include <mgba-util/platform/windows/getopt.h>
27#else
28#include <getopt.h>
29#endif
30
31#include <stdlib.h>
32#include <sys/stat.h>
33#include <sys/types.h>
34
35#define MAX_TEST 200
36#define MAX_JOBS 128
37
38static const struct option longOpts[] = {
39 { "base", required_argument, 0, 'b' },
40 { "diffs", no_argument, 0, 'd' },
41 { "help", no_argument, 0, 'h' },
42 { "jobs", required_argument, 0, 'j' },
43 { "dry-run", no_argument, 0, 'n' },
44 { "outdir", required_argument, 0, 'o' },
45 { "quiet", no_argument, 0, 'q' },
46 { "rebaseline", no_argument, 0, 'r' },
47 { "verbose", no_argument, 0, 'v' },
48 { "version", no_argument, 0, '\0' },
49 { 0, 0, 0, 0 }
50};
51
52static const char shortOpts[] = "b:dhj:no:qrv";
53
54enum CInemaStatus {
55 CI_PASS,
56 CI_FAIL,
57 CI_XPASS,
58 CI_XFAIL,
59 CI_ERROR,
60 CI_SKIP
61};
62
63struct CInemaTest {
64 char directory[MAX_TEST];
65 char filename[MAX_TEST];
66 char name[MAX_TEST];
67 enum CInemaStatus status;
68 unsigned failedFrames;
69 uint64_t failedPixels;
70 unsigned totalFrames;
71 uint64_t totalDistance;
72 uint64_t totalPixels;
73};
74
75struct CInemaImage {
76 void* data;
77 unsigned width;
78 unsigned height;
79 unsigned stride;
80};
81
82DECLARE_VECTOR(CInemaTestList, struct CInemaTest)
83DEFINE_VECTOR(CInemaTestList, struct CInemaTest)
84
85DECLARE_VECTOR(ImageList, void*)
86DEFINE_VECTOR(ImageList, void*)
87
88struct StringBuilder {
89 struct StringList out;
90 struct StringList err;
91};
92
93static bool showVersion = false;
94static bool showUsage = false;
95static char base[PATH_MAX] = {0};
96static char outdir[PATH_MAX] = {'.'};
97static bool dryRun = false;
98static bool diffs = false;
99static bool rebaseline = false;
100static int verbosity = 0;
101
102static struct Table configTree;
103static Mutex configMutex;
104
105static int jobs = 1;
106static size_t jobIndex = 0;
107static Mutex jobMutex;
108static Thread jobThreads[MAX_JOBS];
109static int jobStatus;
110static ThreadLocal stringBuilder;
111
112bool CInemaTestInit(struct CInemaTest*, const char* directory, const char* filename);
113void CInemaTestRun(struct CInemaTest*);
114
115bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* value);
116void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core);
117
118static void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args);
119
120ATTRIBUTE_FORMAT(printf, 2, 3) void CIlog(int minlevel, const char* format, ...) {
121 if (verbosity < minlevel) {
122 return;
123 }
124 va_list args;
125 va_start(args, format);
126#ifdef HAVE_VASPRINTF
127 struct StringBuilder* builder = ThreadLocalGetValue(stringBuilder);
128 if (!builder) {
129 vprintf(format, args);
130 } else {
131 vasprintf(StringListAppend(&builder->out), format, args);
132 }
133#else
134 vprintf(format, args);
135#endif
136 va_end(args);
137}
138
139ATTRIBUTE_FORMAT(printf, 2, 3) void CIerr(int minlevel, const char* format, ...) {
140 if (verbosity < minlevel) {
141 return;
142 }
143 va_list args;
144 va_start(args, format);
145#ifdef HAVE_VASPRINTF
146 struct StringBuilder* builder = ThreadLocalGetValue(stringBuilder);
147 if (!builder) {
148 vfprintf(stderr, format, args);
149 } else {
150 vasprintf(StringListAppend(&builder->err), format, args);
151 }
152#else
153 vfprintf(stderr, format, args);
154#endif
155 va_end(args);
156}
157
158void CIflush(struct StringList* list, FILE* out) {
159 size_t len = 0;
160 size_t i;
161 for (i = 0; i < StringListSize(list); ++i) {
162 len += strlen(*StringListGetPointer(list, i));
163 }
164 char* string = calloc(len + 1, sizeof(char));
165 char* cur = string;
166 for (i = 0; i < StringListSize(list); ++i) {
167 char* brick = *StringListGetPointer(list, i);
168 size_t portion = strlen(brick);
169 memcpy(cur, brick, portion);
170 free(brick);
171 cur += portion;
172 }
173 fputs(string, out);
174 free(string);
175 StringListClear(list);
176}
177
178static bool parseCInemaArgs(int argc, char* const* argv) {
179 int ch;
180 int index = 0;
181 while ((ch = getopt_long(argc, argv, shortOpts, longOpts, &index)) != -1) {
182 const struct option* opt = &longOpts[index];
183 switch (ch) {
184 case '\0':
185 if (strcmp(opt->name, "version") == 0) {
186 showVersion = true;
187 } else {
188 return false;
189 }
190 break;
191 case 'b':
192 strlcpy(base, optarg, sizeof(base));
193 // TODO: Verify path exists
194 break;
195 case 'd':
196 diffs = true;
197 break;
198 case 'h':
199 showUsage = true;
200 break;
201 case 'j':
202 jobs = atoi(optarg);
203 break;
204 case 'n':
205 dryRun = true;
206 break;
207 case 'o':
208 strlcpy(outdir, optarg, sizeof(outdir));
209 // TODO: Make directory
210 break;
211 case 'q':
212 --verbosity;
213 break;
214 case 'r':
215 rebaseline = true;
216 break;
217 case 'v':
218 ++verbosity;
219 break;
220 default:
221 return false;
222 }
223 }
224
225 return true;
226}
227
228static void usageCInema(const char* arg0) {
229 printf("usage: %s [-dhnqv] [-b BASE] [-o DIR] [--version] [test...]\n", arg0);
230 puts(" -b, --base [BASE] Path to the CInema base directory");
231 puts(" -d, --diffs Output image diffs from failures");
232 puts(" -h, --help Print this usage and exit");
233 puts(" -n, --dry-run List all collected tests instead of running them");
234 puts(" -o, --output [DIR] Path to output applicable results");
235 puts(" -q, --quiet Decrease log verbosity (can be repeated)");
236 puts(" -r, --rebaseline Rewrite the baseline for failing tests");
237 puts(" -v, --verbose Increase log verbosity (can be repeated)");
238 puts(" --version Print version and exit");
239}
240
241static bool determineBase(int argc, char* const* argv) {
242 // TODO: Better dynamic detection
243 separatePath(__FILE__, base, NULL, NULL);
244 strncat(base, PATH_SEP ".." PATH_SEP ".." PATH_SEP ".." PATH_SEP "cinema", sizeof(base) - strlen(base) - 1);
245 return true;
246}
247
248static bool collectTests(struct CInemaTestList* tests, const char* path) {
249 CIerr(2, "Considering path %s\n", path);
250 struct VDir* dir = VDirOpen(path);
251 if (!dir) {
252 return false;
253 }
254 struct VDirEntry* entry = dir->listNext(dir);
255 while (entry) {
256 char subpath[PATH_MAX];
257 snprintf(subpath, sizeof(subpath), "%s" PATH_SEP "%s", path, entry->name(entry));
258 if (entry->type(entry) == VFS_DIRECTORY && strncmp(entry->name(entry), ".", 2) != 0 && strncmp(entry->name(entry), "..", 3) != 0) {
259 if (!collectTests(tests, subpath)) {
260 dir->close(dir);
261 return false;
262 }
263 } else if (entry->type(entry) == VFS_FILE && strncmp(entry->name(entry), "test.", 5) == 0) {
264 CIerr(3, "Found potential test %s\n", subpath);
265 struct VFile* vf = dir->openFile(dir, entry->name(entry), O_RDONLY);
266 if (vf) {
267 if (mCoreIsCompatible(vf) != PLATFORM_NONE || mVideoLogIsCompatible(vf) != PLATFORM_NONE) {
268 struct CInemaTest* test = CInemaTestListAppend(tests);
269 if (!CInemaTestInit(test, path, entry->name(entry))) {
270 CIerr(3, "Failed to create test\n");
271 CInemaTestListResize(tests, -1);
272 } else {
273 CIerr(2, "Found test %s\n", test->name);
274 }
275 } else {
276 CIerr(3, "Not a compatible file\n");
277 }
278 vf->close(vf);
279 } else {
280 CIerr(3, "Failed to open file\n");
281 }
282 }
283 entry = dir->listNext(dir);
284 }
285 dir->close(dir);
286 return true;
287}
288
289static int _compareNames(const void* a, const void* b) {
290 const struct CInemaTest* ta = a;
291 const struct CInemaTest* tb = b;
292
293 return strncmp(ta->name, tb->name, sizeof(ta->name));
294}
295
296static void reduceTestList(struct CInemaTestList* tests) {
297 qsort(CInemaTestListGetPointer(tests, 0), CInemaTestListSize(tests), sizeof(struct CInemaTest), _compareNames);
298
299 size_t i;
300 for (i = 1; i < CInemaTestListSize(tests);) {
301 struct CInemaTest* cur = CInemaTestListGetPointer(tests, i);
302 struct CInemaTest* prev = CInemaTestListGetPointer(tests, i - 1);
303 if (strncmp(cur->name, prev->name, sizeof(cur->name)) != 0) {
304 ++i;
305 continue;
306 }
307 CInemaTestListShift(tests, i, 1);
308 }
309}
310
311static void testToPath(const char* testName, char* path) {
312 strlcpy(path, base, PATH_MAX);
313
314 bool dotSeen = true;
315 size_t i;
316 for (i = strlen(path); testName[0] && i < PATH_MAX; ++testName) {
317 if (testName[0] == '.') {
318 dotSeen = true;
319 } else {
320 if (dotSeen) {
321 strlcpy(&path[i], PATH_SEP, PATH_MAX - i);
322 i += strlen(PATH_SEP);
323 dotSeen = false;
324 if (!i) {
325 break;
326 }
327 }
328 path[i] = testName[0];
329 ++i;
330 }
331 }
332 if (i == PATH_MAX) {
333 --i;
334 }
335 path[i] = '\0';
336}
337
338static void _loadConfigTree(struct Table* configTree, const char* testName) {
339 char key[MAX_TEST];
340 strlcpy(key, testName, sizeof(key));
341
342 struct mCoreConfig* config;
343 while (!(config = HashTableLookup(configTree, key))) {
344 char path[PATH_MAX];
345 config = malloc(sizeof(*config));
346 mCoreConfigInit(config, "cinema");
347 testToPath(key, path);
348 strncat(path, PATH_SEP, sizeof(path) - 1);
349 strncat(path, "config.ini", sizeof(path) - 1);
350 mCoreConfigLoadPath(config, path);
351 HashTableInsert(configTree, key, config);
352 char* pos = strrchr(key, '.');
353 if (pos) {
354 pos[0] = '\0';
355 } else if (key[0]) {
356 key[0] = '\0';
357 } else {
358 break;
359 }
360 }
361}
362
363static void _unloadConfigTree(const char* key, void* value, void* user) {
364 UNUSED(key);
365 UNUSED(user);
366 mCoreConfigDeinit(value);
367}
368
369static const char* _lookupValue(struct Table* configTree, const char* testName, const char* key) {
370 _loadConfigTree(configTree, testName);
371
372 char testKey[MAX_TEST];
373 strlcpy(testKey, testName, sizeof(testKey));
374
375 struct mCoreConfig* config;
376 while (true) {
377 config = HashTableLookup(configTree, testKey);
378 if (!config) {
379 continue;
380 }
381 const char* str = ConfigurationGetValue(&config->configTable, "testinfo", key);
382 if (str) {
383 return str;
384 }
385 char* pos = strrchr(testKey, '.');
386 if (pos) {
387 pos[0] = '\0';
388 } else if (testKey[0]) {
389 testKey[0] = '\0';
390 } else {
391 break;
392 }
393 }
394 return NULL;
395}
396
397bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* out) {
398 const char* charValue = _lookupValue(configTree, testName, key);
399 if (!charValue) {
400 return false;
401 }
402 char* end;
403 unsigned long value = strtoul(charValue, &end, 10);
404 if (*end) {
405 return false;
406 }
407 *out = value;
408 return true;
409}
410
411void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core) {
412 _loadConfigTree(configTree, testName);
413
414 char testKey[MAX_TEST] = {0};
415 char* keyEnd = testKey;
416 const char* pos;
417 while (true) {
418 pos = strchr(testName, '.');
419 size_t maxlen = sizeof(testKey) - (keyEnd - testKey) - 1;
420 size_t len;
421 if (pos) {
422 len = pos - testName;
423 } else {
424 len = strlen(testName);
425 }
426 if (len > maxlen) {
427 len = maxlen;
428 }
429 strncpy(keyEnd, testName, len);
430 keyEnd += len;
431
432 struct mCoreConfig* config = HashTableLookup(configTree, testKey);
433 if (config) {
434 core->loadConfig(core, config);
435 }
436 if (!pos) {
437 break;
438 }
439 testName = pos + 1;
440 keyEnd[0] = '.';
441 ++keyEnd;
442 }
443}
444
445bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* filename) {
446 if (strncmp(base, directory, strlen(base)) != 0) {
447 return false;
448 }
449 memset(test, 0, sizeof(*test));
450 strlcpy(test->directory, directory, sizeof(test->directory));
451 strlcpy(test->filename, filename, sizeof(test->filename));
452 directory += strlen(base) + 1;
453 strlcpy(test->name, directory, sizeof(test->name));
454 char* str = strstr(test->name, PATH_SEP);
455 while (str) {
456 str[0] = '.';
457 str = strstr(str, PATH_SEP);
458 }
459 return true;
460}
461
462static bool _loadBaselinePNG(struct VDir* dir, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) {
463 char baselineName[32];
464 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
465 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_RDONLY);
466 if (!baselineVF) {
467 if (*status == CI_PASS) {
468 *status = CI_FAIL;
469 }
470 return false;
471 }
472
473 png_structp png = PNGReadOpen(baselineVF, 0);
474 png_infop info = png_create_info_struct(png);
475 png_infop end = png_create_info_struct(png);
476 if (!png || !info || !end || !PNGReadHeader(png, info)) {
477 PNGReadClose(png, info, end);
478 baselineVF->close(baselineVF);
479 CIerr(1, "Failed to load %s\n", baselineName);
480 *status = CI_ERROR;
481 return false;
482 }
483
484 unsigned pwidth = png_get_image_width(png, info);
485 unsigned pheight = png_get_image_height(png, info);
486 if (pheight != image->height || pwidth != image->width) {
487 PNGReadClose(png, info, end);
488 baselineVF->close(baselineVF);
489 CIlog(1, "Size mismatch for %s, expected %ux%u, got %ux%u\n", baselineName, pwidth, pheight, image->width, image->height);
490 if (*status == CI_PASS) {
491 *status = CI_FAIL;
492 }
493 return false;
494 }
495
496 image->data = malloc(pwidth * pheight * BYTES_PER_PIXEL);
497 if (!image->data) {
498 CIerr(1, "Failed to allocate baseline buffer\n");
499 *status = CI_ERROR;
500 PNGReadClose(png, info, end);
501 baselineVF->close(baselineVF);
502 return false;
503 }
504 if (!PNGReadPixels(png, info, image->data, pwidth, pheight, pwidth) || !PNGReadFooter(png, end)) {
505 CIerr(1, "Failed to read %s\n", baselineName);
506 *status = CI_ERROR;
507 free(image->data);
508 return false;
509 }
510 PNGReadClose(png, info, end);
511 baselineVF->close(baselineVF);
512 image->stride = pwidth;
513 return true;
514}
515
516#ifdef USE_FFMPEG
517struct CInemaStream {
518 struct mAVStream d;
519 struct CInemaImage* image;
520 enum CInemaStatus* status;
521};
522
523static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) {
524 struct CInemaStream* cistream = (struct CInemaStream*) stream;
525 if (height != cistream->image->height || width != cistream->image->width) {
526 CIlog(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height);
527 if (*cistream->status == CI_PASS) {
528 *cistream->status = CI_FAIL;
529 }
530 }
531}
532
533static void _cinemaVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) {
534 struct CInemaStream* cistream = (struct CInemaStream*) stream;
535 cistream->image->stride = stride;
536 size_t bufferSize = cistream->image->stride * cistream->image->height * BYTES_PER_PIXEL;
537 cistream->image->data = malloc(bufferSize);
538 memcpy(cistream->image->data, pixels, bufferSize);
539}
540#endif
541
542static struct VDir* _makeOutDir(const char* testName) {
543 char path[PATH_MAX] = {0};
544 strlcpy(path, outdir, sizeof(path));
545 char* pathEnd = path + strlen(path);
546 const char* pos;
547 while (true) {
548 pathEnd[0] = PATH_SEP[0];
549 ++pathEnd;
550 pos = strchr(testName, '.');
551 size_t maxlen = sizeof(path) - (pathEnd - path) - 1;
552 size_t len;
553 if (pos) {
554 len = pos - testName;
555 } else {
556 len = strlen(testName);
557 }
558 if (len > maxlen) {
559 len = maxlen;
560 }
561 strncpy(pathEnd, testName, len);
562 pathEnd += len;
563
564 mkdir(path, 0777);
565
566 if (!pos) {
567 break;
568 }
569 testName = pos + 1;
570 }
571 return VDirOpen(path);
572}
573
574static void _writeImage(struct VFile* vf, const struct CInemaImage* image) {
575 png_structp png = PNGWriteOpen(vf);
576 png_infop info = PNGWriteHeader(png, image->width, image->height);
577 if (!PNGWritePixels(png, image->width, image->height, image->stride, image->data)) {
578 CIerr(0, "Could not write output image\n");
579 }
580 PNGWriteClose(png, info);
581
582 vf->close(vf);
583}
584
585static void _writeDiff(const char* testName, const struct CInemaImage* image, size_t frame, const char* type) {
586 struct VDir* dir = _makeOutDir(testName);
587 if (!dir) {
588 CIerr(0, "Could not open directory for %s\n", testName);
589 return;
590 }
591 char name[32];
592 snprintf(name, sizeof(name), "%s_%04" PRIz "u.png", type, frame);
593 struct VFile* vf = dir->openFile(dir, name, O_CREAT | O_TRUNC | O_WRONLY);
594 if (!vf) {
595 CIerr(0, "Could not open output file %s\n", name);
596 dir->close(dir);
597 return;
598 }
599 _writeImage(vf, image);
600 dir->close(dir);
601}
602
603static void _writeBaseline(struct VDir* dir, const struct CInemaImage* image, size_t frame) {
604 char baselineName[32];
605 snprintf(baselineName, sizeof(baselineName), "baseline_%04" PRIz "u.png", frame);
606 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_CREAT | O_TRUNC | O_WRONLY);
607 if (baselineVF) {
608 _writeImage(baselineVF, image);
609 } else {
610 CIerr(0, "Could not open output file %s\n", baselineName);
611 }
612}
613
614void CInemaTestRun(struct CInemaTest* test) {
615 unsigned ignore = 0;
616 MutexLock(&configMutex);
617 CInemaConfigGetUInt(&configTree, test->name, "ignore", &ignore);
618 MutexUnlock(&configMutex);
619 if (ignore) {
620 test->status = CI_SKIP;
621 return;
622 }
623
624 struct VDir* dir = VDirOpen(test->directory);
625 if (!dir) {
626 CIerr(0, "Failed to open test directory\n");
627 test->status = CI_ERROR;
628 return;
629 }
630 struct VFile* rom = dir->openFile(dir, test->filename, O_RDONLY);
631 if (!rom) {
632 CIerr(0, "Failed to open test\n");
633 test->status = CI_ERROR;
634 return;
635 }
636 struct mCore* core = mCoreFindVF(rom);
637 if (!core) {
638 CIerr(0, "Failed to load test\n");
639 test->status = CI_ERROR;
640 rom->close(rom);
641 return;
642 }
643 if (!core->init(core)) {
644 CIerr(0, "Failed to init test\n");
645 test->status = CI_ERROR;
646 core->deinit(core);
647 return;
648 }
649 struct CInemaImage image;
650 core->desiredVideoDimensions(core, &image.width, &image.height);
651 ssize_t bufferSize = image.width * image.height * BYTES_PER_PIXEL;
652 image.data = malloc(bufferSize);
653 image.stride = image.width;
654 if (!image.data) {
655 CIerr(0, "Failed to allocate video buffer\n");
656 test->status = CI_ERROR;
657 core->deinit(core);
658 }
659 core->setVideoBuffer(core, image.data, image.stride);
660 mCoreConfigInit(&core->config, "cinema");
661
662 unsigned limit = 9999;
663 unsigned skip = 0;
664 unsigned fail = 0;
665 unsigned video = 0;
666
667 MutexLock(&configMutex);
668 CInemaConfigGetUInt(&configTree, test->name, "frames", &limit);
669 CInemaConfigGetUInt(&configTree, test->name, "skip", &skip);
670 CInemaConfigGetUInt(&configTree, test->name, "fail", &fail);
671 CInemaConfigGetUInt(&configTree, test->name, "video", &video);
672 CInemaConfigLoad(&configTree, test->name, core);
673 MutexUnlock(&configMutex);
674
675 struct VFile* save = VFileMemChunk(NULL, 0);
676 core->loadROM(core, rom);
677 if (!core->loadSave(core, save)) {
678 save->close(save);
679 }
680 core->rtc.override = RTC_FAKE_EPOCH;
681 core->rtc.value = 1200000000;
682 core->reset(core);
683
684 test->status = CI_PASS;
685
686 unsigned minFrame = core->frameCounter(core);
687 size_t frame;
688 for (frame = 0; frame < skip; ++frame) {
689 core->runFrame(core);
690 }
691 core->desiredVideoDimensions(core, &image.width, &image.height);
692
693#ifdef USE_FFMPEG
694 struct FFmpegDecoder decoder;
695 struct FFmpegEncoder encoder;
696 struct CInemaStream stream = {0};
697 if (video) {
698 char fname[PATH_MAX];
699 snprintf(fname, sizeof(fname), "%s" PATH_SEP "baseline.mkv", test->directory);
700 if (rebaseline) {
701 FFmpegEncoderInit(&encoder);
702 FFmpegEncoderSetAudio(&encoder, NULL, 0);
703 FFmpegEncoderSetVideo(&encoder, "png", 0, 0);
704 FFmpegEncoderSetContainer(&encoder, "mkv");
705 FFmpegEncoderSetDimensions(&encoder, image.width, image.height);
706 if (!FFmpegEncoderOpen(&encoder, fname)) {
707 CIerr(1, "Failed to save baseline video\n");
708 } else {
709 core->setAVStream(core, &encoder.d);
710 }
711 } else {
712 FFmpegDecoderInit(&decoder);
713 stream.d.postVideoFrame = _cinemaVideoFrame;
714 stream.d.videoDimensionsChanged = _cinemaDimensionsChanged;
715 stream.status = &test->status;
716 decoder.out = &stream.d;
717
718 if (!FFmpegDecoderOpen(&decoder, fname)) {
719 CIerr(1, "Failed to load baseline video\n");
720 }
721 }
722 }
723#else
724 if (video) {
725 CIerr(0, "Failed to run video test without ffmpeg linked in\n");
726 test->status = CI_ERROR;
727 }
728#endif
729
730 for (frame = 0; limit; ++frame, --limit) {
731 core->runFrame(core);
732 ++test->totalFrames;
733 unsigned frameCounter = core->frameCounter(core);
734 if (frameCounter <= minFrame) {
735 break;
736 }
737 CIlog(3, "Test frame: %u\n", frameCounter);
738 core->desiredVideoDimensions(core, &image.width, &image.height);
739 uint8_t* diff = NULL;
740 struct CInemaImage expected = {
741 .data = NULL,
742 .width = image.width,
743 .height = image.height,
744 .stride = image.width,
745 };
746 bool baselineFound;
747 if (video) {
748 baselineFound = false;
749#ifdef USE_FFMPEG
750 if (!rebaseline && FFmpegDecoderIsOpen(&decoder)) {
751 stream.image = &expected;
752 while (!expected.data) {
753 if (!FFmpegDecoderRead(&decoder)) {
754 CIlog(1, "Failed to read more frames. EOF?\n");
755 test->status = CI_FAIL;
756 break;
757 }
758 }
759 baselineFound = expected.data;
760 }
761#endif
762 } else {
763 baselineFound = _loadBaselinePNG(dir, &expected, frame, &test->status);
764 }
765 if (baselineFound) {
766 uint8_t* testPixels = image.data;
767 uint8_t* expectPixels = expected.data;
768 size_t x;
769 size_t y;
770 int max = 0;
771 bool failed = false;
772 for (y = 0; y < image.height; ++y) {
773 for (x = 0; x < image.width; ++x) {
774 size_t pix = expected.stride * y + x;
775 size_t tpix = image.stride * y + x;
776 int testR = testPixels[tpix * 4 + 0];
777 int testG = testPixels[tpix * 4 + 1];
778 int testB = testPixels[tpix * 4 + 2];
779 int expectR = expectPixels[pix * 4 + 0];
780 int expectG = expectPixels[pix * 4 + 1];
781 int expectB = expectPixels[pix * 4 + 2];
782 int r = expectR - testR;
783 int g = expectG - testG;
784 int b = expectB - testB;
785 if (r | g | b) {
786 failed = true;
787 if (diffs && !diff) {
788 diff = calloc(expected.stride * expected.height, BYTES_PER_PIXEL);
789 }
790 CIlog(3, "Frame %u failed at pixel %" PRIz "ux%" PRIz "u with diff %i,%i,%i (expected %02x%02x%02x, got %02x%02x%02x)\n",
791 frameCounter, x, y, r, g, b,
792 expectR, expectG, expectB,
793 testR, testG, testB);
794 test->status = CI_FAIL;
795 if (r < 0) {
796 r = -r;
797 }
798 if (g < 0) {
799 g = -g;
800 }
801 if (b < 0) {
802 b = -b;
803 }
804
805 if (diff) {
806 if (r > max) {
807 max = r;
808 }
809 if (g > max) {
810 max = g;
811 }
812 if (b > max) {
813 max = b;
814 }
815 diff[pix * 4 + 0] = r;
816 diff[pix * 4 + 1] = g;
817 diff[pix * 4 + 2] = b;
818 }
819
820 test->totalDistance += r + g + b;
821 ++test->failedPixels;
822 }
823 }
824 }
825 if (failed) {
826 ++test->failedFrames;
827 }
828 test->totalPixels += image.height * image.width;
829 if (rebaseline && failed) {
830 _writeBaseline(dir, &image, frame);
831 }
832 if (diff) {
833 if (failed) {
834 struct CInemaImage outdiff = {
835 .data = diff,
836 .width = expected.width,
837 .height = expected.height,
838 .stride = expected.stride,
839 };
840
841 _writeDiff(test->name, &image, frame, "result");
842 _writeDiff(test->name, &expected, frame, "expected");
843 _writeDiff(test->name, &outdiff, frame, "diff");
844
845 for (y = 0; y < outdiff.height; ++y) {
846 for (x = 0; x < outdiff.width; ++x) {
847 size_t pix = outdiff.stride * y + x;
848 diff[pix * 4 + 0] = diff[pix * 4 + 0] * 255 / max;
849 diff[pix * 4 + 1] = diff[pix * 4 + 1] * 255 / max;
850 diff[pix * 4 + 2] = diff[pix * 4 + 2] * 255 / max;
851 }
852 }
853 _writeDiff(test->name, &outdiff, frame, "normalized");
854 }
855 free(diff);
856 }
857 free(expected.data);
858 } else if (test->status == CI_ERROR) {
859 break;
860 } else if (rebaseline && !video) {
861 _writeBaseline(dir, &image, frame);
862 } else if (!rebaseline) {
863 test->status = CI_FAIL;
864 }
865 }
866
867 if (fail) {
868 if (test->status == CI_FAIL) {
869 test->status = CI_XFAIL;
870 } else if (test->status == CI_PASS) {
871 test->status = CI_XPASS;
872 }
873 }
874
875#ifdef USE_FFMPEG
876 if (video) {
877 if (rebaseline) {
878 FFmpegEncoderClose(&encoder);
879 } else {
880 FFmpegDecoderClose(&decoder);
881 }
882 }
883#endif
884
885 free(image.data);
886 mCoreConfigDeinit(&core->config);
887 core->deinit(core);
888 dir->close(dir);
889}
890
891static bool CInemaTask(struct CInemaTestList* tests, size_t i) {
892 bool success = true;
893 struct CInemaTest* test = CInemaTestListGetPointer(tests, i);
894 if (dryRun) {
895 CIlog(-1, "%s\n", test->name);
896 } else {
897 CIlog(1, "%s: ", test->name);
898 fflush(stdout);
899 CInemaTestRun(test);
900 switch (test->status) {
901 case CI_PASS:
902 CIlog(1, "pass\n");
903 break;
904 case CI_FAIL:
905 success = false;
906 CIlog(1, "fail\n");
907 break;
908 case CI_XPASS:
909 CIlog(1, "xpass\n");
910 break;
911 case CI_XFAIL:
912 CIlog(1, "xfail\n");
913 break;
914 case CI_SKIP:
915 CIlog(1, "skip\n");
916 break;
917 case CI_ERROR:
918 success = false;
919 CIlog(1, "error\n");
920 break;
921 }
922 if (test->failedFrames) {
923 CIlog(2, "\tfailed frames: %u/%u (%1.3g%%)\n", test->failedFrames, test->totalFrames, test->failedFrames / (test->totalFrames * 0.01));
924 CIlog(2, "\tfailed pixels: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->failedPixels, test->totalPixels, test->failedPixels / (test->totalPixels * 0.01));
925 CIlog(2, "\tdistance: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->totalDistance, test->totalPixels * 765, test->totalDistance / (test->totalPixels * 7.65));
926 }
927 }
928 return success;
929}
930
931static THREAD_ENTRY CInemaJob(void* context) {
932 struct CInemaTestList* tests = context;
933 struct StringBuilder builder;
934 StringListInit(&builder.out, 0);
935 StringListInit(&builder.err, 0);
936 ThreadLocalSetKey(stringBuilder, &builder);
937
938 bool success = true;
939 while (true) {
940 size_t i;
941 MutexLock(&jobMutex);
942 i = jobIndex;
943 ++jobIndex;
944 MutexUnlock(&jobMutex);
945 if (i >= CInemaTestListSize(tests)) {
946 break;
947 }
948 if (!CInemaTask(tests, i)) {
949 success = false;
950 }
951 CIflush(&builder.out, stdout);
952 CIflush(&builder.err, stderr);
953 }
954 MutexLock(&jobMutex);
955 if (!success) {
956 jobStatus = 1;
957 }
958 MutexUnlock(&jobMutex);
959
960 CIflush(&builder.out, stdout);
961 StringListDeinit(&builder.out);
962
963 CIflush(&builder.err, stderr);
964 StringListDeinit(&builder.err);
965}
966
967void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args) {
968 UNUSED(log);
969 if (verbosity < 0) {
970 return;
971 }
972 int mask = mLOG_FATAL;
973 if (verbosity >= 1) {
974 mask |= mLOG_ERROR;
975 }
976 if (verbosity >= 2) {
977 mask |= mLOG_WARN;
978 }
979 if (verbosity >= 4) {
980 mask |= mLOG_INFO;
981 }
982 if (verbosity >= 5) {
983 mask |= mLOG_ALL;
984 }
985 if (!(mask & level)) {
986 return;
987 }
988
989 char buffer[256];
990 vsnprintf(buffer, sizeof(buffer), format, args);
991 CIerr(0, "[%s] %s\n", mLogCategoryName(category), buffer);
992}
993
994int main(int argc, char** argv) {
995 ThreadLocalInitKey(&stringBuilder);
996 ThreadLocalSetKey(stringBuilder, NULL);
997
998 int status = 0;
999 if (!parseCInemaArgs(argc, argv)) {
1000 status = 1;
1001 goto cleanup;
1002 }
1003
1004 if (showVersion) {
1005 version(argv[0]);
1006 goto cleanup;
1007 }
1008
1009 if (showUsage) {
1010 usageCInema(argv[0]);
1011 goto cleanup;
1012 }
1013
1014 argc -= optind;
1015 argv += optind;
1016
1017 if (!base[0] && !determineBase(argc, argv)) {
1018 CIlog(0, "Could not determine CInema test base. Please specify manually.");
1019 status = 1;
1020 goto cleanup;
1021 }
1022#ifndef _WIN32
1023 char* rbase = realpath(base, NULL);
1024 if (rbase) {
1025 strlcpy(base, rbase, sizeof(base));
1026 free(rbase);
1027 }
1028#endif
1029
1030 struct CInemaTestList tests;
1031 CInemaTestListInit(&tests, 0);
1032
1033 struct mLogger logger = { .log = _log };
1034 mLogSetDefaultLogger(&logger);
1035#ifdef USE_FFMPEG
1036 if (verbosity < 2) {
1037 av_log_set_level(AV_LOG_ERROR);
1038 }
1039#endif
1040
1041 if (argc > 0) {
1042 size_t i;
1043 for (i = 0; i < (size_t) argc; ++i) {
1044 char path[PATH_MAX + 1] = {0};
1045 testToPath(argv[i], path);
1046
1047 if (!collectTests(&tests, path)) {
1048 status = 1;
1049 break;
1050 }
1051 }
1052 } else if (!collectTests(&tests, base)) {
1053 status = 1;
1054 }
1055
1056 if (CInemaTestListSize(&tests) == 0) {
1057 CIlog(1, "No tests found.");
1058 status = 1;
1059 } else {
1060 reduceTestList(&tests);
1061 }
1062
1063 HashTableInit(&configTree, 0, free);
1064 MutexInit(&configMutex);
1065
1066 if (jobs == 1) {
1067 size_t i;
1068 for (i = 0; i < CInemaTestListSize(&tests); ++i) {
1069 bool success = CInemaTask(&tests, i);
1070 if (!success) {
1071 status = 1;
1072 }
1073 }
1074 } else {
1075 MutexInit(&jobMutex);
1076 int i;
1077 for (i = 0; i < jobs; ++i) {
1078 ThreadCreate(&jobThreads[i], CInemaJob, &tests);
1079 }
1080 for (i = 0; i < jobs; ++i) {
1081 ThreadJoin(&jobThreads[i]);
1082 }
1083 MutexDeinit(&jobMutex);
1084 status = jobStatus;
1085 }
1086
1087 MutexDeinit(&configMutex);
1088 HashTableEnumerate(&configTree, _unloadConfigTree, NULL);
1089 HashTableDeinit(&configTree);
1090 CInemaTestListDeinit(&tests);
1091
1092cleanup:
1093 return status;
1094}