src/platform/test/cinema-main.c (view raw)
1/* Copyright (c) 2013-2020 Jeffrey Pfau
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6#include <mgba/core/config.h>
7#include <mgba/core/core.h>
8#include <mgba/core/log.h>
9#include <mgba/core/version.h>
10#include <mgba/feature/commandline.h>
11#include <mgba/feature/video-logger.h>
12
13#include <mgba-util/png-io.h>
14#include <mgba-util/string.h>
15#include <mgba-util/table.h>
16#include <mgba-util/threading.h>
17#include <mgba-util/vector.h>
18#include <mgba-util/vfs.h>
19
20#ifdef USE_FFMPEG
21#include "feature/ffmpeg/ffmpeg-decoder.h"
22#include "feature/ffmpeg/ffmpeg-encoder.h"
23#endif
24
25#ifdef _MSC_VER
26#include <mgba-util/platform/windows/getopt.h>
27#else
28#include <getopt.h>
29#endif
30
31#include <stdlib.h>
32#include <sys/stat.h>
33#include <sys/types.h>
34
35#define MAX_TEST 200
36#define MAX_JOBS 128
37#define LOG_THRESHOLD 1000000
38
39static const struct option longOpts[] = {
40 { "base", required_argument, 0, 'b' },
41 { "diffs", no_argument, 0, 'd' },
42 { "help", no_argument, 0, 'h' },
43 { "jobs", required_argument, 0, 'j' },
44 { "dry-run", no_argument, 0, 'n' },
45 { "outdir", required_argument, 0, 'o' },
46 { "quiet", no_argument, 0, 'q' },
47 { "rebaseline", no_argument, 0, 'r' },
48 { "rebaseline-missing", no_argument, 0, 'R' },
49 { "verbose", no_argument, 0, 'v' },
50 { "xbaseline", no_argument, 0, 'x' },
51 { "version", no_argument, 0, '\0' },
52 { 0, 0, 0, 0 }
53};
54
55static const char shortOpts[] = "b:dhj:no:qRrvx";
56
57enum CInemaStatus {
58 CI_PASS,
59 CI_FAIL,
60 CI_XPASS,
61 CI_XFAIL,
62 CI_ERROR,
63 CI_SKIP
64};
65
66enum CInemaRebaseline {
67 CI_R_NONE = 0,
68 CI_R_FAILING,
69 CI_R_MISSING,
70};
71
72struct CInemaTest {
73 char directory[MAX_TEST];
74 char filename[MAX_TEST];
75 char name[MAX_TEST];
76 enum CInemaStatus status;
77 unsigned failedFrames;
78 uint64_t failedPixels;
79 unsigned totalFrames;
80 uint64_t totalDistance;
81 uint64_t totalPixels;
82};
83
84struct CInemaImage {
85 void* data;
86 unsigned width;
87 unsigned height;
88 unsigned stride;
89};
90
91DECLARE_VECTOR(CInemaTestList, struct CInemaTest)
92DEFINE_VECTOR(CInemaTestList, struct CInemaTest)
93
94DECLARE_VECTOR(ImageList, void*)
95DEFINE_VECTOR(ImageList, void*)
96
97struct StringBuilder {
98 struct StringList lines;
99 struct StringList partial;
100 unsigned repeat;
101
102};
103
104struct CInemaLogStream {
105 struct StringBuilder err;
106 struct StringBuilder out;
107};
108
109static bool showVersion = false;
110static bool showUsage = false;
111static char base[PATH_MAX] = {0};
112static char outdir[PATH_MAX] = {'.'};
113static bool dryRun = false;
114static bool diffs = false;
115static enum CInemaRebaseline rebaseline = CI_R_NONE;
116static enum CInemaRebaseline xbaseline = CI_R_NONE;
117static int verbosity = 0;
118
119static struct Table configTree;
120static Mutex configMutex;
121
122static int jobs = 1;
123static size_t jobIndex = 0;
124static Mutex jobMutex;
125static Thread jobThreads[MAX_JOBS];
126static int jobStatus;
127static ThreadLocal logStream;
128static ThreadLocal currentTest;
129
130bool CInemaTestInit(struct CInemaTest*, const char* directory, const char* filename);
131void CInemaTestRun(struct CInemaTest*);
132
133bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* value);
134void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core);
135
136static void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args);
137
138void CIflush(struct StringBuilder* list, FILE* file);
139
140static char* _compileStringList(struct StringList* list) {
141 size_t len = 0;
142 size_t i;
143 for (i = 0; i < StringListSize(list); ++i) {
144 len += strlen(*StringListGetPointer(list, i));
145 }
146 char* string = calloc(len + 1, sizeof(char));
147 char* cur = string;
148 for (i = 0; i < StringListSize(list); ++i) {
149 char* brick = *StringListGetPointer(list, i);
150 size_t portion = strlen(brick);
151 memcpy(cur, brick, portion);
152 free(brick);
153 cur += portion;
154 }
155 StringListClear(list);
156 return string;
157}
158
159static void _logToStream(FILE* file, const char* format, va_list args) {
160#ifdef HAVE_VASPRINTF
161 struct CInemaLogStream* stream = ThreadLocalGetValue(logStream);
162 if (!stream) {
163 vfprintf(file, format, args);
164 } else {
165 struct StringBuilder* builder = &stream->out;
166 if (file == stderr) {
167 builder = &stream->err;
168 }
169 if (StringListSize(&builder->lines) > LOG_THRESHOLD) {
170 CIflush(builder, file);
171 }
172 char** line = StringListAppend(&builder->partial);
173 vasprintf(line, format, args);
174 size_t len = strlen(*line);
175 if (len && (*line)[len - 1] == '\n') {
176 char* string = _compileStringList(&builder->partial);
177 size_t linecount = StringListSize(&builder->lines);
178 if (linecount && strcmp(string, *StringListGetPointer(&builder->lines, linecount - 1)) == 0) {
179 ++builder->repeat;
180 free(string);
181 } else {
182 if (builder->repeat > 1) {
183 asprintf(StringListAppend(&builder->lines), "The previous message was repeated %u times.\n", builder->repeat);
184 }
185 *StringListAppend(&builder->lines) = string;
186 builder->repeat = 1;
187 }
188 }
189 }
190#else
191 vfprintf(file, format, args);
192#endif
193}
194
195ATTRIBUTE_FORMAT(printf, 2, 3) void CIlog(int minlevel, const char* format, ...) {
196 if (verbosity < minlevel) {
197 return;
198 }
199 va_list args;
200 va_start(args, format);
201 _logToStream(stdout, format, args);
202 va_end(args);
203}
204
205ATTRIBUTE_FORMAT(printf, 2, 3) void CIerr(int minlevel, const char* format, ...) {
206 if (verbosity < minlevel) {
207 return;
208 }
209 va_list args;
210 va_start(args, format);
211 _logToStream(stderr, format, args);
212 va_end(args);
213}
214
215void CIflush(struct StringBuilder* builder, FILE* out) {
216 if (StringListSize(&builder->partial)) {
217 *StringListAppend(&builder->lines) = _compileStringList(&builder->partial);
218 }
219#ifdef HAVE_VASPRINTF
220 if (builder->repeat > 1) {
221 asprintf(StringListAppend(&builder->lines), "The previous message was repeated %u times.\n", builder->repeat);
222 }
223#endif
224
225 char* string = _compileStringList(&builder->lines);
226 builder->repeat = 0;
227 fputs(string, out);
228 free(string);
229}
230
231static bool parseCInemaArgs(int argc, char* const* argv) {
232 int ch;
233 int index = 0;
234 while ((ch = getopt_long(argc, argv, shortOpts, longOpts, &index)) != -1) {
235 const struct option* opt = &longOpts[index];
236 switch (ch) {
237 case '\0':
238 if (strcmp(opt->name, "version") == 0) {
239 showVersion = true;
240 } else {
241 return false;
242 }
243 break;
244 case 'b':
245 strlcpy(base, optarg, sizeof(base));
246 // TODO: Verify path exists
247 break;
248 case 'd':
249 diffs = true;
250 break;
251 case 'h':
252 showUsage = true;
253 break;
254 case 'j':
255 jobs = atoi(optarg);
256 if (jobs > MAX_JOBS) {
257 jobs = MAX_JOBS;
258 }
259 if (jobs < 1) {
260 jobs = 1;
261 }
262 break;
263 case 'n':
264 dryRun = true;
265 break;
266 case 'o':
267 strlcpy(outdir, optarg, sizeof(outdir));
268 // TODO: Make directory
269 break;
270 case 'q':
271 --verbosity;
272 break;
273 case 'r':
274 rebaseline = CI_R_FAILING;
275 break;
276 case 'R':
277 rebaseline = CI_R_MISSING;
278 break;
279 case 'v':
280 ++verbosity;
281 break;
282 case 'x':
283 xbaseline = CI_R_FAILING;
284 break;
285 default:
286 return false;
287 }
288 }
289
290 return true;
291}
292
293static void usageCInema(const char* arg0) {
294 printf("usage: %s [-dhnqrRv] [-j JOBS] [-b BASE] [-o DIR] [--version] [test...]\n", arg0);
295 puts(" -b, --base BASE Path to the CInema base directory");
296 puts(" -d, --diffs Output image diffs from failures");
297 puts(" -h, --help Print this usage and exit");
298 puts(" -j, --jobs JOBS Run a number of jobs in parallel");
299 puts(" -n, --dry-run List all collected tests instead of running them");
300 puts(" -o, --output DIR Path to output applicable results");
301 puts(" -q, --quiet Decrease log verbosity (can be repeated)");
302 puts(" -r, --rebaseline Rewrite the baseline for failing tests");
303 puts(" -R, --rebaseline-missing Write missing baselines tests only");
304 puts(" -v, --verbose Increase log verbosity (can be repeated)");
305 puts(" -x, --xbaseline Rewrite the xfail baselines for failing tests");
306 puts(" --version Print version and exit");
307}
308
309static bool determineBase(int argc, char* const* argv) {
310 // TODO: Better dynamic detection
311 separatePath(__FILE__, base, NULL, NULL);
312 strncat(base, PATH_SEP ".." PATH_SEP ".." PATH_SEP ".." PATH_SEP "cinema", sizeof(base) - strlen(base) - 1);
313 return true;
314}
315
316static bool collectTests(struct CInemaTestList* tests, const char* path) {
317 CIerr(2, "Considering path %s\n", path);
318 struct VDir* dir = VDirOpen(path);
319 if (!dir) {
320 return false;
321 }
322 struct VDirEntry* entry = dir->listNext(dir);
323 while (entry) {
324 char subpath[PATH_MAX];
325 snprintf(subpath, sizeof(subpath), "%s" PATH_SEP "%s", path, entry->name(entry));
326 if (entry->type(entry) == VFS_DIRECTORY && strncmp(entry->name(entry), ".", 2) != 0 && strncmp(entry->name(entry), "..", 3) != 0) {
327 if (!collectTests(tests, subpath)) {
328 dir->close(dir);
329 return false;
330 }
331 } else if (entry->type(entry) == VFS_FILE && strncmp(entry->name(entry), "test.", 5) == 0) {
332 CIerr(3, "Found potential test %s\n", subpath);
333 struct VFile* vf = dir->openFile(dir, entry->name(entry), O_RDONLY);
334 if (vf) {
335 if (mCoreIsCompatible(vf) != PLATFORM_NONE || mVideoLogIsCompatible(vf) != PLATFORM_NONE) {
336 struct CInemaTest* test = CInemaTestListAppend(tests);
337 if (!CInemaTestInit(test, path, entry->name(entry))) {
338 CIerr(3, "Failed to create test\n");
339 CInemaTestListResize(tests, -1);
340 } else {
341 CIerr(2, "Found test %s\n", test->name);
342 }
343 } else {
344 CIerr(3, "Not a compatible file\n");
345 }
346 vf->close(vf);
347 } else {
348 CIerr(3, "Failed to open file\n");
349 }
350 }
351 entry = dir->listNext(dir);
352 }
353 dir->close(dir);
354 return true;
355}
356
357static int _compareNames(const void* a, const void* b) {
358 const struct CInemaTest* ta = a;
359 const struct CInemaTest* tb = b;
360
361 return strncmp(ta->name, tb->name, sizeof(ta->name));
362}
363
364static void reduceTestList(struct CInemaTestList* tests) {
365 qsort(CInemaTestListGetPointer(tests, 0), CInemaTestListSize(tests), sizeof(struct CInemaTest), _compareNames);
366
367 size_t i;
368 for (i = 1; i < CInemaTestListSize(tests);) {
369 struct CInemaTest* cur = CInemaTestListGetPointer(tests, i);
370 struct CInemaTest* prev = CInemaTestListGetPointer(tests, i - 1);
371 if (strncmp(cur->name, prev->name, sizeof(cur->name)) != 0) {
372 ++i;
373 continue;
374 }
375 CInemaTestListShift(tests, i, 1);
376 }
377}
378
379static void testToPath(const char* testName, char* path) {
380 strlcpy(path, base, PATH_MAX);
381
382 bool dotSeen = true;
383 size_t i;
384 for (i = strlen(path); testName[0] && i < PATH_MAX; ++testName) {
385 if (testName[0] == '.') {
386 dotSeen = true;
387 } else {
388 if (dotSeen) {
389 strlcpy(&path[i], PATH_SEP, PATH_MAX - i);
390 i += strlen(PATH_SEP);
391 dotSeen = false;
392 if (!i) {
393 break;
394 }
395 }
396 path[i] = testName[0];
397 ++i;
398 }
399 }
400 if (i == PATH_MAX) {
401 --i;
402 }
403 path[i] = '\0';
404}
405
406static void _loadConfigTree(struct Table* configTree, const char* testName) {
407 char key[MAX_TEST];
408 strlcpy(key, testName, sizeof(key));
409
410 struct mCoreConfig* config;
411 while (!(config = HashTableLookup(configTree, key))) {
412 char path[PATH_MAX];
413 config = malloc(sizeof(*config));
414 mCoreConfigInit(config, "cinema");
415 testToPath(key, path);
416 strncat(path, PATH_SEP, sizeof(path) - 1);
417 strncat(path, "config.ini", sizeof(path) - 1);
418 mCoreConfigLoadPath(config, path);
419 HashTableInsert(configTree, key, config);
420 char* pos = strrchr(key, '.');
421 if (pos) {
422 pos[0] = '\0';
423 } else if (key[0]) {
424 key[0] = '\0';
425 } else {
426 break;
427 }
428 }
429}
430
431static void _unloadConfigTree(const char* key, void* value, void* user) {
432 UNUSED(key);
433 UNUSED(user);
434 mCoreConfigDeinit(value);
435}
436
437static const char* CInemaConfigGet(struct Table* configTree, const char* testName, const char* key) {
438 _loadConfigTree(configTree, testName);
439
440 char testKey[MAX_TEST];
441 strlcpy(testKey, testName, sizeof(testKey));
442
443 struct mCoreConfig* config;
444 while (true) {
445 config = HashTableLookup(configTree, testKey);
446 if (!config) {
447 continue;
448 }
449 const char* str = ConfigurationGetValue(&config->configTable, "testinfo", key);
450 if (str) {
451 return str;
452 }
453 char* pos = strrchr(testKey, '.');
454 if (pos) {
455 pos[0] = '\0';
456 } else if (testKey[0]) {
457 testKey[0] = '\0';
458 } else {
459 break;
460 }
461 }
462 return NULL;
463}
464
465bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* out) {
466 const char* charValue = CInemaConfigGet(configTree, testName, key);
467 if (!charValue) {
468 return false;
469 }
470 char* end;
471 unsigned long value = strtoul(charValue, &end, 10);
472 if (*end) {
473 return false;
474 }
475 *out = value;
476 return true;
477}
478
479void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core) {
480 _loadConfigTree(configTree, testName);
481
482 char testKey[MAX_TEST] = {0};
483 char* keyEnd = testKey;
484 const char* pos;
485 while (true) {
486 pos = strchr(testName, '.');
487 size_t maxlen = sizeof(testKey) - (keyEnd - testKey) - 1;
488 size_t len;
489 if (pos) {
490 len = pos - testName;
491 } else {
492 len = strlen(testName);
493 }
494 if (len > maxlen) {
495 len = maxlen;
496 }
497 strncpy(keyEnd, testName, len);
498 keyEnd += len;
499
500 struct mCoreConfig* config = HashTableLookup(configTree, testKey);
501 if (config) {
502 core->loadConfig(core, config);
503 }
504 if (!pos) {
505 break;
506 }
507 testName = pos + 1;
508 keyEnd[0] = '.';
509 ++keyEnd;
510 }
511}
512
513bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* filename) {
514 if (strncmp(base, directory, strlen(base)) != 0) {
515 return false;
516 }
517 memset(test, 0, sizeof(*test));
518 strlcpy(test->directory, directory, sizeof(test->directory));
519 strlcpy(test->filename, filename, sizeof(test->filename));
520 directory += strlen(base) + 1;
521 strlcpy(test->name, directory, sizeof(test->name));
522 char* str = strstr(test->name, PATH_SEP);
523 while (str) {
524 str[0] = '.';
525 str = strstr(str, PATH_SEP);
526 }
527 return true;
528}
529
530static bool _loadBaselinePNG(struct VDir* dir, const char* type, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) {
531 char baselineName[32];
532 snprintf(baselineName, sizeof(baselineName), "%s_%04" PRIz "u.png", type, frame);
533 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_RDONLY);
534 if (!baselineVF) {
535 if (*status == CI_PASS) {
536 *status = CI_FAIL;
537 }
538 return false;
539 }
540
541 png_structp png = PNGReadOpen(baselineVF, 0);
542 png_infop info = png_create_info_struct(png);
543 png_infop end = png_create_info_struct(png);
544 if (!png || !info || !end || !PNGReadHeader(png, info)) {
545 PNGReadClose(png, info, end);
546 baselineVF->close(baselineVF);
547 CIerr(1, "Failed to load %s\n", baselineName);
548 *status = CI_ERROR;
549 return false;
550 }
551
552 unsigned pwidth = png_get_image_width(png, info);
553 unsigned pheight = png_get_image_height(png, info);
554 if (pheight != image->height || pwidth != image->width) {
555 PNGReadClose(png, info, end);
556 baselineVF->close(baselineVF);
557 CIlog(1, "Size mismatch for %s, expected %ux%u, got %ux%u\n", baselineName, pwidth, pheight, image->width, image->height);
558 if (*status == CI_PASS) {
559 *status = CI_FAIL;
560 }
561 return false;
562 }
563
564 image->data = malloc(pwidth * pheight * BYTES_PER_PIXEL);
565 if (!image->data) {
566 CIerr(1, "Failed to allocate baseline buffer\n");
567 *status = CI_ERROR;
568 PNGReadClose(png, info, end);
569 baselineVF->close(baselineVF);
570 return false;
571 }
572 if (!PNGReadPixels(png, info, image->data, pwidth, pheight, pwidth) || !PNGReadFooter(png, end)) {
573 CIerr(1, "Failed to read %s\n", baselineName);
574 *status = CI_ERROR;
575 free(image->data);
576 return false;
577 }
578 PNGReadClose(png, info, end);
579 baselineVF->close(baselineVF);
580 image->stride = pwidth;
581 return true;
582}
583
584#ifdef USE_FFMPEG
585struct CInemaStream {
586 struct mAVStream d;
587 struct CInemaImage* image;
588 enum CInemaStatus* status;
589};
590
591static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) {
592 struct CInemaStream* cistream = (struct CInemaStream*) stream;
593 if (height != cistream->image->height || width != cistream->image->width) {
594 CIlog(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height);
595 if (*cistream->status == CI_PASS) {
596 *cistream->status = CI_FAIL;
597 }
598 }
599}
600
601static void _cinemaVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) {
602 struct CInemaStream* cistream = (struct CInemaStream*) stream;
603 cistream->image->stride = stride;
604 size_t bufferSize = cistream->image->stride * cistream->image->height * BYTES_PER_PIXEL;
605 cistream->image->data = malloc(bufferSize);
606 memcpy(cistream->image->data, pixels, bufferSize);
607}
608#endif
609
610static struct VDir* _makeOutDir(const char* testName) {
611 char path[PATH_MAX] = {0};
612 strlcpy(path, outdir, sizeof(path));
613 char* pathEnd = path + strlen(path);
614 const char* pos;
615 while (true) {
616 pathEnd[0] = PATH_SEP[0];
617 ++pathEnd;
618 pos = strchr(testName, '.');
619 size_t maxlen = sizeof(path) - (pathEnd - path) - 1;
620 size_t len;
621 if (pos) {
622 len = pos - testName;
623 } else {
624 len = strlen(testName);
625 }
626 if (len > maxlen) {
627 len = maxlen;
628 }
629 strncpy(pathEnd, testName, len);
630 pathEnd += len;
631
632 mkdir(path, 0777);
633
634 if (!pos) {
635 break;
636 }
637 testName = pos + 1;
638 }
639 return VDirOpen(path);
640}
641
642static void _writeImage(struct VFile* vf, const struct CInemaImage* image) {
643 png_structp png = PNGWriteOpen(vf);
644 png_infop info = PNGWriteHeader(png, image->width, image->height);
645 if (!PNGWritePixels(png, image->width, image->height, image->stride, image->data)) {
646 CIerr(0, "Could not write output image\n");
647 }
648 PNGWriteClose(png, info);
649
650 vf->close(vf);
651}
652
653static void _writeDiff(const char* testName, const struct CInemaImage* image, size_t frame, const char* type) {
654 struct VDir* dir = _makeOutDir(testName);
655 if (!dir) {
656 CIerr(0, "Could not open directory for %s\n", testName);
657 return;
658 }
659 char name[32];
660 snprintf(name, sizeof(name), "%s_%04" PRIz "u.png", type, frame);
661 struct VFile* vf = dir->openFile(dir, name, O_CREAT | O_TRUNC | O_WRONLY);
662 if (!vf) {
663 CIerr(0, "Could not open output file %s\n", name);
664 dir->close(dir);
665 return;
666 }
667 _writeImage(vf, image);
668 dir->close(dir);
669}
670
671static void _writeBaseline(struct VDir* dir, const char* type, const struct CInemaImage* image, size_t frame) {
672 char baselineName[32];
673 snprintf(baselineName, sizeof(baselineName), "%s_%04" PRIz "u.png", type, frame);
674 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_CREAT | O_TRUNC | O_WRONLY);
675 if (baselineVF) {
676 _writeImage(baselineVF, image);
677 } else {
678 CIerr(0, "Could not open output file %s\n", baselineName);
679 }
680}
681
682static bool _updateInput(struct mCore* core, size_t frame, const char** input) {
683 if (!*input || !*input[0]) {
684 return false;
685 }
686 char* end;
687 uint32_t start = strtoul(*input, &end, 10);
688 if (end[0] != ':') {
689 return false;
690 }
691 if (start != frame) {
692 return true;
693 }
694 ++end;
695 *input = end;
696 uint32_t keys = strtoul(*input, &end, 16);
697 if (end[0] == ',') {
698 ++end;
699 }
700 *input = end;
701 core->setKeys(core, keys);
702 return true;
703}
704
705static bool _compareImages(struct CInemaTest* restrict test, const struct CInemaImage* restrict image, const struct CInemaImage* restrict expected, int* restrict max, uint8_t** restrict outdiff) {
706 const uint8_t* testPixels = image->data;
707 const uint8_t* expectPixels = expected->data;
708 uint8_t* diff = NULL;
709 size_t x;
710 size_t y;
711 bool failed = false;
712 for (y = 0; y < image->height; ++y) {
713 for (x = 0; x < image->width; ++x) {
714 size_t pix = expected->stride * y + x;
715 size_t tpix = image->stride * y + x;
716 int testR = testPixels[tpix * 4 + 0];
717 int testG = testPixels[tpix * 4 + 1];
718 int testB = testPixels[tpix * 4 + 2];
719 int expectR = expectPixels[pix * 4 + 0];
720 int expectG = expectPixels[pix * 4 + 1];
721 int expectB = expectPixels[pix * 4 + 2];
722 int r = expectR - testR;
723 int g = expectG - testG;
724 int b = expectB - testB;
725 if (r | g | b) {
726 failed = true;
727 if (outdiff && !diff) {
728 diff = calloc(expected->stride * expected->height, BYTES_PER_PIXEL);
729 *outdiff = diff;
730 }
731 test->status = CI_FAIL;
732 if (r < 0) {
733 r = -r;
734 }
735 if (g < 0) {
736 g = -g;
737 }
738 if (b < 0) {
739 b = -b;
740 }
741
742 if (diff) {
743 if (r > *max) {
744 *max = r;
745 }
746 if (g > *max) {
747 *max = g;
748 }
749 if (b > *max) {
750 *max = b;
751 }
752 diff[pix * 4 + 0] = r;
753 diff[pix * 4 + 1] = g;
754 diff[pix * 4 + 2] = b;
755 }
756
757 if (test) {
758 test->totalDistance += r + g + b;
759 ++test->failedPixels;
760 }
761 }
762 }
763 }
764 return !failed;
765}
766
767void _writeDiffSet(struct CInemaImage* expected, const char* name, uint8_t* diff, int frame, int max, bool xfail) {
768 struct CInemaImage outdiff = {
769 .data = diff,
770 .width = expected->width,
771 .height = expected->height,
772 .stride = expected->stride,
773 };
774
775 if (xfail) {
776 _writeDiff(name, expected, frame, "xexpected");
777 _writeDiff(name, &outdiff, frame, "xdiff");
778 } else {
779 _writeDiff(name, expected, frame, "expected");
780 _writeDiff(name, &outdiff, frame, "diff");
781 }
782
783 size_t x;
784 size_t y;
785 for (y = 0; y < outdiff.height; ++y) {
786 for (x = 0; x < outdiff.width; ++x) {
787 size_t pix = outdiff.stride * y + x;
788 diff[pix * 4 + 0] = diff[pix * 4 + 0] * 255 / max;
789 diff[pix * 4 + 1] = diff[pix * 4 + 1] * 255 / max;
790 diff[pix * 4 + 2] = diff[pix * 4 + 2] * 255 / max;
791 }
792 }
793 if (xfail) {
794 _writeDiff(name, &outdiff, frame, "xnormalized");
795 }
796}
797
798void CInemaTestRun(struct CInemaTest* test) {
799 unsigned ignore = 0;
800 MutexLock(&configMutex);
801 CInemaConfigGetUInt(&configTree, test->name, "ignore", &ignore);
802 MutexUnlock(&configMutex);
803 if (ignore) {
804 test->status = CI_SKIP;
805 return;
806 }
807
808 struct VDir* dir = VDirOpen(test->directory);
809 if (!dir) {
810 CIerr(0, "Failed to open test directory\n");
811 test->status = CI_ERROR;
812 return;
813 }
814 struct VFile* rom = dir->openFile(dir, test->filename, O_RDONLY);
815 if (!rom) {
816 CIerr(0, "Failed to open test\n");
817 test->status = CI_ERROR;
818 return;
819 }
820 struct mCore* core = mCoreFindVF(rom);
821 if (!core) {
822 CIerr(0, "Failed to load test\n");
823 test->status = CI_ERROR;
824 rom->close(rom);
825 return;
826 }
827 if (!core->init(core)) {
828 CIerr(0, "Failed to init test\n");
829 test->status = CI_ERROR;
830 core->deinit(core);
831 return;
832 }
833 struct CInemaImage image;
834 core->desiredVideoDimensions(core, &image.width, &image.height);
835 ssize_t bufferSize = image.width * image.height * BYTES_PER_PIXEL;
836 image.data = malloc(bufferSize);
837 image.stride = image.width;
838 if (!image.data) {
839 CIerr(0, "Failed to allocate video buffer\n");
840 test->status = CI_ERROR;
841 core->deinit(core);
842 }
843 core->setVideoBuffer(core, image.data, image.stride);
844 mCoreConfigInit(&core->config, "cinema");
845
846 unsigned limit = 9999;
847 unsigned skip = 0;
848 unsigned fail = 0;
849 unsigned video = 0;
850 const char* input = NULL;
851
852 MutexLock(&configMutex);
853 CInemaConfigGetUInt(&configTree, test->name, "frames", &limit);
854 CInemaConfigGetUInt(&configTree, test->name, "skip", &skip);
855 CInemaConfigGetUInt(&configTree, test->name, "fail", &fail);
856 CInemaConfigGetUInt(&configTree, test->name, "video", &video);
857 input = CInemaConfigGet(&configTree, test->name, "input");
858 CInemaConfigLoad(&configTree, test->name, core);
859 MutexUnlock(&configMutex);
860
861 struct VFile* save = VFileMemChunk(NULL, 0);
862 core->loadROM(core, rom);
863 if (!core->loadSave(core, save)) {
864 save->close(save);
865 }
866 core->rtc.override = RTC_FAKE_EPOCH;
867 core->rtc.value = 1200000000;
868 core->reset(core);
869
870 test->status = CI_PASS;
871
872 unsigned minFrame = core->frameCounter(core);
873 size_t frame;
874 for (frame = 0; frame < skip; ++frame) {
875 core->runFrame(core);
876 }
877 core->desiredVideoDimensions(core, &image.width, &image.height);
878
879#ifdef USE_FFMPEG
880 struct FFmpegDecoder decoder;
881 struct FFmpegEncoder encoder;
882 struct CInemaStream stream = {0};
883
884 char baselineName[PATH_MAX];
885 snprintf(baselineName, sizeof(baselineName), "%s" PATH_SEP "baseline.avi", test->directory);
886 bool exists = access(baselineName, 0) == 0;
887
888 char tmpBaselineName[PATH_MAX];
889 snprintf(tmpBaselineName, sizeof(tmpBaselineName), "%s" PATH_SEP ".baseline.avi", test->directory);
890
891 if (video) {
892 FFmpegEncoderInit(&encoder);
893 FFmpegDecoderInit(&decoder);
894
895 if (rebaseline == CI_R_FAILING || (rebaseline == CI_R_MISSING && !exists)) {
896 FFmpegEncoderSetAudio(&encoder, NULL, 0);
897 FFmpegEncoderSetVideo(&encoder, "zmbv", 0, 0);
898 FFmpegEncoderSetContainer(&encoder, "avi");
899 FFmpegEncoderSetDimensions(&encoder, image.width, image.height);
900
901 const char* usedFname = baselineName;
902 if (exists) {
903 usedFname = tmpBaselineName;
904 }
905 if (!FFmpegEncoderOpen(&encoder, usedFname)) {
906 CIerr(1, "Failed to save baseline video\n");
907 } else {
908 core->setAVStream(core, &encoder.d);
909 }
910 }
911
912 if (exists) {
913 stream.d.postVideoFrame = _cinemaVideoFrame;
914 stream.d.videoDimensionsChanged = _cinemaDimensionsChanged;
915 stream.status = &test->status;
916 decoder.out = &stream.d;
917
918 if (!FFmpegDecoderOpen(&decoder, baselineName)) {
919 CIerr(1, "Failed to load baseline video\n");
920 }
921 } else if (!rebaseline) {
922 test->status = CI_FAIL;
923 }
924 }
925#else
926 if (video) {
927 CIerr(0, "Failed to run video test without ffmpeg linked in\n");
928 test->status = CI_ERROR;
929 }
930#endif
931
932 bool xdiff = false;
933 for (frame = 0; limit; ++frame, --limit) {
934 _updateInput(core, frame, &input);
935 core->runFrame(core);
936 ++test->totalFrames;
937 unsigned frameCounter = core->frameCounter(core);
938 if (frameCounter <= minFrame) {
939 break;
940 }
941 if (test->status == CI_ERROR) {
942 break;
943 }
944 CIlog(3, "Test frame: %u\n", frameCounter);
945 core->desiredVideoDimensions(core, &image.width, &image.height);
946 uint8_t* diff = NULL;
947 struct CInemaImage expected = {
948 .data = NULL,
949 .width = image.width,
950 .height = image.height,
951 .stride = image.width,
952 };
953 bool baselineFound;
954 if (video) {
955 baselineFound = false;
956#ifdef USE_FFMPEG
957 if (FFmpegDecoderIsOpen(&decoder)) {
958 stream.image = &expected;
959 while (!expected.data) {
960 if (!FFmpegDecoderRead(&decoder)) {
961 CIlog(1, "Failed to read more frames. EOF?\n");
962 test->status = CI_FAIL;
963 break;
964 }
965 }
966 baselineFound = expected.data;
967 }
968#endif
969 } else {
970 baselineFound = _loadBaselinePNG(dir, "baseline", &expected, frame, &test->status);
971 }
972 if (test->status == CI_ERROR) {
973 break;
974 }
975 bool failed = false;
976 if (baselineFound) {
977 int max = 0;
978 failed = !_compareImages(test, &image, &expected, &max, diffs ? &diff : NULL);
979 if (failed) {
980 ++test->failedFrames;
981 }
982 test->totalPixels += image.height * image.width;
983 if (rebaseline == CI_R_FAILING && !video && failed) {
984 _writeBaseline(dir, "baseline", &image, frame);
985 }
986 if (diff) {
987 if (failed) {
988 _writeDiff(test->name, &image, frame, "result");
989 _writeDiffSet(&expected, test->name, diff, frame, max, false);
990 }
991 free(diff);
992 diff = NULL;
993 }
994 free(expected.data);
995 } else if (rebaseline && !video) {
996 _writeBaseline(dir, "baseline", &image, frame);
997 } else if (!rebaseline) {
998 test->status = CI_FAIL;
999 }
1000
1001 if (fail && failed) {
1002 if (video) {
1003 // TODO
1004 baselineFound = false;
1005 } else {
1006 baselineFound = _loadBaselinePNG(dir, "xbaseline", &expected, frame, &test->status);
1007 }
1008
1009 if (baselineFound) {
1010 int max = 0;
1011 failed = !_compareImages(test, &image, &expected, &max, diffs ? &diff : NULL);
1012 if (diff) {
1013 if (failed) {
1014 _writeDiffSet(&expected, test->name, diff, frame, max, true);
1015 }
1016 free(diff);
1017 diff = NULL;
1018 }
1019 if (failed) {
1020 if (xbaseline == CI_R_FAILING && !video) {
1021 _writeBaseline(dir, "xbaseline", &image, frame);
1022 }
1023 xdiff = true;
1024 }
1025 free(expected.data);
1026 } else if (xbaseline && !video) {
1027 _writeBaseline(dir, "xbaseline", &image, frame);
1028 }
1029 }
1030 }
1031
1032#ifdef USE_FFMPEG
1033 if (video) {
1034 if (FFmpegEncoderIsOpen(&encoder)) {
1035 FFmpegEncoderClose(&encoder);
1036 if (exists) {
1037 if (test->status == CI_FAIL) {
1038#ifdef _WIN32
1039 MoveFileEx(tmpBaselineName, baselineName, MOVEFILE_REPLACE_EXISTING);
1040#else
1041 rename(tmpBaselineName, baselineName);
1042#endif
1043 } else {
1044 remove(tmpBaselineName);
1045 }
1046 }
1047 }
1048 if (FFmpegDecoderIsOpen(&decoder)) {
1049 FFmpegDecoderClose(&decoder);
1050 }
1051 }
1052#endif
1053
1054 if (fail) {
1055 if (test->status == CI_FAIL && !xdiff) {
1056 test->status = CI_XFAIL;
1057 } else if (test->status == CI_PASS) {
1058 test->status = CI_XPASS;
1059 }
1060 }
1061
1062 free(image.data);
1063 mCoreConfigDeinit(&core->config);
1064 core->deinit(core);
1065 dir->close(dir);
1066}
1067
1068static bool CInemaTask(struct CInemaTestList* tests, size_t i) {
1069 bool success = true;
1070 struct CInemaTest* test = CInemaTestListGetPointer(tests, i);
1071 if (dryRun) {
1072 CIlog(-1, "%s\n", test->name);
1073 } else {
1074 CIlog(1, "%s: ", test->name);
1075 fflush(stdout);
1076 ThreadLocalSetKey(currentTest, test);
1077 CInemaTestRun(test);
1078 ThreadLocalSetKey(currentTest, NULL);
1079
1080 switch (test->status) {
1081 case CI_PASS:
1082 CIlog(1, "pass\n");
1083 break;
1084 case CI_FAIL:
1085 success = false;
1086 CIlog(1, "fail\n");
1087 break;
1088 case CI_XPASS:
1089 CIlog(1, "xpass\n");
1090 break;
1091 case CI_XFAIL:
1092 CIlog(1, "xfail\n");
1093 break;
1094 case CI_SKIP:
1095 CIlog(1, "skip\n");
1096 break;
1097 case CI_ERROR:
1098 success = false;
1099 CIlog(1, "error\n");
1100 break;
1101 }
1102 if (test->failedFrames) {
1103 CIlog(2, "\tfailed frames: %u/%u (%1.3g%%)\n", test->failedFrames, test->totalFrames, test->failedFrames / (test->totalFrames * 0.01));
1104 CIlog(2, "\tfailed pixels: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->failedPixels, test->totalPixels, test->failedPixels / (test->totalPixels * 0.01));
1105 CIlog(2, "\tdistance: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->totalDistance, test->totalPixels * 765, test->totalDistance / (test->totalPixels * 7.65));
1106 }
1107 }
1108 return success;
1109}
1110
1111static THREAD_ENTRY CInemaJob(void* context) {
1112 struct CInemaTestList* tests = context;
1113 struct CInemaLogStream stream;
1114 StringListInit(&stream.out.lines, 0);
1115 StringListInit(&stream.out.partial, 0);
1116 stream.out.repeat = 0;
1117 StringListInit(&stream.err.lines, 0);
1118 StringListInit(&stream.err.partial, 0);
1119 stream.err.repeat = 0;
1120 ThreadLocalSetKey(logStream, &stream);
1121
1122 bool success = true;
1123 while (true) {
1124 size_t i;
1125 MutexLock(&jobMutex);
1126 i = jobIndex;
1127 ++jobIndex;
1128 MutexUnlock(&jobMutex);
1129 if (i >= CInemaTestListSize(tests)) {
1130 break;
1131 }
1132 if (!CInemaTask(tests, i)) {
1133 success = false;
1134 }
1135 CIflush(&stream.out, stdout);
1136 CIflush(&stream.err, stderr);
1137 }
1138 MutexLock(&jobMutex);
1139 if (!success) {
1140 jobStatus = 1;
1141 }
1142 MutexUnlock(&jobMutex);
1143
1144 CIflush(&stream.out, stdout);
1145 StringListDeinit(&stream.out.lines);
1146 StringListDeinit(&stream.out.partial);
1147
1148 CIflush(&stream.err, stderr);
1149 StringListDeinit(&stream.err.lines);
1150 StringListDeinit(&stream.err.partial);
1151}
1152
1153void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args) {
1154 UNUSED(log);
1155 if (level == mLOG_FATAL) {
1156 struct CInemaTest* test = ThreadLocalGetValue(currentTest);
1157 test->status = CI_ERROR;
1158 }
1159 if (verbosity < 0) {
1160 return;
1161 }
1162 int mask = mLOG_FATAL;
1163 if (verbosity >= 1) {
1164 mask |= mLOG_ERROR;
1165 }
1166 if (verbosity >= 2) {
1167 mask |= mLOG_WARN;
1168 }
1169 if (verbosity >= 4) {
1170 mask |= mLOG_INFO;
1171 }
1172 if (verbosity >= 5) {
1173 mask |= mLOG_ALL;
1174 }
1175 if (!(mask & level)) {
1176 return;
1177 }
1178
1179 char buffer[256];
1180 vsnprintf(buffer, sizeof(buffer), format, args);
1181 CIerr(0, "[%s] %s\n", mLogCategoryName(category), buffer);
1182}
1183
1184int main(int argc, char** argv) {
1185 ThreadLocalInitKey(&logStream);
1186 ThreadLocalSetKey(logStream, NULL);
1187
1188 int status = 0;
1189 if (!parseCInemaArgs(argc, argv)) {
1190 status = 1;
1191 goto cleanup;
1192 }
1193
1194 if (showVersion) {
1195 version(argv[0]);
1196 goto cleanup;
1197 }
1198
1199 if (showUsage) {
1200 usageCInema(argv[0]);
1201 goto cleanup;
1202 }
1203
1204 argc -= optind;
1205 argv += optind;
1206
1207 if (!base[0] && !determineBase(argc, argv)) {
1208 CIlog(0, "Could not determine CInema test base. Please specify manually.");
1209 status = 1;
1210 goto cleanup;
1211 }
1212#ifndef _WIN32
1213 char* rbase = realpath(base, NULL);
1214 if (rbase) {
1215 strlcpy(base, rbase, sizeof(base));
1216 free(rbase);
1217 }
1218#endif
1219
1220 struct CInemaTestList tests;
1221 CInemaTestListInit(&tests, 0);
1222
1223 struct mLogger logger = { .log = _log };
1224 mLogSetDefaultLogger(&logger);
1225#ifdef USE_FFMPEG
1226 if (verbosity < 2) {
1227 av_log_set_level(AV_LOG_ERROR);
1228 }
1229#endif
1230
1231 if (argc > 0) {
1232 size_t i;
1233 for (i = 0; i < (size_t) argc; ++i) {
1234 char path[PATH_MAX + 1] = {0};
1235 testToPath(argv[i], path);
1236
1237 if (!collectTests(&tests, path)) {
1238 status = 1;
1239 break;
1240 }
1241 }
1242 } else if (!collectTests(&tests, base)) {
1243 status = 1;
1244 }
1245
1246 if (CInemaTestListSize(&tests) == 0) {
1247 CIlog(1, "No tests found.");
1248 status = 1;
1249 } else {
1250 reduceTestList(&tests);
1251 }
1252
1253 HashTableInit(&configTree, 0, free);
1254 MutexInit(&configMutex);
1255 ThreadLocalInitKey(¤tTest);
1256 ThreadLocalSetKey(currentTest, NULL);
1257
1258 if (jobs == 1) {
1259 size_t i;
1260 for (i = 0; i < CInemaTestListSize(&tests); ++i) {
1261 bool success = CInemaTask(&tests, i);
1262 if (!success) {
1263 status = 1;
1264 }
1265 }
1266 } else {
1267 MutexInit(&jobMutex);
1268 int i;
1269 for (i = 0; i < jobs; ++i) {
1270 ThreadCreate(&jobThreads[i], CInemaJob, &tests);
1271 }
1272 for (i = 0; i < jobs; ++i) {
1273 ThreadJoin(&jobThreads[i]);
1274 }
1275 MutexDeinit(&jobMutex);
1276 status = jobStatus;
1277 }
1278
1279 MutexDeinit(&configMutex);
1280 HashTableEnumerate(&configTree, _unloadConfigTree, NULL);
1281 HashTableDeinit(&configTree);
1282 CInemaTestListDeinit(&tests);
1283
1284cleanup:
1285 return status;
1286}