src/platform/test/cinema-main.c (view raw)
1/* Copyright (c) 2013-2020 Jeffrey Pfau
2 *
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6#include <mgba/core/config.h>
7#include <mgba/core/core.h>
8#include <mgba/core/log.h>
9#include <mgba/core/version.h>
10#include <mgba/feature/commandline.h>
11#include <mgba/feature/video-logger.h>
12
13#include <mgba-util/png-io.h>
14#include <mgba-util/string.h>
15#include <mgba-util/table.h>
16#include <mgba-util/threading.h>
17#include <mgba-util/vector.h>
18#include <mgba-util/vfs.h>
19
20#ifdef USE_FFMPEG
21#include "feature/ffmpeg/ffmpeg-decoder.h"
22#include "feature/ffmpeg/ffmpeg-encoder.h"
23#endif
24
25#ifdef _MSC_VER
26#include <mgba-util/platform/windows/getopt.h>
27#else
28#include <getopt.h>
29#endif
30
31#include <stdlib.h>
32#include <sys/stat.h>
33#include <sys/types.h>
34
35#define MAX_TEST 200
36#define MAX_JOBS 128
37#define LOG_THRESHOLD 1000000
38
39static const struct option longOpts[] = {
40 { "base", required_argument, 0, 'b' },
41 { "diffs", no_argument, 0, 'd' },
42 { "help", no_argument, 0, 'h' },
43 { "jobs", required_argument, 0, 'j' },
44 { "dry-run", no_argument, 0, 'n' },
45 { "outdir", required_argument, 0, 'o' },
46 { "quiet", no_argument, 0, 'q' },
47 { "rebaseline", no_argument, 0, 'r' },
48 { "rebaseline-missing", no_argument, 0, 'R' },
49 { "verbose", no_argument, 0, 'v' },
50 { "xbaseline", no_argument, 0, 'x' },
51 { "version", no_argument, 0, '\0' },
52 { 0, 0, 0, 0 }
53};
54
55static const char shortOpts[] = "b:dhj:no:qRrvx";
56
57enum CInemaStatus {
58 CI_PASS,
59 CI_FAIL,
60 CI_XPASS,
61 CI_XFAIL,
62 CI_ERROR,
63 CI_SKIP
64};
65
66enum CInemaRebaseline {
67 CI_R_NONE = 0,
68 CI_R_FAILING,
69 CI_R_MISSING,
70};
71
72struct CInemaTest {
73 char directory[MAX_TEST];
74 char filename[MAX_TEST];
75 char name[MAX_TEST];
76 enum CInemaStatus status;
77 unsigned failedFrames;
78 uint64_t failedPixels;
79 unsigned totalFrames;
80 uint64_t totalDistance;
81 uint64_t totalPixels;
82};
83
84struct CInemaImage {
85 void* data;
86 unsigned width;
87 unsigned height;
88 unsigned stride;
89};
90
91DECLARE_VECTOR(CInemaTestList, struct CInemaTest)
92DEFINE_VECTOR(CInemaTestList, struct CInemaTest)
93
94DECLARE_VECTOR(ImageList, void*)
95DEFINE_VECTOR(ImageList, void*)
96
97struct StringBuilder {
98 struct StringList lines;
99 struct StringList partial;
100 unsigned repeat;
101
102};
103
104struct CInemaLogStream {
105 struct StringBuilder err;
106 struct StringBuilder out;
107};
108
109static bool showVersion = false;
110static bool showUsage = false;
111static char base[PATH_MAX] = {0};
112static char outdir[PATH_MAX] = {'.'};
113static bool dryRun = false;
114static bool diffs = false;
115static enum CInemaRebaseline rebaseline = CI_R_NONE;
116static enum CInemaRebaseline xbaseline = CI_R_NONE;
117static int verbosity = 0;
118
119static struct Table configTree;
120static Mutex configMutex;
121
122static int jobs = 1;
123static size_t jobIndex = 0;
124static Mutex jobMutex;
125static Thread jobThreads[MAX_JOBS];
126static int jobStatus;
127static ThreadLocal logStream;
128static ThreadLocal currentTest;
129
130bool CInemaTestInit(struct CInemaTest*, const char* directory, const char* filename);
131void CInemaTestRun(struct CInemaTest*);
132
133bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* value);
134void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core);
135
136static void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args);
137
138void CIflush(struct StringBuilder* list, FILE* file);
139
140static char* _compileStringList(struct StringList* list) {
141 size_t len = 0;
142 size_t i;
143 for (i = 0; i < StringListSize(list); ++i) {
144 len += strlen(*StringListGetPointer(list, i));
145 }
146 char* string = calloc(len + 1, sizeof(char));
147 char* cur = string;
148 for (i = 0; i < StringListSize(list); ++i) {
149 char* brick = *StringListGetPointer(list, i);
150 size_t portion = strlen(brick);
151 memcpy(cur, brick, portion);
152 free(brick);
153 cur += portion;
154 }
155 StringListClear(list);
156 return string;
157}
158
159static void _logToStream(FILE* file, const char* format, va_list args) {
160#ifdef HAVE_VASPRINTF
161 struct CInemaLogStream* stream = ThreadLocalGetValue(logStream);
162 if (!stream) {
163 vfprintf(file, format, args);
164 } else {
165 struct StringBuilder* builder = &stream->out;
166 if (file == stderr) {
167 builder = &stream->err;
168 }
169 if (StringListSize(&builder->lines) > LOG_THRESHOLD) {
170 CIflush(builder, file);
171 }
172 char** line = StringListAppend(&builder->partial);
173 vasprintf(line, format, args);
174 size_t len = strlen(*line);
175 if (len && (*line)[len - 1] == '\n') {
176 char* string = _compileStringList(&builder->partial);
177 size_t linecount = StringListSize(&builder->lines);
178 if (linecount && strcmp(string, *StringListGetPointer(&builder->lines, linecount - 1)) == 0) {
179 ++builder->repeat;
180 free(string);
181 } else {
182 if (builder->repeat > 1) {
183 asprintf(StringListAppend(&builder->lines), "The previous message was repeated %u times.\n", builder->repeat);
184 }
185 *StringListAppend(&builder->lines) = string;
186 builder->repeat = 1;
187 }
188 }
189 }
190#else
191 vfprintf(file, format, args);
192#endif
193}
194
195ATTRIBUTE_FORMAT(printf, 2, 3) void CIlog(int minlevel, const char* format, ...) {
196 if (verbosity < minlevel) {
197 return;
198 }
199 va_list args;
200 va_start(args, format);
201 _logToStream(stdout, format, args);
202 va_end(args);
203}
204
205ATTRIBUTE_FORMAT(printf, 2, 3) void CIerr(int minlevel, const char* format, ...) {
206 if (verbosity < minlevel) {
207 return;
208 }
209 va_list args;
210 va_start(args, format);
211 _logToStream(stderr, format, args);
212 va_end(args);
213}
214
215void CIflush(struct StringBuilder* builder, FILE* out) {
216 if (StringListSize(&builder->partial)) {
217 *StringListAppend(&builder->lines) = _compileStringList(&builder->partial);
218 }
219#ifdef HAVE_VASPRINTF
220 if (builder->repeat > 1) {
221 asprintf(StringListAppend(&builder->lines), "The previous message was repeated %u times.\n", builder->repeat);
222 }
223#endif
224
225 char* string = _compileStringList(&builder->lines);
226 builder->repeat = 0;
227 fputs(string, out);
228 free(string);
229}
230
231static bool parseCInemaArgs(int argc, char* const* argv) {
232 int ch;
233 int index = 0;
234 while ((ch = getopt_long(argc, argv, shortOpts, longOpts, &index)) != -1) {
235 const struct option* opt = &longOpts[index];
236 switch (ch) {
237 case '\0':
238 if (strcmp(opt->name, "version") == 0) {
239 showVersion = true;
240 } else {
241 return false;
242 }
243 break;
244 case 'b':
245 strlcpy(base, optarg, sizeof(base));
246 // TODO: Verify path exists
247 break;
248 case 'd':
249 diffs = true;
250 break;
251 case 'h':
252 showUsage = true;
253 break;
254 case 'j':
255 jobs = atoi(optarg);
256 if (jobs > MAX_JOBS) {
257 jobs = MAX_JOBS;
258 }
259 if (jobs < 1) {
260 jobs = 1;
261 }
262 break;
263 case 'n':
264 dryRun = true;
265 break;
266 case 'o':
267 strlcpy(outdir, optarg, sizeof(outdir));
268 // TODO: Make directory
269 break;
270 case 'q':
271 --verbosity;
272 break;
273 case 'r':
274 rebaseline = CI_R_FAILING;
275 break;
276 case 'R':
277 rebaseline = CI_R_MISSING;
278 break;
279 case 'v':
280 ++verbosity;
281 break;
282 case 'x':
283 xbaseline = CI_R_FAILING;
284 break;
285 default:
286 return false;
287 }
288 }
289
290 return true;
291}
292
293static void usageCInema(const char* arg0) {
294 printf("usage: %s [-dhnqrRv] [-j JOBS] [-b BASE] [-o DIR] [--version] [test...]\n", arg0);
295 puts(" -b, --base BASE Path to the CInema base directory");
296 puts(" -d, --diffs Output image diffs from failures");
297 puts(" -h, --help Print this usage and exit");
298 puts(" -j, --jobs JOBS Run a number of jobs in parallel");
299 puts(" -n, --dry-run List all collected tests instead of running them");
300 puts(" -o, --output DIR Path to output applicable results");
301 puts(" -q, --quiet Decrease log verbosity (can be repeated)");
302 puts(" -r, --rebaseline Rewrite the baseline for failing tests");
303 puts(" -R, --rebaseline-missing Write missing baselines tests only");
304 puts(" -v, --verbose Increase log verbosity (can be repeated)");
305 puts(" -x, --xbaseline Rewrite the xfail baselines for failing tests");
306 puts(" --version Print version and exit");
307}
308
309static bool determineBase(int argc, char* const* argv) {
310 // TODO: Better dynamic detection
311 separatePath(__FILE__, base, NULL, NULL);
312 strncat(base, PATH_SEP ".." PATH_SEP ".." PATH_SEP ".." PATH_SEP "cinema", sizeof(base) - strlen(base) - 1);
313 return true;
314}
315
316static bool collectTests(struct CInemaTestList* tests, const char* path) {
317 CIerr(2, "Considering path %s\n", path);
318 struct VDir* dir = VDirOpen(path);
319 if (!dir) {
320 return false;
321 }
322 struct VDirEntry* entry = dir->listNext(dir);
323 while (entry) {
324 char subpath[PATH_MAX];
325 snprintf(subpath, sizeof(subpath), "%s" PATH_SEP "%s", path, entry->name(entry));
326 if (entry->type(entry) == VFS_DIRECTORY && strncmp(entry->name(entry), ".", 2) != 0 && strncmp(entry->name(entry), "..", 3) != 0) {
327 if (!collectTests(tests, subpath)) {
328 dir->close(dir);
329 return false;
330 }
331 } else if (entry->type(entry) == VFS_FILE && strncmp(entry->name(entry), "test.", 5) == 0) {
332 CIerr(3, "Found potential test %s\n", subpath);
333 struct VFile* vf = dir->openFile(dir, entry->name(entry), O_RDONLY);
334 if (vf) {
335 if (mCoreIsCompatible(vf) != PLATFORM_NONE || mVideoLogIsCompatible(vf) != PLATFORM_NONE) {
336 struct CInemaTest* test = CInemaTestListAppend(tests);
337 if (!CInemaTestInit(test, path, entry->name(entry))) {
338 CIerr(3, "Failed to create test\n");
339 CInemaTestListResize(tests, -1);
340 } else {
341 CIerr(2, "Found test %s\n", test->name);
342 }
343 } else {
344 CIerr(3, "Not a compatible file\n");
345 }
346 vf->close(vf);
347 } else {
348 CIerr(3, "Failed to open file\n");
349 }
350 }
351 entry = dir->listNext(dir);
352 }
353 dir->close(dir);
354 return true;
355}
356
357static int _compareNames(const void* a, const void* b) {
358 const struct CInemaTest* ta = a;
359 const struct CInemaTest* tb = b;
360
361 return strncmp(ta->name, tb->name, sizeof(ta->name));
362}
363
364static void reduceTestList(struct CInemaTestList* tests) {
365 qsort(CInemaTestListGetPointer(tests, 0), CInemaTestListSize(tests), sizeof(struct CInemaTest), _compareNames);
366
367 size_t i;
368 for (i = 1; i < CInemaTestListSize(tests);) {
369 struct CInemaTest* cur = CInemaTestListGetPointer(tests, i);
370 struct CInemaTest* prev = CInemaTestListGetPointer(tests, i - 1);
371 if (strncmp(cur->name, prev->name, sizeof(cur->name)) != 0) {
372 ++i;
373 continue;
374 }
375 CInemaTestListShift(tests, i, 1);
376 }
377}
378
379static void testToPath(const char* testName, char* path) {
380 strlcpy(path, base, PATH_MAX);
381
382 bool dotSeen = true;
383 size_t i;
384 for (i = strlen(path); testName[0] && i < PATH_MAX; ++testName) {
385 if (testName[0] == '.') {
386 dotSeen = true;
387 } else {
388 if (dotSeen) {
389 strlcpy(&path[i], PATH_SEP, PATH_MAX - i);
390 i += strlen(PATH_SEP);
391 dotSeen = false;
392 if (!i) {
393 break;
394 }
395 }
396 path[i] = testName[0];
397 ++i;
398 }
399 }
400 if (i == PATH_MAX) {
401 --i;
402 }
403 path[i] = '\0';
404}
405
406static void _loadConfigTree(struct Table* configTree, const char* testName) {
407 char key[MAX_TEST];
408 strlcpy(key, testName, sizeof(key));
409
410 struct mCoreConfig* config;
411 while (!(config = HashTableLookup(configTree, key))) {
412 char path[PATH_MAX];
413 config = malloc(sizeof(*config));
414 mCoreConfigInit(config, "cinema");
415 testToPath(key, path);
416 strncat(path, PATH_SEP, sizeof(path) - 1);
417 strncat(path, "config.ini", sizeof(path) - 1);
418 mCoreConfigLoadPath(config, path);
419 HashTableInsert(configTree, key, config);
420 char* pos = strrchr(key, '.');
421 if (pos) {
422 pos[0] = '\0';
423 } else if (key[0]) {
424 key[0] = '\0';
425 } else {
426 break;
427 }
428 }
429}
430
431static void _unloadConfigTree(const char* key, void* value, void* user) {
432 UNUSED(key);
433 UNUSED(user);
434 mCoreConfigDeinit(value);
435}
436
437static const char* CInemaConfigGet(struct Table* configTree, const char* testName, const char* key) {
438 _loadConfigTree(configTree, testName);
439
440 char testKey[MAX_TEST];
441 strlcpy(testKey, testName, sizeof(testKey));
442
443 struct mCoreConfig* config;
444 while (true) {
445 config = HashTableLookup(configTree, testKey);
446 if (!config) {
447 continue;
448 }
449 const char* str = ConfigurationGetValue(&config->configTable, "testinfo", key);
450 if (str) {
451 return str;
452 }
453 char* pos = strrchr(testKey, '.');
454 if (pos) {
455 pos[0] = '\0';
456 } else if (testKey[0]) {
457 testKey[0] = '\0';
458 } else {
459 break;
460 }
461 }
462 return NULL;
463}
464
465bool CInemaConfigGetUInt(struct Table* configTree, const char* testName, const char* key, unsigned* out) {
466 const char* charValue = CInemaConfigGet(configTree, testName, key);
467 if (!charValue) {
468 return false;
469 }
470 char* end;
471 unsigned long value = strtoul(charValue, &end, 10);
472 if (*end) {
473 return false;
474 }
475 *out = value;
476 return true;
477}
478
479void CInemaConfigLoad(struct Table* configTree, const char* testName, struct mCore* core) {
480 _loadConfigTree(configTree, testName);
481
482 char testKey[MAX_TEST] = {0};
483 char* keyEnd = testKey;
484 const char* pos;
485 while (true) {
486 pos = strchr(testName, '.');
487 size_t maxlen = sizeof(testKey) - (keyEnd - testKey) - 1;
488 size_t len;
489 if (pos) {
490 len = pos - testName;
491 } else {
492 len = strlen(testName);
493 }
494 if (len > maxlen) {
495 len = maxlen;
496 }
497 strncpy(keyEnd, testName, len);
498 keyEnd += len;
499
500 struct mCoreConfig* config = HashTableLookup(configTree, testKey);
501 if (config) {
502 core->loadConfig(core, config);
503 }
504 if (!pos) {
505 break;
506 }
507 testName = pos + 1;
508 keyEnd[0] = '.';
509 ++keyEnd;
510 }
511}
512
513bool CInemaTestInit(struct CInemaTest* test, const char* directory, const char* filename) {
514 if (strncmp(base, directory, strlen(base)) != 0) {
515 return false;
516 }
517 memset(test, 0, sizeof(*test));
518 strlcpy(test->directory, directory, sizeof(test->directory));
519 strlcpy(test->filename, filename, sizeof(test->filename));
520 directory += strlen(base) + 1;
521 strlcpy(test->name, directory, sizeof(test->name));
522 char* str = strstr(test->name, PATH_SEP);
523 while (str) {
524 str[0] = '.';
525 str = strstr(str, PATH_SEP);
526 }
527 return true;
528}
529
530static bool _loadBaselinePNG(struct VDir* dir, const char* type, struct CInemaImage* image, size_t frame, enum CInemaStatus* status) {
531 char baselineName[32];
532 snprintf(baselineName, sizeof(baselineName), "%s_%04" PRIz "u.png", type, frame);
533 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_RDONLY);
534 if (!baselineVF) {
535 if (*status == CI_PASS) {
536 *status = CI_FAIL;
537 }
538 return false;
539 }
540
541 png_structp png = PNGReadOpen(baselineVF, 0);
542 png_infop info = png_create_info_struct(png);
543 png_infop end = png_create_info_struct(png);
544 if (!png || !info || !end || !PNGReadHeader(png, info)) {
545 PNGReadClose(png, info, end);
546 baselineVF->close(baselineVF);
547 CIerr(1, "Failed to load %s\n", baselineName);
548 *status = CI_ERROR;
549 return false;
550 }
551
552 unsigned pwidth = png_get_image_width(png, info);
553 unsigned pheight = png_get_image_height(png, info);
554 if (pheight != image->height || pwidth != image->width) {
555 PNGReadClose(png, info, end);
556 baselineVF->close(baselineVF);
557 CIlog(1, "Size mismatch for %s, expected %ux%u, got %ux%u\n", baselineName, pwidth, pheight, image->width, image->height);
558 if (*status == CI_PASS) {
559 *status = CI_FAIL;
560 }
561 return false;
562 }
563
564 image->data = malloc(pwidth * pheight * BYTES_PER_PIXEL);
565 if (!image->data) {
566 CIerr(1, "Failed to allocate baseline buffer\n");
567 *status = CI_ERROR;
568 PNGReadClose(png, info, end);
569 baselineVF->close(baselineVF);
570 return false;
571 }
572 if (!PNGReadPixels(png, info, image->data, pwidth, pheight, pwidth) || !PNGReadFooter(png, end)) {
573 CIerr(1, "Failed to read %s\n", baselineName);
574 *status = CI_ERROR;
575 free(image->data);
576 return false;
577 }
578 PNGReadClose(png, info, end);
579 baselineVF->close(baselineVF);
580 image->stride = pwidth;
581 return true;
582}
583
584#ifdef USE_FFMPEG
585struct CInemaStream {
586 struct mAVStream d;
587 struct CInemaImage* image;
588 enum CInemaStatus* status;
589};
590
591static void _cinemaDimensionsChanged(struct mAVStream* stream, unsigned width, unsigned height) {
592 struct CInemaStream* cistream = (struct CInemaStream*) stream;
593 if (height != cistream->image->height || width != cistream->image->width) {
594 CIlog(1, "Size mismatch for video, expected %ux%u, got %ux%u\n", width, height, cistream->image->width, cistream->image->height);
595 if (*cistream->status == CI_PASS) {
596 *cistream->status = CI_FAIL;
597 }
598 }
599}
600
601static void _cinemaVideoFrame(struct mAVStream* stream, const color_t* pixels, size_t stride) {
602 struct CInemaStream* cistream = (struct CInemaStream*) stream;
603 cistream->image->stride = stride;
604 size_t bufferSize = cistream->image->stride * cistream->image->height * BYTES_PER_PIXEL;
605 cistream->image->data = malloc(bufferSize);
606 memcpy(cistream->image->data, pixels, bufferSize);
607}
608#endif
609
610static struct VDir* _makeOutDir(const char* testName) {
611 char path[PATH_MAX] = {0};
612 strlcpy(path, outdir, sizeof(path));
613 char* pathEnd = path + strlen(path);
614 const char* pos;
615 while (true) {
616 pathEnd[0] = PATH_SEP[0];
617 ++pathEnd;
618 pos = strchr(testName, '.');
619 size_t maxlen = sizeof(path) - (pathEnd - path) - 1;
620 size_t len;
621 if (pos) {
622 len = pos - testName;
623 } else {
624 len = strlen(testName);
625 }
626 if (len > maxlen) {
627 len = maxlen;
628 }
629 strncpy(pathEnd, testName, len);
630 pathEnd += len;
631
632 mkdir(path, 0777);
633
634 if (!pos) {
635 break;
636 }
637 testName = pos + 1;
638 }
639 return VDirOpen(path);
640}
641
642static void _writeImage(struct VFile* vf, const struct CInemaImage* image) {
643 png_structp png = PNGWriteOpen(vf);
644 png_infop info = PNGWriteHeader(png, image->width, image->height);
645 if (!PNGWritePixels(png, image->width, image->height, image->stride, image->data)) {
646 CIerr(0, "Could not write output image\n");
647 }
648 PNGWriteClose(png, info);
649
650 vf->close(vf);
651}
652
653static void _writeDiff(const char* testName, const struct CInemaImage* image, size_t frame, const char* type) {
654 struct VDir* dir = _makeOutDir(testName);
655 if (!dir) {
656 CIerr(0, "Could not open directory for %s\n", testName);
657 return;
658 }
659 char name[32];
660 snprintf(name, sizeof(name), "%s_%04" PRIz "u.png", type, frame);
661 struct VFile* vf = dir->openFile(dir, name, O_CREAT | O_TRUNC | O_WRONLY);
662 if (!vf) {
663 CIerr(0, "Could not open output file %s\n", name);
664 dir->close(dir);
665 return;
666 }
667 _writeImage(vf, image);
668 dir->close(dir);
669}
670
671static void _writeBaseline(struct VDir* dir, const char* type, const struct CInemaImage* image, size_t frame) {
672 char baselineName[32];
673 snprintf(baselineName, sizeof(baselineName), "%s_%04" PRIz "u.png", type, frame);
674 struct VFile* baselineVF = dir->openFile(dir, baselineName, O_CREAT | O_TRUNC | O_WRONLY);
675 if (baselineVF) {
676 _writeImage(baselineVF, image);
677 } else {
678 CIerr(0, "Could not open output file %s\n", baselineName);
679 }
680}
681
682static bool _updateInput(struct mCore* core, size_t frame, const char** input) {
683 if (!*input || !*input[0]) {
684 return false;
685 }
686 char* end;
687 uint32_t start = strtoul(*input, &end, 10);
688 if (end[0] != ':') {
689 return false;
690 }
691 if (start != frame) {
692 return true;
693 }
694 ++end;
695 *input = end;
696 uint32_t keys = strtoul(*input, &end, 16);
697 if (end[0] == ',') {
698 ++end;
699 }
700 *input = end;
701 core->setKeys(core, keys);
702 return true;
703}
704
705static bool _compareImages(struct CInemaTest* restrict test, const struct CInemaImage* restrict image, const struct CInemaImage* restrict expected, int* restrict max, uint8_t** restrict outdiff) {
706 const uint8_t* testPixels = image->data;
707 const uint8_t* expectPixels = expected->data;
708 uint8_t* diff = NULL;
709 size_t x;
710 size_t y;
711 bool failed = false;
712 for (y = 0; y < image->height; ++y) {
713 for (x = 0; x < image->width; ++x) {
714 size_t pix = expected->stride * y + x;
715 size_t tpix = image->stride * y + x;
716 int testR = testPixels[tpix * 4 + 0];
717 int testG = testPixels[tpix * 4 + 1];
718 int testB = testPixels[tpix * 4 + 2];
719 int expectR = expectPixels[pix * 4 + 0];
720 int expectG = expectPixels[pix * 4 + 1];
721 int expectB = expectPixels[pix * 4 + 2];
722 int r = expectR - testR;
723 int g = expectG - testG;
724 int b = expectB - testB;
725 if (r | g | b) {
726 failed = true;
727 if (outdiff && !diff) {
728 diff = calloc(expected->stride * expected->height, BYTES_PER_PIXEL);
729 *outdiff = diff;
730 }
731 test->status = CI_FAIL;
732 if (r < 0) {
733 r = -r;
734 }
735 if (g < 0) {
736 g = -g;
737 }
738 if (b < 0) {
739 b = -b;
740 }
741
742 if (diff) {
743 if (r > *max) {
744 *max = r;
745 }
746 if (g > *max) {
747 *max = g;
748 }
749 if (b > *max) {
750 *max = b;
751 }
752 diff[pix * 4 + 0] = r;
753 diff[pix * 4 + 1] = g;
754 diff[pix * 4 + 2] = b;
755 }
756
757 if (test) {
758 test->totalDistance += r + g + b;
759 ++test->failedPixels;
760 }
761 }
762 }
763 }
764 return !failed;
765}
766
767void _writeDiffSet(struct CInemaImage* expected, const char* name, uint8_t* diff, int frame, int max, bool xfail) {
768 struct CInemaImage outdiff = {
769 .data = diff,
770 .width = expected->width,
771 .height = expected->height,
772 .stride = expected->stride,
773 };
774
775 if (xfail) {
776 _writeDiff(name, expected, frame, "xexpected");
777 _writeDiff(name, &outdiff, frame, "xdiff");
778 } else {
779 _writeDiff(name, expected, frame, "expected");
780 _writeDiff(name, &outdiff, frame, "diff");
781 }
782
783 size_t x;
784 size_t y;
785 for (y = 0; y < outdiff.height; ++y) {
786 for (x = 0; x < outdiff.width; ++x) {
787 size_t pix = outdiff.stride * y + x;
788 diff[pix * 4 + 0] = diff[pix * 4 + 0] * 255 / max;
789 diff[pix * 4 + 1] = diff[pix * 4 + 1] * 255 / max;
790 diff[pix * 4 + 2] = diff[pix * 4 + 2] * 255 / max;
791 }
792 }
793 if (xfail) {
794 _writeDiff(name, &outdiff, frame, "xnormalized");
795 }
796}
797
798#ifdef USE_FFMPEG
799static void _replayBaseline(struct CInemaTest* test, struct FFmpegEncoder* encoder, const struct CInemaImage* image, int frame) {
800 char baselineName[PATH_MAX];
801 snprintf(baselineName, sizeof(baselineName), "%s" PATH_SEP ".baseline.avi", test->directory);
802
803 if (!FFmpegEncoderOpen(encoder, baselineName)) {
804 CIerr(1, "Failed to save baseline video\n");
805 test->status = CI_ERROR;
806 return;
807 }
808
809 snprintf(baselineName, sizeof(baselineName), "%s" PATH_SEP "baseline.avi", test->directory);
810
811
812 struct CInemaImage buffer = {
813 .data = NULL,
814 .width = image->width,
815 .height = image->height,
816 .stride = image->width,
817 };
818 struct FFmpegDecoder decoder;
819 struct CInemaStream stream = {0};
820 stream.d.postVideoFrame = _cinemaVideoFrame;
821 stream.d.videoDimensionsChanged = _cinemaDimensionsChanged;
822 stream.status = &test->status;
823 stream.image = &buffer;
824
825 FFmpegDecoderInit(&decoder);
826 decoder.out = &stream.d;
827
828 if (!FFmpegDecoderOpen(&decoder, baselineName)) {
829 CIerr(1, "Failed to load baseline video\n");
830 test->status = CI_ERROR;
831 return;
832 }
833
834 int i;
835 for (i = 0; i < frame; ++i) {
836 while (!buffer.data) {
837 if (!FFmpegDecoderRead(&decoder)) {
838 CIlog(1, "Failed to read more frames. EOF?\n");
839 test->status = CI_FAIL;
840 break;
841 }
842 }
843 encoder->d.postVideoFrame(&encoder->d, buffer.data, buffer.stride);
844 free(buffer.data);
845 buffer.data = NULL;
846 }
847 FFmpegDecoderClose(&decoder);
848}
849#endif
850
851void CInemaTestRun(struct CInemaTest* test) {
852 unsigned ignore = 0;
853 MutexLock(&configMutex);
854 CInemaConfigGetUInt(&configTree, test->name, "ignore", &ignore);
855 MutexUnlock(&configMutex);
856 if (ignore) {
857 test->status = CI_SKIP;
858 return;
859 }
860
861 struct VDir* dir = VDirOpen(test->directory);
862 if (!dir) {
863 CIerr(0, "Failed to open test directory\n");
864 test->status = CI_ERROR;
865 return;
866 }
867 struct VFile* rom = dir->openFile(dir, test->filename, O_RDONLY);
868 if (!rom) {
869 CIerr(0, "Failed to open test\n");
870 test->status = CI_ERROR;
871 return;
872 }
873 struct mCore* core = mCoreFindVF(rom);
874 if (!core) {
875 CIerr(0, "Failed to load test\n");
876 test->status = CI_ERROR;
877 rom->close(rom);
878 return;
879 }
880 if (!core->init(core)) {
881 CIerr(0, "Failed to init test\n");
882 test->status = CI_ERROR;
883 core->deinit(core);
884 return;
885 }
886 struct CInemaImage image;
887 core->desiredVideoDimensions(core, &image.width, &image.height);
888 ssize_t bufferSize = image.width * image.height * BYTES_PER_PIXEL;
889 image.data = malloc(bufferSize);
890 image.stride = image.width;
891 if (!image.data) {
892 CIerr(0, "Failed to allocate video buffer\n");
893 test->status = CI_ERROR;
894 core->deinit(core);
895 }
896 core->setVideoBuffer(core, image.data, image.stride);
897 mCoreConfigInit(&core->config, "cinema");
898
899 unsigned limit = 9999;
900 unsigned skip = 0;
901 unsigned fail = 0;
902 unsigned video = 0;
903 const char* input = NULL;
904
905 MutexLock(&configMutex);
906 CInemaConfigGetUInt(&configTree, test->name, "frames", &limit);
907 CInemaConfigGetUInt(&configTree, test->name, "skip", &skip);
908 CInemaConfigGetUInt(&configTree, test->name, "fail", &fail);
909 CInemaConfigGetUInt(&configTree, test->name, "video", &video);
910 input = CInemaConfigGet(&configTree, test->name, "input");
911 CInemaConfigLoad(&configTree, test->name, core);
912 MutexUnlock(&configMutex);
913
914 struct VFile* save = VFileMemChunk(NULL, 0);
915 core->loadROM(core, rom);
916 if (!core->loadSave(core, save)) {
917 save->close(save);
918 }
919 core->rtc.override = RTC_FAKE_EPOCH;
920 core->rtc.value = 1200000000;
921 core->reset(core);
922
923 test->status = CI_PASS;
924
925 unsigned minFrame = core->frameCounter(core);
926 size_t frame;
927 for (frame = 0; frame < skip; ++frame) {
928 core->runFrame(core);
929 }
930 core->desiredVideoDimensions(core, &image.width, &image.height);
931
932#ifdef USE_FFMPEG
933 struct FFmpegDecoder decoder;
934 struct FFmpegEncoder encoder;
935 struct CInemaStream stream = {0};
936
937 char baselineName[PATH_MAX];
938 snprintf(baselineName, sizeof(baselineName), "%s" PATH_SEP "baseline.avi", test->directory);
939 bool exists = access(baselineName, 0) == 0;
940
941 if (video) {
942 FFmpegEncoderInit(&encoder);
943 FFmpegDecoderInit(&decoder);
944
945 FFmpegEncoderSetAudio(&encoder, NULL, 0);
946 FFmpegEncoderSetVideo(&encoder, "zmbv", 0, 0);
947 FFmpegEncoderSetContainer(&encoder, "avi");
948 FFmpegEncoderSetDimensions(&encoder, image.width, image.height);
949
950 if (rebaseline && !exists) {
951 if (!FFmpegEncoderOpen(&encoder, baselineName)) {
952 CIerr(1, "Failed to save baseline video\n");
953 } else {
954 core->setAVStream(core, &encoder.d);
955 }
956 }
957
958 if (exists) {
959 stream.d.postVideoFrame = _cinemaVideoFrame;
960 stream.d.videoDimensionsChanged = _cinemaDimensionsChanged;
961 stream.status = &test->status;
962 decoder.out = &stream.d;
963
964 if (!FFmpegDecoderOpen(&decoder, baselineName)) {
965 CIerr(1, "Failed to load baseline video\n");
966 }
967 } else if (!rebaseline) {
968 test->status = CI_FAIL;
969 }
970 }
971#else
972 if (video) {
973 CIerr(0, "Failed to run video test without ffmpeg linked in\n");
974 test->status = CI_ERROR;
975 }
976#endif
977
978 bool xdiff = false;
979 for (frame = 0; limit; ++frame, --limit) {
980 _updateInput(core, frame, &input);
981 core->runFrame(core);
982 ++test->totalFrames;
983 unsigned frameCounter = core->frameCounter(core);
984 if (frameCounter <= minFrame) {
985 break;
986 }
987 if (test->status == CI_ERROR) {
988 break;
989 }
990 CIlog(3, "Test frame: %u\n", frameCounter);
991 core->desiredVideoDimensions(core, &image.width, &image.height);
992 uint8_t* diff = NULL;
993 struct CInemaImage expected = {
994 .data = NULL,
995 .width = image.width,
996 .height = image.height,
997 .stride = image.width,
998 };
999 bool baselineFound;
1000 if (video) {
1001 baselineFound = false;
1002#ifdef USE_FFMPEG
1003 if (FFmpegDecoderIsOpen(&decoder)) {
1004 stream.image = &expected;
1005 while (!expected.data) {
1006 if (!FFmpegDecoderRead(&decoder)) {
1007 CIlog(1, "Failed to read more frames. EOF?\n");
1008 test->status = CI_FAIL;
1009 break;
1010 }
1011 }
1012 baselineFound = expected.data;
1013 }
1014#endif
1015 } else {
1016 baselineFound = _loadBaselinePNG(dir, "baseline", &expected, frame, &test->status);
1017 }
1018 if (test->status == CI_ERROR) {
1019 break;
1020 }
1021 bool failed = false;
1022 if (baselineFound) {
1023 int max = 0;
1024 failed = !_compareImages(test, &image, &expected, &max, diffs ? &diff : NULL);
1025 if (failed) {
1026 ++test->failedFrames;
1027#ifdef USE_FFMPEG
1028 if (video && exists && rebaseline && !FFmpegEncoderIsOpen(&encoder)) {
1029 _replayBaseline(test, &encoder, &image, frame);
1030 if (test->status == CI_ERROR) {
1031 break;
1032 }
1033 encoder.d.postVideoFrame(&encoder.d, image.data, image.stride);
1034 core->setAVStream(core, &encoder.d);
1035 }
1036#endif
1037 }
1038 test->totalPixels += image.height * image.width;
1039 if (rebaseline == CI_R_FAILING && !video && failed) {
1040 _writeBaseline(dir, "baseline", &image, frame);
1041 }
1042 if (diff) {
1043 if (failed) {
1044 _writeDiff(test->name, &image, frame, "result");
1045 _writeDiffSet(&expected, test->name, diff, frame, max, false);
1046 }
1047 free(diff);
1048 diff = NULL;
1049 }
1050 free(expected.data);
1051 } else if (rebaseline && !video) {
1052 _writeBaseline(dir, "baseline", &image, frame);
1053 } else if (!rebaseline) {
1054 test->status = CI_FAIL;
1055 }
1056
1057 if (fail && failed) {
1058 if (video) {
1059 // TODO
1060 baselineFound = false;
1061 } else {
1062 baselineFound = _loadBaselinePNG(dir, "xbaseline", &expected, frame, &test->status);
1063 }
1064
1065 if (baselineFound) {
1066 int max = 0;
1067 failed = !_compareImages(test, &image, &expected, &max, diffs ? &diff : NULL);
1068 if (diff) {
1069 if (failed) {
1070 _writeDiffSet(&expected, test->name, diff, frame, max, true);
1071 }
1072 free(diff);
1073 diff = NULL;
1074 }
1075 if (failed) {
1076 if (xbaseline == CI_R_FAILING && !video) {
1077 _writeBaseline(dir, "xbaseline", &image, frame);
1078 }
1079 xdiff = true;
1080 }
1081 free(expected.data);
1082 } else if (xbaseline && !video) {
1083 _writeBaseline(dir, "xbaseline", &image, frame);
1084 }
1085 }
1086 }
1087
1088#ifdef USE_FFMPEG
1089 if (video) {
1090 if (FFmpegEncoderIsOpen(&encoder)) {
1091 FFmpegEncoderClose(&encoder);
1092 if (exists && rebaseline) {
1093 char tmpBaselineName[PATH_MAX];
1094 snprintf(tmpBaselineName, sizeof(tmpBaselineName), "%s" PATH_SEP ".baseline.avi", test->directory);
1095#ifdef _WIN32
1096 MoveFileEx(tmpBaselineName, baselineName, MOVEFILE_REPLACE_EXISTING);
1097#else
1098 rename(tmpBaselineName, baselineName);
1099#endif
1100 }
1101 }
1102 if (FFmpegDecoderIsOpen(&decoder)) {
1103 FFmpegDecoderClose(&decoder);
1104 }
1105 }
1106#endif
1107
1108 if (fail) {
1109 if (test->status == CI_FAIL && !xdiff) {
1110 test->status = CI_XFAIL;
1111 } else if (test->status == CI_PASS) {
1112 test->status = CI_XPASS;
1113 }
1114 }
1115
1116 free(image.data);
1117 mCoreConfigDeinit(&core->config);
1118 core->deinit(core);
1119 dir->close(dir);
1120}
1121
1122static bool CInemaTask(struct CInemaTestList* tests, size_t i) {
1123 bool success = true;
1124 struct CInemaTest* test = CInemaTestListGetPointer(tests, i);
1125 if (dryRun) {
1126 CIlog(-1, "%s\n", test->name);
1127 } else {
1128 CIlog(1, "%s: ", test->name);
1129 fflush(stdout);
1130 ThreadLocalSetKey(currentTest, test);
1131 CInemaTestRun(test);
1132 ThreadLocalSetKey(currentTest, NULL);
1133
1134 switch (test->status) {
1135 case CI_PASS:
1136 CIlog(1, "pass\n");
1137 break;
1138 case CI_FAIL:
1139 success = false;
1140 CIlog(1, "fail\n");
1141 break;
1142 case CI_XPASS:
1143 CIlog(1, "xpass\n");
1144 break;
1145 case CI_XFAIL:
1146 CIlog(1, "xfail\n");
1147 break;
1148 case CI_SKIP:
1149 CIlog(1, "skip\n");
1150 break;
1151 case CI_ERROR:
1152 success = false;
1153 CIlog(1, "error\n");
1154 break;
1155 }
1156 if (test->failedFrames) {
1157 CIlog(2, "\tfailed frames: %u/%u (%1.3g%%)\n", test->failedFrames, test->totalFrames, test->failedFrames / (test->totalFrames * 0.01));
1158 CIlog(2, "\tfailed pixels: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->failedPixels, test->totalPixels, test->failedPixels / (test->totalPixels * 0.01));
1159 CIlog(2, "\tdistance: %" PRIu64 "/%" PRIu64 " (%1.3g%%)\n", test->totalDistance, test->totalPixels * 765, test->totalDistance / (test->totalPixels * 7.65));
1160 }
1161 }
1162 return success;
1163}
1164
1165static THREAD_ENTRY CInemaJob(void* context) {
1166 struct CInemaTestList* tests = context;
1167 struct CInemaLogStream stream;
1168 StringListInit(&stream.out.lines, 0);
1169 StringListInit(&stream.out.partial, 0);
1170 stream.out.repeat = 0;
1171 StringListInit(&stream.err.lines, 0);
1172 StringListInit(&stream.err.partial, 0);
1173 stream.err.repeat = 0;
1174 ThreadLocalSetKey(logStream, &stream);
1175
1176 bool success = true;
1177 while (true) {
1178 size_t i;
1179 MutexLock(&jobMutex);
1180 i = jobIndex;
1181 ++jobIndex;
1182 MutexUnlock(&jobMutex);
1183 if (i >= CInemaTestListSize(tests)) {
1184 break;
1185 }
1186 if (!CInemaTask(tests, i)) {
1187 success = false;
1188 }
1189 CIflush(&stream.out, stdout);
1190 CIflush(&stream.err, stderr);
1191 }
1192 MutexLock(&jobMutex);
1193 if (!success) {
1194 jobStatus = 1;
1195 }
1196 MutexUnlock(&jobMutex);
1197
1198 CIflush(&stream.out, stdout);
1199 StringListDeinit(&stream.out.lines);
1200 StringListDeinit(&stream.out.partial);
1201
1202 CIflush(&stream.err, stderr);
1203 StringListDeinit(&stream.err.lines);
1204 StringListDeinit(&stream.err.partial);
1205}
1206
1207void _log(struct mLogger* log, int category, enum mLogLevel level, const char* format, va_list args) {
1208 UNUSED(log);
1209 if (level == mLOG_FATAL) {
1210 struct CInemaTest* test = ThreadLocalGetValue(currentTest);
1211 test->status = CI_ERROR;
1212 }
1213 if (verbosity < 0) {
1214 return;
1215 }
1216 int mask = mLOG_FATAL;
1217 if (verbosity >= 1) {
1218 mask |= mLOG_ERROR;
1219 }
1220 if (verbosity >= 2) {
1221 mask |= mLOG_WARN;
1222 }
1223 if (verbosity >= 4) {
1224 mask |= mLOG_INFO;
1225 }
1226 if (verbosity >= 5) {
1227 mask |= mLOG_ALL;
1228 }
1229 if (!(mask & level)) {
1230 return;
1231 }
1232
1233 char buffer[256];
1234 vsnprintf(buffer, sizeof(buffer), format, args);
1235 CIerr(0, "[%s] %s\n", mLogCategoryName(category), buffer);
1236}
1237
1238int main(int argc, char** argv) {
1239 ThreadLocalInitKey(&logStream);
1240 ThreadLocalSetKey(logStream, NULL);
1241
1242 int status = 0;
1243 if (!parseCInemaArgs(argc, argv)) {
1244 status = 1;
1245 goto cleanup;
1246 }
1247
1248 if (showVersion) {
1249 version(argv[0]);
1250 goto cleanup;
1251 }
1252
1253 if (showUsage) {
1254 usageCInema(argv[0]);
1255 goto cleanup;
1256 }
1257
1258 argc -= optind;
1259 argv += optind;
1260
1261 if (!base[0] && !determineBase(argc, argv)) {
1262 CIlog(0, "Could not determine CInema test base. Please specify manually.");
1263 status = 1;
1264 goto cleanup;
1265 }
1266#ifndef _WIN32
1267 char* rbase = realpath(base, NULL);
1268 if (rbase) {
1269 strlcpy(base, rbase, sizeof(base));
1270 free(rbase);
1271 }
1272#endif
1273
1274 struct CInemaTestList tests;
1275 CInemaTestListInit(&tests, 0);
1276
1277 struct mLogger logger = { .log = _log };
1278 mLogSetDefaultLogger(&logger);
1279#ifdef USE_FFMPEG
1280 if (verbosity < 2) {
1281 av_log_set_level(AV_LOG_ERROR);
1282 }
1283#endif
1284
1285 if (argc > 0) {
1286 size_t i;
1287 for (i = 0; i < (size_t) argc; ++i) {
1288 char path[PATH_MAX + 1] = {0};
1289 testToPath(argv[i], path);
1290
1291 if (!collectTests(&tests, path)) {
1292 status = 1;
1293 break;
1294 }
1295 }
1296 } else if (!collectTests(&tests, base)) {
1297 status = 1;
1298 }
1299
1300 if (CInemaTestListSize(&tests) == 0) {
1301 CIlog(1, "No tests found.");
1302 status = 1;
1303 } else {
1304 reduceTestList(&tests);
1305 }
1306
1307 HashTableInit(&configTree, 0, free);
1308 MutexInit(&configMutex);
1309 ThreadLocalInitKey(¤tTest);
1310 ThreadLocalSetKey(currentTest, NULL);
1311
1312 if (jobs == 1) {
1313 size_t i;
1314 for (i = 0; i < CInemaTestListSize(&tests); ++i) {
1315 bool success = CInemaTask(&tests, i);
1316 if (!success) {
1317 status = 1;
1318 }
1319 }
1320 } else {
1321 MutexInit(&jobMutex);
1322 int i;
1323 for (i = 0; i < jobs; ++i) {
1324 ThreadCreate(&jobThreads[i], CInemaJob, &tests);
1325 }
1326 for (i = 0; i < jobs; ++i) {
1327 ThreadJoin(&jobThreads[i]);
1328 }
1329 MutexDeinit(&jobMutex);
1330 status = jobStatus;
1331 }
1332
1333 MutexDeinit(&configMutex);
1334 HashTableEnumerate(&configTree, _unloadConfigTree, NULL);
1335 HashTableDeinit(&configTree);
1336 CInemaTestListDeinit(&tests);
1337
1338cleanup:
1339 return status;
1340}