tools/perf.py (view raw)
1#!/usr/bin/env python
2from __future__ import print_function
3import argparse
4import csv
5import os
6import shlex
7import signal
8import socket
9import subprocess
10import sys
11import time
12
13class PerfTest(object):
14 EXECUTABLE = 'mgba-perf'
15
16 def __init__(self, rom, renderer='software'):
17 self.rom = rom
18 self.renderer = renderer
19 self.results = None
20 self.name = 'Perf Test: {}'.format(rom)
21
22 def get_args(self):
23 return []
24
25 def wait(self, proc):
26 pass
27
28 def run(self, cwd):
29 args = [os.path.join(os.getcwd(), self.EXECUTABLE), '-P']
30 args.extend(self.get_args())
31 if not self.renderer:
32 args.append('-N')
33 elif self.renderer == 'threaded-software':
34 args.append('-T')
35 args.append(self.rom)
36 env = {}
37 if 'LD_LIBRARY_PATH' in os.environ:
38 env['LD_LIBRARY_PATH'] = os.path.abspath(os.environ['LD_LIBRARY_PATH'])
39 env['DYLD_LIBRARY_PATH'] = env['LD_LIBRARY_PATH'] # Fake it on OS X
40 proc = subprocess.Popen(args, stdout=subprocess.PIPE, cwd=cwd, universal_newlines=True, env=env)
41 try:
42 self.wait(proc)
43 proc.wait()
44 except:
45 proc.kill()
46 raise
47 if proc.returncode:
48 print('Game crashed!', file=sys.stderr)
49 return
50 reader = csv.DictReader(proc.stdout)
51 self.results = next(reader)
52
53class WallClockTest(PerfTest):
54 def __init__(self, rom, duration, renderer='software'):
55 super(WallClockTest, self).__init__(rom, renderer)
56 self.duration = duration
57 self.name = 'Wall-Clock Test ({} seconds, {} renderer): {}'.format(duration, renderer, rom)
58
59 def wait(self, proc):
60 time.sleep(self.duration)
61 proc.send_signal(signal.SIGINT)
62
63class GameClockTest(PerfTest):
64 def __init__(self, rom, frames, renderer='software'):
65 super(GameClockTest, self).__init__(rom, renderer)
66 self.frames = frames
67 self.name = 'Game-Clock Test ({} frames, {} renderer): {}'.format(frames, renderer, rom)
68
69 def get_args(self):
70 return ['-F', str(self.frames)]
71
72class PerfServer(object):
73 ITERATIONS_PER_INSTANCE = 50
74 RETRIES = 5
75
76 def __init__(self, address, root='/', command=None):
77 s = address.rsplit(':', 1)
78 if len(s) == 1:
79 self.address = (s[0], 7216)
80 else:
81 self.address = (s[0], s[1])
82 self.command = None
83 if command:
84 self.command = shlex.split(command)
85 self.iterations = self.ITERATIONS_PER_INSTANCE
86 self.socket = None
87 self.results = []
88 self.reader = None
89 self.root = root
90
91 def _start(self, test):
92 if self.command:
93 server_command = list(self.command)
94 else:
95 server_command = [os.path.join(os.getcwd(), PerfTest.EXECUTABLE)]
96 server_command.extend(['-PD'])
97 if hasattr(test, "frames"):
98 server_command.extend(['-F', str(test.frames)])
99 if not test.renderer:
100 server_command.append('-N')
101 elif test.renderer == 'threaded-software':
102 server_command.append('-T')
103 subprocess.check_call(server_command)
104 time.sleep(3)
105 for backoff in range(self.RETRIES):
106 try:
107 self.socket = socket.create_connection(self.address, timeout=1000)
108 break
109 except OSError as e:
110 print("Failed to connect:", e, file=sys.stderr)
111 if backoff < self.RETRIES - 1:
112 time.sleep(2 ** backoff)
113 else:
114 raise
115 kwargs = {}
116 if sys.version_info[0] >= 3:
117 kwargs["encoding"] = "utf-8"
118 self.reader = csv.DictReader(self.socket.makefile(**kwargs))
119
120 def run(self, test):
121 if not self.socket:
122 self._start(test)
123 self.socket.send(os.path.join(self.root, test.rom).encode("utf-8"))
124 self.results.append(next(self.reader))
125 self.iterations -= 1
126 if self.iterations == 0:
127 self.finish()
128 self.iterations = self.ITERATIONS_PER_INSTANCE
129
130 def finish(self):
131 self.socket.send(b"\n");
132 self.reader = None
133 self.socket.close()
134 time.sleep(5)
135 self.socket = None
136
137class Suite(object):
138 def __init__(self, cwd, wall=None, game=None, renderer='software'):
139 self.cwd = cwd
140 self.tests = []
141 self.wall = wall
142 self.game = game
143 self.renderer = renderer
144 self.server = None
145
146 def set_server(self, server):
147 self.server = server
148
149 def collect_tests(self):
150 roms = []
151 for f in os.listdir(self.cwd):
152 if f.endswith('.gba') or f.endswith('.zip') or f.endswith('.gbc') or f.endswith('.gb'):
153 roms.append(f)
154 roms.sort()
155 for rom in roms:
156 self.add_tests(rom)
157
158 def add_tests(self, rom):
159 if self.wall:
160 self.tests.append(WallClockTest(rom, self.wall, renderer=self.renderer))
161 if self.game:
162 self.tests.append(GameClockTest(rom, self.game, renderer=self.renderer))
163
164 def run(self):
165 results = []
166 sock = None
167 for test in self.tests:
168 print('Running test {}'.format(test.name), file=sys.stderr)
169 last_result = None
170 if self.server:
171 self.server.run(test)
172 last_result = self.server.results[-1]
173 else:
174 try:
175 test.run(self.cwd)
176 except KeyboardInterrupt:
177 print('Interrupted, returning early...', file=sys.stderr)
178 return results
179 if test.results:
180 results.append(test.results)
181 last_result = results[-1]
182 if last_result:
183 print('{:.2f} fps'.format(int(last_result['frames']) * 1000000 / float(last_result['duration'])), file=sys.stderr)
184 if self.server:
185 self.server.finish()
186 results.extend(self.server.results)
187 return results
188
189if __name__ == '__main__':
190 parser = argparse.ArgumentParser()
191 parser.add_argument('-w', '--wall-time', type=float, default=0, metavar='TIME', help='wall-clock time')
192 parser.add_argument('-g', '--game-frames', type=int, default=0, metavar='FRAMES', help='game-clock frames')
193 parser.add_argument('-N', '--disable-renderer', action='store_const', const=True, help='disable video rendering')
194 parser.add_argument('-T', '--threaded-renderer', action='store_const', const=True, help='threaded video rendering')
195 parser.add_argument('-s', '--server', metavar='ADDRESS', help='run on server')
196 parser.add_argument('-S', '--server-command', metavar='COMMAND', help='command to launch server')
197 parser.add_argument('-o', '--out', metavar='FILE', help='output file path')
198 parser.add_argument('-r', '--root', metavar='PATH', type=str, default='/perfroms', help='root path for server mode')
199 parser.add_argument('directory', help='directory containing ROM files')
200 args = parser.parse_args()
201
202 renderer = 'software'
203 if args.disable_renderer:
204 renderer = None
205 elif args.threaded_renderer:
206 renderer = 'threaded-software'
207 s = Suite(args.directory, wall=args.wall_time, game=args.game_frames, renderer=renderer)
208 if args.server:
209 if args.server_command:
210 server = PerfServer(args.server, args.root, args.server_command)
211 else:
212 server = PerfServer(args.server, args.root)
213 s.set_server(server)
214 s.collect_tests()
215 results = s.run()
216 fout = sys.stdout
217 if args.out:
218 fout = open(args.out, 'w')
219 writer = csv.DictWriter(fout, results[0].keys())
220 writer.writeheader()
221 writer.writerows(results)
222 if fout is not sys.stdout:
223 fout.close()