Repository navigation
Expand file tree
/
Copy pathevaluate_pbt.py
More file actions
268 lines (218 loc) · 10.1 KB
/
Copy pathevaluate_pbt.py
File metadata and controls
268 lines (218 loc) · 10.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
""" evaluate round-robin fashion among a set of models
All tables should be read row-wise.
opponent lr=0.1_vw=1 lr=0.1_vw=1.5 lr=0.2_vw=1 lr=0.2_vw=1.5
lr=0.1_vw=1 - b2/3, w1/3 b1/3,w1/3 b1/2(*),w2/3
0.5 0.33
lr=0.1_vw=1.5 - ... ...
lr=0.2_vw=1 - ...
lr=0.2_vw=1.5 -
- each pair needs to play the same number of games as white/black
- it's ok if more games are played in certain pairs. We only care about win-rate in the end.
- should construct the table from eval-sgfs, so that we can tolerate failures, reruns, parallel runs
- tf hangs in metal occasionally, so batch runs need to tolerate this
"""
import itertools
import os
from collections import defaultdict
import subprocess, shlex
from typing import Tuple, List
import logging
import numpy as np
import pandas as pd
from natsort import natsorted
import coords
import utils
from model_config import ModelConfig
from katago.analysis_engine import KataDualNetwork, KataModels
from sgf_wrapper import SGFReader
import myconf
from sgfs_stats import run_tournament_report
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 200)
MIN_NUM_GAMES_SIDED = 3
class RawGameCount:
""" wrap around raw sided game counts df, provides convenience access """
def __init__(self, df: pd.DataFrame):
assert df.shape[0] == df.shape[1]
self.df = df
def count(self, black: str, white: str):
""" black/white should be model_id """
try:
return self.df.loc[black, white]
except KeyError:
return 0
def eye(self):
return pd.DataFrame(np.eye(len(self.df), len(self.df)),
index=self.df.index, columns=self.df.columns)
def format_black_wins(self, df_blackwins: pd.DataFrame) -> pd.DataFrame:
""" format black_wins to include game count """
df = df_blackwins.astype(str) + '/' + self.df.astype(str)
return df.replace('0/0', '-')
def scan_results(sgfs_dir: str, order=None) -> Tuple[RawGameCount, pd.DataFrame]:
""" scan sgfs, build the raw sided stats dfs
todo: when order is specified, only read sgfs played by models in the list
"""
game_counts = defaultdict(lambda: defaultdict(int))
black_wins = defaultdict(lambda: defaultdict(int))
models = set()
for sgf_fname in os.listdir(f'{sgfs_dir}'):
if not sgf_fname.endswith('.sgf'):
continue
reader = SGFReader.from_file_compatible(f'{sgfs_dir}/{sgf_fname}')
black_id = reader.black_name()
white_id = reader.white_name()
result_sign = reader.result()
assert result_sign != 0
models.update([black_id, white_id])
game_counts[black_id][white_id] += 1
black_wins[black_id][white_id] += result_sign == 1
# construct df: sided first
models = natsorted(models) if order is None else order
df_counts_raw = pd.DataFrame(game_counts, index=models, columns=models)
df_counts = df_counts_raw.T.fillna(0).astype(int)
df_blackwins = pd.DataFrame(black_wins, index=models, columns=models).T.fillna(0).astype(int)
df_blackwins.index.name = 'black_id'
return RawGameCount(df_counts), df_blackwins
def verify_and_fold(raw_game_counts: RawGameCount, df_blackwins: pd.DataFrame) -> pd.DataFrame:
""" verify black/white parity, and that a minimum number of games has been played
Example: two models playing each other 16 games total, equally as black/white (we check parity)
df_blackwin: row-id is black, column-id is white
model1 model2
black_id
model1 0 8 (model1 as black won all 8 games)
model2 4 0
dfw: this sums up a model's total win (both as black and white). Again, it's row-major
model1 model2
nwin total wrate nwin total wrate
model1 12 16 0.75
model2 4 16 0.25
"""
df_counts = raw_game_counts.df
# df_eye = raw_game_counts.eye() * MIN_NUM_GAMES_SIDED
# assert (df_counts + df_eye >= MIN_NUM_GAMES_SIDED).all().all()
# make sure df_counts == df_counts.T for black/white parity, as well as min #games is played
pd.testing.assert_frame_equal(df_counts, df_counts.T)
total_num_games = df_counts.sum().sum()
print(f'Found {total_num_games} games')
# now merge sided stats into a single stat: upper-triangle only, ignore the lower half (although it has a natural meaning)
df_counts2 = df_counts + df_counts.T
# totalwin: #wins playing black + #wins as white
df_totalwins = df_blackwins + (df_counts - df_blackwins).T
df_wrate2 = (df_totalwins / df_counts2).fillna('-')
dfw = pd.concat([df_totalwins, df_counts2, df_wrate2], axis=1, keys=['nwin', 'total', 'wrate'])
dfw = dfw.swaplevel(axis=1) #.sort_index(axis=1)
return dfw
def model_fname(i: int, lr, vw) -> str:
return f'model{i}.lr={lr}_vw={vw}.h5'
class Evaluator:
""" encapsulate various eval related settings, so we don't need to pass them around """
def __init__(self, sgfs_dir, num_games_per_side: int = 32):
self.sgfs_dir = sgfs_dir
self.num_games_per_side = num_games_per_side
def start_games(self, black_id, white_id, num_games: int) -> subprocess.Popen:
"""
run one-sided eval in separate process
"""
# evaluate doesn't inject noise, so no need to specify --dirichlet_noise_weight = 0.0125
cmdline = f"""/Users/hyu/anaconda/envs/tf2/bin/python /Users/hyu/PycharmProjects/dlgo/minigo/eval_gating.py
--reduce_symmetry_before_move=3
--softpick_move_cutoff=8
--parallel_readouts=16
--resign_threshold=-1
%s %s %s %d
""" % (black_id, white_id, self.sgfs_dir, num_games)
args = shlex.split(cmdline)
p = subprocess.Popen(args)
# p.wait()
return p
def run_two_sided_eval(self, model1_str, model2_str):
""" play two models against each other for n games, then switch sides """
raw_game_count, _ = scan_results(self.sgfs_dir)
model1, model2 = ModelConfig(model1_str), ModelConfig(model2_str)
procs = []
for black, white in [(model1, model2), (model2, model1)]:
games_to_run = self.num_games_per_side - raw_game_count.count(black.model_id(), white.model_id())
if games_to_run <= 0:
continue
elif games_to_run <= 3:
logging.info(f'starting {black} vs {white}: {games_to_run} games')
procs.append(self.start_games(black, white, games_to_run))
else:
logging.info(f'starting {black} vs {white}: {games_to_run} games, 2 processes')
procs.append(self.start_games(black, white, games_to_run // 2))
procs.append(self.start_games(black, white, games_to_run - games_to_run // 2))
for p in procs:
p.wait()
logging.info(f'Done two-sided: {model1} vs {model2}')
def run_multi_models(self, models: List[str], band_size=1):
""" eval multiple pairs of models, based on two_sided_eval
models: ordered by (hypothesized) strength, strongest first
todo:
- try to be smart in figuring out actual ranking
"""
# all pairs:
# itertools.permutations(models, 2)
for band in range(1, 1 + band_size):
print(f'\n\n===== band {band}')
it_a, it_b = itertools.tee(models)
for _ in range(band):
next(it_b, None)
for model1, model2 in zip(it_a, it_b):
self.run_two_sided_eval(model1, model2)
def main_pbt_eval(self):
raw_game_count, _ = scan_results(self.sgfs_dir)
num_target_games = 5
gen_idx = 14
lrs = [0.003, 0.005, 0.006, 0.008]
vws = [0.6, 0.8, 1, 1.2]
lr_ref, vw_ref = 0.005, 1
# for lr in set(lrs) - {lr_ref}:
# model1 = model_fname(gen_idx, lr, vw_ref)
# modelr = model_fname(gen_idx, lr_ref, vw_ref)
for vw in set(vws) - {vw_ref}:
model1 = model_fname(gen_idx, lr_ref, vw)
modelr = model_fname(gen_idx, lr_ref, vw_ref)
procs = []
for black, white in [(model1, modelr), (modelr, model1)]:
games_to_run = num_target_games - raw_game_count.count(black, white)
if games_to_run <= 0:
continue
p = self.start_games(black, white, games_to_run)
procs.append(p)
for p in procs:
p.wait()
def state_of_the_world(self, order=None):
raw_game_count, df_blackwins = scan_results(self.sgfs_dir, order=order)
print('black_wins:')
print(raw_game_count.format_black_wins(df_blackwins))
dfw = verify_and_fold(raw_game_count, df_blackwins)
print(dfw.swaplevel(axis=1)['wrate'])
pickle_fpath = '/tmp/df.pkl'
dfw.to_pickle(pickle_fpath)
print(f'dfw saved to {pickle_fpath}')
def count_disagreement():
"""
sgfs-200readouts: 2/320
sgfs-100readouts-batch: 4/576
sgfs-50readouts-batch: 13/576
sgfs: model1_epoch5#10-vs-model1_epoch5#1 20/64
sgfs-epoch5-batch: 75/640 many due to #1, #10 readouts
"""
sgfs_dir = f'{myconf.EXP_HOME}/eval_bots/sgfs-epoch5-batch'
# moved to sgfs_stats
game_outcome_review(sgfs_dir)
def main():
sgfs_dir = f'{myconf.EXP_HOME}/eval9/epochs2'
utils.ensure_dir_exists(sgfs_dir)
evaluator = Evaluator(sgfs_dir, 40)
models = ['model7_10#200', f'model9_2#200', 'model8_4#200']
# evaluator.run_two_sided_eval(models[0], models[1])
# models = ['model7_10#200'] + [f'model9_{x}#200' for x in (1,)]
# models = [f'model7_10#{x}' for x in (50, 100, 200, 400)]
evaluator.run_multi_models(models[::-1], band_size=1)
# evaluator.state_of_the_world(order=models)
evaluator.state_of_the_world()
run_tournament_report(f'{sgfs_dir}/*.sgf')
if __name__ == '__main__':
main()
# count_disagreement()