Repository navigation
Expand file tree
/
Copy pathsgfs_stats.py
More file actions
549 lines (454 loc) · 22.1 KB
/
Copy pathsgfs_stats.py
File metadata and controls
549 lines (454 loc) · 22.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
""" analyze a batch of selfplay / eval games for diversity, resign thresh, and other insights
- print sorted list of game moves
- should be easy to embed for live stats & actions
keep all game tree in memory? so we know where exploration happens.
Or just zhash counts in the first cut?
"""
import glob
import logging
import os.path
import random
from collections import defaultdict, Counter, namedtuple
from itertools import islice
from typing import List, Set
import numpy as np
import pandas as pd
import coords
import go
import myconf
from sgf_wrapper import SGFReader
from utils import format_game_summary
pd.set_option('display.max_columns', None)
pd.set_option('display.width', 200)
class StatsItf:
def add_game(self, reader: SGFReader):
pass
def report(self):
pass
class QExtractor:
""" common Q-value related code """
@staticmethod
def _extract_q(comment: str) -> float:
"""
Q=-0.99 raw=-0.99 N=435 path=-
Note this could be the 2nd line
"""
lines = comment.split('\n')
line = lines[0]
if not line.startswith('Q='):
line = lines[1]
flds = line.split()
assert flds[0][:2] == 'Q='
return float(flds[0][2:])
@staticmethod
def extract_q_curve(reader: SGFReader) -> pd.Series:
q_vals = []
for move_idx, (move, comments) in enumerate(reader.iter_comments()):
assert len(comments) == 1
q_val = QExtractor._extract_q(comments[0])
q_vals.append(q_val)
ts = pd.Series(q_vals)
return ts
class QResignAnalysis(StatsItf):
""" see how well extreme Q values correlate with game outcome.
find a good strategy for trade-off accuracy vs savings, also how much savings
"""
def __init__(self, resign_thresh: float = 0.9, run_length: int = 1):
self._thresh = resign_thresh
self._run_length = run_length
self.num_games = 0
self.num_games_resigned = 0
self.num_errors = 0
self.sum_moves = 0
self.sum_saved = 0
self._error_games = []
@staticmethod
def count_consecutive_overage(ts: pd.Series, thresh: float) -> pd.Series:
""" count consecutive #times q has exceeded thresh
When q-value flips between 1 to -1, we reset
"""
consecutive_count = pd.Series(0, index=ts.index)
prev_sign = 5 # just not 1, -1, 0
for i in ts.index:
qval = ts[i]
cur_sign = np.sign(qval)
if abs(qval) >= thresh:
if cur_sign == prev_sign:
consecutive_count[i] += consecutive_count[i - 1] + 1
else:
consecutive_count[i] = 1
else:
consecutive_count[i] = 0
prev_sign = cur_sign
return consecutive_count
def add_game(self, reader: SGFReader):
""" """
winner_sign = reader.result()
game_name_short = os.path.basename(reader.name)
ts = QExtractor.extract_q_curve(reader)
ts_cc = self.count_consecutive_overage(ts, self._thresh)
ts_resign = ts_cc[ts_cc >= self._run_length]
if len(ts_resign) > 0:
resign_idx = ts_resign.index[0]
sign_at_resign = np.sign(ts[resign_idx])
self.num_games_resigned += 1
else: # no resign
resign_idx = len(ts) - 1
sign_at_resign = winner_sign
is_error = sign_at_resign != winner_sign
self.num_games += 1
self.num_errors += is_error
self.sum_moves += len(ts)
self.sum_saved += len(ts) - resign_idx - 1
if is_error:
logging.info(f'{game_name_short} %d moves {reader.result_str()}: resigned at {resign_idx} %.1f',
len(ts), ts[resign_idx])
def report(self):
logging.info(f'Total {self.num_games} games, thresh={self._thresh}, run_len={self._run_length}: '
f'{self.num_games_resigned} resigned = %.1f%%, {self.num_errors} errors = %.1f%%, saved {self.sum_saved} = %.1f%%',
self.num_games_resigned / self.num_games * 100,
self.num_errors / self.num_games * 100, self.sum_saved / self.sum_moves * 100)
class QVDatum(namedtuple('QVDatum', ['zhash', 'move', 'qval', 'outcome'])):
pass
class QVarianceAnalysis(StatsItf):
""" collect Q-values for the same state, variations, how much it agrees with game outcome;
and how it evolves across selfplays
Analysis will be hard to interpret, as game outcome is not ground truth either.
- compare empirical winrate vs Q-values (mean & std)?
"""
def __init__(self, move_indices: List[int] = None):
""" move indices are 1-based, i.e. the index of the first move is 1 """
self._moves_of_interest = move_indices
if not move_indices:
self._moves_of_interest = list(range(1, 8)) + list(range(10, 80, 10))
self.q_curves = dict()
self.records = [] # type: List[QVDatum]
def add_game(self, reader: SGFReader):
""" """
winner_sign = reader.result()
ts = QExtractor.extract_q_curve(reader)
# game_name_short = os.path.basename(reader.name)
# if random.random() < 0.1: self.q_curves[game_name_short] = ts
zhashes = [pwc.position.zobrist_hash for pwc in reader.iter_pwcs()]
for move_idx in self._moves_of_interest:
if move_idx > len(zhashes):
break
move_idx_0_based = move_idx - 1
zhash = zhashes[move_idx_0_based]
self.records.append(QVDatum(zhash, move_idx_0_based, ts[move_idx_0_based], winner_sign))
def get_df(self) -> pd.DataFrame:
df = pd.DataFrame.from_records(self.records, columns=QVDatum._fields)
return df
def test_qvariance():
review_root = '/Users/hyu/PycharmProjects/dlgo/9x9-exp2'
qstat = QVarianceAnalysis([1, 2, 6])
reader = SGFReader.from_file_compatible(f'{review_root}/selfplay11/sgf/full/0-31022166082.sgf')
qstat.add_game(reader)
print(len(qstat.data))
class WinnerStats(StatsItf):
""" this can count games among multiple players """
def __init__(self):
self.num_games = defaultdict(lambda: defaultdict(int))
self.num_wins = defaultdict(lambda: defaultdict(int))
self.players = set()
def add_game(self, reader: SGFReader):
black_id = reader.black_name()
white_id = reader.white_name()
self.num_games[black_id][white_id] += 1
self.players.add(black_id)
self.players.add(white_id)
result = reader.result()
if result > 0:
self.num_wins[black_id][white_id] += 1
elif result < 0:
pass
# self.num_wins[white_id][black_id] += 1
else: # no RE, or tie
logging.warning(f'game {reader.name} has no winner: {reader.result_str()}')
def report(self):
players = sorted(self.players)
df_num_games = pd.DataFrame(self.num_games, index=players, columns=players).T.fillna(0).astype(int)
df_blackwins = pd.DataFrame(self.num_wins, index=players, columns=players).T.fillna(0).astype(int)
df = df_blackwins.astype(str) + '/' + df_num_games.astype(str)
print(df)
return df
# todo integrate this into WinnerStats, or a separate review class?
def game_outcome_review(sgfs_dir):
""" KataEngine reviews final game outcome, to see if games are properly scored/finished
"""
from katago.analysis_engine import KataDualNetwork, KataModels
dnn_kata = KataDualNetwork(KataModels.G170_B6C96)
num_disagree = 0
game_counts = defaultdict(lambda: defaultdict(int))
disagree_cnt = defaultdict(lambda: defaultdict(int))
models = set()
for sgf_fname in os.listdir(f'{sgfs_dir}'):
if not sgf_fname.endswith('.sgf'):
continue
reader = SGFReader.from_file_compatible(f'{sgfs_dir}/{sgf_fname}')
black_id = reader.black_name()
white_id = reader.white_name()
result_sign = reader.result()
assert result_sign != 0
pos = reader.last_pos()
pi, v_expert = dnn_kata.run(pos)
sign_expert = np.sign(v_expert)
if result_sign != sign_expert:
result_str = reader.result_str()
num_disagree += 1
logging.info(f'{sgf_fname} {result_str} kata_v={v_expert}')
models.update([black_id, white_id])
game_counts[black_id][white_id] += 1
disagree_cnt[black_id][white_id] += result_sign != sign_expert
models = sorted(models)
df_counts_raw = pd.DataFrame(game_counts, index=models, columns=models)
df_counts = df_counts_raw.T.fillna(0).astype(int)
df_disagree = pd.DataFrame(disagree_cnt, index=models, columns=models).T.fillna(0).astype(int)
df_disagree.index.name = 'black_id'
df = df_disagree.astype(str) + '/' + df_counts.astype(str)
logging.info(f'Total disagreement: {num_disagree} / %d', df_counts.sum().sum())
print(df.replace('0/0', '-'))
class DiversityStats(StatsItf):
""" #unique states at certain moves, as well as freq of each zhash """
def __init__(self, move_indices: List[int] = None):
""" move indices are 1-based, i.e. the index of the first move is 1 """
self._moves_of_interest = move_indices
if not move_indices:
self._moves_of_interest = list(range(1, 8)) + list(range(10, 80, 10))
self.zhash_by_move = {move_idx: Counter() for move_idx in self._moves_of_interest}
self.num_games = 0
def add_game(self, reader: SGFReader):
zhashes = [pwc.position.zobrist_hash for pwc in reader.iter_pwcs()]
for move_idx, cnter in self.zhash_by_move.items():
# if a game ends early, use the last state for higher move indices
zhash = zhashes[move_idx] if move_idx < len(zhashes) else zhashes[-1]
cnter[zhash] += 1
self.num_games += 1
def report(self) -> pd.DataFrame:
cnt_by_move = {move_idx: len(cnter) for move_idx, cnter in self.zhash_by_move.items()}
ts = pd.Series(cnt_by_move, name='count')
ts.index.name = 'move'
df = pd.DataFrame({'count': ts, 'freq': ts / self.num_games})
print(df.T)
return df
def zhash_count_for_move(self, move_idx: int) -> Counter:
return self.zhash_by_move[move_idx]
def zhash_set_for_move(self, move_idx: int) -> Set:
cnter = self.zhash_by_move[move_idx]
return set(cnter.keys())
class GameSequenceReport(StatsItf):
""" report on move sequences, sorted
"""
def __init__(self, first_n: int = 12, last_n: int = 2):
self.first_n = first_n
self.last_n = last_n
self.game_summary = [] # type: List
def add_game(self, reader: SGFReader):
all_moves = [coords.to_gtp(pwc.next_move) for pwc in reader.iter_pwcs()]
line = format_game_summary(all_moves, reader.result_str(), self.first_n, self.last_n, sgf_fname=reader.name)
self.game_summary.append(line)
def report(self):
self.game_summary.sort()
print('\n'.join(self.game_summary))
class BensonAgreementStats(StatsItf):
""" Benson vs Tromp
"""
def __init__(self):
self.num_games = 0
self.num_benson_non_final = 0
self.num_disagree_with_benson = 0
self.num_disagree_with_tromp = 0
self.num_exceeds_max_moves = 0
def add_game(self, reader: SGFReader):
self.num_games += 1
result_sign = reader.result()
pos = reader.last_pos()
self.num_exceeds_max_moves += pos.n >= 2 * myconf.BOARD_SIZE_SQUARED - 1
self.num_disagree_with_tromp += result_sign != np.sign(pos.score_tromp())
benson_score = pos.score_benson()
if not benson_score.final:
self.num_benson_non_final += 1
self.num_disagree_with_benson += result_sign != np.sign(benson_score.score)
def report(self):
logging.info(f'Total {self.num_games}, of which {self.num_benson_non_final} benson non-final,'
f' {self.num_disagree_with_benson} differs w/ Benson, {self.num_disagree_with_tromp} differs w/ Tromp,'
f' {self.num_exceeds_max_moves} exceeds max game length')
class SgfProcessor:
""" first version
- winner stats
- diversity stats
"""
def __init__(self, stats: List[StatsItf]):
self._stats = stats
def _process_game(self, sgf_fname: str, reader: SGFReader):
for stat in self._stats:
stat.add_game(reader)
def process(self, sgf_glob_pattern, max_num_games: int = 1000_000) -> int:
logging.info(f'Processing {sgf_glob_pattern} ...')
sgf_fnames = glob.glob(sgf_glob_pattern)
cnt = 0
for sgf_fname in sgf_fnames:
if cnt >= max_num_games:
logging.info(f'#processed games exceeded {max_num_games}, break')
break
if not sgf_fname.endswith('.sgf'):
continue
reader = SGFReader.from_file_compatible(f'{sgf_fname}')
self._process_game(sgf_fname, reader)
cnt += 1
return cnt
def run_tournament_report(sgf_pattern):
wstats = WinnerStats()
dstats = DiversityStats()
game_report = GameSequenceReport()
processor = SgfProcessor([wstats, dstats, game_report])
logging.info(f'Scanning {sgf_pattern}')
processor.process(sgf_pattern)
logging.info(f'{sgf_pattern}: found %d games', dstats.num_games)
wstats.report()
logging.info('\tUnique states by move:')
dstats.report()
logging.info('\tgames sorted:')
game_report.report()
def test_basic_report():
sgf_pattern = f'{myconf.EXP_HOME}/eval_bots-model11/model11_2/model10/*.sgf'
sgf_pattern = f'{myconf.EXP_HOME}/eval_gating/model12_2-vs-elo5k/kata*.sgf'
# sgf_pattern = f'{myconf.EXP_HOME}/selfplay/sgf/full/*sgf'
run_tournament_report(sgf_pattern)
def test_state_growth():
""" within a selfplay, how many new states we encounter with each extra game
The question in mind: how much state caching will lead to how much saving in computation
What about noise injection? Do we only inject at root?
Is it just easier to have a NN cache?
"""
def test_review_common_states():
"""
0 shared states at move#60, across all selfplays!
#shared state counts at move 10:
1 2 3 4 5 6 7 8 9 10 11
1 916/2000 34/231 32/168 11/81 23/187 3/24 4/30 1/2 1/2 3/10 2/10
2 -1/-1 1128/2000 74/604 27/144 37/202 5/23 4/19 2/19 3/8 5/13 2/8
3 -1/-1 -1/-1 921/2000 37/266 35/219 14/48 6/22 5/37 2/9 5/17 3/8
4 -1/-1 -1/-1 -1/-1 1275/2000 52/339 14/123 6/31 5/30 4/34 4/54 1/5
5 -1/-1 -1/-1 -1/-1 -1/-1 1503/2000 23/137 12/51 1/7 4/12 5/17 0/0
6 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 1325/2000 53/306 22/211 6/25 11/69 16/143
7 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 1073/2000 25/86 12/35 9/38 16/149
8 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 1217/2000 23/403 18/240 16/108
9 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 1066/2000 32/384 23/313
10 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 1213/2000 25/190
11 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 -1/-1 1004/2000
"""
# review_root = '/Users/hyu/PycharmProjects/dlgo/9x9-exp2/eval_review/kata1_5k'
review_root = '/Users/hyu/PycharmProjects/dlgo/9x9-exp2'
MOVE_OF_INTEREST = 10
MAX_NUM_GAMES_EACH = 2000
# model_ids = [f'model{x}'
# for x in ['1_5', '2_2', '3_3', '4_4', '5_2', '6_2', '7_4', '8_4', '9_4', '10_4', '11_3', '12_2']]
model_ids = range(1, 12)
model_ids = model_ids[:]
zhs_list = [] # type: List[Counter]
for model_id in model_ids:
sgf_pattern = f'{review_root}/selfplay{model_id}/sgf/full/*.sgf'
dstats = DiversityStats()
processor = SgfProcessor([dstats])
num_games = processor.process(sgf_pattern, max_num_games=MAX_NUM_GAMES_EACH)
print(f'\nProcessed {num_games} games for {model_id}')
df = dstats.report()
s60 = dstats.zhash_count_for_move(MOVE_OF_INTEREST)
zhs_list.append(s60)
num_models = len(model_ids)
BAND_SIZE = num_models - 1
unique2d = np.ones((num_models, num_models), dtype=np.int) * -1
cnt2d = np.ones((num_models, num_models), dtype=np.int) * -1
for m1_idx in range(num_models):
cnter1 = zhs_list[m1_idx]
unique2d[m1_idx, m1_idx] = len(cnter1)
cnt2d[m1_idx, m1_idx] = sum(cnter1.values()) # not double-counted
for m2_idx in range(m1_idx + 1, min(m1_idx + BAND_SIZE + 1, num_models)):
cnter2 = zhs_list[m2_idx]
states_in_common = set(cnter1.keys()).intersection(cnter2.keys())
unique2d[m1_idx, m2_idx] = len(states_in_common)
cnt2d[m1_idx, m2_idx] = sum(cnter1[k] + cnter2[k] for k in states_in_common)
dfu = pd.DataFrame(unique2d, index=model_ids, columns=model_ids)
print(f'#shared states at move {MOVE_OF_INTEREST}:')
print(dfu)
dfc = pd.DataFrame(cnt2d, index=model_ids, columns=model_ids)
print(f'#shared state counts at move {MOVE_OF_INTEREST}:')
print(dfu.astype(str) + '/' + dfc.astype(str))
def test_count_consecutive_overage():
ts = pd.Series([0, 0.3, 0.9, 0.8, 0.9, 0.9, 1.0])
ts = pd.Series([0, 0.3, 0.9, 0.8, 0, -0.9, -1.0])
ts = pd.Series([0, 0.3, 0.9, -0.8, 0.9, -1.0, 0.9, 0.9])
ts_cc = QResignAnalysis.count_consecutive_overage(ts, 0.9)
df = pd.DataFrame({'ts': ts, 'count': ts_cc})
print()
print(df)
def test_resign_stats():
"""
selfplay6: restrict on move# should help this
'11:25:23 INFO Total 3376 games, thresh=0.9, run_len=2: 2903 resigned = 86.0%, 294 errors = 8.7%, saved 94525 = 35.6%'
selfplay7:
'11:04:52 INFO Total 3324 games, thresh=0.9, run_len=1: 35 errors = 1.1%, saved 47070 = 18.1%'
'11:23:18 INFO Total 3324 games, thresh=0.9, run_len=2: 1986 resigned = 59.7%, 26 errors = 0.8%, saved 40118 = 15.4%'
'11:06:48 INFO Total 3324 games, thresh=0.95, run_len=1: 15 errors = 0.5%, saved 33322 = 12.8%'
'11:09:37 INFO Total 3324 games, thresh=0.95, run_len=2: 11 errors = 0.3%, saved 27732 = 10.7%'
"""
qstat = QResignAnalysis(0.9, 2)
sgf_fname = f'{myconf.EXP_HOME}/selfplay/sgf/full/0-32080527168.sgf'
sgf_pattern = f'{myconf.EXP_HOME}/selfplay7/sgf/full/*.sgf'
processor = SgfProcessor([qstat])
processor.process(sgf_pattern)
qstat.report()
def test_review_vtarget_variance():
""" review Q value in sgfs
- how much inconsistencies in vnet targets for the same state due to uncertain game outcomes
"""
qvstat = QVarianceAnalysis()
sgf_pattern = f'{myconf.EXP_HOME}/selfplay11/sgf/full/*.sgf'
processor = SgfProcessor([qvstat])
processor.process(sgf_pattern, max_num_games=200)
def test_review_all_selfplay():
""" review state dist change across generations, see how much is shared, and thus revised
- quantify the amount of natural exploration not due to new model: even this depends on the model
bet w/ 2000 games, we are far from saturation, probably except the first n moves
"""
def test_review_game_score():
""" 10/17/23 I disabled Benson scoring in selfplay. They are scored on Tromp only. See how much
this differs from Benson score
"""
"""
In exp2, I used primarily Benson scoring, which is more accurate, and stops the game much earlier.
exp2/selfplay3/ Total 3559, of which 1091 benson non-final, 1 differs w/ Benson, 563 differs w/ Tromp, 2 exceeds max game length'
move 1 5 10 20
count 6.000000 991.000000 1363.000000 1449.000000
freq 0.001686 0.278449 0.382973 0.407137
selfplay5/ Total 3340, of which 358 benson non-final, 2 differs w/ Benson, 611 differs w/ Tromp, 2 exceeds max game length'
count 6.000000 1905.000000 2379.000000 2552.000000
freq 0.001796 0.570359 0.712275 0.764072
exp2/selfplay6/ Total 3376, of which 264 benson non-final, 0 differs w/ Benson, 509 differs w/ Tromp, 1497 exceeds 80 moves, 3 exceeds 160
count 18.000000 1438.000000 2040.000000 2275.000000
freq 0.005332 0.425948 0.604265 0.673874
selfplay6 switched off init_positions, which explains why move#1 increased but diversity decreased later.
exp3/selfplay7f Total 3000, of which 713 benson non-final, 76 differs w/ Benson, 0 differs w/ Tromp, 3 exceeds max game length
move 1 5 10 20
count 6.000 789.000 1275.000 1912.000000
freq 0.002 0.263 0.425 0.637333
selfplay8f Total 3000, of which 388 benson non-final, 38 differs w/ Benson, 0 differs w/ Tromp, 6 exceeds max game length'
selfplay9f Total 3000, of which 286 benson non-final, 40 differs w/ Benson, 0 differs w/ Tromp, 8 exceeds max game length'
selfplay10f Total 3000, of which 296 benson non-final, 50 differs w/ Benson, 0 differs w/ Tromp, 2813 exceeds 80 moves, 3 exceeds 160
count 6.000 1906.000000 2302.000000 2484.000
freq 0.002 0.635333 0.767333 0.828
selfplay12f Total 3000, of which 416 benson non-final, 45 differs w/ Benson, 0 differs w/ Tromp, 6 exceeds max game length'
selfplay13f Total 3000, of which 425 benson non-final, 32 differs w/ Benson, 0 differs w/ Tromp, 4 exceeds max game length'
count 6.000 1538.000000 2124.000 2368.000000
freq 0.002 0.512667 0.708 0.789333
"""
for i in range(5, 6):
stat1 = BensonAgreementStats()
stat2 = DiversityStats([1, 5, 10, 20])
sgf_pattern = f'{myconf.EXP_HOME}/selfplay{i}f/sgf/full/*.sgf'
sgf_pattern = f'{myconf.EXP_HOME}/../9x9-exp2/selfplay{i}/sgf/full/*.sgf'
processor = SgfProcessor([stat1, stat2])
processor.process(sgf_pattern, max_num_games=5000)
stat1.report()
stat2.report()