Repository navigation
Expand file tree
/
Copy pathfsdb.py
More file actions
99 lines (77 loc) · 3.15 KB
/
Copy pathfsdb.py
File metadata and controls
99 lines (77 loc) · 3.15 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Filesystem DB: poor man's worker coordination strategy.
This module works equivalently with local filesystems and GCS.
"""
import os
from absl import flags
from tensorflow import gfile
import re
import shipname
flags.DEFINE_string(
'base_dir', None,
'Root directory if using local FS as the database.'
'Leave blank if using bucket_name.')
flags.DEFINE_string(
'bucket_name', None,
'Bucket name if using GCS as the filesystem DB.'
'Leave blank is using base_dir.')
flags.register_multi_flags_validator(
['base_dir', 'bucket_name'],
lambda flags: bool(flags['base_dir']) != bool(flags['bucket_name']),
'Exactly one of --base_dir, --bucket_name must be set!')
FLAGS = flags.FLAGS
def _with_base(*args):
def inner():
base_dir = FLAGS.base_dir or 'gs://{}'.format(FLAGS.bucket_name)
return os.path.join(base_dir, *args)
return inner
# Functions to compute various important directories, based on FLAGS input.
models_dir = _with_base('models')
selfplay_dir = _with_base('data', 'selfplay')
holdout_dir = _with_base('data', 'holdout')
sgf_dir = _with_base('sgf')
eval_dir = _with_base('sgf', 'eval')
training_chunk_dir = _with_base('data', 'training_chunks')
golden_chunk_dir = _with_base('data', 'golden_chunks')
def get_models():
"""Finds all models, returning a list of model number and names
sorted increasing.
Returns: [(13, 000013-modelname), (17, 000017-modelname), ...etc]
"""
all_models = gfile.Glob(os.path.join(models_dir(), '*.meta'))
model_filenames = [os.path.basename(m) for m in all_models]
model_numbers_names = sorted([
(shipname.detect_model_num(m), shipname.detect_model_name(m))
for m in model_filenames])
return model_numbers_names
def get_latest_model():
"""Finds the latest model, returning its model number and name
Returns: (17, 000017-modelname)
"""
return get_models()[-1]
def get_model(model_num):
"""Given a model number 17, returns its full name 000017-modelname."""
model_names_by_num = dict(get_models())
return model_names_by_num[model_num]
def get_hour_dirs(root=None):
"""Gets the directories under selfplay_dir that match YYYY-MM-DD-HH."""
root = root or selfplay_dir()
return list(filter(lambda s: re.match(r"\d{4}-\d{2}-\d{2}-\d{2}", s),
gfile.ListDirectory(root)))
def game_counts(n_back=20):
"""Prints statistics for the most recent n_back models"""
for _, model_name in get_models[-n_back:]:
games = gfile.Glob(os.path.join(selfplay_dir(), model_name, '*.zz'))
print(model_name, len(games))