8000 minigo/cc/eval.cc at master · Az-mark/minigo · GitHub
[go: up one dir, main page]
Skip to content

eval.cc

Latest commit

 

History

History
302 lines (261 loc) · 10.4 KB

eval.cc

File metadata and controls

302 lines (261 loc) · 10.4 KB
Raw
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <stdio.h>
#include <cmath>
#include <functional>
#include <iostream>
#include <memory>
#include <string>
#include <thread>
#include <utility>
#include <vector>
#include "absl/memory/memory.h"
#include "absl/strings/str_cat.h"
#include "absl/strings/str_format.h"
#include "absl/strings/str_split.h"
#include "absl/time/clock.h"
#include "absl/time/time.h"
#include "cc/constants.h"
#include "cc/dual_net/batching_dual_net.h"
#include "cc/dual_net/factory.h"
#include "cc/file/path.h"
#include "cc/file/utils.h"
#include "cc/game.h"
#include "cc/game_utils.h"
#include "cc/init.h"
#include "cc/logging.h"
#include "cc/mcts_player.h"
#include "cc/random.h"
#include "cc/tf_utils.h"
#include "cc/zobrist.h"
#include "gflags/gflags.h"
// Game options flags.
DEFINE_double(resign_threshold, -0.999, "Resign threshold.");
DEFINE_uint64(seed, 0,
"Random seed. Use default value of 0 to use a time-based seed.");
// Tree search flags.
DEFINE_int32(num_readouts, 100,
"Number of readouts to make during tree search for each move.");
DEFINE_int32(virtual_losses, 8,
"Number of virtual losses when running tree search.");
DEFINE_double(value_init_penalty, 2.0,
"New children value initialization penalty.\n"
"Child value = parent's value - penalty * color, clamped to"
" [-1, 1]. Penalty should be in [0.0, 2.0].\n"
"0 is init-to-parent, 2.0 is init-to-loss [default].\n"
"This behaves similiarly to Leela's FPU \"First Play Urgency\".");
// Inference flags.
DEFINE_string(model, "",
"Path to a minigo model. The format of the model depends on the "
"inference engine. If parallel_games=1, this model is used for "
"black. For engine=tf, the model should be a GraphDef proto. For "
"engine=lite, the model should be .tflite flatbuffer.");
DEFINE_string(model_two, "", "Descriptor for the second model");
DEFINE_int32(parallel_games, 32, "Number of games to play in parallel.");
// Output flags.
DEFINE_string(output_bigtable, "",
"Output Bigtable specification, of the form: "
"project,instance,table. "
"If empty, no examples are written to Bigtable.");
DEFINE_string(sgf_dir, "",
"SGF directory for selfplay and puzzles. If empty in selfplay "
"mode, no SGF is written.");
DEFINE_string(bigtable_tag, "", "Used in Bigtable metadata");
namespace minigo {
namespace {
void ParseOptionsFromFlags(Game::Options* game_options,
MctsPlayer::Options* player_options) {
game_options->resign_threshold = -std::abs(FLAGS_resign_threshold);
game_options->ignore_repeated_moves = true;
player_options->virtual_losses = FLAGS_virtual_losses;
player_options->random_seed = FLAGS_seed;
player_options->num_readouts = FLAGS_num_readouts;
player_options->inject_noise = false;
player_options->soft_pick = false;
player_options->random_symmetry = true;
}
class Evaluator {
class Model {
public:
Model(BatchingDualNetFactory* batcher, const std::string& path)
: batcher_(batcher), path_(path) {}
BatchingDualNetFactory* batcher() { return batcher_; }
std::string name() {
absl::MutexLock lock(&mutex_);
if (name_.empty()) {
// The model's name is lazily initialized the first time we create a
// instance. Make sure it's valid.
NewDualNetImpl();
}
return name_;
}
WinStats GetWinStats() const {
absl::MutexLock lock(&mutex_);
return win_stats_;
}
void UpdateWinStats(const Game& game) {
absl::MutexLock lock(&mutex_);
win_stats_.Update(game);
}
std::unique_ptr<DualNet> NewDualNet() {
absl::MutexLock lock(&mutex_);
return NewDualNetImpl();
}
private:
std::unique_ptr<DualNet> NewDualNetImpl()
EXCLUSIVE_LOCKS_REQUIRED(&mutex_) {
auto dual_net = batcher_->NewDualNet(path_);
if (name_.empty()) {
name_ = dual_net->name();
}
return dual_net;
}
mutable absl::Mutex mutex_;
BatchingDualNetFactory* batcher_ GUARDED_BY(&mutex_);
const std::string path_;
std::string name_ GUARDED_BY(&mutex_);
WinStats win_stats_ GUARDED_BY(&mutex_);
};
public:
Evaluator(ModelDescriptor desc_a, ModelDescriptor desc_b)
: desc_a_(std::move(desc_a)), desc_b_(std::move(desc_b)) {
// Create a batcher for the first model.
batchers_.push_back(absl::make_unique<BatchingDualNetFactory>(
NewDualNetFactory(desc_a_.engine)));
// If the second model requires a different factory, create one & a second
// batcher too.
if (desc_b_.engine != desc_a_.engine) {
batchers_.push_back(absl::make_unique<BatchingDualNetFactory>(
NewDualNetFactory(desc_b_.engine)));
}
}
void Run() {
auto start_time = absl::Now();
Model model_a(batchers_.front().get(), desc_a_.model);
Model model_b(batchers_.back().get(), desc_b_.model);
MG_LOG(INFO) << "DualNet factories created from " << desc_a_ << "\n and "
<< desc_b_ << " in "
<< absl::ToDoubleSeconds(absl::Now() - start_time) << " sec.";
ParseOptionsFromFlags(&game_options_, &player_options_);
int num_games = FLAGS_parallel_games;
for (int thread_id = 0; thread_id < num_games; ++thread_id) {
bool swap_models = (thread_id & 1) != 0;
threads_.emplace_back(std::bind(&Evaluator::ThreadRun, this, thread_id,
swap_models ? &model_b : &model_a,
swap_models ? &model_a : &model_b));
}
for (auto& t : threads_) {
t.join();
}
MG_LOG(INFO) << "Evaluated " << num_games << " games, total time "
<< (absl::Now() - start_time);
MG_LOG(INFO) << FormatWinStatsTable(
{{model_a.name(), model_a.GetWinStats()},
{model_b.name(), model_b.GetWinStats()}});
}
private:
void ThreadRun(int thread_id, Model* black_model, Model* white_model) {
// Only print the board using ANSI colors if stderr is sent to the
// terminal.
const bool use_ansi_colors = FdSupportsAnsiColors(fileno(stderr));
// The player and other_player reference this pointer.
std::unique_ptr<DualNet> dual_net;
std::vector<std::string> bigtable_spec =
absl::StrSplit(FLAGS_output_bigtable, ',');
bool use_bigtable = bigtable_spec.size() == 3;
if (!FLAGS_output_bigtable.empty() && !use_bigtable) {
MG_LOG(FATAL)
<< "Bigtable output must be of the form: project,instance,table";
return;
}
Game game(black_model->name(), white_model->name(), game_options_);
auto player_options = player_options_;
// If an random seed was explicitly specified, make sure we use a
// different seed for each thread.
if (player_options.random_seed != 0) {
player_options.random_seed += 1299283 * thread_id;
}
const bool verbose = thread_id == 0;
auto black = absl::make_unique<MctsPlayer>(black_model->NewDualNet(),
nullptr, &game, player_options);
auto white = absl::make_unique<MctsPlayer>(white_model->NewDualNet(),
nullptr, &game, player_options);
BatchingDualNetFactory::StartGame(black->network(), white->network());
auto* curr_player = black.get();
auto* next_player = white.get();
while (!game.game_over() && !curr_player->root()->at_move_limit()) {
auto move = curr_player->SuggestMove();
if (verbose) {
std::cerr << curr_player->root()->Describe() << "\n";
}
curr_player->PlayMove(move);
if (!game.game_over()) {
next_player->PlayMove(move);
}
if (verbose) {
MG_LOG(INFO) << absl::StreamFormat(
"%d: %s by %s\nQ: %0.4f", curr_player->root()->position.n(),
move.ToGtp(), curr_player->name(), curr_player->root()->Q());
MG_LOG(INFO) << curr_player->root()->position.ToPrettyString(
use_ansi_colors);
}
std::swap(curr_player, next_player);
}
BatchingDualNetFactory::EndGame(black->network(), white->network());
if (game.result() > 0) {
black_model->UpdateWinStats(game);
} else {
white_model->UpdateWinStats(game);
}
if (verbose) {
MG_LOG(INFO) << game.result_string();
MG_LOG(INFO) << "Black was: " << game.black_name();
}
// Write SGF.
std::string output_name = "NO_SGF_SAVED";
if (!FLAGS_sgf_dir.empty()) {
output_name = absl::StrCat(GetOutputName(absl::Now(), thread_id), "-",
black->name(), "-", white->name());
game.AddComment(
absl::StrCat("B inferences: ", black->GetModelsUsedForInference()));
game.AddComment(
absl::StrCat("W inferences: ", white->GetModelsUsedForInference()));
WriteSgf(FLAGS_sgf_dir, output_name, game, true);
}
if (use_bigtable) {
const auto& gcp_project_name = bigtable_spec[0];
const auto& instance_name = bigtable_spec[1];
const auto& table_name = bigtable_spec[2];
tf_utils::WriteEvalRecord(gcp_project_name, instance_name, table_name,
game, output_name, FLAGS_bigtable_tag);
}
MG_LOG(INFO) << "Thread " << thread_id << " stopping";
}
Game::Options game_options_;
MctsPlayer::Options player_options_;
std::vector<std::thread> threads_;
const ModelDescriptor desc_a_;
const ModelDescriptor desc_b_;
std::vector<std::unique_ptr<BatchingDualNetFactory>> batchers_;
};
} // namespace
} // namespace minigo
int main(int argc, char* argv[]) {
minigo::Init(&argc, &argv);
minigo::zobrist::Init(FLAGS_seed * 614944751);
minigo::Evaluator evaluator(minigo::ParseModelDescriptor(FLAGS_model),
minigo::ParseModelDescriptor(FLAGS_model_two));
evaluator.Run();
return 0;
}
0