8000 Add a test to the RandomDesigner to confirm that it respects LOG scaling. by copybara-service[bot] · Pull Request #596 · google/vizier · GitHub
[go: up one dir, main page]
Skip to content

Add a test to the RandomDesigner to confirm that it respects LOG scaling. #596

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Mar 25, 2023
Merged

Add a test to the RandomDesigner to confirm that it respects LOG scaling. #596

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions vizier/_src/algorithms/designers/random_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

"""Tests for random."""

import math

from vizier import pyvizier as vz
from vizier._src.algorithms.designers import random
from vizier._src.algorithms.testing import test_runners
Expand All @@ -42,6 +44,40 @@ def test_reproducible_random(self):
t2 = designer.suggest(10)
self.assertEqual(t1, t2)

def test_log_scaling(self):
"""Confirm that LINEAR and LOG scaling give the right distribution."""
config = vz.ProblemStatement(test_studies.flat_space_with_all_types())
designer = random.RandomDesigner(config.search_space, seed=None)
trials = test_runners.run_with_random_metrics(
designer, config, iters=1000, batch_size=1
)
# LINEAR scaling.
sum_lineardouble = 0.0
for t in trials:
sum_lineardouble += t.parameters['lineardouble'].value
avg_lineardouble = sum_lineardouble / len(trials)
# Delta=0.16 corresponds to ~6 standard deviations, so the probability of
# accidental test failure is ~1e-6.
self.assertAlmostEqual(avg_lineardouble, 0.5 * (-1 + 2), delta=0.16)
# LOG scaling
sum_log_logdouble = 0.0
sum_logdouble = 0.0
for t in trials:
sum_logdouble += t.parameters['logdouble'].value
sum_log_logdouble += math.log(t.parameters['logdouble'].value)
avg_logdouble = sum_logdouble / len(trials)
avg_log_logdouble = sum_log_logdouble / len(trials)
# If the distribution were LINEAR, we'd expect avg_logdouble = 50;
# if the distribution were LOG, we'd expect avg_logdouble = 7.2, so
# we check that avg_logdouble is closer to the LOG expected value.
self.assertLess(abs(avg_logdouble - 7.2), abs(avg_logdouble - 50))
# The average of log(value) should be close to the average of
# the logs of the endpoints. $delta is set at 6 standard deviations to
# yield a false failure rate of about 1e-6.
self.assertAlmostEqual(
avg_log_logdouble, 0.5 * (math.log(1e-4) + math.log(1e2)), delta=0.76
)


if __name__ == '__main__':
absltest.main()
0