Repository navigation
Expand file tree
/
Copy pathgp.py
More file actions
1285 lines (1092 loc) · 46.8 KB
/
Copy pathgp.py
File metadata and controls
1285 lines (1092 loc) · 46.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# Copyright 2020 The PyMC Developers
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import warnings
import aesara.tensor as at
import numpy as np
from aesara.tensor.nlinalg import eigh
import pymc as pm
from pymc.gp.cov import Constant, Covariance
from pymc.gp.mean import Zero
from pymc.gp.util import (
JITTER_DEFAULT,
cholesky,
conditioned_vars,
replace_with_values,
solve_lower,
solve_upper,
stabilize,
)
from pymc.math import cartesian, kron_diag, kron_dot, kron_solve_lower, kron_solve_upper
__all__ = ["Latent", "Marginal", "TP", "MarginalApprox", "LatentKron", "MarginalKron"]
_noise_deprecation_warning = (
"The 'noise' parameter has been been changed to 'sigma' "
"in order to standardize the GP API and will be "
"deprecated in future releases."
)
def _handle_sigma_noise_parameters(sigma, noise):
"""Helper function for transition of 'noise' parameter to be named 'sigma'."""
if (sigma is None and noise is None) or (sigma is not None and noise is not None):
raise ValueError("'sigma' argument must be specified.")
if sigma is None:
warnings.warn(_noise_deprecation_warning, FutureWarning)
return noise
return sigma
class Base:
R"""
Base class.
"""
def __init__(self, *, mean_func=Zero(), cov_func=Constant(0.0)):
self.mean_func = mean_func
self.cov_func = cov_func
def __add__(self, other):
same_attrs = set(self.__dict__.keys()) == set(other.__dict__.keys())
if not isinstance(self, type(other)) or not same_attrs:
raise TypeError("Cannot add different GP types")
mean_total = self.mean_func + other.mean_func
cov_total = self.cov_func + other.cov_func
return self.__class__(mean_func=mean_total, cov_func=cov_total)
def prior(self, name, X, *args, **kwargs):
raise NotImplementedError
def marginal_likelihood(self, name, X, *args, **kwargs):
raise NotImplementedError
def conditional(self, name, Xnew, *args, **kwargs):
raise NotImplementedError
def predict(self, Xnew, point=None, given=None, diag=False, model=None):
raise NotImplementedError
@conditioned_vars(["X", "f"])
class Latent(Base):
R"""
Latent Gaussian process.
The `gp.Latent` class is a direct implementation of a GP. No additive
noise is assumed. It is called "Latent" because the underlying function
values are treated as latent variables. It has a `prior` method and a
`conditional` method. Given a mean and covariance function the
function :math:`f(x)` is modeled as,
.. math::
f(x) \sim \mathcal{GP}\left(\mu(x), k(x, x')\right)
Use the `prior` and `conditional` methods to actually construct random
variables representing the unknown, or latent, function whose
distribution is the GP prior or GP conditional. This GP implementation
can be used to implement regression on data that is not normally
distributed. For more information on the `prior` and `conditional` methods,
see their docstrings.
Parameters
----------
cov_func: None, 2D array, or instance of Covariance
The covariance function. Defaults to zero.
mean_func: None, instance of Mean
The mean function. Defaults to zero.
Examples
--------
.. code:: python
# A one dimensional column vector of inputs.
X = np.linspace(0, 1, 10)[:, None]
with pm.Model() as model:
# Specify the covariance function.
cov_func = pm.gp.cov.ExpQuad(1, ls=0.1)
# Specify the GP. The default mean function is `Zero`.
gp = pm.gp.Latent(cov_func=cov_func)
# Place a GP prior over the function f.
f = gp.prior("f", X=X)
...
# After fitting or sampling, specify the distribution
# at new points with .conditional
Xnew = np.linspace(-1, 2, 50)[:, None]
with model:
fcond = gp.conditional("fcond", Xnew=Xnew)
"""
def __init__(self, *, mean_func=Zero(), cov_func=Constant(0.0)):
super().__init__(mean_func=mean_func, cov_func=cov_func)
def _build_prior(self, name, X, reparameterize=True, jitter=JITTER_DEFAULT, **kwargs):
mu = self.mean_func(X)
cov = stabilize(self.cov_func(X), jitter)
if reparameterize:
size = np.shape(X)[0]
v = pm.Normal(name + "_rotated_", mu=0.0, sigma=1.0, size=size, **kwargs)
f = pm.Deterministic(name, mu + cholesky(cov).dot(v), dims=kwargs.get("dims", None))
else:
f = pm.MvNormal(name, mu=mu, cov=cov, **kwargs)
return f
def prior(self, name, X, reparameterize=True, jitter=JITTER_DEFAULT, **kwargs):
R"""
Returns the GP prior distribution evaluated over the input
locations `X`.
This is the prior probability over the space
of functions described by its mean and covariance function.
.. math::
f \mid X \sim \text{MvNormal}\left( \mu(X), k(X, X') \right)
Parameters
----------
name: string
Name of the random variable
X: array-like
Function input values.
reparameterize: bool
Reparameterize the distribution by rotating the random
variable by the Cholesky factor of the covariance matrix.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to distribution constructor.
"""
f = self._build_prior(name, X, reparameterize, jitter, **kwargs)
self.X = X
self.f = f
return f
def _get_given_vals(self, given):
if given is None:
given = {}
if "gp" in given:
cov_total = given["gp"].cov_func
mean_total = given["gp"].mean_func
else:
cov_total = self.cov_func
mean_total = self.mean_func
if all(val in given for val in ["X", "f"]):
X, f = given["X"], given["f"]
else:
X, f = self.X, self.f
return X, f, cov_total, mean_total
def _build_conditional(self, Xnew, X, f, cov_total, mean_total, jitter):
Kxx = cov_total(X)
Kxs = self.cov_func(X, Xnew)
L = cholesky(stabilize(Kxx, jitter))
A = solve_lower(L, Kxs)
v = solve_lower(L, f - mean_total(X))
mu = self.mean_func(Xnew) + at.dot(at.transpose(A), v)
Kss = self.cov_func(Xnew)
cov = Kss - at.dot(at.transpose(A), A)
return mu, cov
def conditional(self, name, Xnew, given=None, jitter=JITTER_DEFAULT, **kwargs):
R"""
Returns the conditional distribution evaluated over new input
locations `Xnew`.
Given a set of function values `f` that
the GP prior was over, the conditional distribution over a
set of new points, `f_*` is
.. math::
f_* \mid f, X, X_* \sim \mathcal{GP}\left(
K(X_*, X) K(X, X)^{-1} f \,,
K(X_*, X_*) - K(X_*, X) K(X, X)^{-1} K(X, X_*) \right)
Parameters
----------
name: string
Name of the random variable
Xnew: array-like
Function input values.
given: dict
Can optionally take as key value pairs: `X`, `y`,
and `gp`. See the section in the documentation on additive GP
models in PyMC for more information.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to `MvNormal` distribution
constructor.
"""
givens = self._get_given_vals(given)
mu, cov = self._build_conditional(Xnew, *givens, jitter)
return pm.MvNormal(name, mu=mu, cov=cov, **kwargs)
@conditioned_vars(["X", "f", "nu"])
class TP(Latent):
r"""
Student's T process prior.
The usage is nearly identical to that of `gp.Latent`. The differences
are that it must be initialized with a degrees of freedom parameter, and
TP is not additive. Given a mean and covariance function, and a degrees of
freedom parameter, the function :math:`f(x)` is modeled as,
.. math::
f(X) \sim \mathcal{TP}\left( \mu(X), k(X, X'), \nu \right)
Parameters
----------
scale_func : None, 2D array, or instance of Covariance
The scale function. Defaults to zero.
mean_func : None, instance of Mean
The mean function. Defaults to zero.
nu : float
The degrees of freedom
References
----------
- Shah, A., Wilson, A. G., and Ghahramani, Z. (2014). Student-t
Processes as Alternatives to Gaussian Processes. arXiv preprint arXiv:1402.4306.
"""
def __init__(self, *, mean_func=Zero(), scale_func=Constant(0.0), cov_func=None, nu=None):
if nu is None:
raise ValueError("Student's T process requires a degrees of freedom parameter, 'nu'")
if cov_func is not None:
warnings.warn(
"Use the scale_func argument to specify the scale function."
"cov_func will be removed in future versions.",
FutureWarning,
)
scale_func = cov_func
self.nu = nu
super().__init__(mean_func=mean_func, cov_func=scale_func)
def __add__(self, other)
BD0
:
raise TypeError("Student's T processes aren't additive")
def _build_prior(self, name, X, reparameterize=True, jitter=JITTER_DEFAULT, **kwargs):
mu = self.mean_func(X)
cov = stabilize(self.cov_func(X), jitter)
if reparameterize:
size = np.shape(X)[0]
v = pm.StudentT(name + "_rotated_", mu=0.0, sigma=1.0, nu=self.nu, size=size, **kwargs)
f = pm.Deterministic(name, mu + cholesky(cov).dot(v), dims=kwargs.get("dims", None))
else:
f = pm.MvStudentT
10000
(name, nu=self.nu, mu=mu, scale=cov, **kwargs)
return f
def prior(self, name, X, reparameterize=True, jitter=JITTER_DEFAULT, **kwargs):
R"""
Returns the TP prior distribution evaluated over the input
locations `X`.
This is the prior probability over the space
of functions described by its mean and covariance function.
Parameters
----------
name: string
Name of the random variable
X: array-like
Function input values.
reparameterize: bool
Reparameterize the distribution by rotating the random
variable by the Cholesky factor of the covariance matrix.
**kwargs
Extra keyword arguments that are passed to distribution constructor.
"""
f = self._build_prior(name, X, reparameterize, jitter, **kwargs)
self.X = X
self.f = f
return f
def _build_conditional(self, Xnew, X, f, jitter):
Kxx = self.cov_func(X)
Kxs = self.cov_func(X, Xnew)
Kss = self.cov_func(Xnew)
L = cholesky(stabilize(Kxx, jitter))
A = solve_lower(L, Kxs)
cov = Kss - at.dot(at.transpose(A), A)
v = solve_lower(L, f - self.mean_func(X))
mu = self.mean_func(Xnew) + at.dot(at.transpose(A), v)
beta = at.dot(v, v)
nu2 = self.nu + X.shape[0]
covT = (self.nu + beta - 2) / (nu2 - 2) * cov
return nu2, mu, covT
def conditional(self, name, Xnew, jitter=JITTER_DEFAULT, **kwargs):
R"""
Returns the conditional distribution evaluated over new input
locations `Xnew`.
Given a set of function values `f` that
the TP prior was over, the conditional distribution over a
set of new points, `f_*` is
Parameters
----------
name: string
Name of the random variable
Xnew: array-like
Function input values.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to `MvNormal` distribution
constructor.
"""
X = self.X
f = self.f
nu2, mu, cov = self._build_conditional(Xnew, X, f, jitter)
return pm.MvStudentT(name, nu=nu2, mu=mu, scale=cov, **kwargs)
@conditioned_vars(["X", "y", "sigma"])
class Marginal(Base):
R"""
Marginal Gaussian process.
The `gp.Marginal` class is an implementation of the sum of a GP
prior and additive noise. It has `marginal_likelihood`, `conditional`
and `predict` methods. This GP implementation can be used to
implement regression on data that is normally distributed. For more
information on the `prior` and `conditional` methods, see their docstrings.
Parameters
----------
cov_func: None, 2D array, or instance of Covariance
The covariance function. Defaults to zero.
mean_func: None, instance of Mean
The mean function. Defaults to zero.
Examples
--------
.. code:: python
# A one dimensional column vector of inputs.
X = np.linspace(0, 1, 10)[:, None]
with pm.Model() as model:
# Specify the covariance function.
cov_func = pm.gp.cov.ExpQuad(1, ls=0.1)
# Specify the GP. The default mean function is `Zero`.
gp = pm.gp.Marginal(cov_func=cov_func)
# Place a GP prior over the function f.
sigma = pm.HalfCauchy("sigma", beta=3)
y_ = gp.marginal_likelihood("y", X=X, y=y, sigma=sigma)
...
# After fitting or sampling, specify the distribution
# at new points with .conditional
Xnew = np.linspace(-1, 2, 50)[:, None]
with model:
fcond = gp.conditional("fcond", Xnew=Xnew)
"""
def _build_marginal_likelihood(self, X, noise_func, jitter):
mu = self.mean_func(X)
Kxx = self.cov_func(X)
Knx = noise_func(X)
cov = Kxx + Knx
return mu, stabilize(cov, jitter)
def marginal_likelihood(
self, name, X, y, sigma=None, noise=None, jitter=JITTER_DEFAULT, is_observed=True, **kwargs
):
R"""
Returns the marginal likelihood distribution, given the input
locations `X` and the data `y`.
This is integral over the product of the GP prior and a normal likelihood.
.. math::
y \mid X,\theta \sim \int p(y \mid f,\, X,\, \theta) \, p(f \mid X,\, \theta) \, df
Parameters
----------
name: string
Name of the random variable
X: array-like
Function input values. If one-dimensional, must be a column
vector with shape `(n, 1)`.
y: array-like
Data that is the sum of the function with the GP prior and Gaussian
noise. Must have shape `(n, )`.
sigma: scalar, Variable, or Covariance
Standard deviation of the Gaussian noise. Can also be a Covariance for
non-white noise.
noise: scalar, Variable, or Covariance
Previous parameterization of `sigma`.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to `MvNormal` distribution
constructor.
"""
sigma = _handle_sigma_noise_parameters(sigma=sigma, noise=noise)
noise_func = sigma if isinstance(sigma, Covariance) else pm.gp.cov.WhiteNoise(sigma)
mu, cov = self._build_marginal_likelihood(X=X, noise_func=noise_func, jitter=jitter)
self.X = X
self.y = y
self.sigma = noise_func
if is_observed:
return pm.MvNormal(name, mu=mu, cov=cov, observed=y, **kwargs)
else:
warnings.warn(
"The 'is_observed' argument has been deprecated. If the GP is "
"unobserved use gp.Latent instead.",
FutureWarning,
)
return pm.MvNormal(name, mu=mu, cov=cov, **kwargs)
def _get_given_vals(self, given):
if given is None:
given = {}
if "gp" in given:
cov_total = given["gp"].cov_func
mean_total = given["gp"].mean_func
else:
cov_total = self.cov_func
mean_total = self.mean_func
if "noise" in given:
warnings.warn(_noise_deprecation_warning, FutureWarning)
given["sigma"] = given["noise"]
if all(val in given for val in ["X", "y", "sigma"]):
X, y, sigma = given["X"], given["y"], given["sigma"]
noise_func = sigma if isinstance(sigma, Covariance) else pm.gp.cov.WhiteNoise(sigma)
else:
X, y, noise_func = self.X, self.y, self.sigma
return X, y, noise_func, cov_total, mean_total
def _build_conditional(
self, Xnew, pred_noise, diag, X, y, noise_func, cov_total, mean_total, jitter
):
Kxx = cov_total(X)
Kxs = self.cov_func(X, Xnew)
Knx = noise_func(X)
rxx = y - mean_total(X)
L = cholesky(stabilize(Kxx, jitter) + Knx)
A = solve_lower(L, Kxs)
v = solve_lower(L, rxx)
mu = self.mean_func(Xnew) + at.dot(at.transpose(A), v)
if diag:
Kss = self.cov_func(Xnew, diag=True)
var = Kss - at.sum(at.square(A), 0)
if pred_noise:
var += noise_func(Xnew, diag=True)
return mu, var
else:
Kss = self.cov_func(Xnew)
cov = Kss - at.dot(at.transpose(A), A)
if pred_noise:
cov += noise_func(Xnew)
return mu, cov if pred_noise else stabilize(cov, jitter)
def conditional(
self, name, Xnew, pred_noise=False, given=None, jitter=JITTER_DEFAULT, **kwargs
):
R"""
Returns the conditional distribution evaluated over new input
locations `Xnew`.
Given a set of function values `f` that the GP prior was over, the
conditional distribution over a set of new points, `f_*` is:
.. math::
f_* \mid f, X, X_* \sim \mathcal{GP}\left(
K(X_*, X) [K(X, X) + K_{n}(X, X)]^{-1} f \,,
K(X_*, X_*) - K(X_*, X) [K(X, X) + K_{n}(X, X)]^{-1} K(X, X_*) \right)
Parameters
----------
name: string
Name of the random variable
Xnew: array-like
Function input values. If one-dimensional, must be a column
vector with shape `(n, 1)`.
pred_noise: bool
Whether or not observation noise is included in the conditional.
Default is `False`.
given: dict
Can optionally take as key value pairs: `X`, `y`, `sigma`,
and `gp`. See the section in the documentation on additive GP
models in PyMC for more information.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to `MvNormal` distribution
constructor.
"""
givens = self._get_given_vals(given)
mu, cov = self._build_conditional(Xnew, pred_noise, False, *givens, jitter)
return pm.MvNormal(name, mu=mu, cov=cov, **kwargs)
def predict(
self,
Xnew,
point=None,
diag=False,
pred_noise=False,
given=None,
jitter=JITTER_DEFAULT,
model=None,
):
R"""
Return the mean vector and covariance matrix of the conditional
distribution as numpy arrays, given a `point`, such as the MAP
estimate or a sample from a `trace`.
Parameters
----------
Xnew: array-like
Function input values. If one-dimensional, must be a column
vector with shape `(n, 1)`.
point: pymc.model.Point
A specific point to condition on.
diag: bool
If `True`, return the diagonal instead of the full covariance
matrix. Default is `False`.
pred_noise: bool
Whether or not observation noise is included in the conditional.
Default is `False`.
given: dict
Same as `conditional` method.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
"""
if given is None:
given = {}
mu, cov = self._predict_at(Xnew, diag, pred_noise, given, jitter)
return replace_with_values([mu, cov], replacements=point, model=model)
def _predict_at(self, Xnew, diag=False, pred_noise=False, given=None, jitter=JITTER_DEFAULT):
R"""
Return the mean vector and covariance matrix of the conditional
distribution as symbolic variables.
Parameters
----------
Xnew: array-like
Function input values. If one-dimensional, must be a column
vector with shape `(n, 1)`.
diag: bool
If `True`, return the diagonal instead of the full covariance
matrix. Default is `False`.
pred_noise: bool
Whether or not observation noise is included in the conditional.
Default is `False`.
given: dict
Same as `conditional` method.
"""
givens = self._get_given_vals(given)
mu, cov = self._build_conditional(Xnew, pred_noise, diag, *givens, jitter)
return mu, cov
@conditioned_vars(["X", "Xu", "y", "sigma"])
class MarginalApprox(Marginal):
R"""
Approximate marginal Gaussian process.
The `gp.MarginalApprox` class is an implementation of the sum of a GP
prior and additive noise. It has `marginal_likelihood`, `conditional`
and `predict` methods. This GP implementation can be used to
implement regression on data that is normally distributed. The
available approximations are:
- DTC: Deterministic Training Conditional
- FITC: Fully independent Training Conditional
- VFE: Variational Free Energy
Parameters
----------
cov_func: None, 2D array, or instance of Covariance
The covariance function. Defaults to zero.
mean_func: None, instance of Mean
The mean function. Defaults to zero.
approx: string
The approximation to use. Must be one of `VFE`, `FITC` or `DTC`.
Default is VFE.
Examples
--------
.. code:: python
# A one dimensional column vector of inputs.
X = np.linspace(0, 1, 10)[:, None]
# A smaller set of inducing inputs
Xu = np.linspace(0, 1, 5)[:, None]
with pm.Model() as model:
# Specify the covariance function.
cov_func = pm.gp.cov.ExpQuad(1, ls=0.1)
# Specify the GP. The default mean function is `Zero`.
gp = pm.gp.MarginalApprox(cov_func=cov_func, approx="FITC")
# Place a GP prior over the function f.
sigma = pm.HalfCauchy("sigma", beta=3)
y_ = gp.marginal_likelihood("y", X=X, Xu=Xu, y=y, sigma=sigma)
...
# After fitting or sampling, specify the distribution
# at new points with .conditional
Xnew = np.linspace(-1, 2, 50)[:, None]
with model:
fcond = gp.conditional("fcond", Xnew=Xnew)
References
----------
- Quinonero-Candela, J., and Rasmussen, C. (2005). A Unifying View of
Sparse Approximate Gaussian Process Regression.
- Titsias, M. (2009). Variational Learning of Inducing Variables in
Sparse Gaussian Processes.
- Bauer, M., van der Wilk, M., and Rasmussen, C. E. (2016). Understanding
Probabilistic Sparse Gaussian Process Approximations.
"""
_available_approx = ("FITC", "VFE", "DTC")
def __init__(self, approx="VFE", *, mean_func=Zero(), cov_func=Constant(0.0)):
if approx not in self._available_approx:
raise NotImplementedError(approx)
self.approx = approx
super().__init__(mean_func=mean_func, cov_func=cov_func)
def __add__(self, other):
new_gp = super().__add__(other)
if not self.approx == other.approx:
raise TypeError("Cannot add GPs with different approximations")
new_gp.approx = self.approx
return new_gp
def _build_marginal_likelihood_loglik(self, y, X, Xu, sigma, jitter):
sigma2 = at.square(sigma)
Kuu = self.cov_func(Xu)
Kuf = self.cov_func(Xu, X)
Luu = cholesky(stabilize(Kuu, jitter))
A = solve_lower(Luu, Kuf)
Qffd = at.sum(A * A, 0)
if self.approx == "FITC":
Kffd = self.cov_func(X, diag=True)
Lamd = at.clip(Kffd - Qffd, 0.0, np.inf) + sigma2
trace = 0.0
elif self.approx == "VFE":
Lamd = at.ones_like(Qffd) * sigma2
trace = (1.0 / (2.0 * sigma2)) * (
at.sum(self.cov_func(X, diag=True)) - at.sum(at.sum(A * A, 0))
)
else: # DTC
Lamd = at.ones_like(Qffd) * sigma2
trace = 0.0
A_l = A / Lamd
L_B = cholesky(at.eye(Xu.shape[0]) + at.dot(A_l, at.transpose(A)))
r = y - self.mean_func(X)
r_l = r / Lamd
c = solve_lower(L_B, at.dot(A, r_l))
constant = 0.5 * X.shape[0] * at.log(2.0 * np.pi)
logdet = 0.5 * at.sum(at.log(Lamd)) + at.sum(at.log(at.diag(L_B)))
quadratic = 0.5 * (at.dot(r, r_l) - at.dot(c, c))
return -1.0 * (constant + logdet + quadratic + trace)
def marginal_likelihood(
self, name, X, Xu, y, sigma=None, noise=None, jitter=JITTER_DEFAULT, **kwargs
):
R"""
Returns the approximate marginal likelihood distribution, given the input
locations `X`, inducing point locations `Xu`, data `y`, and white noise
standard deviations `sigma`.
Parameters
----------
name: string
Name of the random variable
X: array-like
Function input values. If one-dimensional, must be a column
vector with shape `(n, 1)`.
Xu: array-like
The inducing points. Must have the same number of columns as `X`.
y: array-like
Data that is the sum of the function with the GP prior and Gaussian
noise. Must have shape `(n, )`.
sigma: scalar, Variable
Standard deviation of the Gaussian noise.
noise: scalar, Variable
Previous parameterization of `sigma`
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to `MvNormal` distribution
constructor.
"""
self.X = X
self.Xu = Xu
self.y = y
self.sigma = _handle_sigma_noise_parameters(sigma=sigma, noise=noise)
approx_loglik = self._build_marginal_likelihood_loglik(
y=self.y, X=self.X, Xu=self.Xu, sigma=self.sigma, jitter=jitter
)
pm.Potential(f"marginalapprox_loglik_{name}", approx_loglik, **kwargs)
def _build_conditional(
self, Xnew, pred_noise, diag, X, Xu, y, sigma, cov_total, mean_total, jitter
):
sigma2 = at.square(sigma)
Kuu = cov_total(Xu)
Kuf = cov_total(Xu, X)
Luu = cholesky(stabilize(Kuu, jitter))
A = solve_lower(Luu, Kuf)
Qffd = at.sum(A * A, 0)
if self.approx == "FITC":
Kffd = cov_total(X, diag=True)
Lamd = at.clip(Kffd - Qffd, 0.0, np.inf) + sigma2
else: # VFE or DTC
Lamd = at.ones_like(Qffd) * sigma2
A_l = A / Lamd
L_B = cholesky(at.eye(Xu.shape[0]) + at.dot(A_l, at.transpose(A)))
r = y - mean_total(X)
r_l = r / Lamd
c = solve_lower(L_B, at.dot(A, r_l))
Kus = self.cov_func(Xu, Xnew)
As = solve_lower(Luu, Kus)
mu = self.mean_func(Xnew) + at.dot(at.transpose(As), solve_upper(at.transpose(L_B), c))
C = solve_lower(L_B, As)
if diag:
Kss = self.cov_func(Xnew, diag=True)
var = Kss - at.sum(at.square(As), 0) + at.sum(at.square(C), 0)
if pred_noise:
var += sigma2
return mu, var
else:
cov = self.cov_func(Xnew) - at.dot(at.transpose(As), As) + at.dot(at.transpose(C), C)
if pred_noise:
cov += sigma2 * at.identity_like(cov)
return mu, cov if pred_noise else stabilize(cov, jitter)
def _get_given_vals(self, given):
if given is None:
given = {}
if "gp" in given:
cov_total = given["gp"].cov_func
mean_total = given["gp"].mean_func
else:
cov_total = self.cov_func
mean_total = self.mean_func
if all(val in given for val in ["X", "Xu", "y", "sigma"]):
X, Xu, y, sigma = given["X"], given["Xu"], given["y"], given["sigma"]
else:
X, Xu, y, sigma = self.X, self.Xu, self.y, self.sigma
return X, Xu, y, sigma, cov_total, mean_total
def conditional(
self, name, Xnew, pred_noise=False, given=None, jitter=JITTER_DEFAULT, **kwargs
):
R"""
Returns the approximate conditional distribution of the GP evaluated over
new input locations `Xnew`.
Parameters
----------
name: string
Name of the random variable
Xnew: array-like
Function input values. If one-dimensional, must be a column
vector with shape `(n, 1)`.
pred_noise: bool
Whether or not observation noise is included in the conditional.
Default is `False`.
given: dict
Can optionally take as key value pairs: `X`, `Xu`, `y`, `sigma`,
and `gp`. See the section in the documentation on additive GP
models in PyMC for more information.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to `MvNormal` distribution
constructor.
"""
givens = self._get_given_vals(given)
mu, cov = self._build_conditional(Xnew, pred_noise, False, *givens, jitter)
return pm.MvNormal(name, mu=mu, cov=cov, **kwargs)
@conditioned_vars(["X", "Xu", "y", "sigma"])
class MarginalSparse(MarginalApprox):
def __init__(self, approx="VFE", *, mean_func=Zero(), cov_func=Constant(0.0)):
warnings.warn(
"gp.MarginalSparse has been renamed to gp.MarginalApprox.",
FutureWarning,
)
super().__init__(mean_func=mean_func, cov_func=cov_func, approx=approx)
@conditioned_vars(["Xs", "f"])
class LatentKron(Base):
R"""
Latent Gaussian process whose covariance is a tensor product kernel.
The `gp.LatentKron` class is a direct implementation of a GP with a
Kronecker structured covariance, without reference to any noise or
specific likelihood. The GP is constructed with the `prior` method,
and the conditional GP over new input locations is constructed with
the `conditional` method. `conditional` and method. For more
information on these methods, see their docstrings. This GP
implementation can be used to model a Gaussian process whose inputs
cover evenly spaced grids on more than one dimension. `LatentKron`
is relies on the `KroneckerNormal` distribution, see its docstring
for more information.
Parameters
----------
cov_funcs: list of Covariance objects
The covariance functions that compose the tensor (Kronecker) product.
Defaults to [zero].
mean_func: None, instance of Mean
The mean function. Defaults to zero.
Examples
--------
.. code:: python
# One dimensional column vectors of inputs
X1 = np.linspace(0, 1, 10)[:, None]
X2 = np.linspace(0, 2, 5)[:, None]
Xs = [X1, X2]
with pm.Model() as model:
# Specify the covariance functions for each Xi
cov_func1 = pm.gp.cov.ExpQuad(1, ls=0.1) # Must accept X1 without error
cov_func2 = pm.gp.cov.ExpQuad(1, ls=0.3) # Must accept X2 without error
# Specify the GP. The default mean function is `Zero`.
gp = pm.gp.LatentKron(cov_funcs=[cov_func1, cov_func2])
# ...
# After fitting or sampling, specify the distribution
# at new points with .conditional
# Xnew need not be on a full grid
Xnew1 = np.linspace(-1, 2, 10)[:, None]
Xnew2 = np.linspace(0, 3, 10)[:, None]
Xnew = np.concatenate((Xnew1, Xnew2), axis=1) # Not full grid, works
Xnew = pm.math.cartesian(Xnew1, Xnew2) # Full grid, also works
with model:
fcond = gp.conditional("fcond", Xnew=Xnew)
"""
def __init__(self, *, mean_func=Zero(), cov_funcs=(Constant(0.0))):
try:
self.cov_funcs = list(cov_funcs)
except TypeError:
self.cov_funcs = [cov_funcs]
cov_func = pm.gp.cov.Kron(self.cov_funcs)
super().__init__(mean_func=mean_func, cov_func=cov_func)
def __add__(self, other):
raise TypeError("Additive, Kronecker-structured processes not implemented")
def _build_prior(self, name, Xs, jitter, **kwargs):
self.N = int(np.prod([len(X) for X in Xs]))
mu = self.mean_func(cartesian(*Xs))
chols = [cholesky(stabilize(cov(X), jitter)) for cov, X in zip(self.cov_funcs, Xs)]
v = pm.Normal(name + "_rotated_", mu=0.0, sigma=1.0, size=self.N, **kwargs)
f = pm.Deterministic(name, mu + at.flatten(kron_dot(chols, v)))
return f
def prior(self, name, Xs, jitter=JITTER_DEFAULT, **kwargs):
"""
Returns the prior distribution evaluated over the input
locations `Xs`.
Parameters
----------
name: string
Name of the random variable
Xs: list of array-like
Function input values for each covariance function. Each entry
must be passable to its respective covariance without error. The
total covariance function is measured on the full grid
`cartesian(*Xs)`.
jitter: scalar
A small correction added to the diagonal of positive semi-definite
covariance matrices to ensure numerical stability.
**kwargs
Extra keyword arguments that are passed to the `KroneckerNormal`
distribution constructor.
"""
if len(Xs) != len(self.cov_funcs):
raise ValueError("Must provide a covariance function for each X")
f = self._build_prior(name, Xs, jitter, **kwargs)
self.Xs = Xs
self.f = f
return f
def _build_conditional(self, Xnew, jitter):
Xs, f = self.Xs, self.f
X = cartesian(*Xs)
delta = f - self.mean_func(X)
covs = [stabilize(cov(Xi), jitter) for cov, Xi in zip(self.cov_funcs, Xs)]
chols = [cholesky(cov) for cov in covs]
cholTs = [at.transpose(chol) for chol in chols]
Kss = self.cov_func(Xnew)
Kxs = self.cov_func(X, Xnew)
Ksx = at.transpose(Kxs)
alpha = kron_solve_lower(chols, delta)
alpha = kron_solve_upper(cholTs, alpha)
mu = at.dot(Ksx, alpha).ravel() + self.mean_func(Xnew)