From a8483bcbcf91f47e1854115493c9109f81c1b77b Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 15:59:29 +0000 Subject: [PATCH 01/22] added gauss-legendre quadrature --- .../fourier_features/__init__.py | 11 +- .../fourier_features/quadrature/__init__.py | 4 +- .../fourier_features/quadrature/gaussian.py | 118 ++++++++++++------ .../fourier_features/random/__init__.py | 6 - 4 files changed, 84 insertions(+), 55 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/__init__.py b/gpflux/layers/basis_functions/fourier_features/__init__.py index 42c09d47..11b54eed 100644 --- a/gpflux/layers/basis_functions/fourier_features/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/__init__.py @@ -18,16 +18,11 @@ :class:`gpflux.sampling.KernelWithFeatureDecomposition` """ -from gpflux.layers.basis_functions.fourier_features.quadrature import QuadratureFourierFeatures +from gpflux.layers.basis_functions.fourier_features.quadrature import ( + HermiteQuadratureFourierFeatures, +) from gpflux.layers.basis_functions.fourier_features.random import ( OrthogonalRandomFeatures, RandomFourierFeatures, RandomFourierFeaturesCosine, ) - -__all__ = [ - "QuadratureFourierFeatures", - "OrthogonalRandomFeatures", - "RandomFourierFeatures", - "RandomFourierFeaturesCosine", -] diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py index bd3e54fb..fbc6967e 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py @@ -15,7 +15,5 @@ # """ A kernel's features and coefficients using quadrature Fourier features (QFF). """ from gpflux.layers.basis_functions.fourier_features.quadrature.gaussian import ( - QuadratureFourierFeatures, + HermiteQuadratureFourierFeatures, ) - -__all__ = ["QuadratureFourierFeatures"] diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index 2391bb13..b61afbe6 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -21,37 +21,43 @@ import warnings from typing import Mapping, Tuple, Type +import numpy as np import tensorflow as tf +from scipy.stats import multivariate_normal, multivariate_t import gpflow -from gpflow.base import TensorType -from gpflow.quadrature.gauss_hermite import ndgh_points_and_weights -from gpflux.layers.basis_functions.fourier_features.base import FourierFeaturesBase -from gpflux.layers.basis_functions.fourier_features.utils import _bases_concat +# from gpflow.config import default_float +from gpflow.quadrature.gauss_hermite import ndgh_points_and_weights, repeat_as_list, reshape_Z_dZ + +from gpflux.layers.basis_functions.fourier_features.quadrature.base import ( + QuadratureFourierFeatures, + TanTransform, +) +from gpflux.layers.basis_functions.fourier_features.utils import _matern_number from gpflux.types import ShapeType -""" -Kernels supported by :class:`QuadratureFourierFeatures`. -Currently we only support the :class:`gpflow.kernels.SquaredExponential` kernel. -For Matern kernels please use :class:`RandomFourierFeatures` -or :class:`RandomFourierFeaturesCosine`. -""" -QFF_SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( - gpflow.kernels.SquaredExponential, -) +class GaussianQuadratureFourierFeatures(QuadratureFourierFeatures): + pass + +class GaussHermiteQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): + + SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( + gpflow.kernels.SquaredExponential, + ) -class QuadratureFourierFeatures(FourierFeaturesBase): def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): - assert isinstance(kernel, QFF_SUPPORTED_KERNELS), "Unsupported Kernel" + assert isinstance( + kernel, GaussHermiteQuadratureFourierFeatures.SUPPORTED_KERNELS + ), "Unsupported Kernel" if tf.reduce_any(tf.less(kernel.lengthscales, 1e-1)): warnings.warn( - "Quadrature Fourier feature approximation of kernels " + "Gauss-Hermite Quadrature Fourier feature approximation of kernels " "with small lengthscale lead to unexpected behaviors!" ) - super(QuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) + super(GaussHermiteQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) def build(self, input_shape: ShapeType) -> None: """ @@ -60,33 +66,69 @@ def build(self, input_shape: ShapeType) -> None: `_. """ input_dim = input_shape[-1] - abscissa_value, omegas_value = ndgh_points_and_weights( + # (L^D, D), (L^D, 1) + abscissa_value, factors_value = ndgh_points_and_weights( dim=input_dim, n_gh=self.n_components ) - omegas_value = tf.squeeze(omegas_value, axis=-1) + factors_value = tf.squeeze(factors_value, axis=-1) # (L^D,) - # Quadrature node points - self.abscissa = tf.Variable(initial_value=abscissa_value, trainable=False) # (M^D, D) - # Gauss-Hermite weights - self.factors = tf.Variable(initial_value=omegas_value, trainable=False) # (M^D,) - super(QuadratureFourierFeatures, self).build(input_shape) + # Gauss-Christoffel nodes (L^D, D) + self.abscissa = tf.Variable(initial_value=abscissa_value, trainable=False) + # Gauss-Christoffel weights (L^D,) + self.factors = tf.Variable(initial_value=factors_value, trainable=False) + super(GaussHermiteQuadratureFourierFeatures, self).build(input_shape) - def _compute_output_dim(self, input_shape: ShapeType) -> int: - input_dim = input_shape[-1] - return 2 * self.n_components ** input_dim - def _compute_bases(self, inputs: TensorType) -> tf.Tensor: - """ - Compute basis functions. +class GaussLegendreQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): - :return: A tensor with the shape ``[N, 2M^D]``. - """ - return _bases_concat(inputs, self.abscissa) + SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( + gpflow.kernels.SquaredExponential, + gpflow.kernels.Matern12, + gpflow.kernels.Matern32, + gpflow.kernels.Matern52, + ) - def _compute_constant(self) -> tf.Tensor: - """ - Compute normalizing constant for basis functions. + def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): + assert isinstance( + kernel, GaussLegendreQuadratureFourierFeatures.SUPPORTED_KERNELS + ), "Unsupported Kernel" + super(GaussLegendreQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) - :return: A tensor with the shape ``[2M^D,]`` + def build(self, input_shape: ShapeType) -> None: + """ + Creates the variables of the layer. + See `tf.keras.layers.Layer.build() + `_. """ - return tf.tile(tf.sqrt(self.kernel.variance * self.factors), multiples=[2]) + input_dim = input_shape[-1] + + if isinstance(self.kernel, gpflow.kernels.SquaredExponential): + dist = multivariate_normal(loc=np.zeros(input_dim)) + else: + p = _matern_number(self.kernel) + nu = 2.0 * p + 1.0 # degrees of freedom + dist = multivariate_t(loc=np.zeros(input_dim), df=nu) + + # raw 1-dimensional quadrature nodes and weights (L,) (L,) + abscissa_value_flat, factors_value_flat = np.polynomial.legendre.leggauss( + deg=self.n_components + ) + + # transformed 1-dimensional quadrature nodes and weights + transform = TanTransform() + abscissa_value_flat = transform(abscissa_value_flat) # (L,) + factors_value_flat *= transform.multiplier(abscissa_value_flat) # (L,) + + # transformed D-dimensional quadrature nodes and weights + abscissa_value_rep = repeat_as_list(abscissa_value_flat, n=input_dim) # (L, ..., L) + factors_value_rep = repeat_as_list(factors_value_flat, n=input_dim) # (L, ..., L) + # (L^D, D), (L^D, 1) + abscissa_value, factors_value = reshape_Z_dZ(abscissa_value_rep, factors_value_rep) + factors_value *= dist.pdf(abscissa_value) # (L^D, 1) + factors_value = tf.squeeze(factors_value, axis=-1) # (L^D,) + + # Gauss-Christoffel nodes (L^D, D) + self.abscissa = tf.Variable(initial_value=abscissa_value, trainable=False) + # Gauss-Christoffel weights (L^D,) + self.factors = tf.Variable(initial_value=factors_value, trainable=False) + super(GaussLegendreQuadratureFourierFeatures, self).build(input_shape) diff --git a/gpflux/layers/basis_functions/fourier_features/random/__init__.py b/gpflux/layers/basis_functions/fourier_features/random/__init__.py index ee8337be..162e18cf 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/random/__init__.py @@ -22,9 +22,3 @@ from gpflux.layers.basis_functions.fourier_features.random.orthogonal import ( OrthogonalRandomFeatures, ) - -__all__ = [ - "OrthogonalRandomFeatures", - "RandomFourierFeatures", - "RandomFourierFeaturesCosine", -] From 6e10cd0496ed94b76ee8b936b3840b20e45d1a79 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 17:40:10 +0000 Subject: [PATCH 02/22] basic implementation completed --- .../fourier_features/__init__.py | 2 +- .../fourier_features/quadrature/__init__.py | 2 +- .../fourier_features/quadrature/base.py | 82 +++++++++++++++++++ .../fourier_features/quadrature/gaussian.py | 15 +++- 4 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 gpflux/layers/basis_functions/fourier_features/quadrature/base.py diff --git a/gpflux/layers/basis_functions/fourier_features/__init__.py b/gpflux/layers/basis_functions/fourier_features/__init__.py index 11b54eed..ec68b5d1 100644 --- a/gpflux/layers/basis_functions/fourier_features/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/__init__.py @@ -19,7 +19,7 @@ """ from gpflux.layers.basis_functions.fourier_features.quadrature import ( - HermiteQuadratureFourierFeatures, + GaussHermiteQuadratureFourierFeatures, ) from gpflux.layers.basis_functions.fourier_features.random import ( OrthogonalRandomFeatures, diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py index fbc6967e..931a8243 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py @@ -15,5 +15,5 @@ # """ A kernel's features and coefficients using quadrature Fourier features (QFF). """ from gpflux.layers.basis_functions.fourier_features.quadrature.gaussian import ( - HermiteQuadratureFourierFeatures, + GaussHermiteQuadratureFourierFeatures, ) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/base.py b/gpflux/layers/basis_functions/fourier_features/quadrature/base.py new file mode 100644 index 00000000..f25445b6 --- /dev/null +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/base.py @@ -0,0 +1,82 @@ +from abc import ABC, abstractmethod + +import numpy as np +import tensorflow as tf + +from gpflow.base import TensorType + +from gpflux.layers.basis_functions.fourier_features.base import FourierFeaturesBase +from gpflux.layers.basis_functions.fourier_features.utils import _bases_concat +from gpflux.types import ShapeType + + +class Transform(ABC): + r""" + This class encapsulates functions :math:`g(x) = u` and :math:`h(x)` such that + .. math:: + \int_{g(a)}^{g(b)} z(u) f(u) du + = \int_a^b w(x) h(x) f(g(x)) dx + for some integrand :math:`f(u)` and weight function :math:`z(u)`. + """ + + @abstractmethod + def __call__(self, x): + pass + + @abstractmethod + def multiplier(self, x): + pass + + +class TanTransform(Transform): + r""" + This class encapsulates functions :math:`g(x) = u` and :math:`h(x) = du/dx` + such that + .. math:: + \int_{-\infty}^{\infty} f(u) du + = \int_{-1}^{1} f(g(x)) h(x) dx + """ + CONST = 0.5 * np.pi + + def __call__(self, x): + return tf.tan(TanTransform.CONST * x) + + def multiplier(self, x): + return TanTransform.CONST / tf.square(tf.cos(TanTransform.CONST * x)) + + +class NormalWeightTransform(Transform): + r""" + This class encapsulates functions :math:`g(x) = u` and :math:`h(x)` such that + .. math:: + \int_{-\infty}^{\infty} \mathcal{N}(u|0,1) f(u) du + = \int_{-infty}^{infty} e^{-x^2} f(g(x)) h(x) dx + """ + + def __call__(self, x): + return tf.sqrt(2.0) * x + + def multiplier(self, x): + return tf.rsqrt(np.pi) + + +class QuadratureFourierFeatures(FourierFeaturesBase): + def _compute_output_dim(self, input_shape: ShapeType) -> int: + input_dim = input_shape[-1] + return 2 * self.n_components ** input_dim + + def _compute_bases(self, inputs: TensorType) -> tf.Tensor: + """ + Compute basis functions. + + :return: A tensor with the shape ``(N, 2L^D)``. + """ + return _bases_concat(inputs, self.abscissa) + + def _compute_constant(self) -> tf.Tensor: + """ + Compute normalizing constant for basis functions. + + :return: A tensor with the shape ``(2L^D,)`` + """ + return tf.tile(tf.sqrt(self.kernel.variance * self.factors), multiples=[2]) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index b61afbe6..cf262913 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -103,7 +103,7 @@ def build(self, input_shape: ShapeType) -> None: input_dim = input_shape[-1] if isinstance(self.kernel, gpflow.kernels.SquaredExponential): - dist = multivariate_normal(loc=np.zeros(input_dim)) + dist = multivariate_normal(mean=np.zeros(input_dim)) else: p = _matern_number(self.kernel) nu = 2.0 * p + 1.0 # degrees of freedom @@ -116,19 +116,28 @@ def build(self, input_shape: ShapeType) -> None: # transformed 1-dimensional quadrature nodes and weights transform = TanTransform() - abscissa_value_flat = transform(abscissa_value_flat) # (L,) factors_value_flat *= transform.multiplier(abscissa_value_flat) # (L,) + abscissa_value_flat = transform(abscissa_value_flat) # (L,) # transformed D-dimensional quadrature nodes and weights abscissa_value_rep = repeat_as_list(abscissa_value_flat, n=input_dim) # (L, ..., L) factors_value_rep = repeat_as_list(factors_value_flat, n=input_dim) # (L, ..., L) # (L^D, D), (L^D, 1) abscissa_value, factors_value = reshape_Z_dZ(abscissa_value_rep, factors_value_rep) - factors_value *= dist.pdf(abscissa_value) # (L^D, 1) + factors_value = tf.squeeze(factors_value, axis=-1) # (L^D,) + factors_value *= dist.pdf(abscissa_value) # (L^D,) # Gauss-Christoffel nodes (L^D, D) self.abscissa = tf.Variable(initial_value=abscissa_value, trainable=False) # Gauss-Christoffel weights (L^D,) self.factors = tf.Variable(initial_value=factors_value, trainable=False) + super(GaussLegendreQuadratureFourierFeatures, self).build(input_shape) + + +class QuadratureFourierFeatures(GaussLegendreQuadratureFourierFeatures): + """ + Alias for GaussLegendreQuadratureFourierFeatures. + """ + pass From 61992ee84c8d31f797f3646e7c580431f5d3b8c4 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 17:47:53 +0000 Subject: [PATCH 03/22] formatting and added matern degrees of freedom utility method --- .../fourier_features/quadrature/base.py | 48 +++++++++---------- .../fourier_features/quadrature/gaussian.py | 6 +-- .../fourier_features/random/base.py | 5 +- .../basis_functions/fourier_features/utils.py | 8 ++++ 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/base.py b/gpflux/layers/basis_functions/fourier_features/quadrature/base.py index f25445b6..c5af5049 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/base.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/base.py @@ -10,6 +10,28 @@ from gpflux.types import ShapeType +class QuadratureFourierFeatures(FourierFeaturesBase): + def _compute_output_dim(self, input_shape: ShapeType) -> int: + input_dim = input_shape[-1] + return 2 * self.n_components ** input_dim + + def _compute_bases(self, inputs: TensorType) -> tf.Tensor: + """ + Compute basis functions. + + :return: A tensor with the shape ``(N, 2L^D)``. + """ + return _bases_concat(inputs, self.abscissa) + + def _compute_constant(self) -> tf.Tensor: + """ + Compute normalizing constant for basis functions. + + :return: A tensor with the shape ``(2L^D,)`` + """ + return tf.tile(tf.sqrt(self.kernel.variance * self.factors), multiples=[2]) + + class Transform(ABC): r""" This class encapsulates functions :math:`g(x) = u` and :math:`h(x)` such that @@ -20,11 +42,11 @@ class Transform(ABC): """ @abstractmethod - def __call__(self, x): + def __call__(self, x: TensorType) -> tf.Tensor: pass @abstractmethod - def multiplier(self, x): + def multiplier(self, x: TensorType) -> tf.Tensor: pass @@ -58,25 +80,3 @@ def __call__(self, x): def multiplier(self, x): return tf.rsqrt(np.pi) - - -class QuadratureFourierFeatures(FourierFeaturesBase): - def _compute_output_dim(self, input_shape: ShapeType) -> int: - input_dim = input_shape[-1] - return 2 * self.n_components ** input_dim - - def _compute_bases(self, inputs: TensorType) -> tf.Tensor: - """ - Compute basis functions. - - :return: A tensor with the shape ``(N, 2L^D)``. - """ - return _bases_concat(inputs, self.abscissa) - - def _compute_constant(self) -> tf.Tensor: - """ - Compute normalizing constant for basis functions. - - :return: A tensor with the shape ``(2L^D,)`` - """ - return tf.tile(tf.sqrt(self.kernel.variance * self.factors), multiples=[2]) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index cf262913..169814ae 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -34,7 +34,7 @@ QuadratureFourierFeatures, TanTransform, ) -from gpflux.layers.basis_functions.fourier_features.utils import _matern_number +from gpflux.layers.basis_functions.fourier_features.utils import _matern_dof from gpflux.types import ShapeType @@ -105,8 +105,7 @@ def build(self, input_shape: ShapeType) -> None: if isinstance(self.kernel, gpflow.kernels.SquaredExponential): dist = multivariate_normal(mean=np.zeros(input_dim)) else: - p = _matern_number(self.kernel) - nu = 2.0 * p + 1.0 # degrees of freedom + nu = _matern_dof(self.kernel) # degrees of freedom dist = multivariate_t(loc=np.zeros(input_dim), df=nu) # raw 1-dimensional quadrature nodes and weights (L,) (L,) @@ -140,4 +139,5 @@ class QuadratureFourierFeatures(GaussLegendreQuadratureFourierFeatures): """ Alias for GaussLegendreQuadratureFourierFeatures. """ + pass diff --git a/gpflux/layers/basis_functions/fourier_features/random/base.py b/gpflux/layers/basis_functions/fourier_features/random/base.py index 82ae4aa9..544d73d4 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/base.py +++ b/gpflux/layers/basis_functions/fourier_features/random/base.py @@ -25,7 +25,7 @@ from gpflux.layers.basis_functions.fourier_features.utils import ( _bases_concat, _bases_cosine, - _matern_number, + _matern_dof, ) from gpflux.types import ShapeType @@ -101,8 +101,7 @@ def _weights_init(self, shape: TensorType, dtype: Optional[DType] = None) -> Ten if isinstance(self.kernel, gpflow.kernels.SquaredExponential): return tf.random.normal(shape, dtype=dtype) else: - p = _matern_number(self.kernel) - nu = 2.0 * p + 1.0 # degrees of freedom + nu = _matern_dof(self.kernel) # degrees of freedom return _sample_students_t(nu, shape, dtype) @staticmethod diff --git a/gpflux/layers/basis_functions/fourier_features/utils.py b/gpflux/layers/basis_functions/fourier_features/utils.py index 71372e9e..9c88873c 100644 --- a/gpflux/layers/basis_functions/fourier_features/utils.py +++ b/gpflux/layers/basis_functions/fourier_features/utils.py @@ -34,6 +34,14 @@ def _matern_number(kernel: gpflow.kernels.Kernel) -> int: return p +def _matern_dof(kernel: gpflow.kernels.Kernel) -> float: + """ + Degrees of freedom corresponding to a kernel from the Matern family. + """ + p = _matern_number(kernel) + return 2.0 * p + 1.0 # degrees of freedom + + def _bases_cosine(X: TensorType, W: TensorType, b: TensorType) -> TensorType: """ Feature map for random Fourier features (RFF) as originally prescribed From 8c745b0f6bbb330da499c7b4dbc0fa71ddf15c3c Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 18:03:46 +0000 Subject: [PATCH 04/22] fixed formatting and type annotations --- Makefile | 3 +++ .../basis_functions/fourier_features/base.py | 8 +++++- .../fourier_features/quadrature/base.py | 26 +++++++++++++++---- .../fourier_features/quadrature/gaussian.py | 21 ++++----------- .../fourier_features/random/base.py | 25 ++++++------------ .../fourier_features/random/orthogonal.py | 18 ++----------- 6 files changed, 46 insertions(+), 55 deletions(-) diff --git a/Makefile b/Makefile index ffab011e..f0ef2583 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,9 @@ LINT_FILE_IGNORES = "$(LIB_NAME)/__init__.py:F401,F403 \ $(LIB_NAME)/initializers/__init__.py:F401 \ $(LIB_NAME)/layers/__init__.py:F401 \ $(LIB_NAME)/layers/basis_functions/__init__.py:F401 \ + $(LIB_NAME)/layers/basis_functions/fourier_features/__init__.py:F401 \ + $(LIB_NAME)/layers/basis_functions/fourier_features/random/__init__.py:F401 \ + $(LIB_NAME)/layers/basis_functions/fourier_features/quadrature/__init__.py:F401 \ $(LIB_NAME)/models/__init__.py:F401 \ $(LIB_NAME)/optimization/__init__.py:F401 \ $(LIB_NAME)/sampling/__init__.py:F401 \ diff --git a/gpflux/layers/basis_functions/fourier_features/base.py b/gpflux/layers/basis_functions/fourier_features/base.py index 80461c80..cf9952b6 100644 --- a/gpflux/layers/basis_functions/fourier_features/base.py +++ b/gpflux/layers/basis_functions/fourier_features/base.py @@ -16,7 +16,7 @@ """ Shared functionality for stationary kernel basis functions. """ from abc import ABC, abstractmethod -from typing import Mapping +from typing import Mapping, Tuple, Type import tensorflow as tf @@ -27,6 +27,9 @@ class FourierFeaturesBase(ABC, tf.keras.layers.Layer): + + SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] + def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): """ :param kernel: kernel to approximate using a set of Fourier bases. @@ -34,6 +37,9 @@ def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: M quadrature nodes, etc.) used to numerically approximate the kernel. """ super(FourierFeaturesBase, self).__init__(**kwargs) + assert isinstance( + kernel, self.SUPPORTED_KERNELS + ), f"Only the following kernels are supported: {self.SUPPORTED_KERNELS}" self.kernel = kernel self.n_components = n_components if kwargs.get("input_dim", None): diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/base.py b/gpflux/layers/basis_functions/fourier_features/quadrature/base.py index c5af5049..5b7f5ef1 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/base.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/base.py @@ -1,3 +1,19 @@ +# +# Copyright (c) 2021 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + from abc import ABC, abstractmethod import numpy as np @@ -10,7 +26,7 @@ from gpflux.types import ShapeType -class QuadratureFourierFeatures(FourierFeaturesBase): +class QuadratureFourierFeaturesBase(FourierFeaturesBase): def _compute_output_dim(self, input_shape: ShapeType) -> int: input_dim = input_shape[-1] return 2 * self.n_components ** input_dim @@ -60,10 +76,10 @@ class TanTransform(Transform): """ CONST = 0.5 * np.pi - def __call__(self, x): + def __call__(self, x: TensorType) -> tf.Tensor: return tf.tan(TanTransform.CONST * x) - def multiplier(self, x): + def multiplier(self, x: TensorType) -> tf.Tensor: return TanTransform.CONST / tf.square(tf.cos(TanTransform.CONST * x)) @@ -75,8 +91,8 @@ class NormalWeightTransform(Transform): = \int_{-infty}^{infty} e^{-x^2} f(g(x)) h(x) dx """ - def __call__(self, x): + def __call__(self, x: TensorType) -> tf.Tensor: return tf.sqrt(2.0) * x - def multiplier(self, x): + def multiplier(self, x: TensorType) -> tf.Tensor: return tf.rsqrt(np.pi) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index 169814ae..08fd752d 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -19,7 +19,7 @@ """ import warnings -from typing import Mapping, Tuple, Type +from typing import Mapping import numpy as np import tensorflow as tf @@ -31,27 +31,22 @@ from gpflow.quadrature.gauss_hermite import ndgh_points_and_weights, repeat_as_list, reshape_Z_dZ from gpflux.layers.basis_functions.fourier_features.quadrature.base import ( - QuadratureFourierFeatures, + QuadratureFourierFeaturesBase, TanTransform, ) from gpflux.layers.basis_functions.fourier_features.utils import _matern_dof from gpflux.types import ShapeType -class GaussianQuadratureFourierFeatures(QuadratureFourierFeatures): +class GaussianQuadratureFourierFeatures(QuadratureFourierFeaturesBase): pass class GaussHermiteQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): - SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( - gpflow.kernels.SquaredExponential, - ) + SUPPORTED_KERNELS = (gpflow.kernels.SquaredExponential,) def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): - assert isinstance( - kernel, GaussHermiteQuadratureFourierFeatures.SUPPORTED_KERNELS - ), "Unsupported Kernel" if tf.reduce_any(tf.less(kernel.lengthscales, 1e-1)): warnings.warn( "Gauss-Hermite Quadrature Fourier feature approximation of kernels " @@ -81,19 +76,13 @@ def build(self, input_shape: ShapeType) -> None: class GaussLegendreQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): - SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( + SUPPORTED_KERNELS = ( gpflow.kernels.SquaredExponential, gpflow.kernels.Matern12, gpflow.kernels.Matern32, gpflow.kernels.Matern52, ) - def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): - assert isinstance( - kernel, GaussLegendreQuadratureFourierFeatures.SUPPORTED_KERNELS - ), "Unsupported Kernel" - super(GaussLegendreQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) - def build(self, input_shape: ShapeType) -> None: """ Creates the variables of the layer. diff --git a/gpflux/layers/basis_functions/fourier_features/random/base.py b/gpflux/layers/basis_functions/fourier_features/random/base.py index 544d73d4..1c73d51c 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/base.py +++ b/gpflux/layers/basis_functions/fourier_features/random/base.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from typing import Mapping, Optional, Tuple, Type +from typing import Optional import numpy as np import tensorflow as tf @@ -29,19 +29,6 @@ ) from gpflux.types import ShapeType -""" -Kernels supported by :class:`RandomFourierFeatures`. - -You can build RFF for shift-invariant stationary kernels from which you can -sample frequencies from their power spectrum, following Bochner's theorem. -""" -RFF_SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( - gpflow.kernels.SquaredExponential, - gpflow.kernels.Matern12, - gpflow.kernels.Matern32, - gpflow.kernels.Matern52, -) - def _sample_students_t(nu: float, shape: ShapeType, dtype: DType) -> TensorType: """ @@ -73,9 +60,13 @@ def _sample_students_t(nu: float, shape: ShapeType, dtype: DType) -> TensorType: class RandomFourierFeaturesBase(FourierFeaturesBase): - def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): - assert isinstance(kernel, RFF_SUPPORTED_KERNELS), "Unsupported Kernel" - super(RandomFourierFeaturesBase, self).__init__(kernel, n_components, **kwargs) + + SUPPORTED_KERNELS = ( + gpflow.kernels.SquaredExponential, + gpflow.kernels.Matern12, + gpflow.kernels.Matern32, + gpflow.kernels.Matern52, + ) def build(self, input_shape: ShapeType) -> None: """ diff --git a/gpflux/layers/basis_functions/fourier_features/random/orthogonal.py b/gpflux/layers/basis_functions/fourier_features/random/orthogonal.py index 395da743..13f78d94 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/orthogonal.py +++ b/gpflux/layers/basis_functions/fourier_features/random/orthogonal.py @@ -14,7 +14,7 @@ # limitations under the License. # -from typing import Mapping, Optional, Tuple, Type +from typing import Optional import numpy as np import tensorflow as tf @@ -25,18 +25,6 @@ from gpflux.layers.basis_functions.fourier_features.random.base import RandomFourierFeatures from gpflux.types import ShapeType -""" -Kernels supported by :class:`OrthogonalRandomFeatures`. - -This random matrix sampling scheme only applies to the :class:`gpflow.kernels.SquaredExponential` -kernel. -For Matern kernels please use :class:`RandomFourierFeatures` -or :class:`RandomFourierFeaturesCosine`. -""" -ORF_SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( - gpflow.kernels.SquaredExponential, -) - def _sample_chi_squared(nu: float, shape: ShapeType, dtype: DType) -> TensorType: """ @@ -69,9 +57,7 @@ class OrthogonalRandomFeatures(RandomFourierFeatures): efficient and accurate kernel approximations than :class:`RandomFourierFeatures`. """ - def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): - assert isinstance(kernel, ORF_SUPPORTED_KERNELS), "Unsupported Kernel" - super(OrthogonalRandomFeatures, self).__init__(kernel, n_components, **kwargs) + SUPPORTED_KERNELS = (gpflow.kernels.SquaredExponential,) def _weights_init(self, shape: TensorType, dtype: Optional[DType] = None) -> TensorType: n_components, input_dim = shape # M, D From 55d989a05afbdaa8cbfa12ece8f4344f64b6ddd1 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 18:05:16 +0000 Subject: [PATCH 05/22] fixed formatting and type annotations --- gpflux/layers/basis_functions/fourier_features/random/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/random/base.py b/gpflux/layers/basis_functions/fourier_features/random/base.py index 1c73d51c..eacd5f5e 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/base.py +++ b/gpflux/layers/basis_functions/fourier_features/random/base.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from typing import Optional +from typing import Optional, Tuple, Type import numpy as np import tensorflow as tf @@ -61,7 +61,7 @@ def _sample_students_t(nu: float, shape: ShapeType, dtype: DType) -> TensorType: class RandomFourierFeaturesBase(FourierFeaturesBase): - SUPPORTED_KERNELS = ( + SUPPORTED_KERNELS: Tuple[Type[gpflow.kernels.Stationary], ...] = ( gpflow.kernels.SquaredExponential, gpflow.kernels.Matern12, gpflow.kernels.Matern32, From 5b4786442f73372cda5c1399678e8e29a88dfee9 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 18:07:30 +0000 Subject: [PATCH 06/22] updated tests --- .../layers/basis_functions/fourier_features/test_quadrature.py | 2 +- .../layers/basis_functions/fourier_features/test_random.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py b/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py index 2784cf53..0209f6be 100644 --- a/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py +++ b/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py @@ -61,7 +61,7 @@ def test_throw_for_unsupported_kernel(): kernel = gpflow.kernels.Constant() with pytest.raises(AssertionError) as excinfo: QuadratureFourierFeatures(kernel, n_components=1) - assert "Unsupported Kernel" in str(excinfo.value) + assert "Only the following kernels are supported" in str(excinfo.value) def test_quadrature_fourier_features_can_approximate_kernel_multidim( diff --git a/tests/gpflux/layers/basis_functions/fourier_features/test_random.py b/tests/gpflux/layers/basis_functions/fourier_features/test_random.py index 3211a0d5..df8c04f8 100644 --- a/tests/gpflux/layers/basis_functions/fourier_features/test_random.py +++ b/tests/gpflux/layers/basis_functions/fourier_features/test_random.py @@ -79,7 +79,7 @@ def test_throw_for_unsupported_kernel(basis_func_cls): kernel = gpflow.kernels.Constant() with pytest.raises(AssertionError) as excinfo: basis_func_cls(kernel, n_components=1) - assert "Unsupported Kernel" in str(excinfo.value) + assert "Only the following kernels are supported" in str(excinfo.value) def test_random_fourier_features_can_approximate_kernel_multidim( From bb09fe8121955bd4781024b2baca2b52c14e8d14 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 18:10:21 +0000 Subject: [PATCH 07/22] updated tests --- .../basis_functions/fourier_features/quadrature/gaussian.py | 4 ++-- .../basis_functions/fourier_features/test_quadrature.py | 3 +-- .../layers/basis_functions/fourier_features/test_random.py | 3 +-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index 08fd752d..6c891c09 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -124,9 +124,9 @@ def build(self, input_shape: ShapeType) -> None: super(GaussLegendreQuadratureFourierFeatures, self).build(input_shape) -class QuadratureFourierFeatures(GaussLegendreQuadratureFourierFeatures): +class QuadratureFourierFeatures(GaussHermiteQuadratureFourierFeatures): """ - Alias for GaussLegendreQuadratureFourierFeatures. + Alias """ pass diff --git a/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py b/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py index 0209f6be..e15618bd 100644 --- a/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py +++ b/tests/gpflux/layers/basis_functions/fourier_features/test_quadrature.py @@ -24,7 +24,6 @@ from gpflow.utilities.ops import difference_matrix from gpflux.layers.basis_functions.fourier_features.quadrature import QuadratureFourierFeatures -from gpflux.layers.basis_functions.fourier_features.quadrature.gaussian import QFF_SUPPORTED_KERNELS @pytest.fixture(name="n_dims", params=[1, 2, 3]) @@ -52,7 +51,7 @@ def _batch_size_fixture(request): return request.param -@pytest.fixture(name="kernel_cls", params=list(QFF_SUPPORTED_KERNELS)) +@pytest.fixture(name="kernel_cls", params=list(QuadratureFourierFeatures.SUPPORTED_KERNELS)) def _kernel_cls_fixture(request): return request.param diff --git a/tests/gpflux/layers/basis_functions/fourier_features/test_random.py b/tests/gpflux/layers/basis_functions/fourier_features/test_random.py index df8c04f8..1c0e8226 100644 --- a/tests/gpflux/layers/basis_functions/fourier_features/test_random.py +++ b/tests/gpflux/layers/basis_functions/fourier_features/test_random.py @@ -26,7 +26,6 @@ RandomFourierFeatures, RandomFourierFeaturesCosine, ) -from gpflux.layers.basis_functions.fourier_features.random.base import RFF_SUPPORTED_KERNELS @pytest.fixture(name="n_dims", params=[1, 2, 3, 5, 10, 20]) @@ -54,7 +53,7 @@ def _n_features_fixture(request): return request.param -@pytest.fixture(name="kernel_cls", params=list(RFF_SUPPORTED_KERNELS)) +@pytest.fixture(name="kernel_cls", params=list(RandomFourierFeatures.SUPPORTED_KERNELS)) def _kernel_cls_fixture(request): return request.param From 52e438178890dc589bb8d1cd7f27b733476bf31d Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 18:15:13 +0000 Subject: [PATCH 08/22] added minimal scipy requirements --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9108d9ff..e8d16d6f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ "deprecated", "gpflow>=2.1", "numpy", - "scipy", + "scipy>=1.6.0", "tensorflow>=2.5.0,<2.6.0", "tensorflow-probability>=0.12.0,<0.14.0", ] From 3f05fd844ed5aff3248f7aca328389d00febd70b Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 15 Dec 2021 18:32:06 +0000 Subject: [PATCH 09/22] disable support for 3.6 --- .github/workflows/quality-check.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/quality-check.yaml b/.github/workflows/quality-check.yaml index e5bd665e..0971f8f7 100644 --- a/.github/workflows/quality-check.yaml +++ b/.github/workflows/quality-check.yaml @@ -21,7 +21,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.7, 3.8] tensorflow: ["~=2.5.0"] name: Python-${{ matrix.python-version }} tensorflow${{ matrix.tensorflow }} env: From 0dcf4c6832804ccb9cb7de775d5a6c58b9e6835c Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 26 Jan 2022 20:50:32 +0000 Subject: [PATCH 10/22] added missing imports --- .../basis_functions/fourier_features/quadrature/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py index 931a8243..5bd43e5e 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py @@ -15,5 +15,7 @@ # """ A kernel's features and coefficients using quadrature Fourier features (QFF). """ from gpflux.layers.basis_functions.fourier_features.quadrature.gaussian import ( + QuadratureFourierFeatures, GaussHermiteQuadratureFourierFeatures, + GaussLegendreQuadratureFourierFeatures ) From 031dc9a34aa4cb25174c17f0a5528237da41977a Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 26 Jan 2022 20:53:48 +0000 Subject: [PATCH 11/22] formatting --- .../basis_functions/fourier_features/quadrature/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py index 5bd43e5e..63998901 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py @@ -15,7 +15,7 @@ # """ A kernel's features and coefficients using quadrature Fourier features (QFF). """ from gpflux.layers.basis_functions.fourier_features.quadrature.gaussian import ( - QuadratureFourierFeatures, GaussHermiteQuadratureFourierFeatures, - GaussLegendreQuadratureFourierFeatures + GaussLegendreQuadratureFourierFeatures, + QuadratureFourierFeatures, ) From e465f35a56d03baf426f53233c7f1864878d0adc Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 26 Jan 2022 21:18:46 +0000 Subject: [PATCH 12/22] make sure to call parent initializer first for input argument checks --- .../basis_functions/fourier_features/quadrature/gaussian.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index 6c891c09..2559ccc2 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -47,12 +47,12 @@ class GaussHermiteQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): SUPPORTED_KERNELS = (gpflow.kernels.SquaredExponential,) def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): + super(GaussHermiteQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) if tf.reduce_any(tf.less(kernel.lengthscales, 1e-1)): warnings.warn( "Gauss-Hermite Quadrature Fourier feature approximation of kernels " "with small lengthscale lead to unexpected behaviors!" ) - super(GaussHermiteQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) def build(self, input_shape: ShapeType) -> None: """ @@ -126,7 +126,7 @@ def build(self, input_shape: ShapeType) -> None: class QuadratureFourierFeatures(GaussHermiteQuadratureFourierFeatures): """ - Alias + Alias for `GaussHermiteQuadratureFourierFeatures`. """ pass From 58da313286852b70c2edb522037f79f694e9e669 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Fri, 28 Jan 2022 00:22:55 +0000 Subject: [PATCH 13/22] fixed imports --- gpflux/layers/basis_functions/fourier_features/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gpflux/layers/basis_functions/fourier_features/__init__.py b/gpflux/layers/basis_functions/fourier_features/__init__.py index ec68b5d1..796bf3d3 100644 --- a/gpflux/layers/basis_functions/fourier_features/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/__init__.py @@ -19,7 +19,9 @@ """ from gpflux.layers.basis_functions.fourier_features.quadrature import ( + QuadratureFourierFeatures, GaussHermiteQuadratureFourierFeatures, + GaussLegendreQuadratureFourierFeatures, ) from gpflux.layers.basis_functions.fourier_features.random import ( OrthogonalRandomFeatures, From 0cf22ff186f93ebad17309183cdc735a7483b1ff Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Sun, 30 Jan 2022 21:05:51 +0000 Subject: [PATCH 14/22] fixed imports --- gpflux/layers/basis_functions/fourier_features/__init__.py | 3 ++- .../layers/basis_functions/fourier_features/random/__init__.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gpflux/layers/basis_functions/fourier_features/__init__.py b/gpflux/layers/basis_functions/fourier_features/__init__.py index 796bf3d3..36fde400 100644 --- a/gpflux/layers/basis_functions/fourier_features/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/__init__.py @@ -19,12 +19,13 @@ """ from gpflux.layers.basis_functions.fourier_features.quadrature import ( - QuadratureFourierFeatures, GaussHermiteQuadratureFourierFeatures, GaussLegendreQuadratureFourierFeatures, + QuadratureFourierFeatures, ) from gpflux.layers.basis_functions.fourier_features.random import ( OrthogonalRandomFeatures, + QuasiRandomFourierFeatures, RandomFourierFeatures, RandomFourierFeaturesCosine, ) diff --git a/gpflux/layers/basis_functions/fourier_features/random/__init__.py b/gpflux/layers/basis_functions/fourier_features/random/__init__.py index 162e18cf..a5014438 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/random/__init__.py @@ -22,3 +22,4 @@ from gpflux.layers.basis_functions.fourier_features.random.orthogonal import ( OrthogonalRandomFeatures, ) +from gpflux.layers.basis_functions.fourier_features.random.quasi import QuasiRandomFourierFeatures From abd375a454664259ae974bf113a2b3e3bd8acf6b Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 2 Feb 2022 11:22:08 +0000 Subject: [PATCH 15/22] added new modules --- .../fourier_features/__init__.py | 1 + .../fourier_features/quadrature/__init__.py | 3 + .../quadrature/newton_cotes.py | 100 ++++++++++++++++++ .../fourier_features/random/quasi.py | 37 +++++++ 4 files changed, 141 insertions(+) create mode 100644 gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py create mode 100644 gpflux/layers/basis_functions/fourier_features/random/quasi.py diff --git a/gpflux/layers/basis_functions/fourier_features/__init__.py b/gpflux/layers/basis_functions/fourier_features/__init__.py index 36fde400..3a9d84b6 100644 --- a/gpflux/layers/basis_functions/fourier_features/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/__init__.py @@ -22,6 +22,7 @@ GaussHermiteQuadratureFourierFeatures, GaussLegendreQuadratureFourierFeatures, QuadratureFourierFeatures, + SimpsonQuadratureFourierFeatures ) from gpflux.layers.basis_functions.fourier_features.random import ( OrthogonalRandomFeatures, diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py index 63998901..b6902eca 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py @@ -19,3 +19,6 @@ GaussLegendreQuadratureFourierFeatures, QuadratureFourierFeatures, ) +from gpflux.layers.basis_functions.fourier_features.quadrature.newton_cotes import ( + SimpsonQuadratureFourierFeatures +) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py b/gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py new file mode 100644 index 00000000..6a426c87 --- /dev/null +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py @@ -0,0 +1,100 @@ +# +# Copyright (c) 2021 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Kernel decompositon into features and coefficients based on Newton-Cotes quadrature. +""" +import numpy as np +import tensorflow as tf +from scipy.stats import multivariate_normal, multivariate_t + +import gpflow +from gpflow.quadrature.gauss_hermite import repeat_as_list, reshape_Z_dZ + +from gpflux.layers.basis_functions.fourier_features.quadrature.base import ( + QuadratureFourierFeaturesBase, + TanTransform, +) +from gpflux.layers.basis_functions.fourier_features.utils import _matern_dof +from gpflux.types import ShapeType + + +class SimpsonQuadratureFourierFeatures(QuadratureFourierFeaturesBase): + + SUPPORTED_KERNELS = ( + gpflow.kernels.SquaredExponential, + gpflow.kernels.Matern12, + gpflow.kernels.Matern32, + gpflow.kernels.Matern52, + ) + + def _compute_output_dim(self, input_shape: ShapeType) -> int: + input_dim = input_shape[-1] + n_abscissa = 2 * self.n_components + 1 + return 2 * n_abscissa ** input_dim + + def build(self, input_shape: ShapeType) -> None: + """ + Creates the variables of the layer. + See `tf.keras.layers.Layer.build() + `_. + """ + input_dim = input_shape[-1] + + if isinstance(self.kernel, gpflow.kernels.SquaredExponential): + dist = multivariate_normal(mean=np.zeros(input_dim)) + else: + nu = _matern_dof(self.kernel) # degrees of freedom + dist = multivariate_t(loc=np.zeros(input_dim), df=nu) + + stop = 1. + start = -1. + + # `n_components` denotes half the desired number of intervals + n_abscissa = 2 * self.n_components + 1 + width = np.true_divide(stop - start, n_abscissa - 1) + + # raw 1-dimensional quadrature nodes (L,) + abscissa_value_flat = np.linspace(start, stop, n_abscissa) + + alpha = np.atleast_2d(4.) + beta = np.atleast_2d(2.) + a = np.hstack([beta, alpha]) + + factors_value_flat = np.append(np.tile(a, reps=self.n_components), beta, axis=-1) + factors_value_flat *= width + factors_value_flat /= 3. + factors_value_flat[..., [0, -1]] /= 2. # halve first and last weight + + # transformed 1-dimensional quadrature nodes and weights + transform = TanTransform() + factors_value_flat *= transform.multiplier(abscissa_value_flat) # (L,) + abscissa_value_flat = transform(abscissa_value_flat) # (L,) + + # transformed D-dimensional quadrature nodes and weights + abscissa_value_rep = repeat_as_list(abscissa_value_flat, n=input_dim) # (L, ..., L) + factors_value_rep = repeat_as_list(factors_value_flat, n=input_dim) # (L, ..., L) + # (L^D, D), (L^D, 1) + abscissa_value, factors_value = reshape_Z_dZ(abscissa_value_rep, factors_value_rep) + + factors_value = tf.squeeze(factors_value, axis=-1) # (L^D,) + factors_value *= dist.pdf(abscissa_value) # (L^D,) + + # Quadrature nodes (L^D, D) + self.abscissa = tf.Variable(initial_value=abscissa_value, trainable=False) + # Quadrature weights (L^D,) + self.factors = tf.Variable(initial_value=factors_value, trainable=False) + + super(SimpsonQuadratureFourierFeatures, self).build(input_shape) diff --git a/gpflux/layers/basis_functions/fourier_features/random/quasi.py b/gpflux/layers/basis_functions/fourier_features/random/quasi.py new file mode 100644 index 00000000..c0bf0c72 --- /dev/null +++ b/gpflux/layers/basis_functions/fourier_features/random/quasi.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2021 The GPflux Contributors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Optional + +import numpy as np +from scipy.stats.qmc import MultivariateNormalQMC + +from gpflow.base import DType, TensorType + +from gpflux.layers.basis_functions.fourier_features.random.base import RandomFourierFeatures + + +class QuasiRandomFourierFeatures(RandomFourierFeatures): + + r""" + Quasi-random Fourier features (ORF) :cite:p:`yang2014quasi` for more + efficient and accurate kernel approximations than random Fourier features. + """ + + def _weights_init(self, shape: TensorType, dtype: Optional[DType] = None) -> TensorType: + n_components, input_dim = shape # M, D + sampler = MultivariateNormalQMC(mean=np.zeros(input_dim)) + return sampler.random(n=n_components) # shape [M, D] From 8d04e69d39c74cd2bd53552fd3372643a13d0fd0 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Fri, 4 Feb 2022 12:37:26 +0000 Subject: [PATCH 16/22] fixed class hierarchy --- .../fourier_features/quadrature/gaussian.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index 2559ccc2..6129aff1 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -39,21 +39,21 @@ class GaussianQuadratureFourierFeatures(QuadratureFourierFeaturesBase): - pass - - -class GaussHermiteQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): - - SUPPORTED_KERNELS = (gpflow.kernels.SquaredExponential,) def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): - super(GaussHermiteQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) + super(GaussianQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) if tf.reduce_any(tf.less(kernel.lengthscales, 1e-1)): warnings.warn( - "Gauss-Hermite Quadrature Fourier feature approximation of kernels " - "with small lengthscale lead to unexpected behaviors!" + "Fourier feature approximation of kernels with small " + "lengthscales using Gaussian quadrature can have " + "unexpected behaviors!" ) + +class GaussHermiteQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): + + SUPPORTED_KERNELS = (gpflow.kernels.SquaredExponential,) + def build(self, input_shape: ShapeType) -> None: """ Creates the variables of the layer. From 4173ff017396af9811ce3a22db66ebdbe3861155 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Fri, 4 Feb 2022 14:57:14 +0000 Subject: [PATCH 17/22] fixed formatting --- .../basis_functions/fourier_features/__init__.py | 2 +- .../fourier_features/quadrature/__init__.py | 2 +- .../fourier_features/quadrature/gaussian.py | 3 +-- .../fourier_features/quadrature/newton_cotes.py | 12 ++++++------ 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/__init__.py b/gpflux/layers/basis_functions/fourier_features/__init__.py index 3a9d84b6..7cdd85da 100644 --- a/gpflux/layers/basis_functions/fourier_features/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/__init__.py @@ -22,7 +22,7 @@ GaussHermiteQuadratureFourierFeatures, GaussLegendreQuadratureFourierFeatures, QuadratureFourierFeatures, - SimpsonQuadratureFourierFeatures + SimpsonQuadratureFourierFeatures, ) from gpflux.layers.basis_functions.fourier_features.random import ( OrthogonalRandomFeatures, diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py index b6902eca..3bb0a3f5 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/__init__.py @@ -20,5 +20,5 @@ QuadratureFourierFeatures, ) from gpflux.layers.basis_functions.fourier_features.quadrature.newton_cotes import ( - SimpsonQuadratureFourierFeatures + SimpsonQuadratureFourierFeatures, ) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index 6129aff1..e37cd036 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -39,12 +39,11 @@ class GaussianQuadratureFourierFeatures(QuadratureFourierFeaturesBase): - def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): super(GaussianQuadratureFourierFeatures, self).__init__(kernel, n_components, **kwargs) if tf.reduce_any(tf.less(kernel.lengthscales, 1e-1)): warnings.warn( - "Fourier feature approximation of kernels with small " + "Fourier feature approximation of kernels with small " "lengthscales using Gaussian quadrature can have " "unexpected behaviors!" ) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py b/gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py index 6a426c87..a9143067 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/newton_cotes.py @@ -59,8 +59,8 @@ def build(self, input_shape: ShapeType) -> None: nu = _matern_dof(self.kernel) # degrees of freedom dist = multivariate_t(loc=np.zeros(input_dim), df=nu) - stop = 1. - start = -1. + stop = 1.0 + start = -1.0 # `n_components` denotes half the desired number of intervals n_abscissa = 2 * self.n_components + 1 @@ -69,14 +69,14 @@ def build(self, input_shape: ShapeType) -> None: # raw 1-dimensional quadrature nodes (L,) abscissa_value_flat = np.linspace(start, stop, n_abscissa) - alpha = np.atleast_2d(4.) - beta = np.atleast_2d(2.) + alpha = np.atleast_2d(4.0) + beta = np.atleast_2d(2.0) a = np.hstack([beta, alpha]) factors_value_flat = np.append(np.tile(a, reps=self.n_components), beta, axis=-1) factors_value_flat *= width - factors_value_flat /= 3. - factors_value_flat[..., [0, -1]] /= 2. # halve first and last weight + factors_value_flat /= 3.0 + factors_value_flat[..., [0, -1]] /= 2.0 # halve first and last weight # transformed 1-dimensional quadrature nodes and weights transform = TanTransform() From 8d2e9a7fd84adcf7b52c60fe176a83f4d48ba092 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Tue, 8 Feb 2022 18:12:43 +0000 Subject: [PATCH 18/22] added re-weighted version --- .../fourier_features/random/quasi.py | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/gpflux/layers/basis_functions/fourier_features/random/quasi.py b/gpflux/layers/basis_functions/fourier_features/random/quasi.py index c0bf0c72..3135047e 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/quasi.py +++ b/gpflux/layers/basis_functions/fourier_features/random/quasi.py @@ -17,11 +17,21 @@ from typing import Optional import numpy as np +import tensorflow as tf +import tensorflow_probability as tfp + +from scipy.stats import multivariate_normal, multivariate_t from scipy.stats.qmc import MultivariateNormalQMC +import gpflow from gpflow.base import DType, TensorType from gpflux.layers.basis_functions.fourier_features.random.base import RandomFourierFeatures +from gpflux.layers.basis_functions.fourier_features.utils import _matern_dof +from gpflux.types import ShapeType + + +tfd = tfp.distributions class QuasiRandomFourierFeatures(RandomFourierFeatures): @@ -35,3 +45,47 @@ def _weights_init(self, shape: TensorType, dtype: Optional[DType] = None) -> Ten n_components, input_dim = shape # M, D sampler = MultivariateNormalQMC(mean=np.zeros(input_dim)) return sampler.random(n=n_components) # shape [M, D] + + +class ReweightedQuasiRandomFourierFeatures(QuasiRandomFourierFeatures): + + SUPPORTED_KERNELS = ( + gpflow.kernels.SquaredExponential, + gpflow.kernels.Matern12, + gpflow.kernels.Matern32, + gpflow.kernels.Matern52, + ) + + def _compute_constant(self) -> tf.Tensor: + """ + Compute normalizing constant for basis functions. + + :return: A tensor with the shape ``[]`` (i.e. a scalar). + """ + return ( + tf.tile(tf.sqrt(self.factors), multiples=[2]) + * super(ReweightedQuasiRandomFourierFeatures, self)._compute_constant() + ) + + def build(self, input_shape: ShapeType) -> None: + """ + Creates the variables of the layer. + See `tf.keras.layers.Layer.build() + `_. + """ + super(ReweightedQuasiRandomFourierFeatures, self).build(input_shape) + + input_dim = input_shape[-1] + + if isinstance(self.kernel, gpflow.kernels.SquaredExponential): + dist = multivariate_normal(mean=np.zeros(input_dim)) + else: + nu = _matern_dof(self.kernel) # degrees of freedom + dist = tfd.MultivariateStudentTLinearOperator(df=nu, loc=np.zeros(input_dim, dtype=self.dtype), scale=tf.eye(input_dim, dtype=self.dtype)) + + print("DIST!!!", dist.prob(self.W)) + + factors_value = tf.ones(self.n_components, dtype=self.dtype) # dist.pdf(self.W) + self.factors = tf.Variable(initial_value=factors_value, trainable=False) + + print(f"CALLED!, {self.W} {self.factors}") From 39e432e9f00c244bb4e26e8bbd2bd67735abcdcd Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Wed, 9 Feb 2022 15:21:29 +0000 Subject: [PATCH 19/22] complete subclass implementation --- .../fourier_features/__init__.py | 1 + .../fourier_features/random/__init__.py | 5 ++++- .../fourier_features/random/quasi.py | 17 ++++++++--------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/__init__.py b/gpflux/layers/basis_functions/fourier_features/__init__.py index 7cdd85da..6f191b50 100644 --- a/gpflux/layers/basis_functions/fourier_features/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/__init__.py @@ -29,4 +29,5 @@ QuasiRandomFourierFeatures, RandomFourierFeatures, RandomFourierFeaturesCosine, + ReweightedQuasiRandomFourierFeatures, ) diff --git a/gpflux/layers/basis_functions/fourier_features/random/__init__.py b/gpflux/layers/basis_functions/fourier_features/random/__init__.py index a5014438..0e2bfaa5 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/__init__.py +++ b/gpflux/layers/basis_functions/fourier_features/random/__init__.py @@ -22,4 +22,7 @@ from gpflux.layers.basis_functions.fourier_features.random.orthogonal import ( OrthogonalRandomFeatures, ) -from gpflux.layers.basis_functions.fourier_features.random.quasi import QuasiRandomFourierFeatures +from gpflux.layers.basis_functions.fourier_features.random.quasi import ( + QuasiRandomFourierFeatures, + ReweightedQuasiRandomFourierFeatures, +) diff --git a/gpflux/layers/basis_functions/fourier_features/random/quasi.py b/gpflux/layers/basis_functions/fourier_features/random/quasi.py index 3135047e..435bd5ab 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/quasi.py +++ b/gpflux/layers/basis_functions/fourier_features/random/quasi.py @@ -19,7 +19,6 @@ import numpy as np import tensorflow as tf import tensorflow_probability as tfp - from scipy.stats import multivariate_normal, multivariate_t from scipy.stats.qmc import MultivariateNormalQMC @@ -30,7 +29,6 @@ from gpflux.layers.basis_functions.fourier_features.utils import _matern_dof from gpflux.types import ShapeType - tfd = tfp.distributions @@ -78,14 +76,15 @@ def build(self, input_shape: ShapeType) -> None: input_dim = input_shape[-1] if isinstance(self.kernel, gpflow.kernels.SquaredExponential): - dist = multivariate_normal(mean=np.zeros(input_dim)) + factors_value = tf.ones(self.n_components, dtype=self.dtype) else: nu = _matern_dof(self.kernel) # degrees of freedom - dist = tfd.MultivariateStudentTLinearOperator(df=nu, loc=np.zeros(input_dim, dtype=self.dtype), scale=tf.eye(input_dim, dtype=self.dtype)) + q = tfd.MultivariateNormalDiag(loc=tf.zeros(input_dim, dtype=self.dtype)) + p = tfd.MultivariateStudentTLinearOperator( + df=nu, + loc=tf.zeros(input_dim, dtype=self.dtype), + scale=tf.linalg.LinearOperatorLowerTriangular(tf.eye(input_dim, dtype=self.dtype)), + ) + factors_value = tf.exp(p.log_prob(self.W) - q.log_prob(self.W)) - print("DIST!!!", dist.prob(self.W)) - - factors_value = tf.ones(self.n_components, dtype=self.dtype) # dist.pdf(self.W) self.factors = tf.Variable(initial_value=factors_value, trainable=False) - - print(f"CALLED!, {self.W} {self.factors}") From 383df2a66a693f7a3f7af11327187270f5debdf1 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Mon, 14 Feb 2022 11:37:41 +0000 Subject: [PATCH 20/22] added reweighted version --- .../basis_functions/fourier_features/random/quasi.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/random/quasi.py b/gpflux/layers/basis_functions/fourier_features/random/quasi.py index 435bd5ab..5cbcd02c 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/quasi.py +++ b/gpflux/layers/basis_functions/fourier_features/random/quasi.py @@ -34,7 +34,7 @@ class QuasiRandomFourierFeatures(RandomFourierFeatures): - r""" + """ Quasi-random Fourier features (ORF) :cite:p:`yang2014quasi` for more efficient and accurate kernel approximations than random Fourier features. """ @@ -61,7 +61,7 @@ def _compute_constant(self) -> tf.Tensor: :return: A tensor with the shape ``[]`` (i.e. a scalar). """ return ( - tf.tile(tf.sqrt(self.factors), multiples=[2]) + tf.tile(tf.sqrt(self.importance_weight), multiples=[2]) * super(ReweightedQuasiRandomFourierFeatures, self)._compute_constant() ) @@ -85,6 +85,7 @@ def build(self, input_shape: ShapeType) -> None: loc=tf.zeros(input_dim, dtype=self.dtype), scale=tf.linalg.LinearOperatorLowerTriangular(tf.eye(input_dim, dtype=self.dtype)), ) - factors_value = tf.exp(p.log_prob(self.W) - q.log_prob(self.W)) + importance_weight_value = tf.exp(p.log_prob(self.W) - q.log_prob(self.W)) - self.factors = tf.Variable(initial_value=factors_value, trainable=False) + self.importance_weight = tf.Variable(initial_value=importance_weight_value, + trainable=False) From e9cbc77b699dd01e63c3fdbead35eaf5249b3813 Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Mon, 14 Feb 2022 16:04:44 +0000 Subject: [PATCH 21/22] fixed small bug --- .../layers/basis_functions/fourier_features/random/quasi.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gpflux/layers/basis_functions/fourier_features/random/quasi.py b/gpflux/layers/basis_functions/fourier_features/random/quasi.py index 5cbcd02c..bc2cdcfa 100644 --- a/gpflux/layers/basis_functions/fourier_features/random/quasi.py +++ b/gpflux/layers/basis_functions/fourier_features/random/quasi.py @@ -74,10 +74,9 @@ def build(self, input_shape: ShapeType) -> None: super(ReweightedQuasiRandomFourierFeatures, self).build(input_shape) input_dim = input_shape[-1] + importance_weight_value = tf.ones(self.n_components, dtype=self.dtype) - if isinstance(self.kernel, gpflow.kernels.SquaredExponential): - factors_value = tf.ones(self.n_components, dtype=self.dtype) - else: + if not isinstance(self.kernel, gpflow.kernels.SquaredExponential): nu = _matern_dof(self.kernel) # degrees of freedom q = tfd.MultivariateNormalDiag(loc=tf.zeros(input_dim, dtype=self.dtype)) p = tfd.MultivariateStudentTLinearOperator( From 66137365659b0af76ac4464feddb5a4440ae0cde Mon Sep 17 00:00:00 2001 From: Louis Tiao Date: Fri, 11 Mar 2022 10:17:11 +0000 Subject: [PATCH 22/22] added importance sampling for quadrature --- .../fourier_features/quadrature/gaussian.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py index e37cd036..ee408d24 100644 --- a/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py +++ b/gpflux/layers/basis_functions/fourier_features/quadrature/gaussian.py @@ -23,6 +23,8 @@ import numpy as np import tensorflow as tf +import tensorflow_probability as tfp + from scipy.stats import multivariate_normal, multivariate_t import gpflow @@ -37,6 +39,8 @@ from gpflux.layers.basis_functions.fourier_features.utils import _matern_dof from gpflux.types import ShapeType +tfd = tfp.distributions + class GaussianQuadratureFourierFeatures(QuadratureFourierFeaturesBase): def __init__(self, kernel: gpflow.kernels.Kernel, n_components: int, **kwargs: Mapping): @@ -73,6 +77,51 @@ def build(self, input_shape: ShapeType) -> None: super(GaussHermiteQuadratureFourierFeatures, self).build(input_shape) +class ReweightedGaussHermiteQuadratureFourierFeatures(GaussHermiteQuadratureFourierFeatures): + + SUPPORTED_KERNELS = ( + gpflow.kernels.SquaredExponential, + gpflow.kernels.Matern12, + gpflow.kernels.Matern32, + gpflow.kernels.Matern52, + ) + + def _compute_constant(self) -> tf.Tensor: + """ + Compute normalizing constant for basis functions. + + :return: A tensor with the shape ``[]`` (i.e. a scalar). + """ + return ( + tf.tile(tf.sqrt(self.importance_weight), multiples=[2]) + * super(ReweightedGaussHermiteQuadratureFourierFeatures, self)._compute_constant() + ) + + def build(self, input_shape: ShapeType) -> None: + """ + Creates the variables of the layer. + See `tf.keras.layers.Layer.build() + `_. + """ + super(ReweightedGaussHermiteQuadratureFourierFeatures, self).build(input_shape) + + input_dim = input_shape[-1] + importance_weight_value = tf.ones(self.abscissa.shape[0], dtype=self.dtype) + + if not isinstance(self.kernel, gpflow.kernels.SquaredExponential): + nu = _matern_dof(self.kernel) # degrees of freedom + q = tfd.MultivariateNormalDiag(loc=tf.zeros(input_dim, dtype=self.dtype)) + p = tfd.MultivariateStudentTLinearOperator( + df=nu, + loc=tf.zeros(input_dim, dtype=self.dtype), + scale=tf.linalg.LinearOperatorLowerTriangular(tf.eye(input_dim, dtype=self.dtype)), + ) + importance_weight_value = tf.exp(p.log_prob(self.abscissa) - q.log_prob(self.abscissa)) + + self.importance_weight = tf.Variable(initial_value=importance_weight_value, + trainable=False) + + class GaussLegendreQuadratureFourierFeatures(GaussianQuadratureFourierFeatures): SUPPORTED_KERNELS = (