From 28670f033e1d687429f7c9314fb2c70d44d4ff6e Mon Sep 17 00:00:00 2001 From: bial_fr Date: Tue, 15 Sep 2026 10:09:41 +0200 Subject: [PATCH 1/5] Robust optimization v 1.0 --- .../robust_optimization/rosenbrock.py | 99 +++++ sb_arch_opt/stochastic_problem.py | 145 +++++++ sb_arch_opt/tests/conftest.py | 97 ++++- sb_arch_opt/tests/test_stochastic_problem.py | 367 ++++++++++++++++++ sb_arch_opt/uncertainty.py | 346 +++++++++++++++++ 5 files changed, 1053 insertions(+), 1 deletion(-) create mode 100644 sb_arch_opt/problems/robust_optimization/rosenbrock.py create mode 100644 sb_arch_opt/stochastic_problem.py create mode 100644 sb_arch_opt/tests/test_stochastic_problem.py create mode 100644 sb_arch_opt/uncertainty.py diff --git a/sb_arch_opt/problems/robust_optimization/rosenbrock.py b/sb_arch_opt/problems/robust_optimization/rosenbrock.py new file mode 100644 index 0000000..73a15d6 --- /dev/null +++ b/sb_arch_opt/problems/robust_optimization/rosenbrock.py @@ -0,0 +1,99 @@ +""" +MIT License + +Copyright: (c) 2023, Deutsches Zentrum fuer Luft- und Raumfahrt e.V. +Contact: jasper.bussemaker@dlr.de + +Stochastic version of the Rosenbrock problem: a robust optimization test problem. +""" +from typing import List + +import numpy as np +import openturns as ot +from pymoo.core.variable import Real + +from sb_arch_opt.stochastic_problem import StochasticArchOptProblem +from sb_arch_opt.uncertainty import * + +__all__ = ['StochasticRosenbrock'] + + +class StochasticRosenbrock(StochasticArchOptProblem): + """ + The Rosenbrock function where the location of the valley is uncertain: + + f(x, u) = sum_i [ 100*(x_{i+1} - x_i^2)^2 + (u_i - x_i)^2 ] + + with one uncertain parameter u_i ~ N(mean, std) per term. + + With the default MEAN reduction the expected objective is minimized; since E[(u-x)^2] = (E[u]-x)^2 + Var[u], + the optimum stays at x_i = mean and the optimal value is the sum of the parameter variances rather than 0. + + Note that only `_arch_evaluate_sample` is implemented, and that it is vectorized over all design points: the + loop over uncertain-parameter samples is owned by `StochasticArchOptProblem`. + + A UQ method is configured independently and receives the parameter space when drawing samples / processing results: + + ```python + method = PolynomialChaos(n_evaluations=40, degree=3, seed=42) + problem = StochasticRosenbrock(n_var=2, uq_method=method) + ``` + """ + + def __init__(self, n_var=2, mean=1., std=.05, n=100, seed=42, + uq_method: UQMethod = None, obj_scalar: List[Scalarization] = None): + if n_var < 2: + raise ValueError('Need at least 2 design variables') + self.mean = mean + self.std = std + self._n_param = n_var - 1 + + # The problem and its UQ method must be defined over the same parameter space; when a method is supplied + # it already carries the space it was built over, so take that one rather than building a second + if uq_method is None: + uq_method = MonteCarlo(n_evaluations=n, seed=seed) + + super().__init__( + [Real(bounds=(-2.048, 2.048)) for _ in range(n_var)], + param_space=self.get_parameter_space(n_var, mean, std), + uq_method=uq_method, + n_obj=1, obj_scalar=obj_scalar + ) + + @staticmethod + def get_parameter_space(n_var=2, mean=1., std=.05) -> StochasticParameterSpace: + """The uncertain valley location: one parameter per Rosenbrock term. Build this first if you want to + construct the UQ method yourself, since a method is defined over a parameter space.""" + parameters = [] + for i in range(n_var-1): + parameters.append(StochasticParameter(f'u{i}', ot.Normal(mean, std))) + param_space = StochasticParameterSpace(parameters) + return param_space + + def _is_conditionally_active(self) -> List[bool]: + return [False]*self.n_var + + def _correct_x(self, x: np.ndarray, is_active: np.ndarray): + pass + + def _get_n_valid_discrete(self) -> int: + return 1 + + def might_have_hidden_constraints(self): + return False + + def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, *args, **kwargs): + f = np.zeros((x.shape[0],)) + for i in range(self._n_param): + f += 100 * (x[:, i+1] - x[:, i]**2) ** 2 + (parameters[i] - x[:, i]) ** 2 + f_out[:, 0] = f + + def _calc_pareto_front(self, *args, **kwargs): + # E[(u-x)^2] is minimized at x = E[u], leaving the parameter variance + return np.array([[self._n_param * self.std**2]]) + + def _calc_pareto_set(self, *args, **kwargs): + return np.full((1, self.n_var), self.mean) + + def __repr__(self): + return f'{self.__class__.__name__}(n_var={self.n_var}, std={self.std}, n={self.uq_method.n_evaluations})' \ No newline at end of file diff --git a/sb_arch_opt/stochastic_problem.py b/sb_arch_opt/stochastic_problem.py new file mode 100644 index 0000000..e8c46df --- /dev/null +++ b/sb_arch_opt/stochastic_problem.py @@ -0,0 +1,145 @@ +""" +MIT License + +Copyright: (c) 2024, Deutsches Zentrum fuer Luft- und Raumfahrt e.V. +Contact: jasper.bussemaker@dlr.de + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" +from typing import Union, List, Optional + +import numpy as np +from pymoo.core.variable import Variable + +from sb_arch_opt.design_space import ArchDesignSpace +from sb_arch_opt.problem import ArchOptProblemBase +from sb_arch_opt.uncertainty import * + +__all__ = ['StochasticArchOptProblem'] + + +class StochasticArchOptProblem(ArchOptProblemBase): + """ + Base class for a stochastic (robust) architecture optimization problem. + """ + + def __init__(self, des_vars: Union[List[Variable], ArchDesignSpace], + param_space: StochasticParameterSpace, + uq_method: UQMethod, n_obj=1, n_ieq_constr=0, n_eq_constr=0, + obj_scalar: List[Scalarization] = None, + ieq_constr_scalar: List[Scalarization] = None, + eq_constr_scalar: List[Scalarization] = None, + **kwargs): + + self.obj_scalar = self.check_scalars(obj_scalar, n_obj) + self.ieq_constr_scalar = self.check_scalars(ieq_constr_scalar, n_ieq_constr) + self.eq_constr_scalar = self.check_scalars(eq_constr_scalar, n_eq_constr) + self.scalars = self.obj_scalar + self.ieq_constr_scalar + self.eq_constr_scalar + + if uq_method is None: + raise ValueError('A UQ method must be specified, e.g. uq_method=MonteCarlo(n_evaluations=100, seed=42)') + if not isinstance(uq_method, UQMethod): + raise ValueError(f'uq_method should be a UQMethod instance, got: {uq_method!r}') + + if param_space is None or param_space.n_parameters == 0: + raise ValueError('param_space must contain valid parameter space') + + self.param_space = param_space + self.uq_method = uq_method + # List for storing stochastic results object for each design point + self.stochastic_results: List[StochasticResults] = [] + + super().__init__(des_vars, n_obj=n_obj, n_ieq_constr=n_ieq_constr, n_eq_constr=n_eq_constr, **kwargs) + + @staticmethod + def check_scalars(scalars: Optional[List[Scalarization]], n: int) -> List[Scalarization]: + """Default unspecified responses to the expected value, and check the count""" + if scalars is None: + # Mean is the default robust scalar type + return [Mean() for _ in range(n)] + + if len(scalars) != n: + raise ValueError(f'scalars should have {n} Scalarization entries, got {len(scalars)}') + for scalar in scalars: + if not isinstance(scalar, Scalarization): + raise ValueError(f'scalars should contain Scalarization instances, got: {scalar!r}') + return list(scalars) + + def _evaluate(self, x, out, *args, **kwargs): + # The pymoo outputs are processed by the parent method + super()._evaluate(x, out, *args, **kwargs) + + # Add stochastic result to pymoo out dictionary. + if len(self.stochastic_results) == len(out['X']): + out['stochastic'] = list(self.stochastic_results) + + def _arch_evaluate(self, x: np.ndarray, is_active_out: np.ndarray, f_out: np.ndarray, g_out: np.ndarray, + h_out: np.ndarray, *args, **kwargs): + """ + Evaluate architecture for the provided design vectors and samples. + Implement _arch_evaluate_sample to evaluate architecture for single samples realizations. + """ + + if not self.design_space.is_explicit(): + self._correct_x_impute(x, is_active_out) + + # Get samples and include deterministic parameter values for evaluation + samples = self.uq_method.get_samples(self.param_space) + # parameter_values = self.uq_method.param_space.include_deterministic_values(samples) + + n_x, n_s = x.shape[0], samples.shape[0] + + f_s = np.zeros((n_x, n_s, self.n_obj))*np.nan + g_s = np.zeros((n_x, n_s, self.n_ieq_constr))*np.nan + h_s = np.zeros((n_x, n_s, self.n_eq_constr))*np.nan + + # Evaluate all design vectors for each realization of the uncertain parameters + for i in range(n_s): + self._arch_evaluate_sample( + x, is_active_out, f_s[:, i, :], g_s[:, i, :], h_s[:, i, :], samples[i, :],*args, **kwargs) + + # Evaluate the stochastic result for all the evaluated design vectors and samples + self.stochastic_results = [] + for x_i in range(n_x): + results = self.uq_method.process_results(np.concatenate([f_s[x_i], g_s[x_i], h_s[x_i]], axis=1), self.param_space) + self.stochastic_results.append(results) + + # Reduce the sampled responses of each design point to the values the optimizer sees + n_f, n_g = self.n_obj, self.n_ieq_constr + for f_i, output in enumerate(results.outputs[:n_f]): + f_out[x_i, f_i] = output.reduce(self.obj_scalar[f_i]) + for g_i, output in enumerate(results.outputs[n_f:n_f+n_g]): + g_out[x_i, g_i] = output.reduce(self.ieq_constr_scalar[g_i]) + for h_i, output in enumerate(results.outputs[n_f+n_g:]): + h_out[x_i, h_i] = output.reduce(self.eq_constr_scalar[h_i]) + + def _arch_evaluate_sample(self, x: np.ndarray, is_active: np.ndarray, f_out: np.ndarray, g_out: np.ndarray, + h_out: np.ndarray, parameters: np.ndarray, *args, **kwargs): + """ + Implement evaluation and write results in the provided output matrices: + - x (design vectors): discrete variables have integer values, imputed design vectors can be output here (except + if using an explicit design space definition) + - is_active (activeness): vector specifying for each design variable whether it was active or not + - f (objectives): written as a minimization + - g (inequality constraints): written as "<= 0" + - h (equality constraints): written as "= 0" + - sample: realization of single sample + """ + + raise NotImplementedError \ No newline at end of file diff --git a/sb_arch_opt/tests/conftest.py b/sb_arch_opt/tests/conftest.py index e9a6a6f..270139f 100644 --- a/sb_arch_opt/tests/conftest.py +++ b/sb_arch_opt/tests/conftest.py @@ -1,9 +1,12 @@ import pytest import itertools import numpy as np +import openturns as ot from typing import Optional, Tuple from sb_arch_opt.sampling import * +from sb_arch_opt.uncertainty import * from sb_arch_opt.problems.problems_base import * +from sb_arch_opt.stochastic_problem import StochasticArchOptProblem from pymoo.core.variable import Real, Integer, Choice from pymoo.problems.multi.zdt import ZDT1 @@ -100,7 +103,99 @@ def failing_problem(): return DummyProblem(fail=True) +def make_space(*distributions) -> StochasticParameterSpace: + return StochasticParameterSpace([StochasticParameter(f'u{i}', dist) for i, dist in enumerate(distributions)]) + + +class VectorizedProblem(StochasticArchOptProblem): + """f = (u - x0)^2 + x1^2, evaluated for all design points at once""" + + def __init__(self, n=100, seed=42, uq_method=None, fail=False, **kwargs): + self.fail = fail + self.seen_parameters = [] + super().__init__([Real(bounds=(-2., 2.)), Real(bounds=(-2., 2.))], + param_space=make_space(ot.Normal(1., .05)), + uq_method=uq_method if uq_method is not None else MonteCarlo(n, seed=seed), + n_obj=1, **kwargs) + + def _is_conditionally_active(self): + return [False, False] + + def _correct_x(self, x, is_active): + pass + + def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, *args, **kwargs): + self.seen_parameters.append(np.asarray(parameters).copy()) + f_out[:, 0] = (parameters[0] - x[:, 0])**2 + x[:, 1]**2 + if self.fail and parameters[0] > 1.02: # some samples fail to evaluate + f_out[:, 0] = np.nan + + +class HierarchicalProblem(StochasticArchOptProblem): + """Implicit (hierarchical) design space, a constraint, and the per-row evaluation pattern""" + + def __init__(self, n=50, seed=1, **kwargs): + super().__init__([Choice(options=['electric', 'hybrid']), Real(bounds=(.2, 1.)), Real(bounds=(.1, .4))], + param_space=StochasticParameterSpace([ + StochasticParameter('payload', ot.Normal(2., .3)), + StochasticParameter('bsfc', ot.Normal(.42, .075)), + ]), + uq_method=MonteCarlo(n, seed=seed), n_obj=1, n_ieq_constr=1, **kwargs) + + def _is_conditionally_active(self): + return [False, False, True] + + def _correct_x(self, x, is_active): + is_active[:, 2] = x[:, 0] == 1 # fuel fraction only active for the hybrid branch + + def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, *args, **kwargs): + payload, bsfc = parameters + for i in range(x.shape[0]): + mass = .5 + x[i, 1] + payload + energy = x[i, 2]*130./(bsfc+.1) if x[i, 0] == 1 else x[i, 1]*40. + f_out[i, 0] = mass**1.5 / energy + g_out[i, 0] = mass - 3.5 + + +class AllResponseKindsProblem(StochasticArchOptProblem): + """One objective, one inequality and one equality constraint, each with its own scalar""" + + def __init__(self, **kwargs): + kwargs.setdefault('obj_scalar', [Mean()]) + kwargs.setdefault('ieq_constr_scalar', [Mean()]) + kwargs.setdefault('eq_constr_scalar', [Margin(k=3.)]) + super().__init__([Real(bounds=(0., 1.))], param_space=make_space(ot.Normal(1., .2)), + uq_method=MonteCarlo(200, seed=5), + n_obj=1, n_ieq_constr=1, n_eq_constr=1, **kwargs) + + def _is_conditionally_active(self): + return [False] + + def _correct_x(self, x, is_active): + pass + + def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, *args, **kwargs): + f_out[:, 0] = parameters[0] + x[:, 0] + g_out[:, 0] = parameters[0] - x[:, 0] + h_out[:, 0] = parameters[0] * x[:, 0] + + +@pytest.fixture +def stochastic_problem(): + return VectorizedProblem() + + +@pytest.fixture +def hierarchical_problem(): + return HierarchicalProblem() + + +@pytest.fixture +def all_response_kinds_problem(): + return AllResponseKindsProblem() + + def pytest_sessionstart(session): from sb_arch_opt.util import _prevent_capture print('PREVENT CAPTURE') - _prevent_capture() + _prevent_capture() \ No newline at end of file diff --git a/sb_arch_opt/tests/test_stochastic_problem.py b/sb_arch_opt/tests/test_stochastic_problem.py new file mode 100644 index 0000000..19d7a66 --- /dev/null +++ b/sb_arch_opt/tests/test_stochastic_problem.py @@ -0,0 +1,367 @@ +import pytest +import numpy as np +import openturns as ot +from sb_arch_opt.uncertainty import * +from sb_arch_opt.stochastic_problem import StochasticArchOptProblem +from sb_arch_opt.problems.robust_optimization.rosenbrock import StochasticRosenbrock +from sb_arch_opt.tests.conftest import VectorizedProblem, HierarchicalProblem, make_space +from pymoo.core.variable import Real + + +def _output(values): + return StochasticOutput(ot.Sample(np.array(values, dtype=float).reshape((-1, 1)))) + + +def _n_occupied_strata(space, samples, i_param): + # An LHS puts exactly one point in each of the n equiprobable strata of every marginal + n = samples.shape[0] + marginal = space.joint_dist.getMarginal(i_param) + cdf = np.array([marginal.computeCDF(v) for v in samples[:, i_param]]) + return len(np.unique(np.floor(cdf*n).astype(int))) + + +def test_parameter_space(): + space = StochasticParameterSpace([ + StochasticParameter('a', ot.Normal(0., 1.)), + StochasticParameter('b', ot.Uniform(0., 1.)), + ]) + + assert space.n_parameters == 2 + assert space.parameter_names == ['a', 'b'] + assert space.joint_dist.getDimension() == 2 + + for samples in [space.get_random_samples(20), space.get_lhs_samples(20)]: + assert samples.shape == (20, 2) + assert np.all(np.isfinite(samples)) + + # The two draws differ in how evenly they cover each marginal + assert all(_n_occupied_strata(space, space.get_lhs_samples(40), i) == 40 for i in range(2)) + assert any(_n_occupied_strata(space, space.get_random_samples(40), i) < 40 for i in range(2)) + + +def test_scalars(): + out = _output([1., 2., 3., 4., 5.]) + mean, std = out.mean, out.std + + assert out.reduce(Mean()) == pytest.approx(mean) + assert out.reduce(Quantile(q=.9)) == pytest.approx(out.quantile(.9)) + assert out.reduce(Margin(k=2.)) == pytest.approx(out.margin(2.)) + + # The margin is the conservative value, so which tail that is depends on the direction + assert out.reduce(Margin(k=2.)) == pytest.approx(mean + 2.*std) + assert out.reduce(Margin(k=2., direction=-1)) == pytest.approx(mean + 2.*std) + assert out.reduce(Margin(k=2., direction=1)) == pytest.approx(mean - 2.*std) + + # Each instance carries its own parameters + assert out.reduce(Quantile(q=.5)) != out.reduce(Quantile(q=.95)) + assert out.reduce(Margin(k=1.)) != out.reduce(Margin(k=3.)) + + with pytest.raises(ValueError): + Quantile(q=2.) + + +def test_scalars_penalize_spread(): + wide, narrow = _output([1., 3., 5.]), _output([2.5, 3., 3.5]) + assert wide.mean == pytest.approx(narrow.mean) + + assert narrow.reduce(Margin(k=2.)) < wide.reduce(Margin(k=2.)) # minimized: lower is better + assert narrow.reduce(Margin(k=2., direction=1)) > wide.reduce(Margin(k=2., direction=1)) # maximized + + +def test_custom_scalar(): + class WorstCase(Scalarization): + def reduce(self, samples): + return float(samples.getMax()[0]) + + assert _output([1., 5., 3.]).reduce(WorstCase()) == pytest.approx(5.) + + +@pytest.mark.parametrize('method_class', [MonteCarlo, PolynomialChaos]) +def test_uq_method_samples(method_class): + space = make_space(ot.Normal(0., 1.), ot.Normal(0., 1.)) + method = method_class(n_evaluations=40, seed=42) + + samples = method.get_samples(space) + assert samples.shape == (40, 2) + + # Drawn once and reused, so every design point sees the same realizations (common random numbers) + assert np.all(method.get_samples(space) == samples) + + # Both methods use an LHS rather than a plain random draw + assert all(_n_occupied_strata(space, samples, i) == 40 for i in range(2)) + + method.resample() + assert method.get_samples(space).shape == (40, 2) + + with pytest.raises(ValueError): + method.get_samples(None) + + +def test_uq_method_requires_n_evaluations(): + with pytest.raises(ValueError): + MonteCarlo(n_evaluations=None) + + +def test_parameter_realization(): + space = StochasticParameterSpace([ + StochasticParameter('a', ot.Normal(0., 1.)), + StochasticParameter('b', ot.Uniform(2., 4.)), + ]) + samples = MonteCarlo(n_evaluations=5, seed=42).get_samples(space) + + for i in range(5): + parameters = space.param_realization(samples, i) + + assert [parameter.name for parameter in parameters] == ['a', 'b'] + # Column j of the design belongs to parameter j + assert [parameter.sample for parameter in parameters] == pytest.approx(list(samples[i, :])) + assert all(isinstance(parameter.sample, float) for parameter in parameters) + + +def test_process_results(): + space = make_space(ot.Normal(0., 1.)) + method = MonteCarlo(n_evaluations=3, seed=42) + + result = method.process_results(np.array([[1., 10., 20.], [2., 11., 21.], [3., 12., 22.]]), space) + + assert isinstance(result, StochasticResults) + assert len(result.outputs) == 3 # one output per response column + assert [output.mean for output in result.outputs] == pytest.approx([2., 11., 21.]) + assert result.method_result is None # Monte Carlo has nothing beyond the samples + +def test_user_only_implements_arch_evaluate_sample(stochastic_problem): + out = stochastic_problem.evaluate(np.array([[1., 0.], [0., 0.]]), return_as_dictionary=True) + + assert out['F'].shape == (2, 1) + assert np.all(np.isfinite(out['F'])) + # E[(u-x0)^2] + x1^2 = (E[u]-x0)^2 + var + x1^2 + assert out['F'][0, 0] == pytest.approx(.05**2, abs=2e-3) + assert out['F'][1, 0] == pytest.approx(1. + .05**2, abs=2e-2) + + +def test_evaluate_sample_receives_one_realization_per_sample(): + problem = VectorizedProblem(n=25) + problem.evaluate(np.array([[1., 0.], [0., 0.]]), return_as_dictionary=True) + + seen = problem.seen_parameters + assert len(seen) == 25 # once per sample, not once per design point + assert all(realization.shape == (1,) for realization in seen) # one row, not the whole design + assert len({tuple(realization) for realization in seen}) == 25 # a different row each time + assert np.allclose(np.array(seen), problem.uq_method.get_samples(problem.param_space)) + + +def test_multiple_design_points_at_once(stochastic_problem): + x = np.random.RandomState(0).uniform(-2., 2., size=(7, 2)) + out = stochastic_problem.evaluate(x, return_as_dictionary=True) + + assert out['F'].shape == (7, 1) + assert np.all(np.isfinite(out['F'])) + assert len(np.unique(out['F'])) == 7 + + +def test_evaluation_is_deterministic_and_seeded(): + x = np.array([[.5, .5]]) + f_a = VectorizedProblem(n=50, seed=3).evaluate(x, return_as_dictionary=True)['F'] + f_b = VectorizedProblem(n=50, seed=3).evaluate(x, return_as_dictionary=True)['F'] + f_c = VectorizedProblem(n=50, seed=4).evaluate(x, return_as_dictionary=True)['F'] + + assert np.allclose(f_a, f_b) + assert not np.allclose(f_a, f_c) + + +def test_resample_draws_new_samples(): + problem = VectorizedProblem(n=50, seed=3) + x = np.array([[.5, .5]]) + f_a = problem.evaluate(x, return_as_dictionary=True)['F'].copy() + + problem.uq_method.resample() + problem.uq_method.seed = None + assert not np.allclose(f_a, problem.evaluate(x, return_as_dictionary=True)['F']) + + +def test_correction_runs_for_implicit_design_space(hierarchical_problem): + # Correction only happens up front for explicit design spaces, so the loop has to do it: otherwise the + # evaluation function sees an all-True activeness matrix and un-imputed design vectors + x = np.array([[0, .5, .3], [1, .5, .3]]) + out = hierarchical_problem.evaluate(x, return_as_dictionary=True) + + assert np.all(out['is_active'] == [[True, True, False], [True, True, True]]) + assert out['X'][0, 2] == pytest.approx(.25) # imputed to the middle of its bounds + assert np.all(np.isfinite(out['F'])) + + +def test_hierarchical_problem_with_constraint(hierarchical_problem): + x = np.array([[0, .5, .3], [1, .9, .35]]) + out = hierarchical_problem.evaluate(x, return_as_dictionary=True) + + assert out['F'].shape == (2, 1) + assert out['G'].shape == (2, 1) + assert np.all(np.isfinite(out['F'])) + assert np.all(np.isfinite(out['G'])) + + +def test_response_kinds_use_their_own_scalar(all_response_kinds_problem): + problem = all_response_kinds_problem + out = problem.evaluate(np.array([[.5]]), return_as_dictionary=True) + result = out['stochastic'][0] + + assert len(result.outputs) == 3 + assert out['F'][0, 0] == pytest.approx(result.outputs[0].reduce(Mean())) + assert out['G'][0, 0] == pytest.approx(result.outputs[1].reduce(Mean())) + assert out['H'][0, 0] == pytest.approx(result.outputs[2].reduce(Margin(k=3.))) + + # The equality constraint uses a margin, so it is above its own mean + assert out['H'][0, 0] > result.outputs[2].mean + + +def test_scalar_counts_checked_per_response_kind(): + with pytest.raises(ValueError): + HierarchicalProblem(ieq_constr_scalar=[Mean(), Mean()]) + with pytest.raises(ValueError): + HierarchicalProblem(obj_scalar=[Mean(), Mean()]) + with pytest.raises(ValueError): + HierarchicalProblem(ieq_constr_scalar=['quantile']) + + HierarchicalProblem(ieq_constr_scalar=[Mean()]) # a matching count is accepted + + +def test_problem_configuration_checks(): + space = make_space(ot.Normal(0., 1.)) + + with pytest.raises(ValueError): + StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=space, uq_method=None, n_obj=1) + with pytest.raises(ValueError): + StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=space, uq_method='monte carlo', n_obj=1) + with pytest.raises(ValueError): + StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=None, uq_method=MonteCarlo(10), n_obj=1) + with pytest.raises(ValueError): + StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=StochasticParameterSpace([]), + uq_method=MonteCarlo(10), n_obj=1) + + +def test_statistics_available_per_design_point(stochastic_problem): + out = stochastic_problem.evaluate(np.array([[1., 0.], [0., 0.]]), return_as_dictionary=True) + + assert len(out['stochastic']) == 2 + for result in out['stochastic']: + assert isinstance(result, StochasticResults) + assert len(result.outputs) == 1 + assert len(result.outputs[0].to_numpy()) == 100 + + # The realizations really reach the model, so the response scatters + values = out['stochastic'][0].outputs[0].to_numpy() + assert len(set(values.tolist())) == 100 + assert values.std() > 0. + + +def test_reported_statistics_reproduce_the_reduced_value(): + problem = VectorizedProblem(n=200, obj_scalar=[Margin(k=2.)]) + out = problem.evaluate(np.array([[1., 0.], [.5, .5]]), return_as_dictionary=True) + + for i, result in enumerate(out['stochastic']): + output = result.outputs[0] + assert out['F'][i, 0] == pytest.approx(output.reduce(Margin(k=2.))) + assert out['F'][i, 0] == pytest.approx(output.mean + 2.*output.std) + + +def test_stochastic_output_statistics(): + out = _output(np.linspace(0., 10., 101)) + + assert out.mean == pytest.approx(5.) + assert out.var == pytest.approx(out.std**2) + assert out.quantile(.5) == pytest.approx(5., abs=.1) + assert out.prob_exceeds(5.) == pytest.approx(.5, abs=.02) + assert len(out.to_numpy()) == 101 + assert out.to_distribution.computeCDF(5.) == pytest.approx(.5, abs=.05) + + +def test_a_failed_sample_fails_the_design_point(): + # Any non-finite sample discards the design point: with a single failed evaluation the model is not + # trustworthy at that point, and SBArchOpt treats NaN as a hidden-constraint violation + problem = VectorizedProblem(n=50, fail=True) + out = problem.evaluate(np.array([[.5, .5]]), return_as_dictionary=True) + + values = out['stochastic'][0].outputs[0].to_numpy() + assert np.any(~np.isfinite(values)) + assert not np.isfinite(out['F'][0, 0]) + assert problem.get_failed_points(out)[0] + + +def test_pce_is_more_accurate_than_monte_carlo(): + # Both methods draw the same kind of LHS design, so this compares fitting an expansion against averaging + # over it. f = (u-x0)^2 + x1^2 is quadratic in u, so the expansion is essentially exact. + x = np.array([[1., 0.]]) + exact = .05**2 # E[(u-x)^2] at x = E[u] is Var[u] + + f_mc = VectorizedProblem(n=20, obj_scalar=[Mean()]).evaluate(x, return_as_dictionary=True)['F'][0, 0] + f_pce = VectorizedProblem(uq_method=PolynomialChaos(20, seed=42), obj_scalar=[Mean()]) \ + .evaluate(x, return_as_dictionary=True)['F'][0, 0] + + assert abs(f_pce - exact) < abs(f_mc - exact) + assert f_pce == pytest.approx(exact, abs=1e-4) + + +def test_pce_through_the_problem(): + # Regression: the problem called process_results without the parameter space, which PCE requires + problem = VectorizedProblem(uq_method=PolynomialChaos(50, seed=42), obj_scalar=[Mean()]) + x = np.array([[.5, .5]]) + out = problem.evaluate(x, return_as_dictionary=True) + + assert np.all(np.isfinite(out['F'])) + assert np.allclose(out['F'], problem.evaluate(x, return_as_dictionary=True)['F']) + + # Statistics come from the cheap metamodel, not from the 50 expensive evaluations + assert len(out['stochastic'][0].outputs[0].to_numpy()) == problem.uq_method.n_metamodel_samples + + +def test_pce_terms_and_sample_requirement(): + space = make_space(*[ot.Normal(0., 1.)]*3) + + assert PolynomialChaos(200, degree=8).n_terms(space) == 165 # degree 8 in 3 dimensions + assert PolynomialChaos(200, degree=8).n_terms(make_space(ot.Normal(0., 1.))) == 9 # ... in 1 dimension + + with pytest.raises(ValueError): + PolynomialChaos(10, degree=8).get_samples(space) # 165 terms cannot be fitted from 10 samples + + +def test_pce_provides_chaos_result_for_sensitivity_analysis(): + problem = VectorizedProblem(uq_method=PolynomialChaos(50, seed=42), obj_scalar=[Mean()]) + out = problem.evaluate(np.array([[.5, .5]]), return_as_dictionary=True) + + chaos_results = out['stochastic'][0].method_result + assert len(chaos_results) == 1 + assert isinstance(chaos_results[0], ot.FunctionalChaosResult) + + sobol = ot.FunctionalChaosSobolIndices(chaos_results[0]) + assert sobol.getSobolIndex(0) == pytest.approx(1., abs=1e-6) # a single parameter explains everything + + +def test_pce_falls_back_to_raw_samples_when_evaluations_fail(): + # A response with failed evaluations cannot be fitted; the design point then fails as usual + problem = VectorizedProblem(uq_method=PolynomialChaos(50, seed=42), obj_scalar=[Mean()], fail=True) + out = problem.evaluate(np.array([[.5, .5]]), return_as_dictionary=True) + + assert out['stochastic'][0].method_result[0] is None + assert not np.isfinite(out['F'][0, 0]) + + +def test_stochastic_rosenbrock(): + problem = StochasticRosenbrock(n_var=3, std=.05, n=500) + assert problem.n_var == 3 + assert problem.param_space.n_parameters == 2 + assert repr(problem) + + # At the optimum the expected objective equals the sum of the parameter variances + out = problem.evaluate(problem.pareto_set(), return_as_dictionary=True) + assert out['F'][0, 0] == pytest.approx(problem.pareto_front()[0, 0], abs=1e-3) + + # Moving away from the optimum makes it worse + f_off = problem.evaluate(np.array([[.5, .5, .5]]), return_as_dictionary=True)['F'] + assert f_off[0, 0] > out['F'][0, 0] + + +def test_stochastic_rosenbrock_with_pce(): + problem = StochasticRosenbrock(n_var=3, uq_method=PolynomialChaos(40, seed=42, degree=2)) + out = problem.evaluate(problem.pareto_set(), return_as_dictionary=True) + + assert out['F'][0, 0] == pytest.approx(problem.pareto_front()[0, 0], abs=1e-3) \ No newline at end of file diff --git a/sb_arch_opt/uncertainty.py b/sb_arch_opt/uncertainty.py new file mode 100644 index 0000000..5c5b6ca --- /dev/null +++ b/sb_arch_opt/uncertainty.py @@ -0,0 +1,346 @@ +""" +MIT License + +Copyright: (c) 2024, Deutsches Zentrum fuer Luft- und Raumfahrt e.V. +Contact: jasper.bussemaker@dlr.de + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" +from typing import List, Optional, Dict + +import openturns as ot +import numpy as np + +__all__ = ['Scalarization', 'Mean', 'Margin', 'Quantile', 'StochasticParameter', 'StochasticParameterSpace', + 'StochasticOutput', 'StochasticResults', 'UQMethod', 'MonteCarlo', 'PolynomialChaos'] + + + +class Scalarization: + """ + Parent class for all robust scalars etc. Margin, Mean, Quantile + """ + + def reduce(self, samples: ot.Sample) -> float: + """Reduce an (n_samples x 1) sample of one response to a single value""" + raise NotImplementedError + + +class Mean(Scalarization): + """The expected value of the response""" + + def reduce(self, samples: ot.Sample) -> float: + return float(samples.computeMean()[0]) + +class Margin(Scalarization): + """Assumes the response is Gaussian""" + + def __init__(self, k: float = 1.645, direction: int =-1): + self.k = k + self.direction = direction + + def reduce(self, samples: ot.Sample) -> float: + return float(samples.computeMean()[0] - self.k * np.sign(self.direction) * samples.computeStandardDeviation()[0]) + +class Quantile(Scalarization): + """The q quantile of the response for arbitrary distribution""" + + def __init__(self, q: float = 0.95): + if not 0. <= q <= 1.: + raise ValueError(f'Quantile should be between 0 and 1: {q}') + self.q = q + + def reduce(self, samples: ot.Sample) -> float: + return float(samples.computeQuantile(self.q)[0]) + + +class StochasticParameter: + """A stochastic parameter used to evaluate architecture objectives or constraints.""" + + def __init__(self, name, value: ot.DistributionImplementation): + self.name = name + self.value = value + self._sample = None + + @property + def sample(self) -> float: + return self._sample + + @sample.setter + def sample(self, value: float): + self._sample = value + + + + +class StochasticParameterSpace: + """The joint distribution of all stochastic parameters of a problem.""" + + def __init__(self, parameters: List[StochasticParameter]): + self._parameters = parameters + + @property + def n_parameters(self) -> int: + return len(self._parameters) + + @property + def parameter_names(self) -> List[str]: + return [parameter.name for parameter in self._parameters] + + @property + def joint_dist(self) -> ot.JointDistribution: + """ Joint distribution of independent variables is chosen """ + return ot.JointDistribution([parameter.value for parameter in self._parameters], + ot.IndependentCopula(self.n_parameters)) + + def param_realization(self, samples: np.ndarray, i_realization: int) -> List[StochasticParameter]: + for j, param in enumerate(self._parameters): + param.sample = samples[i_realization, j] + return self._parameters + + def get_random_samples(self, n_samples: int) -> np.ndarray: + """ Draw n samples of the stochastic parameters; returns an n x n_parameters matrix """ + result = self.joint_dist.getSample(n_samples) + return np.array(result) + + def get_lhs_samples(self, n_samples: int) -> np.ndarray: + """ Draw n samples of the stochastic parameters; returns an n x n_parameters matrix built with LHS""" + lhs = ot.LHSExperiment(self.joint_dist, n_samples) + result = lhs.generate() + return np.array(result) + + + +class StochasticOutput: + """The sampled values of one response (objective or constraint) for one design point.""" + + def __init__(self, output_samples: ot.Sample): + self.output_samples = output_samples # ot.Sample of shape (n, 1) + + @classmethod + def from_results(cls, results: ot.Sample, index: int) -> 'StochasticOutput': + """Extract one response's column from the full (n_samples, n_outputs) results of a single design point.""" + return cls(output_samples=results.getMarginal(index)) + + @property + def mean(self) -> float: + return self.output_samples.computeMean()[0] + + @property + def std(self) -> float: + return self.output_samples.computeStandardDeviation()[0] + + @property + def var(self) -> float: + return self.output_samples.computeVariance()[0] + + def quantile(self, q: float) -> float: + return self.output_samples.computeQuantile(q)[0] + + def prob_exceeds(self, threshold: float) -> float: + arr = self.to_numpy() + return float(np.mean(arr > threshold)) + + def margin(self, k: float = 1.645, direction: int =-1) -> float: + """mean + k*sigma - the Gaussian-assumption margin formulation.""" + return self.mean - np.sign(direction) * k * self.std + + @property + def to_distribution(self) -> ot.Distribution: + """Fit a continuous distribution if you need PDF/CDF rather than raw samples.""" + return ot.KernelSmoothing().build(self.output_samples) + + def to_numpy(self) -> np.ndarray: + return np.array(self.output_samples).flatten() + + def reduce(self, scalar: Scalarization) -> float: + """ + Reduce the sampled values to the single value the optimizer sees, by applying the given scalar. + """ + values = self.to_numpy() + + if values.size == 0 or not np.all(np.isfinite(values)): + return np.nan + samples = self.output_samples + + return scalar.reduce(samples) + + def __str__(self): + if self.std / self.mean < 1e-6: + return f'{self.mean:.4g}' + return f"(mean = {self.mean:.4g}, sigma = {self.std:.4g})" + + + +class StochasticResults: + """ + All sampled responses of a single design point. + + `method_result` optionally carries whatever the UQ method produced beyond the samples themselves; for polynomial + chaos that is the list of `ot.FunctionalChaosResult`s, from which for example Sobol indices can be obtained. + """ + + def __init__(self, outputs: List[StochasticOutput], method_result=None): + self.outputs = outputs + self.method_result = method_result + + +class UQMethod: + """ + Base class for an uncertainty propagation method. + """ + + def __init__(self, n_evaluations: int, seed: int = None): + if n_evaluations is None: + raise ValueError('n_evaluations must be specified: it is the number of expensive evaluations ' + 'per design point') + self.n_evaluations = n_evaluations + self.seed = seed + + self._samples: Optional[np.ndarray] = None + + def get_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: + if param_space is None: + raise ValueError('No parameter space to sample') + if self._samples is None: + if self.seed is not None: + ot.RandomGenerator.SetSeed(self.seed) + self._samples = self._draw_samples(param_space) + return self._samples + + def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: + """Draw the design of experiments in the parameter space; override to use a different design""" + return param_space.get_random_samples(self.n_evaluations) + + def resample(self): + """Draw a new design on the next evaluation""" + self._samples = None + + def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace) -> StochasticResults: + """ + Turn the responses of ONE design point (an n_samples x (n_obj+n_ieq_constr+n_eq_constr) matrix) into the + stochastic result object that contains list of stochastic outputs. + """ + raise NotImplementedError + + +class MonteCarlo(UQMethod): + """ + Monte Carlo uncertainty propagation + """ + + def __init__(self, n_evaluations: int = 100, seed: int = 42): + super().__init__(n_evaluations, seed) + + def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: + # Draw samples with LHS method + samples = param_space.get_lhs_samples(self.n_evaluations) + return samples + + def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace = None) -> StochasticResults: + results = np.asarray(results, dtype=float) + + sample = ot.Sample(results) + outputs = [StochasticOutput.from_results(sample, i) for i in range(results.shape[1])] + return StochasticResults(outputs) + + +class PolynomialChaos(UQMethod): + """ + Polynomial chaos expansion (PCE): a surrogate of each response as a function of the uncertain parameters is + fitted from n_evaluations evaluations of the expensive model, and the statistics are then taken from that + surrogate. + + The expensive model is evaluated `n_evaluations` times per design point, exactly like Monte Carlo, but the + statistics come from `n_metamodel_samples` evaluations of the cheap fitted expansion instead of from the + expensive ones. For a response that the expansion represents well this gives far more accurate quantiles and + tail statistics for the same number of expensive evaluations. + + The fitted `ot.FunctionalChaosResult` of each response is kept on the `StochasticResult` (`method_result`), so + Sobol sensitivity indices are available for free through `ot.FunctionalChaosSobolIndices`. + """ + + def __init__(self, n_evaluations: int, seed: int = None, degree: int = 3, n_metamodel_samples: int = 10000): + self.degree = degree + self.n_metamodel_samples = n_metamodel_samples + self._metamodel_input: Optional[ot.Sample] = None + super().__init__(n_evaluations, seed) + + def n_terms(self, param_space: StochasticParameterSpace) -> int: + """Number of terms in the expansion, i.e. the minimum number of samples needed to fit it""" + enumerate_function = ot.LinearEnumerateFunction(param_space.n_parameters) + return int(enumerate_function.getStrataCumulatedCardinal(self.degree)) + + def _validate(self, param_space: StochasticParameterSpace): + n_terms = self.n_terms(param_space) + if self.n_evaluations < n_terms: + raise ValueError(f'A degree-{self.degree} expansion in {param_space.n_parameters} ' + f'parameters has {n_terms} terms, so it needs at least that many samples to fit: ' + f'n_evaluations = {self.n_evaluations}') + + def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: + self._validate(param_space) + # Draw samples with LHS method + samples = param_space.get_lhs_samples(self.n_evaluations) + return samples + + def _get_metamodel_input(self, param_space: StochasticParameterSpace) -> ot.Sample: + if self._metamodel_input is None: + self._metamodel_input = param_space.joint_dist.getSample(self.n_metamodel_samples) + return self._metamodel_input + + def _build_algorithm(self, param_space: StochasticParameterSpace, input_sample: ot.Sample, output_sample: ot.Sample) -> ot.FunctionalChaosAlgorithm: + distribution = param_space.joint_dist + dimension = distribution.getDimension() + + polynomials = [ot.StandardDistributionPolynomialFactory(distribution.getMarginal(i)) + for i in range(dimension)] + enumerate_function = ot.LinearEnumerateFunction(dimension) + basis = ot.OrthogonalProductPolynomialFactory(polynomials, enumerate_function) + + adaptive_strategy = ot.FixedStrategy(basis, self.n_terms(param_space)) + projection_strategy = ot.LeastSquaresStrategy() + return ot.FunctionalChaosAlgorithm(input_sample, output_sample, distribution, + adaptive_strategy, projection_strategy) + + def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace) -> StochasticResults: + results = np.asarray(results, dtype=float) + + input_sample = ot.Sample(self.get_samples(param_space)) + metamodel_input = self._get_metamodel_input(param_space) + + outputs, chaos_results = [], [] + for i_out in range(results.shape[1]): + values = results[:, i_out] + + # A response with failed evaluations cannot be fitted: pass the raw samples through instead, so that + # the problem's nan_policy decides what happens to this design point + if not np.all(np.isfinite(values)): + samples = ot.Sample(values.reshape((-1, 1))) + chaos_results.append(None) + + else: + algorithm = self._build_algorithm(param_space, input_sample, ot.Sample(values.reshape((-1, 1)))) + algorithm.run() + chaos_result = algorithm.getResult() + chaos_results.append(chaos_result) + samples = chaos_result.getMetaModel()(metamodel_input) + + outputs.append(StochasticOutput(samples)) + return StochasticResults(outputs, method_result=chaos_results) \ No newline at end of file From d9155be23773158512ac217690b51ed911c569db Mon Sep 17 00:00:00 2001 From: bial_fr Date: Thu, 24 Sep 2026 14:36:11 +0200 Subject: [PATCH 2/5] Robust optimization 1.1 --- .github/workflows/tests.yml | 2 +- docs/algo/stochastic.md | 159 +++++++ docs/api/robust.md | 44 ++ docs/index.md | 6 + docs/tutorial_stochastic.ipynb | 424 +++++++++++++++++++ mkdocs.yml | 3 + sb_arch_opt/stochastic_problem.py | 100 ++++- sb_arch_opt/tests/conftest.py | 41 +- sb_arch_opt/tests/test_stochastic_problem.py | 192 ++++----- sb_arch_opt/uncertainty.py | 307 ++++++++------ setup.py | 3 + 11 files changed, 1013 insertions(+), 268 deletions(-) create mode 100644 docs/algo/stochastic.md create mode 100644 docs/api/robust.md create mode 100644 docs/tutorial_stochastic.ipynb diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d3b3f40..0fc6bcc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,7 +37,7 @@ jobs: python -m pip install --upgrade pip setuptools pip install -r requirements-tests.txt pip install -r requirements-assignment.txt - pip install -e .[arch_sbo,rocket] + pip install -e .[arch_sbo,rocket,uncertainty] - name: Test with pytest run: pytest -v sb_arch_opt --durations=10 diff --git a/docs/algo/stochastic.md b/docs/algo/stochastic.md new file mode 100644 index 0000000..9a0e833 --- /dev/null +++ b/docs/algo/stochastic.md @@ -0,0 +1,159 @@ +# Stochastic Optimization + +Architecture evaluations often depend on parameters that are not chosen by the optimizer: manufacturing tolerances, +operating conditions, material properties. Optimizing against their nominal values yields a design that is optimal on +paper, but that might perform poorly in reality. Stochastic optimization +instead optimizes a statistic of the response distribution, and therefore looks for a design that is insensitive to +the uncertainties present in the system. + +SBArchOpt supports this by evaluating each design point for many realizations of the uncertain parameters, and reducing +the resulting response distributions to the single values the optimizer sees. +The problem is defined with a `StochasticArchOptProblem`, a subclass of`ArchOptProblemBase`, so +a stochastic problem can be solved with pymoo, ArchSBO, or any of the connected optimization frameworks. +Each design points costs additional `n_evaluations` evaluations of the underlying model, +which makes Surrogate-Based Optimization (SBO) especially attractive to reduce the number of design points . + +To facilitate uncertainty quantification, sampling and distribution storage, SBArchOpt uses +the objects defined in [OpenTURNS](https://openturns.github.io/). + +Three things are defined on top of the design variables: + +- A `StochasticParameterSpace`: the joint distribution of the uncertain parameters, assumed to be independent. They + influence the evaluation but are not chosen by the optimizer, and therefore are NOT design variables. +- A `UQMethod` (`MonteCarlo` or `PolynomialChaos`): handles the uncertainty propagation, + and turning the responses into statistics. It is given a budget of `n_evaluations` expensive evaluations per design point. +- A `Scalarization` per objective and constraint (`Mean`, `Margin` or `Quantile`): scalarizes the output distribution to + a single number that optimizer sees. If not specified, responses default to `Mean`. + +## Installation + +``` +pip install sb-arch-opt[uncertainty] +``` + +## Usage + +[API Reference](../api/robust.md) + +A stochastic problem is a subclass of `StochasticArchOptProblem`, which extends `ArchOptProblemBase`. Compared to a +deterministic problem, evaluation is implemented in `_arch_evaluate_sample` instead of `_arch_evaluate`. This method +is called once per realization, receives it in `parameters`, and writes the responses for that +realization. The class automatically handles the loop over the sample realizations, output distribution construction +and the scalarization to `f`, `g` and `h` for the the optimizer. + +Since the result is an `ArchOptProblemBase`, the problem is then handed to an algorithm like any other. See the +[Stochastic Optimization Tutorial](../tutorial_stochastic.ipynb) for a worked example comparing the robust Pareto front +found by NSGA2 and by SBO to the deterministic one. + +### Implementing a Stochastic Problem + +Implement `_arch_evaluate_sample` rather than `_arch_evaluate`; everything else works as described in +[implementing an architecture optimization problem](../index.md). Uncertain parameters are stored in the parameter space, +and their samples are reused for all design vectors in `x`. + +```python +import openturns as ot +from pymoo.core.variable import Real +from sb_arch_opt.stochastic_problem import StochasticArchOptProblem +from sb_arch_opt.uncertainty import MonteCarlo, Mean, StochasticParameter, \ + StochasticParameterSpace + + +class MyStochasticProblem(StochasticArchOptProblem): + + def __init__(self): + # The uncertain parameters: not design variables + param_space = StochasticParameterSpace([ + StochasticParameter('u', ot.Normal(1., .05)), + ]) + + super().__init__( + [Real(bounds=(-2., 2.))], + param_space=param_space, + uq_method=MonteCarlo(n_evaluations=1000, seed=42), + n_obj=1, obj_scalar=[Mean()], + ) + + def _is_conditionally_active(self): + return [False] + + def _correct_x(self, x, is_active): + pass + + def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, + *args, **kwargs): + # Called once per realization; parameters contains one realization of the parameter space + f_out[:, 0] = (parameters[0] - x[:, 0])**2 +``` + +All design points share the same sample realizations (common random numbers) during the evaluation, +ensuring the outputs remain directly comparable. For reproducibility during development, +the UQ method can be run with a fixed seed so that identical realizations are generated across independent runs. + +A failed evaluation (NaN) in any realization fails the whole design point: the response has no distribution and the +optimizer sees NaN, which is how SBArchOpt treats hidden constraint violations elsewhere. + +### Uncertainty Quantification Methods + +Both methods draw samples from the parameter space using Latin Hypercube Sampling (LHS) +and use the same number of expensive model evaluations per design point. +They differ in how the statistics are obtained: Monte Carlo computes them directly from the sampled model outputs, +while PCE first fits a polynomial chaos expansion to those outputs and derives the statistics by performing +Monte Carlo on the trained cheap surrogate model. In general Monte Carlo requires more simulation code executions +to compute the statistics compared to PCE. More UQ methods can be implemented by creating a new subclass of `UQMethod`. + +- `MonteCarlo`: the statistics are computed from the `n_evaluations` responses directly. It is the simplest method to + implement, but slowest to converge. +- `PolynomialChaos`: a polynomial chaos expansion of each response as a function of the uncertain parameters is fitted + from the `n_evaluations` responses, and the statistics are then taken from `n_metamodel_samples` cheap evaluations. + Sampling the trained surrogate model is much cheaper than running the real simulation code, + which makes this method attractive for expensive evaluation codes. The PCE surrogate model is trained + with the least-squares method that requires at least `P` samples, but it is commonly recommended to provide + between `2P` and `3P`, where `P = (n+d)! / (n! d!)` is the number of polynomial terms for `n` parameters and polynomial + degree `d` (Blatman, 2009). + +### Scalarization + +Each objective and constraint is reduced to one value by the selected `Scalarization`. +The resulting `float` is passed to the chosen optimization algorithm. + +- `Mean`: the expected value of the response --> `min(E[F(x)])` +- `Margin`: `min(E[F(x)] + k*sigma[F(x)])`, assuming a Gaussian response. The default `k=1.645` is the one-sided 95% + interval of a normal distribution; use `direction=1` for a response that is maximized, so that the margin is + subtracted rather than added. +- `Quantile`: `min(F_q(x))`, the `q` quantile of the response. Makes no assumption about the shape of the distribution. + +Custom scalarizations are implemented by extending `Scalarization` and implementing `scalarize`, which receives the +whole `StochasticOutput` and can therefore use anything the fitted distribution offers. + +### Sampled Responses + +After each evaluation the fitted response distributions are available in the pymoo output dictionary as +`StochasticOutput` objects, in `out['f_stochastic']`, `out['g_stochastic']` and `out['h_stochastic']`. A response that +turns out not to depend on the uncertain parameters is a plain value instead, and a response that failed is NaN. + +```python +import numpy as np + +problem = ... # Subclass of StochasticArchOptProblem + +out = problem.evaluate(np.array([[.5]]), return_as_dictionary=True) +output = out['f_stochastic'][0, 0] + +print(output.mean, output.std, output.quantile(.95)) +``` + +For polynomial chaos each output also carries the fitted expansion in `method_results`, from which Sobol sensitivity +indices are available to see which uncertain parameters drive the spread of a response: + +```python +import openturns as ot + +sobol = ot.FunctionalChaosSobolIndices(output.method_results) +first_order_u0 = sobol.getSobolIndex(0) +``` + +## References +- Blatman, G. (2009). *Adaptive sparse polynomial chaos expansions for uncertainty + propagation and sensitivity analysis*. PhD thesis, Université Blaise Pascal, + Clermont-Ferrand, France. \ No newline at end of file diff --git a/docs/api/robust.md b/docs/api/robust.md new file mode 100644 index 0000000..0cd3152 --- /dev/null +++ b/docs/api/robust.md @@ -0,0 +1,44 @@ +# Stochastic Problem Definition + +Stochastic optimization uses [OpenTURNS](https://openturns.github.io/) for the uncertainty propagation, +which is an optional dependency: + +``` +pip install sb-arch-opt[uncertainty] +``` + +::: sb_arch_opt.stochastic_problem.StochasticArchOptProblem + handler: python + +::: sb_arch_opt.uncertainty.EvaluationOutput + handler: python + +::: sb_arch_opt.uncertainty.Scalarization + handler: python + +::: sb_arch_opt.uncertainty.Mean + handler: python + +::: sb_arch_opt.uncertainty.Margin + handler: python + +::: sb_arch_opt.uncertainty.Quantile + handler: python + +::: sb_arch_opt.uncertainty.StochasticParameter + handler: python + +::: sb_arch_opt.uncertainty.StochasticParameterSpace + handler: python + +::: sb_arch_opt.uncertainty.StochasticOutput + handler: python + +::: sb_arch_opt.uncertainty.UQMethod + handler: python + +::: sb_arch_opt.uncertainty.MonteCarlo + handler: python + +::: sb_arch_opt.uncertainty.PolynomialChaos + handler: python \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index d101b7b..8cc5a6d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -64,6 +64,7 @@ DOI: [10.2514/6.2024-4401](https://arc.aiaa.org/doi/10.2514/6.2024-4401) See also the tutorial(s): - [SBArchOpt Tutorial](tutorial.ipynb): optimization, implementing new problems +- [Stochastic Optimization Tutorial](tutorial_stochastic.ipynb): optimizing a problem with stochastic parameters ### Connecting to Optimization Frameworks @@ -86,6 +87,11 @@ Following proprietary frameworks are also integrated: Optimization framework dependencies (except pymoo) are always optional! Therefore, when installing SBArchOpt, the interfaces to optimization frameworks are probably not working yet. +### Stochastic Optimization + +Problems with uncertain parameters can be optimized for a statistic of the response distribution rather than for its +nominal value, see: [Stochastic Optimization](algo/stochastic.md) + ### Test Problems For an overview of the available test problems and how to use them, see: diff --git a/docs/tutorial_stochastic.ipynb b/docs/tutorial_stochastic.ipynb new file mode 100644 index 0000000..76a3f20 --- /dev/null +++ b/docs/tutorial_stochastic.ipynb @@ -0,0 +1,424 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Stochastic Optimization Tutorial\n", + "\n", + "Try it out yourself:\n", + "[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/jbussemaker/SBArchOpt/HEAD?labpath=docs%2Ftutorial_stochastic.ipynb)\n", + "\n", + "This tutorial takes `MDMOHimmelblau`, the mixed-discrete multi-objective Himmelblau test problem, and turns it into a stochastic problem by assuming that the continuous design variable cannot be measured accurately. This uncertainty is modeled by adding a perturbation to the variable, represented by a zero-mean Gaussian distribution. The tutorial then compares the robust Pareto fronts found by NSGA2 and SBO with the deterministic one. For more background information, refer to the [documentation](../algo/stochastic).\n", + "\n", + "**Note**: make sure you have installed the `uncertainty` and `arch_sbo` dependencies:\n", + "```\n", + "pip install sb-arch-opt[uncertainty,arch_sbo]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "end_time": "2026-09-23T13:01:12.136622700Z", + "start_time": "2026-09-23T13:01:12.092456200Z" + }, + "execution": { + "iopub.execute_input": "2026-09-24T12:05:33.295694Z", + "iopub.status.busy": "2026-09-24T12:05:33.294696Z", + "iopub.status.idle": "2026-09-24T12:05:33.307190Z", + "shell.execute_reply": "2026-09-24T12:05:33.304172Z" + } + }, + "outputs": [], + "source": [ + "import warnings\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## The Stochastic Problem Class\n", + "\n", + "Stochastic optimization problems in SBArchOpt are defined by extending the `StochasticArchOptProblem` class, which itself extends `ArchOptProblemBase`. Three things are defined on top of the design variables:\n", + "- A `StochasticParameterSpace`: the joint distribution of the uncertain parameters, assumed to be independent. They\n", + " influence the evaluation but are not chosen by the optimizer, and therefore are NOT design variables.\n", + "- A `UQMethod` (`MonteCarlo` or `PolynomialChaos`): handles the uncertainty propagation,\n", + " and turning the responses into statistics. It is given a budget of `n_evaluations` expensive evaluations per design point.\n", + "- A `Scalarization` per objective and constraint (`Mean`, `Margin` or `Quantile`): scalarizes the output distribution to\n", + " a single number that optimizer sees. If not specified, responses default to `Mean`.\n", + "\n", + "Evaluation is implemented in `_arch_evaluate_sample` instead of `_arch_evaluate`. This function is called once per realization of the uncertain parameters, receives it in `parameters`, and writes the responses for that realization only.\n", + "\n", + "Below, `MDMOHimmelblau` is wrapped in such a class. Its continuous design variable is realized with a normally-distributed deviation from its nominal value, and the underlying problem is then evaluated at the realized design vector. Since the design space of the underlying problem is reused, the design variables and the correction of design vectors are inherited as-is and `_correct_x` does not need to be implemented." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "end_time": "2026-09-24T09:25:09.783638200Z", + "start_time": "2026-09-24T09:25:09.763161600Z" + }, + "execution": { + "iopub.execute_input": "2026-09-24T12:05:33.312192Z", + "iopub.status.busy": "2026-09-24T12:05:33.312192Z", + "iopub.status.idle": "2026-09-24T12:05:34.996078Z", + "shell.execute_reply": "2026-09-24T12:05:34.996078Z" + } + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "import openturns as ot\n", + "from sb_arch_opt.problems.md_mo import MDMOHimmelblau\n", + "from sb_arch_opt.stochastic_problem import StochasticArchOptProblem\n", + "from sb_arch_opt.uncertainty import PolynomialChaos, StochasticParameter, StochasticParameterSpace\n", + "\n", + "\n", + "class StochasticMDMOHimmelblau(StochasticArchOptProblem):\n", + " \"\"\"MDMOHimmelblau where the continuous design variable is realized with a manufacturing tolerance\"\"\"\n", + "\n", + " def __init__(self, std=.3, n=10, seed=42, obj_scalar=None):\n", + " self._problem = MDMOHimmelblau()\n", + " self.std = std\n", + "\n", + " # One uncertain parameter per continuous design variable: the deviation from its nominal value\n", + " self._i_cont = np.where(~self._problem.is_discrete_mask)[0]\n", + " param_space = StochasticParameterSpace([\n", + " StochasticParameter(f'dx{i}', ot.Normal(0., std)) for i in self._i_cont])\n", + "\n", + " super().__init__(\n", + " self._problem.design_space, # Same design space, so the same variables and correction\n", + " param_space=param_space,\n", + " uq_method=PolynomialChaos(n_evaluations=n, seed=seed, degree=3, n_metamodel_samples=1000),\n", + " n_obj=self._problem.n_obj, obj_scalar=obj_scalar,\n", + " )\n", + "\n", + " def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, *args, **kwargs):\n", + " # Called once per realization: apply the deviations and evaluate the underlying problem\n", + " x_realized = x.copy()\n", + " x_realized[:, self._i_cont] += parameters\n", + "\n", + " f_out[:, :] = self._problem.evaluate(x_realized, return_as_dictionary=True)['F']\n", + "\n", + " def __repr__(self):\n", + " return f'{self.__class__.__name__}(std={self.std}, n={self.uq_method.n_evaluations})'" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "end_time": "2026-09-24T09:25:12.218295500Z", + "start_time": "2026-09-24T09:25:12.171654800Z" + }, + "execution": { + "iopub.execute_input": "2026-09-24T12:05:34.998092Z", + "iopub.status.busy": "2026-09-24T12:05:34.998092Z", + "iopub.status.idle": "2026-09-24T12:05:35.001389Z", + "shell.execute_reply": "2026-09-24T12:05:35.001389Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "problem: StochasticMDMOHimmelblau(std=0.3, n=12)\n", + "n_discr: 1\n", + "n_cont : 1\n", + "n_obj : 2\n", + "n_con : 0\n", + "MD : True\n", + "MO : True\n", + "HIER : False\n", + "n_valid_discr: 10\n", + "imp_ratio : 1.00 (discr.: 1.00; cont.: 1.00)\n", + "corr_ratio : 1.00 (discr.: 1.00; cont.: 1.00; fraction of imp_ratio: 0.0%)\n", + "stochastic : True\n", + "n_params : 1\n", + "uq_method : Polynomial Chaos Expansion\n", + "n_evaluations: 12\n", + "obj : [Mean, Mean]\n", + "ieq_constr : []\n", + "eq_constr : []\n" + ] + } + ], + "source": [ + "# Instantiate the problem class and print some statistics\n", + "# Sample size for degree d PCE regression: N = 2P to 3P, where\n", + "# P = (n+d)! / (n! d!) is the number of polynomial terms for n parameters.\n", + "problem = StochasticMDMOHimmelblau(std=.3, n=12)\n", + "problem.print_stats()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The resulting problem has the same design variables and objectives as the deterministic one, only the evaluation has changed. Note that one design point now costs 12 evaluations of the underlying problem to train the PCE surrogate. Then the cheap PCE model is evaluated with 1000 samples to construct the distribution. Note that the uncertainty propagation requires overall more model evaluation which makes Surrogate-Based Optimization (SBO) especially attractive for stochastic problems." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Solving with NSGA2\n", + "\n", + "The stochastic problem is an `ArchOptProblemBase` subclass, so the algorithms in SBArchOpt can be used without modification. The code below runs the pre-configured NSGA2 and compares the resulting robust Pareto front to the known deterministic Pareto front of `MDMOHimmelblau`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "start_time": "2026-09-24T09:25:16.824124100Z" + }, + "execution": { + "iopub.execute_input": "2026-09-24T12:05:35.031443Z", + "iopub.status.busy": "2026-09-24T12:05:35.031443Z", + "iopub.status.idle": "2026-09-24T12:06:07.456866Z", + "shell.execute_reply": "2026-09-24T12:06:07.456866Z" + } + }, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjIsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvgI3uAAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAW8FJREFUeJzt3QucTPX/P/D3ui2LdS2bS24pJKQbFekiIipJl1+F+pZCqHwL1Zf+KlQqXaivilBKVCh0F+WSXHJLIclt+7ruuqzrzv/x+syecebsnDPnzJ6ZnZnzej4e+1gzc2bmzOyY8z6fz/v9/qT4fD6fEBEREcVIkVg9ERERERGDDyIiIoo5jnwQERFRTDH4ICIiophi8EFEREQxxeCDiIiIYorBBxEREcUUgw8iIiKKqWISh3Jzc2XHjh1StmxZSUlJKezdISIiIhvQt/TAgQNStWpVKVKkSGIFHwg8atSoUdi7QURERBHYunWrVK9ePbGCD4x4aDufnp5e2LtDRERENmRnZ6vBA+04nlDBhzbVgsCDwQcREVFiCZcywYRTIiIiiikGH0RERBRTDD6IiIgopuIy54OITjl58qQcP36cbwkRFbrixYtL0aJFC/w4DD6I4rhePjMzU/bv31/Yu0JEFFC+fHnJyMgoUB8uBh9EcUoLPE4//XRJS0tjwz0iKvQTosOHD8v//vc/dfmMM86I+LEYfBDF6VSLFnhUqlSpsHeHiEgpVaqU+o0ABN9PkU7BMOGUKA5pOR4Y8SAiiifa91JBctEYfBDFMa5tRETJ+L3E4MOu3JMimxeIrJ7m/43LROQJs2fPlnvuuSfq93Hz/sni999/lwcffFA6dOggb7/9dmHvDrmEOR92rJspMvdxkewdp65LryrSbqRIw05u/S2IEt6+ffvk9ttvD5wdlS5dWqpVqyaXX365dOrUSVJTUx093pYtW6Rnz54yZcoUqVChghSWv//+W+bPnx+1+8yaNUtmzpwp48aNK9Bz6v3www8yfPjwwN8C1QmXXXaZdO/eXYoVi+5X/3fffacChQ8++KDAj9W+fXu5+uqrpU+fPlKnTh2JtmeffVYt6/HQQw9F/bm8jMGHncBj6t3I8w2+Pnun//quExmAEOU5evSofPnll/LMM8/IBRdcIDk5OerM9amnnpLBgwerg2z9+vVtv19YmhuPh8ctTDjrPvfcc6N2HwRZCxYsKPBz6u3cuVO9dwhqEGxs3LhR/Q3mzJkj06dPl2ivTI4ApKCQ1Pjnn3/KF1984ehzUxArVqyQypUrx+S5vIzBhxVMrWDEwxh4KLguRWTuQJH6HUSKFLzpClE0nMz1yc+b98r/DhyR08uWlItrV5SiRQo+Z2sFgUe7du0Cl/v3768ud+7cWVavXh3IkEfC2rvvvqsOVDhAtmzZUu677z51O0ZRHnjgAbXdHXfcISVKlJBLL71U/vOf/1jeDzZs2KDOXEePHi1jxoxRB7ChQ4fKX3/9pe6D/fj000/VQbJ169bqrBrXT5o0SY4cOSI33XST3HbbbYH9xz5PmzZNPQ/g4I3t7777bpk8ebIqi8aoAp5T2wfjfRBAYWTjp59+UiNAOKPv2rWrGqHAPm7bti3wnuF14/Xq7w+7d++WsWPHqgMkyhwxHdGoUSPLv0WbNm2kZMmS6t84o8fIB1YMf/jhh+XgwYNqf7EKKfblqquuClqdFNdhJODjjz+WdevWqf3CfqMSC/u8dOlSqVixolx//fXqPYNly5bJCy+8oP5+2uu588471Q+e99VXX5U//vhDVUrce++90rx585D7/eOPP8oTTzyh/n3//ferJMfXXntN/S0xotKvXz+1Dwiy8DfAyNi8efNk4sSJsmfPHhW4YZsqVaoEHnPIkCFStWpV1acCgdmxY8fklltukRtuuEHdjsfH8+K91/YdzxGLERevYfBhZcvC4KmWfHwi2dv929U+9QVBFC/mrtkpT89aJzuzjgSuO6NcSRnSsaG0axR5jb5TOPhhNAQHUpzh44CPcmIctNA3AAcXHARffvll+eqrr+STTz5RUzZ33XWXOlgjsMDBBQeScPeDrKwsdXDBgRcHIBxIatasKd9//72MHz9elixZog6+OEAOHDhQPvvsM/V4ffv2lX/++UcdoPH8HTt2DDkFsnnzZpkwYYJ6HNznxIkT6kCJ+2pTHcb7YD/wWv7973+rgAnPiTP7Ll26yJVXXqlGKBCkQYMGDdQIhf7+CE5woD777LOlR48eKnDAfmK70047zdbfAe8B7Nq1SwUuCOLwfiJQQvCA4AgBB+DAjPdw+fLl0qtXLzX9dd5558nevXvVfjRr1kxNseE9xH7jMRAY4jnatm2rAg3t9Zx11lnqvUFQigASweTPP/8srVq1UqMa+DsZ1atXT+W84D3A60Wwhb8/grWPPvpI3f/RRx+V6tWrq78VAkLsz+OPPy7XXnutvPPOO3LRRRfJmjVrAqujIzB65ZVX1PPdeuutKgjC+//555+rfcb0ztSpU4OmXey+t+SQLw5lZWVhWEH9LlSrPvb5hqSH/8F2RC7KycnxrVu3Tv2O1JzVO3y1Hv/cV9PwUyvvB7e7befOner/7pw5c/LdduTIEXXba6+9pi5PnjzZV6NGjaDXuGvXLl/x4sV9v/zyi7q8evVqdR88rsbO/ZYuXaruN2nSpKB9eOGFF3ypqam+zMzMwHX/+te/fKVLl/bt2bMncN1tt93mu/vuuwOXx44d66tbt67l44wePdpXp04d0/vg3xMnTgzan/3796vfeE/OOeecoNuM9+/evbuvSZMmvpMnTwa9p0ePHvWFMmXKFPUeaO9Tbm6ur2fPnr4yZcr4Dh48mG/7l156yXfxxRcHvae4/7Bhw4K2GzBggK9NmzZB1/3www/q/T98+LC6jPe9SpUqQdv069fP16hRI7UfGuwPXpOZzZs3q33YsGFD4Lpx48ap6/DZ0KtZs6bviSeeCFzG+4LPyZAhQwLXdejQwde0adOgfbjpppt899xzT+DyzTffrPaLIvt+snv85siHlTJV3N2OKIZTLRjxsJgwVLe3aZgR9SkYDaYaMEqBEQb4+uuv1dk1zjzRORE/gG3Wrl2rzpJDcXK/K664It/9MYSuH4o/88wz1WgCpg/012Fqw4rxcWrXrq2mAMxgtGDEiBFqPzG9gQTQcuXKiV3ffvutmvYoUuRUkaKdBF4k+uI+mK7AqAVGbDBSgMsY6UAuCHJrMBqCaSkj43uI9x9/Q4w+ae8//h4YRcF0V+PGjUPux+LFi9V99GWaN954o/z3v/9VU13a1JAd+Fvpp5sw7YW8GW2kCjB1gikijE7ptWjRImgf8Hf77bffbD83uYPBh5Ual4ikFBHx5Zpvk1LUvx1RHEGOh36qxQiHa9yO7VrUjU0HVQy7Y4gfB11tagRD68i30MM0hlWipZP7lS1bNt/9cVDSw4Eo1HW5uRb/700ex+o+mAbAgRYHf0wX4eD51ltvSZMmTcQOTLNEUvGD6QPsK3IskLSJDpWYNsK0CQ7+mG5BDgQO0iNHjsx3f+N7iPcf0yVaVZNmwIABgWmdUDA9Ywy2cBnBC25z0qrbuE+4v/Z4xsdHwKUXyd+a3Mfgw8rWJdaBB/hO+rdjzgfFESSXurmdG5CPgTNw7UwaZ5yYj8dcu1nTolDX27lfPMIoBQIB/KAKCMmqyLtYuHChrdeB142kT6f0Caca5DggyRSBkMbu2T/2A4GQPqHY7t8NIyN6yLlAMKQFpJGqVauWek48vr4qBo+P53UikT5TiYxNxqwc/Mfd7YhiBFUtbm5XUCixRWInRiu0s2MkEWKo/LnnngtshzNQJISiWkGf7IdhdY2d+8UjVN4g6AAccJEoiZEg7XVi2sPqDByVIXiNSJrUIFkzkteMyhE8HypaYPv27aoKxQ6M2iBZFgmyGkzDoFJEg9eDihh9iTSSh5HMqQU5GEF56aWXVBVMQQ/4eD9vvvlmef755wPTeqjEQZCFIM8J7Lv+80bRwZEPK8z5oASFclpUtWRmHQmZ94Gv+oxy/rLbaHjyySdVVQHm8nH2iYMLrnvkkUcC26By4sMPP1R5DGhIhWZkOHNFXoA2pI+pAszjY+4e26OcFRUV4e4Xj1AmizP0unXrqpEDHPxRtQGozkAeBl4jRiS0EmM9jJIgTwMN2zC9hMdArgoqNJzC+4QpINwfU1gYUbnkkkvU1Fg4KEFGJQ8eA6MKZcqUUbkivXv3DmyDqiYEmXg9yI1BgPF///d/qtLpwgsvVFNN+Judc845geogN4I7fFbwfNivX3/9VVU04bPjBCpxMKqDzxqmd1hqGx0pyDqVOINoHHN1iIy1EqlC6/PxSiN/QzGzr3B0Ou2/mn0+yFU4aGNeHl+iThLxjGW2D05erv6t//Rq55hj72zmerktEg+15lIIOLQzfLwOs7Nb3AcHCpwl46AaKq8BZZw4G0Xzp/PPPz/s/fAdgumMa665JqibJw6S6O2Bck8NDugoedX3m0DAhMfAgRJQNor7aj03Qj0OgomVK1cGykaN9wEEDHgtmIJBzoc+/wB/81WrVqn8BUwdYIrKeH/teZBYi/cVJaxm8H5hfxDY6JNUNRhlwfuH71nsCw4F+v1HAimSXLWDsBFeC+6P14D337gIIv4+eD0YmUHApe0r3hcEHkjWDddEDSNFKK1Fjon2+Cg53rRpU8hkYrwGPCfyPPAeGvNIUDaMx9FPzaAJHl6LPlEZ+4wRGlxv9vq97IjF95Pd4zeDD9sdTk2+wtnhlOI0+IinPh9ElDyOuBB8cNolHKzdggAj5NouI9haneIaAgyU08a6wykRkRUGH3YDELRQRydTJJciF6TmpZxqoYSAQCNW5bRERHYw+LALa7ewnJaIiKjAWGpLREREMcWRD6fVL5x6ISIiKhAGH06qXkImnY5k0ikREZEDnHZxUm6rDzwA/T9wPW4nIiIiWxh82JlqwYiH6fqgaKYw0L8dERERhcXgIxzkeBhHPIL4RLK3+7cjItJZtGiRTJs2LW7ekzlz5siKFSskUaGj6uTJk117vF9++UWmTJkiieKXGOzvjz/+qLrKRhtzPsLh4nJEtmFRL6wZgs6G99xzT9BtWIgM3RDR7luDdt5oh120aFG56KKL1FojoaDd+c8//6zaXZ955pmqxbrZEuzz589XbbSxloi2MJ2xrTi+YNFWG49x1VVX5WsN7hYs/Ibn6tKli0QL3hd0m7z11lstt8N6LFjcDe3dNVj8De8XWrY3aNAgqF28HlrV4++EtXauu+461TY/WttYwdowb775plorxg3ffPONCg7jeU2ggu6v3c+HBt1JsSwB/n+UL19eooUjH+FwcTki29BaGYt5YQXWr7/+Oug2BCUffPCB+jdWc8UB+corr5Qvv/xSfal27txZOnXqFFjpFbDmChYGQ2Dy6aefqgMXDj44SJqtVooF2AYMGKAWnTNCUIJ1TJ566in1pTx06FC15giCoET11Vdfycsvvxx2u2eeeUa951rQhgXTmjZtqt4nBAVYFRaBmLbyrgZ/SyzY9tNPP6n3q3Hjxmpdm2hsE2v4XGEhuURxUQT7a/fzocFigFjTCAtDRhNHPsJBJ9NSFURy9plvU6qifzuieFQIJeJYMOzxxx9XZ1ChFpTD0PGsWbPUAmMYydCPFGhrXZ44cUKdHeP+WPytUqVTXVoRoCAYCTXNgUXHEFy8++67MnDgwKDnL168uMyYMUOt5Ap4LgQ8CFhw31AQACFo6tmzp8ybN0/tC1bQxaqtWHwNwRNGFfA62rZtq57DCAuVYYQBK9viPcGaGBosxIcDfocOHQLXYWoEi8fpz/BxGQETVpHFomoYPcAoBvYbi8hpBwuMLDVs2DBfUIjpCgR5GrwHuL+2aBpeJ0Y/Xn31VfW3AyxJP2HCBLU/CBawiB9Wv8XtkyZNcnWbUA4dOqQ+J1ik7uKLL7YcEcH7k5GRIa1bt853xh7qvdPO8o0jaBhdw3Pib4vnxL7ifdL+Fni/t2/frv6O2t8UKw2bjdrp74Ng2+xzAOE+T8b9DbcvVp8Ps/cE8FofffRRtYJ0qEUJ3cCRD1u4DgYlKFRiYWXm964XmX6v/zcuR7lC68knn1RTJdpIhxEChIoVKwYFHoADsLYK7SeffKJGKjDSoQ88ANM0oaYycBaPEZT+/furs2rj3DXO6rTAAxCY4ICAIWYzOBBgNAfbvfTSS+p1IVjAyqlYCfWRRx5Rc/HYBpdxvR6GvHEmOXXqVPWacIBHkKXB9ePHjw+6D/Z7xIgRgcsjR45Uq6siUPnss8/UwQIHmwMHDqhVcHGAxAq4+MHB0whBE+gP4Fi9Vr9aKw4+CBr17wX2rUWLFipgAKzGi+k0vA4Eh25uY4SpIIzMDBs2TO0/PhsIYIzTfHgd//rXv2TZsmUyduxY9f7i3+HeO0AwNmrUqMC2OEhjHzFKhOdEkIkRG/3fAn87HJjxej7++GMVqOA+yKcxo93H6nOw18bnybi/4fbF7PNh9Z4AAridO3dGNT+IIx/h4IwxJ/jLJB/cju3Yfp3iSWBFZl/oEvEorsiMM1B8eSIIueWWW4KWjgccyJ9++mnp1q2b9OjRQw0nly5dOmib77//XgUd2rL24eBLFV/s+ALGGWLXrl3lnXfeUV+kVjASYuc5rr76ann22WcDlxHg4MCJL2jsO54fj4ODpX6YG8ELzvy1kY3BgwdLnz591BloqFGSUEaPHq0OrNpcP0YytmzZooIpTEvh8a2GyTFdhWXttcAuFJxB46B32223Ba5bt26dCgD0sBw9Dvp4fjymW9sYIQDAWTnO3JErhIMwztgrV64c2AbTNxgtwFm89tpwto6RLJzVW713oeBvh8+O9pw4cCMgQ6Csh8AWgTHef+jVq5c8//zzaqTOTLjPwf/7f//P1ufJyGpfMD0Z6vNx4403Wr4nGF3B+4DpSARA0cCRj3CYcEqJKA5KxB977DE1bI7cAqNWrVqpMy6MgGDIGF90OBPDdRoMIVevXj3ofjjI4EtU+9HnDHz44YfqSxNncXD//ffL9OnTVVKlGRzgcJDCl3U4CJSMCbQ4K9aCJhwocRnX69WoUSNoSqV3797qDBQBgV044H777bfqYAhI6NUONnbgwI332AzOjJGQiOF6vAYNzpyNUxgVKlQI3ObmNqGCQgSm2pLtCACMiZaYvsOIDUbHXnvtNTVlhMfDyAeWfXf63uEgrX9O7KM+GNPg/vrHQPCMz7KVcJ+DmTY/T27si533BJ8XTBdGC4OPcNIqu7sdkUdKxDGkj5EPHOCzsrLy3Y5cC1SC4DYMJ1erVk1uuukmlSCnfflhGFwPBxZ8YaPkEiMryL/QT7ngMXCmi8BkyZIl6mz4/fffD7l/b7zxhjqrxHC18aw8FP2cOGzbti1fcIQDzNatW4Ouwz7pVa1aVc2jY6TBLuRr4MwU98XBZfjw4fkSQ63g4GJ2kEc+BZJN//nnH5k9e7aaEtGgCsh4P5wla7e5uY0RAkvje2d8v/EeIsDduHGjOuDi84A8noceekiNiDh970I9p/Gy9n7qYeQC76OVcJ+DbTY/T27si533JFTA6CZOu4QTIlmuQNsReWjE7oEHHlDBAOaYzeBMD1MjGLFAwIAzPQxFYyQE0yZIStXyNJo3b65+cLDRz/9jRARnu9rZpAaPi8fA9XoYcsbcOgIPzOtHokqVKiovQQ9Jm5hy0jNugxEdlPvi/oAASV/hAzig6mEeH9VDuB7TUdh35JKggsgOTFcgXwXPq08gxEEK02Lr169XOQ44GOlhJMR4Fo3LOMDVqlXL1W1CBXuh3l89jIa0bNlSJRabcfLe4Tnx99Ez7kOkwn0Oqtj8PLkh3HuCkTKMihgTl93EkY9wDu1ydzsiD5WII9cDowvGKZLffvstX2IkzrRwNqx9GWO4GwcmBA6hkij1MOqB4EU/JYOfcePGqVESzIlr8AWLURPkh2D0JVKYLsKoCg4ggAACl5EbYjzIavkH2lkn+o9ow9w4u0VOhFblg99z584NbI/LuF0L1BAsIalWuw5nvsidsIIcG5zZ6kuKMTKAvBgEbjgAGc+64YYbblCJiMhXALxW7D/yCbQ8Hre2McL7iKk07X3B1JCxYRv+fm+99Va+UZU1a9bYeu+MUGr80UcfBZ4TwRmSQ90Q7nNwjc3Pk1PGz4ed9wTVQxj1QEVStHDkI0G+xIkcQTktFj5EcmnIvI8U/+0xKBHHPD0y9BEAaNUOKAFs166dChjOOeccdQaGUQiceSJZENCACln7+GJEZQC+JBGMIEBBOSL6cyDHAwcIfJEPGTIk33MjkMEXKEY/mjVrph4PozEYWcGZnj4JD0P1qKKxC0EVqkdwYMcBAlNHGDpHnokeDjAYXUDTM+SfICBCjoI23YCyxueee04doLGvOCNFoKY14MLBAu9hnTp1VPIfzkgRQCG/ATBChGAKOTYYuQhVaot9wJQWDmZ4H7TXi1EmnPXq9xnvMRIStQDwvffeU68R/SWWLl2qDlLo1aFxaxsj/D0xJYDXg9ENJBJrB2YNcnXw3jdp0kTlrGBUB1N5SGBFqXW49y7Uc+JvihJXPCeqSRCkaTkgBRHuczDM5ufJKePnA0EO9sHqPUEuTffu3W0nREeCwUcCfYkT2YY+HlhxWVW7pBg+u3lThO1GuN7vA2dS/fr1CzqLRjkrkk7xhaaVeuKMG1/wSPDDKAi+gFHOiDNZ/RceKiIQqOCAjAMWhoMxd45tcZaKgw2mYPBliryFUJ544glVxQF4nr59+6p/66dnQDvbDRXA4DXpcyEArxFn2Cgnxvw5XhP2Q0ukBFQbYNgcB1C8VsAoA3oxaPB6MDqDM24EUqgCgsWLF6vfeI0I3HDwxdQSDmI4wOKACziAYMoECYTYD7NRIlSB4D1DDxSc1aLZGt4LVFjo3wt911E8N/JAsG/YR5S1YrpLPxXg1jZGODgiGXPixInqfUGlEf5++oAFiZP42yJRGVUiSNJEIIcDrp33DsGNfuQFAS32D71HEHSgAgXb429m/JvqIThGgrMVBIQ4yJt9Dqrb+DwZ99fOvhg/HxgFsXpPsA3+vzlJiI5Eis/sf1whwpkNks2QiGZMpinckkUJ/SUexZJF8iZk6uPMHE2ICnTWhc8uql70yafp1fyBBz+znoPpKRyctIMz5f9/hykcrS0/AjMcvBEsvfjiixG/XUi8xgFe67cSzzA6iPcAwU8k3092j98c+bADX9IIMPJ9iVfllzjF/2e3foeYdzil+IRmXGQOeRaY8sAIEUYcUO6LkSRMTXnFdRa9StzE4MPpl/jmBSJbfvQPgKCpWK1Tw2ZEcQmBBhvgEdmaNsT0BBJbUQKLaSmMABR04cFQ0yNex+DDifVfBI9+LHghb/RjJIewiYiSAKZctKRnt6DLKHk0+Mg5dlKem71O/tpzWGpVSpPB7RtKqRJFE6JVNRERUTLxRPBx38Sl8vW6U81pFmwQmbT4b2nT8HQZd/dFLrSqTvG3qsa0DOfSiYiIvN1kzBh46OF63J4IrarJm4x9DYiIkuF7qViyT7WYBR4a3I7tLKdg4qRVNXkHavnRowDNpjAHjcvol0FEVFjQmQM9V9AGHt9PZt1poxJ8oNHPDz/8oBqwoOEJuqXpIVMYdcJ6aFqD7m2xhhwPu9sNu9FihUh2OaUYw39s1NDv3LkzqC05EVFhQ/XPmWeeGbROUFSDD7T/xSqTaDuLNsRok4ulsdEWWTsrW7RokbqM1r0aN1rTRgLJpa5sxy6nVAhwVoH/4Gh0ZFx4jIioMODYj8UQCzoSW8xplzaMdmjuuusutSgO1lnAOg0arLcwYMAAKWyoakFyqZ3t7LWqvstkA19UWlUT4T842o1Hc40FIqJYczRmog88tF71ODvD4jfGZYCxPgGW0db3xI81lNO6uR0REREVcrULljtG7odxnYCKFSuqpBQsn4yFotAhzmoJGSyVjH7w+h83IIkU5bRWcHvYfh+BUlszeaW22I6IiIiiE3ysXbtWevXqpaZXsFCRBstDY5VBrED4xhtvyMKFC9WKg1gl0Mzw4cPVQjTaT40aNcQt6ONhFoDY7vPBUlsiIqLCXdX2jz/+kNatW6uV/saPHx8247VFixZq+eZx48aZjnzgR4ORDwQgbq5qW6AOp6uniUy/N/x2N78jcl6XAu8rERFRIoraqrYbNmyQK6+8UpXY2gk8ANn6+uDCKDU1Vf1EEwINy3JaKyy1JSIiKpxpl40bN6oRDyw5PGHChJCBx7x584IuL1iwQJYvX65GSRKWVmobzuE9sdgbIiIi70y71KlTR3U2u++++4ICj2uvvVb9AHp/IEg5//zzZffu3fLFF1/IPffcI6+//rrthiR2h21ias1nItO6WW+TXk2k/2qW3BIRkSdlR2PapW/fvmoKxahMmTKBf3/wwQeyevVqWbJkieqChmTSs88+WxJe6Urht9HWd6ndMhZ7RERElJAcBR/9+/e3tR0aj+EnqXB9FyIiIlck/aq2rmHSKRERkSsYfDhJOi1VwXqbUhX92xEREZF7pbaJ7GSuT37evFf+d+CInF62pFxcu6IULeJkcZww2548VtBdJCIiSnqeCT7mrtkpT89aJzuzjgSuO6NcSRnSsaG0a3RG+AdAImnOXuttjh0Umf+iSGurVuxERETeVsQrgceDk5cHBR6QmXVEXY/bXUs4XTKWa7wQERF5OfjAVAtGPEI1M9Guw+3YzpWE05x9/lESIiIi8mbwgRwP44iHHkIO3I7tCpxw6nSUhIiIyIOSPvhAcqkr2xUpKnLJg/ae1O4oCRERkQclffCBqhbXtrv84fAVLylFRWpcYnPviIiIvCfpgw+U06KqxSxkwPW4HduFtXWJLlPEhO9k3nZERETkyeADfTxQTmsFt9vq98EW60RERAWW9MEHoI/H/a1qizG+wGVcb6vPB7DFOhERUYF5IvhAH4//zt8sxmpan0/U9bb6fGgVL+lVLfI+UkTSq7HFOhERkZeDD9f6fGgVL+1G5l0wBiB5l9uN8G9HRERE3gw+XOvzoWnYSaTrRJF0w1QNeoC0HiRSv0MB95iIiCi5JX3w4VqfD2MA0n+NSOvBIqXK+6/Dui/znhN5pZHIupkR7i0REVHyS/rgw9U+H3rrvxCZN1wkZ3/w9dk7RabezQCEiIjIq8GHq30+NLknReZi5VqLTJK5A7nAHBERkdf7fJikiNrv86HBwnHZOyw28Ilkb+cCc0RERF4MPgB9PMbe2UwyygVPreAyrrfd50PDZmNEREQRKyYegQCjTcMMWbxpjyz6c7ca92hRt5I0r1PJ+YOx2RgREVHEPBN8wNfrMlVPD6309vXvN6p8D0y7OBr90JqNIbnUbK2XUhXZbIyIiMir0y6ALqYPTl6er+dHZtYRdb3tLqdBzcYsGpOh9BYVMUREROS94MPVLqcaNBPD6IapFFa8EBEReTX4cL3LqVbxgtENq0dlxQsREZE3g4+odDllxQsREVFEPBF8RKXLKSteiIiIIuKJ4EPrcmrFcZfTGpeIpNh4+w7usv+YREREHuCJ4APdSzs1sS6lxe2OupxuXSLiyw2/3Zx/s806ERGR14IPVLHM/NW6lBa3O6p2sZvzcXg326wTERF5LfgIV+0ikVS72M35cBKoEBEReYAngo+oVLugy2laZfcDFSIioiTnieAjKtUu6HLaYVT47dhmnYiIyLvVLmbppCmRVLvAuTeKXNrXehu2WSciIvJe8IEqFiweB8YARLuM2x1Vu2iuGSpSsrzFBmyzTkRE5LngA7Bq7dg7m0mGod9HhdLF5Y07zne2qq2xzfqR/RYbsM06ERGRJ4MPQIDxVIeGUrF0icB1ew8dl2Ff/OZsVVs9tlknIiJyxFPBBwKM3h8sl72HjgVdn5l1RB6cvDyyAIRt1omIiBzxTPCBBmJPz1qnVrA10q7D7Y4ajWklt+lVQ2ST6LDihYiIyHvBR7hGY75IGo1pJbftRupCmBBY8UJEROS94CMqjcY09TuIlKpgsQErXoiIiDwXfESl0Zi+4iVnn8UGrHghIiLyXPARtUZjwIoXIiIi2zwTfES10RgrXoiIiGzzTPBh1WgMl3F9xI3Gwla8pIikV/NvR0RE5HHFxGMQYLRpmKGqWpBcihwPTLVENOJhrHiZendeAKKvfMl73HYj/NsRERF5nOeCD0Cg0aJuJXcftGEnka4TReY+LpK949T1GBFB4IHbiYiIyFnwcfLkSfnkk0/khx9+kOPHj8tFF10k3bp1k+LFiwdtt3XrVhk7dqxs2bJF6tWrJ3369JHKlSsn/9uNAANlt6h+QRJqWmWRlBSRQ7tENi/wT7tw9IOIiDzOUc7HZZddJtOmTZP69etL06ZN5cUXX5QrrrhCjh071a78r7/+kmbNmslvv/0mrVu3lgULFsiFF14oe/bsEU9AcFG7pUjREiKfPSAysZPI9HtF3rte5OVzRdbNLOw9JCIiKlQpPp/Pdj/xbdu2SfXq1QOX//77b6lZs6Z8+umncuONN6rr7r33Xlm+fLksW7ZMihQpIkeOHJGzzjpLunfvLs8884yt58nOzpZy5cpJVlaWpKenS8JBgDH1LvPbu07iNAwRESUdu8dvRyMf+sADMjIy1JTLvn2nGmzNnj1bOnfurAIPKFmypHTs2FFd7wm5J0Vm9bXeZlY//3ZEREQeVKBS2zfffFP9vvLKK9XvnJwcyczMlDPPPDNoO1z+888/TR/n6NGjKlrS/8QCFpFbtGmPzFi5Xf12vKhcKMjtsOx2mrfWC7YjIiLyoIirXZB0+u9//1uef/55qVWrViCIgLS0tKBty5Qpo6ZfzAwfPlyefvppiaW5a3aqVWz1i82hwykajUXc7wO2/Gh/u7qtI38eIiIiL418LFy4UE2lPPbYY9K/f/+gIKNo0aJB0zCAZNPy5cubPt6gQYPU/JD2g2qZaAceD05enm+V28ysI+p63B4xu4MnLgyyEBEReSL4WLRokbRr104eeughGTZsWNBtxYoVkwYNGsiqVauCrsflxo0bmz5mamqqSkzR/0QLplYw4hHq2K9dh9sjnoJBpYsdLLklIiKPchR8LFmyRAUeffv2lWeffTbkNnfeeadMnTpVVcbA2rVrZc6cOer6eIDOpsYRDz2EHLgd20Wk1uUipSqE3275RCadEhGRJznK+cBUS25urqxfv166dOkSuL5r167qBx5++GE1LdOkSRPVC2Tp0qVy++23y113WZSexhBaqru5XcgRjUseFJn3nPV2B3b4m5HZHSkhIiLyYvDx1ltvqS6nRg0b+leLhRIlSsiMGTPk119/VX1A0OMDUzHxAmu5uLldSJXq2tvu99kMPoiIyHMcBR833XST7W0x8oGfeINF5FDVguRSs6wOrDG379Cprq2Olalib7tVU0WufYb5H0RE5CkF6vORqIvKoZzWCnJNe39QgKoXrOGSZmPhusO7/VMvREREHuK54APQx+ONO85XIxxmfAWpekHeR+Nb7W2LBeiIiIg8xJPBB1QonapGOKwUqOrlnPbuTtEQERElCc8GH5lZOa5uF3LqJb0q1u4z3yaliMghj6z2S0RE5PXgY6/NhFK724Wcemk30nobX67ItO7+VXCJiIg8wrPBR8Uyqa5uF1LDTiK3TPCPcFiZO5ANx4iIyDM8G3xkpJd0dTtTqHrBCIcpn0j2dla9EBGRZ3g2+ND6fVjB7diuQOxWs7DqhYiIPMKzwYfW7yMlREqodh1ux3YFYreaZc+mgj0PERFRgvBs8KH1+xh7ZzPJMIyA4DKux+0FZqfqBeYNZ+IpERF5QorP54tw7fjoyc7OlnLlyklWVpakp6dH/fnQSAz9PLCYHNZ0wVRLgUc89FDNMjXcwnop/iCl/2q2WyciooRk9/jt6ZEPDQKNFnUryQ1Nq6nfrgYeWtVL68FhNmLiKREReQODj1ixu9ItE0+JiCjJMfiIFbuJp2y3TkRESY7BR6yETTxFzkc1/3ZERERJjMFHzN5pfbv1UMW9KL8ZwWRTIiJKegw+QlS+LNq0R2as3K5+47KriaddJ4qkG0p4MSKC63E7ERFRkitW2DsQT+au2SlPz1onO7OOBHU5RbMxV3p+AAKM+h387dSRXFr6NBFUOx/eLbJ5gX/aBaMkRERESYp9PnSBx4OTl6PgNfgNyvvtWtMxY/+PuY+LZO8IHgXB9AxHQYiIKMGwz4cDmFrBiEeoCRbtOtzu6hSMajx2d3DgAdk7/dfjdiIioiTEnA8R1d1UP9VihJADt2M7V+Se9I94WIU7cwf6tyMiIkoyDD5EVFt1N7cLC/kexhGPIOx2SkREyYvBh4haz8XN7VzrYvr7bHeej4iIKI4w+BBRC8mhqsWi/Ze6Hdu5wm4X08VjmPtBRERJh8FH3sJyKKe1aP+lbndtwbmw3U51mPtBRERJhsFHHpTRopw2o1zw1EqF0sXlnstqSblSJdyrdgl0O7XxeNnb/TkiRERESYJ9PgwQYKCq5et1mfLZyh2y99Cx6DUcmzvIP7USzs3viJzXxZ3nJCIiihL2+YgQplayco7J+J/+Cgo8JK/cFo3I0JDMFee0t7fdnk3uPB8REVEc4LSLg4Zj4HOz4Zjd3I95w5l4SkRESYPBh8OGY+JmwzHbuR8+kTmPs+kYERElBQYfBplZOa5uFxbWcGk9OPx2B3aIzH/RneckIiIqRAw+DIx5HgXdzpZKde1tN+85Tr8QEVHCY/BhULFMqqvbudp0DNj3g4iIEhyDD4OMdHst1P/ec9i9v0Ig8dQG9v0gIqIEx+DDpNV6OK9884d7JbeBxFObuOYLERElMAYfFq3Ww3Gt5NZJ4imsmsrKFyIiSlgMPkJAB9OHr6ln+cb53Cy51bQaIJJWKfx2h3ez5ToRESUsBh8malUubesN/N+BIy7+NYqKNL7V3raceiEiogTF4MPE6WVLurqd6y3XsSbMupnuPjcREVEMMPgIk3hq1vgc1+N2bOcqJ5Uvs/ox94OIiBIOgw8biacpJjkfT1xXX23n7l/EQeVLzl52PSUiooTD4CNM4unYO5tJhknpbd+PVsrw2evc/6ug8qV5L3vbLhnL0Q8iIkooDD5sBCDXNz4j5G2osn1r/uboBCB2cz9y9rHyhYiIEgqDjzCOnciVd37cbLnNuAWb1Xau536UqmBv2wMuNTsjIiKKAQYfYUxa9Jca4bCC27Gd67kflzxof70XVr4QEVGCYPARxpa9h13dznHTMTujH4f3iEy9iwEIERElBAYfYdSsmGbrjTx89IS4DqMfHV+1vz1Lb4mIKBmDj9WrV0uvXr2kefPm8uWXX+a7fcKECeo2/U+bNm0kUd3VopbYqaadvny7ewvNGStfuk4SSascfluW3hIRUQIo5mTjsWPHypgxY6Rnz57q37t27cq3zbZt2+TgwYPy9ttvn3qSYo6eJq6UKFZE7mtZW1W12Flork3DDPd7fyAAOXFE5JP77JXeYroGoyZERESJPvJxxx13qJGPPn36WG5XpkyZoJGPCy+8UBLZoPYNpWPjjNgvNKdXNnS5b8jS2/kvRmcfiIiIYh18lCtXztZ2mzZtUlMtHTt2lGHDhqmRkER3TUPr4CMqC83lK70tb2/bec8x+ZSIiLyTcFq8eHE1QvLII4+o39OnT5cLLrhADh06ZHqfo0ePSnZ2dtBPvCm0heaCSm9tdj3Vym9zT0ZnX4iIiOIp+Ojfv7+MHj1arrvuOrn99tvlm2++kX/++UfliJgZPny4GlXRfmrUqCHxptAWmstXemvz8bO3s/MpERF5I/hITU0Nuly5cmVp2rSp/Prrr6b3GTRokGRlZQV+tm7dKom00Jx2Gbe7nmyar/R2tP3t2fmUiIi82ucjMzNTSpcubRmwpKenB/0k0kJzuIzrcXvUofKl9WB727LzKRERxSHXa2BHjhwpvXv3VhUvPp9PRo0aJb///rvltEsiQYCBclpUtSC5FDkemGqJ6ohHqOmXZePDj2yozqd3i3Sd6A9aiIiIEi34WLp0qTz00EOBy0OHDpXXX39dbrjhBjV1AiVLlpR69epJxYoVZe/evZKSkiLvv/++XHnllZIsEGi0qFup8HYA0y/XPe8PLBRf+BGQ+h3Y+4OIiOJCig/DEzahCmXduvzLx59++ulSp06dwOWTJ0/Khg0bJC0tTapXry5Fijib3cHzIPEU+R/xOgUTF7CY3OcPixzeHX7bbp+L1G4Zi70iIiKPyrZ5/HY08oEHQtOwcIoWLSr169d38tAU7c6nTD4lIqI4wYXlEp3dzqdzB7HxGBERxQUGH4kOnU/Tq4YoADZJPsVUDRERUSFi8JHokHzabqSNDfNSe9j5lIiIChmDD5edzPXJok17ZMbK7eo3Lsck9wPltGnhKnB8/s6nmxdEf5+IiIhMJO5a93Fo7pqd8vSsdWp1Ww1arqPzadQbkDlJPp3WXaTjq+z9QUREhYIjHy4GHg9OXh4UeAAuPzB5ubo9bpJPc/Yx/4OIiAoNgw8XYGoFIx5WEywDP1kd/SkYu8mnGuZ/EBFRIWDw4QK0WjeOeBjtP3xcXv9uo8RH8qku/2PLwujuExERkQGDDxdgjRc7xi/cHP3RDy35tFR5e9v/+YNI7sno7hMREZEOgw8XYHE5OzD6gVGSqEMAcstEe9sueEHklUbs/0FERDHD4MMFWNW2fKnitrb9el2mxESty+3nf2TvZAIqERHFDIMPl1a57XFZLVvbfrZie2x6fwTlf4QLQNiAjIiIYofBh0v6XFVPSqcWDbvd3lgknhrzP9LtlOAyAZWIiGKDwYeLox+3XVjD1rYvf/NHbPp+aAFI/zUirf5tb/vfZ0d7j4iIyOMYfLjomoYZtreNSd8P/RRM7Svsbbt4jMi8kayAISKiqGHw4XLiKdqp2xGTvh+RNiCb9xwrYIiIKGoYfLg89YJ1XOyKSd+PfAmoNp8ve4fI1LtYgktERK5j8OEyLCD38DX14qvvhz7/o3kvZ/eZ1Y9TMERE5CoGH1GqfClns++H3e6orjmnvbPtc/aK/PB8tPaGiIg8iMFHlKZf7rHZ98Nud9RCW3wOfhgpsuazaO4VERF5CIOPKI5+lE8zH/3AoR/JqUhSjSlHi89pfCLTuomsZQBCREQFx+AjiqMfIzqfF3J8QbsOyanYLua05mNl7TQf05nWgyMgRERUYAw+opx8OvbOZvnKbzPKlVTX4/ZCgwDk4bUiTf7P/n18uf4RkHUzo7lnRESU5FJ8Pl+Maj3ty87OlnLlyklWVpakp6dLokM5LapakFyKHA9MtRTKiEcouSdFXjjLn1hqV3o1kf6r/VM4REREDo/fxUxvIdcg0GhRt1J8BiUIIDqO9vf0sCt7u8iWhSK1W0Zzz4iIKEkx+CgkWNtl6My1kpl9NHBdRnqqDO10buynYzAFc8t7/pwOTK3YXQOGwQcREUWAOR+FFHg8MHl5UOABuIzrY7bonN65N4rcPN7+9lwDhoiIIsTgI8Yw1YJF5eJm0Tm9Rjf6R0BSbH4suAYMERFFgMFHjC3etEe1VbeC27FdoXA6AqLWgLmbFTBERGQbg48YW/Tnble3i9oIiKM1YHwicwdyDRgiIrKFwUfM2a1mKeRSXKdrwKACZv6L0dobIiJKIgw+YixUyW1BtourNWCQAzJvJEdAiIjIEoOPGGtep5Llmi9QIa242q5QRbQGDJNQiYgoPAYfhbTmi5Xhnc+Ljw6oka4Bo5JQ7+I6MEREFBKDj0KAJmJv3tlMMtKD13zBGjBvFvaaL2ZrwLQe7Py+03twJVwiIsqHa7sUorhor+4E8jmQ1+FUl/f8FTRERJTUuLZLAq/5EreBSasBIsvGixzY6XwERHJFGnWO1p4REVEC4doucQjt1Z+etU52Zh0JmpIZ0rFh4U7JIAn1uuf9TcXQ28MurBeDdWN2rBC5dlg095CIiBIAcz7iMPB4cPLyoMADMrOOqOsLZd0XN5JQYeGrTEIlIiIGH/EEUy0Y8Qg1pqBdh9sLZd0Xt5JQZz/KPiBERB7HkY84ghwP44iHHkIO3I7tCh2mYFo/7k8mtbsQHRzeLbJlYTT3jIiI4hxzPuIIkkvd3C4mUMWCPNiPu9m/z28zT3VRRRBDRESewpGPOIKqFje3i+lKuE5GQH7+r8h714u80oir4RIReRCDjziCclpUtZgV1OJ63I7t4g5GQG5+J7JOqFwPhojIUxh8xBH08UA5LRgDEO3yUx0aqJyPGSu3y6JNewo/+VQPfTwu7ev8fmhc9vK5HAUhIvIIdjhNoD4fnZqcITN/3Rl//T+M1nwmMvsRkcN7HN4xxV/Gi2oaIiJK2g6nDD7ilLHD6b5Dx6T3B8vzleFqIyJj421NmNyT/qoWJJcix8OutMoij/wmUqxENPeOiIgKMfhwNO1y8uRJmTVrlrRv314yMjJk+vTpIbebM2eOtGrVSmrWrCnXXHONLFzI0spIW6/f0LSayvEY9kUC9P/QQxVL7ZYiDRyOYqAU96UGnIIhIkpijoKP0aNHy1tvvSW9evWSf/75R3JycvJt89NPP0mnTp3k+uuvl7lz50qzZs1UALJ+/Xo399tTEqr/hxHKadOrhshiCROAoIX7urySXCIi8m7w0b9/f/n8889VYGFm+PDhKth47LHHpEGDBvL888/LWWedJaNGjXJjfz0pIft/6EdA2o2M4I4+kVn9RDbNY0dUIiIvBx9FioTffMGCBdKmTZug69q2bSvz5893vneU2P0/jOvBqBEQB3L2iky6gf1AiIiSjKultgcOHFDJJlWqVAm6Hpd37Nhher+jR4+q++l/KEn6f+gDkP5rIlsPhv1AiIiSiqvBh8/nT3gsViy4azsuI1nVaqoG2bHaT40aNdzcLU/0/9BuR++PuOwBol8PpuskkbRKzu/PfiBEREnB1eCjTJkyUrJkSdm9e3fQ9bt27ZLTTz/d9H6DBg1SZTnaz9atW93craSAMlqU02aUC55awWVcD5eP/E5uH7dY+n24Uv3GZfQMictRkEfW+8tqnTqwk8moREQJztWF5ZATcuGFF8qPP/4ovXv3DlyPfI+LL77Y9H6pqanqh8IHIG0aZgT1/8BUy9frMuXByfl7gGRmHVHXx10PEEAfj+tf9gcSipNRGp/I5w+LnDgiUvYMLlBHROT19up9+vSRTz/9VL766is1DTNlyhTV50MfjJA7/T/wW+vxYdYDBD9DZ66NvymYoETUCAIjlON+ch8XqCMiSvbgA5UsaC6GHy3QwL8fffTRwDa33nqrDBs2TLp27SppaWnSr18/GTdunFxxxRXu7z2F7QECmdlH5dVvN8R3IurdM0VKlY/sMbSEVLR1JyKiuOeovfqxY8dk7978jawQZBjbqCLBVGuzaqdEN5L2rCQquRQ5Hnbc17KWPNHh3Ph929BUTE3DRDhKk1JEpMt4kXNvdHvPiIjIxeO3o5yPEiVKBEY9wilatKhUqFDBycNTBJz09hi34C/B7MtT18dpAKJNw8x93D+a4ZQvV+TjbiIpk7g4HRGRl3I+qHB6gNj1zo9/ybNfrJOk7AeiYWdUIqK4xuAjiXqA2DVuwWaZvSoOS3BD9QNx2hUV2BmViCiuMfhIAiijffiaeo7u89SMNfFZARNqFKTb5yKdxznvC5LNniBERPGIwUeS6HNVPamQVtz29nsOHZMJP22O/wAEoyC1W4o07urvC+JI3mubO9C/QN3qaSKbF3ChOiKiRKp2iRVWu0Rm9qod0uuDFY7ug3wRTNvEXRMyMyinnd7Dn1waKYygdBjFqhgiokI6fnPkI4m0b1xVldM6oXVBjcs27KE0utFfTlsQaFCGqpivnnJrr4iIyAEGH0kGfTzuvdx+AKINe6FLatxPwWjQxyPSZFS9ha+yMRkRUSFg8JGE0Mfjvpa1bW+PkANdUtEtNWG40RkVZj/KHBAiohhj8JGknujQUMbc0Uwqli5h+z5YrC6hIBm1zhUiHV9D+lLeTwRTMFsW+gMQJKMyKZWIKLFWtaX40r7xGdK2UYaqahn2xW+udktNqs6ov88W+fT+4PtiSqfdSHZKJSKKAla7eAByOS4f+Z1KLg2V1YHxgirpqTKqa1PZffCoCkLQORUNzBIKRi8winHwH5HSp4l83EMkZ0+ED5b32hHUILghIiLXql0YfHgEqllQ1QI+wyEWl8unFZf9h48nbgluKGs/81e1hFuMzrRsN8U/AtJ/tX+Kh4iILLHUloIgiBh7ZzPJMKwDUy6vMZk+8EjIElyzqphL+1pvY9kvxCeSvd0/mkJERK5hzofHApA2DTNUVQuSSyuXTpVHP/5VRIIDD8kbDUnJK8HFfRJuCkZz7TCRqheIzH5E5LBuCia9mkjDG0QWjwn/GJjG0U/plKkiUvNSjoYQEUWIwYfHIIhoUbeS+veiTXskM/uIrRJc7T4JCY3JGnbMHzzgsp3gY88mkVcaMSGViMglDD48zG5pbcKV4FqtEaOHAAQ5HViAziwVt1QFkXnD89+uFq27S6T1YJFKdTkaQkTkAIMPD7NbWpuwJbh2AhKU0069W5d6q9FfDhWY5F0377lTV7E8l4jIFjYZ8zCU06KqxSybA9fjdmyHcl1M08xYuV39TphW7HZ7hKQbqnoQSGBUI8dB11c1GnK3yLqZru8mEVEyYamtx1mV4AIqZCQv8RT5H0lViqsXKqF07aci0+91+EAszyUi78rmqrZUkBJcXNYCDwQn+sADcPmByctl9qoIOorGIy0n5Lwu/t+4jCDEsbzyXLRqZ7t2IqKQOPJBCqZRtBJcrcMpoDOqMfDQQwXu67c3U63ckw5GQ1SVi1lCqgUkqubsO3WZ+SBE5AHZ7HBKBYXcjtvHLba17cPX1JM+V9VL3H4gZpC/oRJSoSB5LnnvS+tBrI4hoqTF4IMKDMml/T5caXv7jPSSMrRTEuWB6AOQSBets4LRkGbdGYwQUdJg8EExHfnQn98jVyTpAhB9QiqajqneH+Bi1Q+nZogowTHhlFwrxXUKlTFJU4obKiG19eOhy3OR51EQLNUlIo9gwimFLcVFVYtTU+5rntgt2SMpz8UidRM7FfBBdaW6wPVkiCgJRz7Y4ZQsYfpkzB3nS58pK8TJYEZmVo6attFXzyRdMqqxZTuCEct27Q5Kdee/KLJ8AteTIaKkxJEPsmX2qp3S6wP7IyAV0orLvsOnVsvNSE+VoZ3OTb5ckKhVx4SS1/Kd68kQUZxiwilFZQpm6My1kpl9NOLHeDMZk1FjVR0TCpNUiSiOMPigqEAi6evfbZSXv/kjovuXTysuy55sk3xTMGGrY54LsXidG/LeRyTAYp0aIqJCxGoXigoEDf2uqadGMIyVMBXSwqcQ7T98XBZv2pP8f5181TGT8lfH5BNJQJYXzMwdKHLiGFu6E1FCYM4HudaS/ceNu+SN7zeFvV+fK+vKgLb1vffOG0dDlo0XOYDk1Dzp1USadcsbJYlAWiWRw7rAjk3MiCjGWO1CMRkF0ZfT/rRxt817JvmUi93qmFYD8pfSgqpyiaBiRh94AHJO9IEM80OIKE4UKewdoORht69H0vf/KMhKuvhpNzI6QVqoJmYYjeHqu0QUY+zzQa5pXqeSSihFXodVCS62IwtIHEUCqesVMxhJSfHnh9TvILL+i/zPwdERIooB5nxQTDuimpXaGvNHkrIpWTytJ4NeIerxjI/F6hkiihxLbamQ+4Gsk8zsI4HrUBkzpGPoFW+xPdaD2Zl1avvSqUXlvstry0NXn80gxKp/SFplkcN2c210SpUXydkfvsU7poG0IAjJsYd2iZQ+TaTsGf4cFdxORJSHwQcVKrsjGQg8Hpy83PRcvnSJojKqa5Pkb0wW6XoyNS4RebVJAVu6m+j2uUjOPvPpH07REJEBgw9KiADl8pHfBY14eLozakxauqf4V9/N2Rv+cZv3Elk8NsxjpojcMsFf5ouACKMiPp9/NEar4OHoCJFnZHNhOYp3GBmxE3gApmXaNMzgFEyBElTzRp4uecBeL5FVU20EMz6RaT38K/qGwtERIgqBpbZUaDAlYxeCFAQrZBGA9F/jnyq5+R1/QinyMoyBAIIU9BfBv01LeVOc5ZKYBR5m5b1E5HkstaVCg1wQp8EKq2IK2MRMmwJBLxE1VWNcbyYvIGncVWTxGHG9vDfUFIwxj4VTNURJj8EHFRokoVYsXVz2HjLvC6L31+7D+XJErKpoPM8YjNiZqlHTJCP8eSGuBB/gE8ne7g8wjPsTqoKHUzVESY99PqhQzV61Q3p9sCLsdmhelnX4uFlXChnLhNTImI064PpXGrnb5AzTQejmmi9R1kGvEY6SEMU1JpxSQmjfuKr03LZf3pq/Oey2PvNBfSakuj06orV5DxkcRAjBjT6IwIiH1V/VOFVjNUqC7Th1Q5QwOO1ChW5Q+4bSpHoFeWz6Kjl49ES+duzdL60lL3+zwfT+Pl1CKteNiWEVTUoR62TTUxv6gwRt4TxAoGA5qmKYqjEbJVEJrXeJlKoYXD5snLrhiAlRcgcfPp9PTp48GXRdSkqKFC3KTohkrn3jM6RtowxZvGmPLPoTVRb+FXOxDsznq3Y4rp5hYqpLcPDWRhWMHU4P7RGZ1j1vQ7PRkbwpFOSR6JNNMc1jB7YLO0oi+fuWaFU2CJ6AeSVEyR18PPvss/LUU08FBRvly5eX3bsjaAFNnoIOqJfVq6x+IqmK0bYL1a69fKni0uOyWtLnqnrsFeJm4mqRMP1FtARWY+6GfgrGCrYLO0oi5lM3s/qFbqimD06M+6YXbsSEIypE8TPtcskll8jixYuj8dDk0aoYVLVkZh0Jee6Lc+uMcv4W7mbt2vfnHFdTN+MX/iUjOp/H6phojIw46XCK6xGYmLaF103VrP00wp3zWXRytVECHK4SJ+RaO5VE2r8k0ujGCPeZyBvYZIwSYkQE5bQSoi2Wdlm7HSMeVumR+w8fV6vuosqGXKKNjKCSpc4VInVb+/+N68xaq2sJrUF/RZOpGrujJI7p8kqMtBwT44iLNmLy1VOhbz+Mqahu/tutYMRk8wKR1dP8v3E5nEjuQ+Sl4GPlypVSqlQpqVChgrRt21ZdJioI9PFAOS1GOPRwWSuzddKuvc+UFTJ7Fc66qdATWtNNOrFq0yHaKIlpR9YCMuafhM0x8Ykset26CmjhqyJrPwt9GwIblDG/d73I9Hv9v3HZqgtsuPtEKzCJZsDDYMrTXO/z8e6770rZsmWlXbt2sn//fhk8eLDMmDFDVq9eLTVr1gx5n6NHj6offZ1wjRo1JCsrS9LT093cPUpwVomkM1Zul34fOgt0uWBdHLCTN+Fo8TyH0JJen9OCgywO7gWFFvUD/gh+LZH0Ngl3n0sfElkzzf1GbdFsAMfmckkrbla1PX78uNSuXVt69Oghw4YNC7nN0KFD5emnn853PYMPcmLRpj1y+zhnuUbIJfnx8auYhJoIQh2wAiW2Jm3iS5UXydlvnVfSf3VwgICzfIwuuB3YhG3cFmJ/Im72ZhHM2BFJkFTgx9atpnxOe7bZT/LgI+o5H8WLF5ezzz5bNm7caLrNoEGD1I5qP1u3bo32blESJ6Y6YVywDiMrCGIwioLfuExxungefv97o0jXSeZTNx1ftZdXoudmjol+SsdJbxPb97F4LEBCrdOpEjulzZE8btjHzoO2/namoiihRb3J2LFjx2T9+vXSrFkz021SU1PVD5EbialIKHVC6w8SqkSXa8ckQNmvseLGOHVjtYZNqLP3sJU4EmKkxYQ+kHHS28TpfZyuqWPFaQM4J5wEU1py7y0T/FVEXHgwqbgefHTu3Fn69Okj559/vurt8eSTT6phmPvvv9/tpyLKB4mnyOMY+MlqVdliB3JHzEp0Ud6L67l2TIIvouek/XpQa3mT6ZwWvfOSTi2kVwvu6uqkt4nT+1hxGsBEEiRFZV/y3vdpPYI76XLhwaTg+rTL448/LqNGjZJ69erJ1Vdfrbqd/vzzz2rqhShWAciyJ9tIv6vrWdZHpOSNbFxQs4Jpia52HW7nFEySlANblQDbrcRp+6zIpX0tHiAl/5RO2KqdlPwBixuVPk4DmEiCpGjtCxhb+Gst9eeNZNlxAuOqtpTUUE7b64P80zD61XDLlSphK1F1yn3NuXaM14SrxEE57ReP+puqaRBAmE3pmFbt2Kl2Md4nHJOE2nACSa5hGsA5fVxbjx0htPu/oIdIpbrhR7bIG9Uu0dx5IjvC5XLYLdEdfVtTuaFpNb7pFMxpi/WQZaYWAYvVfRrdLLLwNWfBjB2RBEkFfmwXcWqm0DD4ILLZH8Ruia7VyAcXsiNHIlkTxuw+kQQzdkTrcc0e21V5QRKTVWOOwQeRTQgcLh/5Xdi1Y8z6gXAhOyp00VrgLpoL52mP/ftsf3mt3QoiJ1KK5E9WvXa4SGlWz0QLgw8iB7RqF5NBZtNqF7MqGU35tOJcyI4okpEQY+AQLQhI2g5nOa9LGHwQOeS0z4c2YhJuPRkEMCzVJXI4ynIIi/R1z7sxxqmJzBmJGIMPogg4yd1w0s5d38ad+SFE8ZIbYiZEYm00p6A8GHxEvcMpUSJBcGCWVGrWGdVJG/esnGPsokpkl7FB3J5NIvOGx2A0BI+d4m8jj+df/0XoLrnMH4kYgw+iCGFkxImv12XK+J/+Mu2i+sYd50uF0qm2Rl2IPNu99vQGMRoNyWsjP//FvIDH8D8Xzz+tm/V0jdloSS5HUdjngyhCdnM+NBVLl5C9h46Z3o44Q7+OHdeVITKhP3hjNGT5hOglq5aqIJKzz/l0DYQaLWnURWTNtBBrDemCFggXoEQawEQ58GHOB1GMklTDLWSHr6MKpYvL3kP21prR3w+YrEqUQMmq+J8bCFh8keWYrAvVY0UXoIS73Uyk93OAwQdRDAMQs4XstADinstqyTs//eX4sc16jDBplSjayaoIIso7GPUoqJS8PJLn8gInY+CS9///0ofyutqa3G7WfTbQWdbh/Rxi8EEUQwgGXv9uo4z/abPszzmeb+rE7voxdrqrOi0JJvIs44jI4T0iXw6yEZDkHZBbDxKZ95zEVFol/36GlCKSkmIxpWSy7k5gTR2z112A9XoMWO1CFEMYleh3TT3pc9VZIUt1EZwgQDDromq3ssasqZmWtMopGiKLZFVo0DH/FM1Xg0JMRYzwV7qofBKXF8KzYhp4gE/Ecjm2vCRZvD7968Zly4DL5H5RxGoXohiU6uJ6jEwgQIikiTQCGQQwGPHwmRcGqtvbNMxQ12lBUOXSqerG3QePsoqGKFRA0rCjeRIm8iHUdEUU2r9HC16H1WW794siBh9EMYIpEYxMGKdMjFUuoXI+MIKCYMKqsgYPgdsx/fPh0r9Nt+UUDZGNgESDPAjkQ9jKH8kLUEpVdJhwCil5Uy67pcAQQFldtnu/KGLwQRTjAAQjE/qpmX2HjknvD8zXlcGICUZO7DY1e/mbPyxv5xQNUQGbnZnlj2jTNeBotCTF/6vDqLzHNJvmsZnzgZEbPVzG9VaPG+p+UcTggygOpmbGFsk/IpJhSCJ12tTMjHGKho3MiFzKH9FP14QaLUmvJtLoZpM+HyP8QQ56lIQMXPIClBZ98qpdJPTteBxj0igum04fWdwvithkjChOhCuf1ZqaRZq0Gq6KhohcFmmH03Wh+nFUOxWghLvdTKT3c4CltkRJSKt2kRDnLpEEJKNvayo3NK0WcZDEZFaiKMlN7g6nnHYhSoKkVUzR3HZRDXn5mw2OHs/pVE6oHiN6TGYlikESrJ3b3b6fyxh8ECVB0iqmaODDpVttTcvoq2jsMusxosdkViKyo4itrYgoLpNWMWWC37is9RIBq7VwjVU0dlj1GNHTbse2uA8RUSgMPoiScFoGoxpmcJvTTqjheoyE6jeC+1hBcLJo0x6ZsXK7+s1ghcg7OO1ClOTTMm4khdrtMWL3PlyfhsjbGHwQeajNe6Qi6TFidh+uT0NEnHYhorAwWoJKFjvjJdjmDJNk1nDr04TLF+FUDVFy4MgHEYVld2G8cMmsdtenwXbGkRtO1RAlD458EFHMklnt5o4Yt9OmaoyBi1bai9uJKHFw5IOIYpbMajd3RL9duKkarlNDlHgYfBBRzJJZtdwRs0ZooZqf2Z2qWbxpjxTJW/030qoeIooNBh9EFBe5I2b5Inananp/sFz25xwPXGard6L4xZwPIoqL3BGzfBG7UzX6wAOYD0IUvzjyQURxsz5NqGmScFM1ZpgPQhS/OPJBRHGzPo3ZdnbWrClIq3ciii0GH0SUsFM15UsVj1p7eDY0I4oeTrsQUcJO1eT6fPJ/by9xvT08+oYMnblWMrOPBq7LSE+VoZ3OdbQgHxGFxuCDiBK2zBejE05Ld+0EHg9MXp7vegQiuP5NhysCE1F+nHYhooRllQ8SrtV7KAhmBn6y2nIb3G629gwR2cPgg4g8VbprBY3K9h8OLtk1wu3YzgrzRYiscdqFiDxVumtl0Z+7bW93Wb3KIW9zsgAegpSC7jNRImLwQUTi9bbvp6QUaDttATzjpIzW8Ew/EsNVesnLOO1CRJTHbvASartwC+ABbsd2XKWXvI7BBxFRnuZ1Kkn5NOveIRXSiqvtjJwsgGc3SAmHuSWUqDjtQkSkm7oZ0fm8kKW2muGdzwuZl2G3kRnyRewEKQhmrEZi7EzbRCunhLkqVFAMPoiIdHDgRi+PoTPXSWZ2+KRR543M7B38rYIZO7klYDfx1QnmqpAbUnw+X9wVrGdnZ0u5cuUkKytL0tPTC3t3iMiDnJ7dY/vLR34XtuHZi7c0sdWVdcp9zU1zS/A8ZqMneJ5yacUl6/DxfPuh7b3TEuRwQU9BH5eSh93jN0c+iIhcqJ7RGp7h4IyDsc+k4RnyRQrSldVObolZr5KCrPQbLqHWzuMaA7oLalaQZVv2Ra3UWHu+zKwc2XvomFQskyoZ6SxpjgdRCz6OHTsme/bskdNOO02KFWOMQ0TeaXhmnO7IMEx32AlSzA7CkSySF0lOSaQJtWaPG2q6Bi9Rn1frxrSQ1fNF43kojqpdhg0bJhUrVpT69eur4GPs2LHReBoioriDA9qPj1+lpk1G39ZU/cZl/YGuIF1ZnS6SZ8ZpEGN3+1DbmZUWGwt6tJwVbF8QZs+n2enS81hhJZI114ck3n//fRk+fLjMnTtXWrVqJZ988ol07dpVzj77bLn66qvdfjoiooScsom0Kyu2sZq2sctpEGN3e+N2VtM1bk4LRfJ8BXkeK0zKLYSRjzFjxkjnzp1V4AH49+WXX87RDyIikyDlhqbV1G87B0E7i+mhV4nZI+H6Mxyu9KsPepw+brjpGqvpm0jYfb6CPo8ZNpArhOAjNzdXli1bJpdeemnQ9Qg+li5d6uZTERF5ltW0DcqE0avErZV+C7qCcKQ5KrG6X0FzaCLtcut1rk67HDhwQI4ePSqVKgUPN1auXFl27dplej/cBz/6Uh0iIop82sZO4mu0EmrdyFGJ1f3cyqFxIynXS1wNPooU8Q+knDhxIuj648ePS9GiRU3vhxyRp59+2s1dISLydG6JWyv9FvRxneaohCs1Dkd7vnBTLwV9HreTcr3G1WmXsmXLquYimZmZQdfjcrVq1UzvN2jQINWQRPvZunWrm7tFRORJkeSUuP24VtM14uK0kPH57Ny7IM/jZlKuF7mecHrFFVfIl19+GXQdKl9wvZnU1FTVCU3/Q0REyZ2jYjzu2yk1dvJ8GAEJBddHoxtrpEm5XuR6e/UlS5ZIy5Yt5cknn5SOHTvKhAkT5J133pEVK1ZIvXr1bD0G26sTESUfL3Q41apdxKSBXLK3oM+22V49Kmu7fPfddyqP4++//1YBx5AhQ+Siiy6yfX8GH0RElKi83OcjuzCDj4Ji8EFERF5amDBZcGE5IiKiBFmY0GuisrYLERERkRkGH0RERBRTDD6IiIgophh8EBERUUwx+CAiIqKYYvBBREREMcXgg4iIiBh8EBERUfIqJnFIa7qKTmlERESUGLTjdrjm6XEZfBw4cED9rlGjRmHvChEREUVwHMcaLwm1tktubq7s2LFDypYtKykpKQkdASKA2rp1q+UCO17F94fvDT87/H/F753k+k5GSIHAo2rVqlKkSJHEGvnADlevXl2SBf6IDD74/vCzw/9b/N6JH/xejt57YzXioWG1CxEREcUUgw8iIiKKKQYfUZSamipDhgxRv4nvDz87/L8VC/ze4fuTCJ+duEw4JSIiouTFkQ8iIiKKKQYfREREFFMMPoiIiCim4rLPR6L75ZdfZOPGjUHXlS9fXtq1aydetXTpUtm0aZO0bdtWKlSokO/2kydPyuLFi2XXrl3SpEkTqV27tnjFvn375Ntvv5Vq1apJixYtgm7bv3+/zJ07N999rrnmGqlcubIku507d8rKlStVw8Hzzz9fSpcunW+bY8eOycKFCyUrK0suuugi1dzIC/C6V6xYIbt375YGDRpInTp1gm7PzMyUefPm5bvf9ddfL2XKlJFkh/dl2bJlUrx4cfWdUqlSpXzb5OTkyI8//ihHjhxR//e88H9K+7799ddfZfv27VKrVi0577zzRG/z5s2yZMkSMerSpYsUK+ZO2MDgIwrefvtt+eKLL+Syyy4LXIeucV4MPvA+/Oc//1Ed7zZs2KCCkAsvvDDfl0SbNm1k7969Uq9ePVm0aJEMHDhQnnrqKUlmCCweeeQRFVycOHFCrrrqqnzBx19//SW333673HTTTVKiRInA9TgQJ/MXJQKJ+++/XwUV+GJEEIKux++9917Q/yO8P/jsABoT4gvzpZdekgceeECS2eTJk9X/q9NOO019DubPny833nijTJgwQYoWLaq2QdB2xx13SNeuXYPue+WVVyZ98NG3b1/57LPP1GcH/8/wXowYMUIeeuihwDarVq2S6667Tp0M4eQQ24wfP15uueUWSWbz58+XXr16qUC+SpUq6vv27LPPllmzZknFihXVNj/88IPaplOnTkH3xfeQW8EHWqGSy3r27Om7+eab+b76fL6pU6f6fvnlF99vv/2Gqirf0qVL870v3bp18zVu3Nh38OBBdXnu3Llq20WLFiX1e7ht2zbf22+/7Tt06JCvQ4cOvltvvTXfNitWrFDvxa5du3xesnPnTt9HH33kO3nyZOC6Rx991FeuXDnf0aNHA9e1bdvW17p1a9/x48fV5fHjx/uKFSvm27Bhgy+Zffjhh74dO3YELuP1li5d2vfGG28ErpszZ46vaNGiPi+aNGmS78SJE4HLY8aM8aWkpPh2794duK5Jkya+Ll26+HJzc9Xl4cOH+8qUKZP0/9e+/fZb399//x24vH//fl/VqlV9gwcPDlyH/0fVqlWL6n4w5yOKQ+kzZ85UESQib6/CWcQFF1xgOXQ8depU6dmzZ2BIHVMzDRs2lPfff1+SGaZZ7r33XklLSwu77YIFC+Tzzz9Xo0dekJGRoc7Y9WtDdO7cWY2IYLQD/ve//8lXX32lzma1s7G77rpLncl+9NFHksxuvfVWOeOMMwKXzzrrLGncuLE6ezf65ptvZPbs2bJlyxbxijvvvDMwAgTnnHOOWnPk0KFD6vKaNWvUtMPDDz8cWD+sd+/ecvz4cZkxY4Yks6uuuipo0Va0QscIiLagqwbvBUZlv/zySzXq6DZOu0QJPthvvvmmmlPDl+Xo0aOle/fu0Xq6hIU8EMy7NmrUKOh6DJeuXr260PYrnuBL9Pnnn1dD5ZiGuPbaa9Wwe6j8h2SGgyheM+aoYe3ateqAov/s4L1C4Oq1zw5ypTCNcNtttwVdj/dj2LBhKojD8Dqm8MaNG+fe0Hkcw2cAnxEcOP/73/+qaaozzzwzcBvoPzvIK8JnywufnePHj8v06dNVrgv+XyEHZMCAAUHbIBh54YUX1Anizz//LH369JFRo0a5tg/J/wksBPgCePnll6VUqVLq8iuvvCL33XefGgEwJvZ4Hc5kQZtr1CA5bP369eJ1mM9HUqH2ucHZ68UXXyyDBw9WAa1XIJdj+PDh8txzzwVyX6w+O14abcSB4+6771aJthhJ0yBp+7fffgskouKgeumll6r5/UGDBkmyw2v/9NNP1f8ZHGz1gQY+OwjMjIuneeWzc/z4cZUTc/DgQZWH17FjR5X3okGC7p9//qlGILWRV4yY4D3s0aOHK/vAaZcoaN26dSDwgP79+6uhrTlz5kTj6RKa1sYX/wn0cLlkyZLidUii1AesNWvWlH/9618qOcwrcNDs0KGDOrBimFzDz45Ibm6udOvWTY16YGpFPxqGqQZ9BQw+R5jK8spnB68V02+oosPoD04KMd2ifXYQtOHM34vfO2lpafLhhx+qqdw//vhDfvrpp6D/W0ho1wIPaNmypRpxdfOzw+AjRjCkh6FRCoazM8y5/v3330HX42zFWDpIfjhb88pnCYEHzriQO/Taa68F3Va3bl3126ufHQQeOAv97rvv5Pvvv1d5H+F46bOjh8ADIx2YtjT77OD93LZtmyc+O3o4MUZVC0Y3YvnZYfDhMnyAkQinh1pzfCGiBwEFw1AfhoI//vjjwHXIk0EkjrNdr0OJqfHzheFSL3yWMF9/9dVXq94CY8aMCSQGajB9gAOu/rODXKvff/896T87+Bzcc8898vXXX6vAA+9FuM/O0aNH1ehIsn92ULKPPAU9jAxhqkFLtGzevLmartN/dvBe4r7t27eXZLbT8LmA5cuXByWhGrfBVBQ+Z25+dpjz4TIM5V1xxRWqfvzcc89VkTXO2PBlePPNN4vXoDoDwZeWLY3qBDRgw7AehoUBSU3oPYDpBFyPRN1LLrkkX/JcMsKXHz4zeH8w3IuhUEzZ3XDDDer2N954Q1UwoKkYch2mTJmi3lO8j8kMX34IPDAH36pVq6DqFYyEnH766erfyK3Seg9gFA35VXjvcN9khuTAiRMnytChQ1VOEH60CioMkQNWJ0VuAy4jWEEPkMOHD6spiGSGXAV8l6DvCRJIkfCP7xR8B6OSDvB/6cUXX1RVdkh4R4UUkrpx2Zj8nmx69uyp+sMgkMDnAlMpyKnSf6c8+OCD6sQQQRo+M2+99Za6jP5LbuGqtlGAci40Q8IXAj7U+M+PhB4vwgf63XffDVkKh06L+rNcbLdnzx4VgOA/iBfmXpEoaDxLw39yfFlqMKyOZm34XNWvX19VTemTw5IRAtQnn3wy5G2oWkBFiwYJc6j+yc7OVk3aMCKQ7NUcCDpCJWQ3a9ZMHnvsscBlHFhQzYAmdjgZwmfHTml3osP0CYIzfI5woMXJTagmj+gAi1J/jAohqEVTNuMIW7LJzc2VadOmqWZj+DdGzfA9pE/c1rZBqwhMVyEBFWXs+kaHBcXgg4iIiGKKOR9EREQUUww+iIiIKKYYfBAREVFMMfggIiKimGLwQURERDHF4IOIiIhiisEHERERxRSDDyIiIoopBh9EREQUUww+iIiIKKYYfBAREVFMMfggIiIiiaX/D9Dm8HnznAUUAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pymoo.optimize import minimize\n", + "from sb_arch_opt.algo.pymoo_interface import plot, get_nsga2\n", + "\n", + "# Create the pre-configured NSGA2 algorithm\n", + "nsga2 = get_nsga2(pop_size=100)\n", + "\n", + "# Run the optimization: 20 generations (20 x 100 = 2000 design points, each with 50 realizations)\n", + "# NOTE: remove \"seed\" argument when using in production!\n", + "result_nsga2 = minimize(problem, nsga2, termination=('n_gen', 20), seed=42)\n", + "\n", + "# Compare the robust Pareto front to the deterministic one\n", + "pf_deterministic = MDMOHimmelblau().pareto_front()\n", + "plot(pf_deterministic, result_nsga2.opt.get('F'),\n", + " labels=['Deterministic Pareto front', 'NSGA2 robust (2000 design points)'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The robust Pareto front lies above the deterministic one, which clealry shows that the perturbation in design vector has effect on the objectives. The deterministic front is therefore not achievable in practice, and optimizing towards it would select designs that are more sensitive to the perturbations." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Solving with SBO\n", + "\n", + "Because every design point requires 12 evaluations for each sample realization, the number of design points for evaluation should be kept low. Surrogate-Based Optimization (SBO) in general requires fewer evaluations than genetic algorithms to find the Pareto front, therefore they are well suited for the stochastic optimization problems." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "end_time": "2026-09-23T13:01:42.293559800Z", + "start_time": "2026-09-23T13:01:31.942026500Z" + }, + "execution": { + "iopub.execute_input": "2026-09-24T12:06:07.458881Z", + "iopub.status.busy": "2026-09-24T12:06:07.458881Z", + "iopub.status.idle": "2026-09-24T12:06:18.583246Z", + "shell.execute_reply": "2026-09-24T12:06:18.583246Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:09,799 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (20 real unique, 20 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:09,896 sb_arch_opt.sbo : Infill: 4 new (eval 20 real unique, 20 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:10,904 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (24 real unique, 24 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:11,024 sb_arch_opt.sbo : Infill: 4 new (eval 24 real unique, 24 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:11,901 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (28 real unique, 28 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:12,037 sb_arch_opt.sbo : Infill: 4 new (eval 28 real unique, 28 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:12,977 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (32 real unique, 32 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:13,694 sb_arch_opt.sbo : Infill: 4 new (eval 32 real unique, 32 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:14,763 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (36 real unique, 36 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:15,028 sb_arch_opt.sbo : Infill: 4 new (eval 36 real unique, 36 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:15,909 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (40 real unique, 40 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:16,001 sb_arch_opt.sbo : Infill: 4 new (eval 40 real unique, 40 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:16,977 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (44 real unique, 44 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:17,086 sb_arch_opt.sbo : Infill: 4 new (eval 44 real unique, 44 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:18,011 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (48 real unique, 48 eval)\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO 2026-09-24 14:06:18,408 sb_arch_opt.sbo : Infill: 4 new (eval 48 real unique, 48 eval)\n" + ] + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjExLjIsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvgI3uAAAAAAlwSFlzAAAPYQAAD2EBqD+naQAAb1BJREFUeJztnQmcTXX/x7+zMAzGbsYWKpEkTZGEUiOyFqXlaaOnDUVUaPnTo0I9aU89nkqlzRNFhTJPiWSr0YO0ShLGMpjB2Gbm/F+f351znXvmLufeOffOvfd83q/XfY177u+ce5br/D7nuyZomqYJIYQQQkiESIzUFxFCCCGEUHwQQgghJOLQ8kEIIYSQiELxQQghhJCIQvFBCCGEkIhC8UEIIYSQiELxQQghhJCIQvFBCCGEkIiSLFFISUmJbN++XWrUqCEJCQkVvTuEEEIIsQDqlh44cEAaNWokiYmJsSU+IDyaNm1a0btBCCGEkBDYunWrNGnSJLbEBywe+s6npaVV9O4QQgghxAIFBQXKeKDP4zElPnRXC4QHxQchhBASWwQKmWDAKSGEEEIiCsUHIYQQQiIKxQchhBBCIkpUxnwQQk5QXFwsx48f5ykhhFQ4lSpVkqSkpHJvh+KDkCjOl8/NzZX9+/dX9K4QQoibWrVqSUZGRrnqcFF8EBKl6MKjQYMGkpqayoJ7hJAKfyAqLCyUXbt2qfcNGzYMeVsUH4REqatFFx5169at6N0hhBBF1apV1V8IENyfQnXBMOCUkChEj/GAxYMQQqIJ/b5Unlg0ig9Cohj2NiKExON9ieIjECXFIpuXiaz/wPUX7wkhjmLBggUydOjQsK9j5/rxws8//yx33nmn9OnTR/79739X9O4Qm2DMhz82zhdZNFakYPuJZWmNRHpNFWnT365rQEjcsG/fPrn22mvdT0fVqlWTxo0bS5cuXaR///6SkpIS1Pa2bNkit99+u7z77rtSu3ZtqSj+/PNPWbp0adjW+fjjj2X+/PkyY8aMcn2nka+++komT57svhbITrjgggvk5ptvluTk8N76v/jiCyUU3nnnnXJvq3fv3nLJJZfIiBEj5OSTT5Zw89hjj6m2HnfddVfYv8vJUHz4Ex6zb0R8r+fygh2u5YPfpAAhxMTRo0fls88+k0cffVTOOeccOXz4sHpyffjhh+WBBx5Qk2zr1q0tnze05sb2sN2KBE/dZ5xxRtjWgchatmxZub/TyI4dO9S5g6iB2Pjtt9/UNVi4cKHMmTNHwt2ZHAKkvCCo8ffff5dPP/00qN9NeVi7dq3Uq1cvIt/lZCg+vAHXCiweZuGhwLIEkUXjRFr3EUksf7EVQsJJcYkmqzfvlV0HjkiDGlWkY4s6kpRYfp+tPyA8evXq5X4/atQo9X7gwIGyfv16d4Q8AtZee+01NVFhguzatavceuut6nNYUe644w417rrrrpPKlStL586d5f/+7//8rgd+/fVX9eT67LPPyksvvaQmsIkTJ8off/yh1sF+fPjhh2qSvOiii9RTNZa/9dZbcuTIEbniiivkmmuuce8/9vmDDz5Q3wMweWP8jTfeKLNmzVJp0bAq4Dv1fTCvAwEFy8by5cuVBQhP9IMHD1YWCuzjX3/95T5nOG4cr3F9sGfPHpk+fbqaIJHmCHdE27Zt/V6LHj16SJUqVdS/8UQPywc6ht9zzz1y8OBBtb/oQop9ufjiiz26k2IZLAH/+c9/ZOPGjWq/sN/IxMI+r1mzRurUqSN9+/ZV5wx899138uSTT6rrpx/P9ddfr1743ueee05++eUXlSlxyy23SKdOnbzu99dffy0PPvig+vdtt92mghyff/55dS1hURk5cqTaB4gsXANYxpYsWSJvvvmm5OXlKeGGMenp6e5tTpgwQRo1aqTqVECYHTt2TK666ioZMGCA+hzbx/fi3Ov7ju+IhMXFaVB8eGPLN56uljJoIgXbXONanLgxEBJtLNqwQx75eKPsyD/iXtawZhWZ0K+N9Gobeo5+sGDygzUEEyme8DHhI50YkxbqBmBywST49NNPy+effy5z585VLpsbbrhBTdYQFphcMJEEWg/k5+eryQUTLyYgTCTNmjWTL7/8Ul5//XVZtWqVmnwxQY4bN04++ugjtb27775bdu7cqSZofH+/fv28ukA2b94sM2fOVNvBOkVFRWqixLq6q8O8DvYDx3LfffcpwYTvxJP9lVdeKd27d1cWCog0cPrppysLhXF9iBNM1KeddpoMGTJECQfsJ8bVr1/f0nXAOQC7d+9WwgUiDucTQgniAeIIggNgYsY5zMnJkWHDhin315lnnil79+5V+5GZmalcbDiH2G9sA8IQ39GzZ08lNPTjOfXUU9W5gSiFgISYXL16tXTr1k1ZNXCdzLRs2VLFvOAc4HghtnD9Idbef/99tf6YMWOkSZMm6lpBEGJ/xo4dK5deeqm8+uqr0qFDB9mwYYO7OzqE0TPPPKO+7+qrr1YiCOf/k08+UfsM987s2bM93C5Wzy0JEi0Kyc/Ph3lB/a0Q1v1H0yakBX5hHCFh4PDhw9rGjRvV31BZuH671nzsJ1oz06t56Quf282OHTvU/92FCxeW+ezIkSPqs+eff169nzVrlta0aVOPY9y9e7dWqVIl7dtvv1Xv169fr9bBdnWsrLdmzRq13ltvveWxD08++aSWkpKi5ebmupf9/e9/16pVq6bl5eW5l11zzTXajTfe6H4/ffp07ZRTTvG7nWeffVY7+eSTfa6Df7/55pse+7N//371F+ekVatWHp+Z17/55pu1s846SysuLvY4p0ePHtW88e6776pzoJ+nkpIS7fbbb9eqV6+uHTx4sMz4adOmaR07dvQ4p1h/0qRJHuPuvfderUePHh7LvvrqK3X+CwsL1Xuc9/T0dI8xI0eO1Nq2bav2Qwf7g2PyxebNm9U+/Prrr+5lM2bMUMvw2zDSrFkz7cEHH3S/x3nB72TChAnuZX369NHat2/vsQ9XXHGFNnToUPf7QYMGqf0iod2frM7ftHx4o/oJM50t4wipAFcLLB5+HIfq8x5tMsLugtGBqwFWClgYwOLFi9XTNZ48UTkRL4AxP/zwg3pK9kYw61144YVl1ocJ3WiKP+mkk5Q1Ae4D4zK4Nvxh3k6LFi2UC8AXsBZMmTJF7SfcGwgArVmzpljlv//9r3J7JCaeSFK0EsCLQF+sA3cFrBaw2MBSgPewdCAWBLE1sIbALWXGfA5x/nENYX3Szz+uB6wocHe1a9fO636sXLlSrWNM07z88svlX//6l3J16a4hK+BaGd1NcHshbka3VAG4TuAignXKyPnnn++xD7huP/74o+XvJvZA8eGNpueJJCSKaCW+z1xCkmscIVEIYjyMrhYzmK7xOcadf0pkKqjC7A4TPyZd3TUC0zriLYzAjeEv0DKY9WrUqFFmfUxKRjAReVtWUuLn/7+P7fhbB24ATLSY/OEuwuT5yiuvyFlnnSVWgJsllIwfuA+wr4ixQNAmKlTCbQS3CSZ/uFsQA4FJeurUqWXWN59DnH+4S/SsJp17773X7dbxBtwzZrGF9xAv+CyYUt3mfcL6+vbM24fgMhLKtSb2Q/Hhja2r/AsPoBW7xjHmg0QhCC61c5wdIB4DT+D6kzSeOOGPh6/dV9Eib8utrBeNwEoBIYAXsoAQrIq4i2+++cbSceC4EfQZLMaAUx3EOCDIFEJIx+rTP/YDQsgYUGz1usEyYgQxFxBDuiANlebNm6vvxPaNWTHYPr43GGLpNxXLsMiYNw7utHccIREGWS12jisvSLFFYCesFfrTMYIIYSp//PHH3ePwBIqAUGQrGIP9YFbXsbJeNILMG4gOgAkXgZKwBOnHCbeHvydwZIbgGBE0qYNgzVCOGZkj+D5ktIBt27apLBQrwGqDYFkEyOrADYNMER0cDzJijCnSCB5GMKcucmBBmTZtmsqCKe+Ej/M5aNAgeeKJJ9xuPWTiQGRB5AUD9t34eyPhgZYPbzDmg8Q4SKdFVktu/hGvcR+41WfUdKXdhoOHHnpIZRXAl4+nT0wuWDZ69Gj3GGROvPfeeyqOAQWpUIwMT66IC9BN+nAVwI8P3z3GI50VGRWB1otGkCaLJ/RTTjlFWQ4w+SNrAyA7A3EYOEZYJPQUYyOwkiBOAwXb4F7CNhCrggyNYMF5ggsI68OFBYvKeeedp1xjgUAKMjJ5sA1YFapXr65iRYYPH+4eg6wmiEwcD2JjIDD+9re/qUync889V7macM1atWrlzg6yQ9zht4Lvw37973//UxlN+O0EAzJxYNXBbw3uHabahocERJ1KlAE1Dl8dlLGeIhXxOh/PtHUVFPN160al01HrWeeDhAVM2vDL4yYaTCCeOc32zlk56t/GX7H+jDn9+kzb020ReKgXl4Lg0J/wcRy+nm6xDiYKPCVjUvUW14A0TjyNovjT2WefHXA93EPgzsjKyvKo5olJErU9kO6pgwkdKa/GehMQTNgGJkqAtFGsq9fc8LYdiInvv//enTZqXgdAMOBY4IJBzIcx/gDXfN26dSp+Aa4DuKjM6+vfg8BanFeksPoC5wv7A2FjDFLVgZUF5w/3WewLpgLj/iOAFEGu+iRsBseC9XEMOP/mJoi4PjgeWGYguPR9xXmB8ECwbqAiarAUIbUWMSb69pFyvGnTJq/BxDgGfCfiPHAOzXEkSBvGdoyuGRTBw7EYA5Wxz7DQYLmv43cyR/zcn6zO3xQfASuc+rh1s8IpiXLxEU11Pggh8cMRG8QH3S6+QO8WCAyvvV2msLQ6iQkgMJBOG+kKp4QQ4g+Kj0ACBCXUUckUwaWIBWnWma4WElNAaEQqnZYQQqxA8REI9G5hOi0hhBBiG0y1JYQQQkhEoeXDSuYL3S6EEEKIbVB8BMp48RpwOpUBp4QQQkiI0O0SINW2uGC7rKmSIguqpaq/xaj9gRRcfE4IIYSQoKHlw5erZdFYyU6tIlPq1padhiJF6UVFMi5vv2QtGufKhEFAKiGEEEIsQ8uHN7Z8I9lF+2R0g3qyM8lTXOxKSpLRDepKdtFeVywIIYT4YMWKFfLBBx9EzflZuHChrF27VmIVVFSdNWuWbdv79ttv5d1335VY4dsI7O/XX3+tqsqGG1o+vFB8YIeyeKi6pqaS0FpCAsrCytS6taX7gR1CuwchJ0BTL/QMQWXDoUOHepwaNCJDNUSU+9ZBOW+Uw05KSpIOHTqoXiPeQLnz1atXq3LXJ510kiqx7qsF+9KlS1UZbfQS0RvTmcuK4waLstrYxsUXX1ymNLhdoPEbvuvKK6+UcIHzgmqTV199td9x6MeC5m4o766D5m84XyjZfvrpp3uUizeCUvW4Tui1c9lll6my+eEa4w/0hnn55ZdVrxg7yM7OVuIwmnsClXd/rf4+dFCdFG0J8P+jVq1aEi5o+fBCTvEBl6vFRy8KCJDc5GQ1jhDiWVoZzbzQgXXx4sUepwai5J133lH/RjdXTMjdu3eXzz77TN1UBw4cKP3793d3egXouYLGYBAmH374oZq4MPlgkvTVrRQN2O69917VdK7M/+2cHNXH5OGHH1Y35YkTJ6qeIxBBscrnn38uTz/9dMBxjz76qDrnumhDw7T27dur8wRRgK6wEGJ6510dXEs0bFu+fLk6X+3atVN9bcIxJtLgd4VGcrFChxD21+rvQwfNANHTCI0hwwktH17YnZZu6eTtPrDN7utBSFyki6Nh2NixY9UTlLeGcjAdf/zxx6rBGCwZRkuB3uuyqKhIPR1jfTR/q1v3RJVWCBSIEW9uDjQdg7h47bXXZNy4cR7fX6lSJZk3b57q5ArwXRA8ECxY1xsQQBBNt99+uyxZskTtCzroomsrmq9BPMGqgOPo2bOn+g4zaFQGCwM62+KcoCeGDhrxYcLv06ePexlcI2geZ3zCx3sIJnSRRVM1WA9gxcB+o4mcPlnAstSmTZsyohDuCog8HZwDrK83TcNxwvrx3HPPqWsH0JJ+5syZan8gFtDED91v8flbb71l6xhvHDp0SP1O0KSuY8eOfi0iOD8ZGRly0UUXlXli93bu9Kd8swUN1jV8J64tvhP7ivOkXwuc723btqnrqF9TdBr2ZbUzrgOx7et3AAL9nsz7G2hf/P0+fJ0TgGMdM2aM6iDtrSmhHdDy4YX61ayJj/rfv+e6sRMSrSArCx2a3+grMucW11+8D3O21kMPPaRcJbqlwwwEQp06dTyEB8AErHehnTt3rrJUwNJhFB4Abhpvrgw8xcOCMmrUKPVUbfZd46lOFx4AwgQTAkzMvsBEAGsOxk2bNk0dF8QCOqeiE+ro0aOVLx5j8B7LjcDkjSfJ2bNnq2PCBA+RpYPlr7/+usc62O8pU6a430+dOlV1V4VQ+eijj9RkgcnmwIEDqgsuJkh0wMULk6cZiCZgnMDRvdbYrRWTD0Sj8Vxg384//3wlGAC68cKdhuOAOLRzjBm4gmCZmTRpktp//DYgYMxuPhzH3//+d/nuu+9k+vTp6vzi34HOHYAYe+qpp9xjMUljH2ElwndCZMJiY7wWuHaYmHE8//nPf5RQwTqIp/GFvo6/38FeC78n8/4G2hdfvw9/5wRAwO3YsSOs8UG0fHghs0GmpFeuJbuO7lMuFjOI+UgvLpbMvdtdT5Qsv06iEXdnZmNXZjwGl6aLh7EzM55AcfOECLnqqqs8WscDTOSPPPKI3HTTTTJkyBBlTq5WrZrHmC+//FKJDr2tfSBwU8WNHTdgPCEOHjxYXn31VXUj9QcsIVa+45JLLpHHHnvM/R4CBxMnbtDYd3w/toPJ0mjmhnjBk79u2XjggQdkxIgR6gnUm5XEG88++6yaWHVfPywZW7ZsUWIKbils35+ZHO4qtLXXhZ038ASNSe+aa65xL9u4caMSAEbQjh6TPr4f27RrjBkIADyV48kdsUKYhPHEXq9ePfcYuG9gLcBTvH5seFqHJQtP9f7OnTdw7fDb0b8TEzcEGYSyEQhbCGOcfzBs2DB54oknlKXOF4F+B//4xz8s/Z7M+NsXuCe9/T4uv/xyv+cE1hWcB7gjIYDCAS0fXkhKTJJxjS91Cw0j+vuxeftcwaYwZRMSpeniZYSHonQZ0sXDaLm7//77ldkcsQVmunXrpp64YAGByRg3OjyJYZkOTMhNmjTxWA+TDG6i+ssYM/Dee++pmyae4sBtt90mc+bMUUGVvsAEh0kKN+tAQCiZA2jxVKyLJkyUeI/lRpo2berhUhk+fLh6AoUgsAom3P/+979qMgQI6NUnGytg4sY59gWejBGQCHM9jkEHT85mF0bt2rXdn9k5xpsohDDVW7ZDAJgDLeG+g8UG1rHnn39euYywPVg+0PY92HOHSdr4ndhHoxjTwfrGbUA847fsj0C/g/kWf0927IuVc4LfC9yF4YLiwwdZjbvKtF17pIEh+A3A4oHlWYWlQVmpJ1Q4IVEDLHLGyrxl0EQKtoU1XRwmfVg+MMHn5+eX+RyxFsgEwWcwJzdu3FiuuOIKFSCn3/xgBjeCiQU3bKRcwrKC+AujywXbwJMuhMmqVavU0/Dbb7/tdf9efPFF9VQJc7X5qdwbRp84+Ouvv8qII0wwW7du9ViGfTLSqFEj5UeHpcEqiNfAkynWxeQyefLkMoGh/sDk4muSRzwFgk137twpCxYsUC4RHWQBmdfDU7L+mZ1jzEBYms+d+XzjHELg/vbbb2rCxe8BcTx33XWXsogEe+68faf5vX4+jcBygfPoj0C/g78s/p7s2Bcr58SbYLQTul18kZCgBEb3wsOSUyVFdiclSX24Wo4c9Uyv9ZERQ0iFYtUiF2bL3R133KHEAHzMvsCTHlwjsFhAMOBJD6ZoWELgNkFQqh6n0alTJ/XCZGP0/8Migqdd/WlSB9vFNrDcCEzO8K1DeMCvHwrp6ekqLsEIgjbhcjJiHgOLDtJ9sT6AQDJm+ABMqEbgx0f2EJbDHYV9RywJMoisAHcF4lXwvcYAQkxScIv99NNPKsYBk5ERWELMT9F4jwmuefPmto7xJva8nV8jsIZ07dpVBRb7Iphzh+/E9TFi3odQCfQ7SLf4e7KDQOcEljJYRcyBy3ZCy4cvDrl+BBAaHY4cld6HCtXfJB/jCIkqkNVi57gQQawHrAtmF8mPP/5YJjAST1p4GtZvxjB3Y2KCcPAWRGkEVg+IF6NLBq8ZM2YoKwl84jq4wcJqgvgQWF9CBe4iWFUwgQAICLxHbIh5ktXjD/SnTtQf0c3ceLpFTISe5YO/ixYtco/He3yuCzWIJQTV6svw5IvYCX8gxgZPtsaUYlgGEBcD4YYJyPzUDQYMGKACERGvAHCs2H/EE+hxPHaNMYPzCFeafl7gGjIXbMP1e+WVV8pYVTZs2GDp3JlBqvH777/v/k6IMwSH2kGg30GWxd9TsJh/H1bOCbKHYPVARlK4oOUjym/ehIQE0mnRBBHBpV7jPhJcn2NcmIGfHhH6EAB6tgNSAHv16qUEQ6tWrdQTGKwQePJEsCBAASpE7ePGiMwA3CQhRiBQkI6I+hyI8cAEgRv5hAkTynw3hAxuoLB+ZGZmqu3BGgPLCp70jEF4MNUji8YqEFXIHsHEjgkCriOYzhFnYgQTDKwLKHqG+BMIIsQo6O4GpDU+/vjjaoLGvuKJFEJNL8CFyQLn8OSTT1bBf3gihYBCfAOAhQhiCjE2sFx4S7XFPsClhckM50E/XliZ8NRr3GecYwQk6gLwjTfeUMeI+hJr1qxRkxRqdejYNcYMridcAjgeWDcQSKxPzDqI1cG5P+uss1TMCqw6cOUhgBWp1oHOnbfvxDVFiiu+E9kkEGl6DEh5CPQ7mGTx9xQs5t8HRA72wd85QSzNzTffbDkgOhQoPmLg5k1I0KCOB7ovq2yXBNNvuNRV2GuK7fU+8CQ1cuRIj6dopLMi6BQ3ND3VE0/cuMEjwA9WENyAkc6IJ1njDQ8ZERAqmJAxYcEcDN85xuIpFZMNXDC4mSJuwRsPPvigyuIA+J67775b/dvongH60643AYNjMsZCABwjnrCRTgz/OY4J+6EHUgJkG8BsjgkUxwpgZUAtBh0cD6wzeOKGkEIWEFi5cqX6i2OEcMPkC9cSJjFMsJhwASYQuEwQQIj98GUlQhYIzhlqoOCpFsXWcC6QYWE8F8aqo/huxIFg37CPSGuFu8voCrBrjBlMjgjGfPPNN9V5QaYRrp9RsCBwEtcWgcrIEkGQJoQcJlwr5w7ixmh5gaDF/qH2CEQHMlAwHtfMfE2NQBwjwNkfEISY5H39DppY+D2Z99fKvph/H7CC+DsnGIP/b8EERIdCgubrf1wFgicbBJshEM0cTFMxqYri/eYdxlRF4mwQqY8ncxQhKtdTF37DyHoxBp+mNXYJD/52HQfcU5ic9MmZlP1/BxeOXpYfwgyTN8TSP//5z5BPFwKvMcHr9VaiGVgHcQ4gfkK5P1mdv2n58AduzhAYZW7ejXjzJrHzG0b35QhXOCXRCYpxEd8gzgIuD1iIYHFAui8sSXBNOYXL/NQqsROKDys379N6iayZIbLvD5HazUU63CqS7D1IipCoA0KDhfAIseQ2hHsCga1IgYVbChaA8jYe9OYecToUH6GYrVe84PKn02xNCCFxBVwuetCzXaDKKHGo+Dh8rFgeX7BR/sgrlOZ1U+WB3m2kauWkqC1PTQghhMQrjhAft765RhZvPFGcZtmvIm+t/FN6tGkgM27sEGJ56gRXeWr40+k/J4QQQiyT6DThYQTL8Xm0lqcmhBBC4pHEeHe1+BIeOvgc46K1PDUhhBAiTne7oNDPV199pQqwoOAJqqUZQaQw8oSNoGgNqrdFGsR4WB036XJTl0NWOCWEEEIq3vKB8r8jRoxQnSZRHhZlcq+88kqPyoArVqxQZZKRVqS/9F4NkQbBpSGP0yuc6gXFvFY4bcwKp4TEGeg/Ym4uVpGgmFOg3i3hACXe0enWLlAq3K4mbZEgEvv7+++/B9Wd2LGWD1Rpg7VD54YbblBNcdBnAX0adNBv4d5775WKBlktCC61Ms5/eWpvaCJtBzHYlBAfoCMneqXUrVu3TCEndKrVQTl19JwwlvU2g/Laer+TcD/M4KEKFt2JEyeG7TvQJh2VIfVKmr5AiWvcW3/55Rf1HqLILIzQFRdlwc2gXw6uAc6tuTS8FVDjAg+PL7zwgthV4Awl3ctTKTSShLK/Vq+rDjo+4+HdV6+ZeCYoy4dReOi16lFnHgrRCH7w6E+ANtrGmviRBum05RqHNNrOd/le8ZvnXem4hBA3aJiFBxDcH3DzRpMv9HbRLaR4mkSJb5SsRgMz/EWvEfwbJZmNoJcLGlyh2uRFF12k1kPPD/TeiGWGDBmimu0F4r777lNNwdCzBKARHhrD4VzpL2zLCM4z1oPo69Spk5oIcf4rGnTvRePAWCGU/bV6XXUeeOABVfIeHW+dRrkCTtHuGLEf5j4BderUUU8qMF+iURQqxPlrIYNWyagHb3zZAep4IJ3WH/jcZ70PpNtu8GzhXAak22IcIVFKcUmxrMldIwt+X6D+4n24QO8KdI3Fkxye0Hfs2KEeQNA8DvcKs0j56aeflOkZL7QbRyMv45M7ut5iDBpuoflZXl6eahyGxll4avQFRIveLA0uYlgOjPcg/XP07vAHjsGb6R2uEGzXCCpimt0UOGbcB3E/1IEFB24U7AOODS/zuQE4Zzh3N910k8dyNOfT18ML7c+NoMU8XnCBY3/QmA0dbK30FcExmNvTm8F18XfuvB2z3jHWLJQAHl717zS7eowuMJwvdGANhHEdX9fPyu/AvL+B9sXfdfV1TtLT05X4jgZxGHG0ENmwYYNWs2ZN7b777vNYvnnzZo/369at0ypVqqS98cYbPrc1YcIE3BXKvPLz8zU7+Psbq7VmYz8p88Jyv/y+VNMmpAV+YRwhNnL48GFt48aN6m95WPzHYu2S2ZdobWe2db/wHsvDwcSJE7XGjRv7HbNjxw71/3vhwoUeyy+99FLtyiuvdL9/7LHH1L3j999/L7ONBx98UKtataq2a9cur9/x5JNPai1bttSuvfZarXbt2lqrVq20o0ePqvE9e/bUKleurDVo0ECrW7euNnPmTI91L7jgArUfbdq00TIyMrTk5GRt0KBBWmFhoXsMtvf88897rDdgwABt+PDh7vfPPvusVqNGDe2kk05S98rbb79dbWP06NFaamqqVqdOHbUdvP76668yxzBp0iStQ4cOZY77wgsv1LZu3arl5eV5Pfb27dtrt956q8eyzp07a9dcc43m75qcd955WkpKila/fn2te/fuWpcuXTyO59ChQ9rNN9+sVatWTR0T/o4bN04rKSkJeMwA533MmDHusTiGc8891+934lpcddVV2tlnn61+Vxjbu3dv7ciRIz6Pxcr1s/I7MO9voH3xdV39nRPw4osvas2aNdPi5f6EedvK/B2S5QNPEVBrCEBFa2sjzZs393iPmBB0BTSrcyPjx49X5lb9Bb+ZnaCQ2I//6CU3dDpJurasp/7ivc8CYzpMtyUxTPaWbBm9ZLTsLPR8Gt9VuEstx+d2AxcLrB3vvvtuQKsCnnjxdLh+/XplekbXT8Rb6KBNOpp8oXOmmVtvvVUF6iHezBeIK2nZsqWyluB74CJGwDyeTLGPeMKGP/+WW26RH374wWPduXPnKvM5xv32228qyw9uZKtgvVGjRqm25WhRjifmc889Vy3Hds8//3x1DPoTcuPGjctsY9WqVdK+ffsyy5FtiG2hBftpp53mcQ5gRcb5hLvF3FsE7dN9MXz4cBV7AysB3OZ9+vRR18PIsGHD1P7jCR7HhHOG64w28YGO2dd3Ij5C/05Yyc3fCZA9ie/A7wUWJ1yLV199VfwR6PpZ/R0Esy/ermtiYmLAc5KZmak+g+XESQQtPvAfunv37iog6/XXX1cnNxC4CeE/hS8QDIXWu8aX3cC1gnTat245T/0NWFodMN2WxChwrUxZPUU0LxV69WVTV0+13QUD8QC3y4033uiO00CaPTLkzCAuDDELV1xxhZoM+vXrpx5qdHBDhpjx5Y/HZKm7VryBtt4PP/ywJCS4MtYgQmbPnq32B65hgHgSPBzpE6gOgjz1IPpmzZrJmDFjyozxB9wIcPPowbEICkUAI+JVrIIJql69eh7LIEYQhIrziQc1Pe4DLhoAVxACes3r4b2vDB4s//DDD5XLq0aNGmoZ4kxw3MYxiLO5/fbb1aSNeQD3dFwzTPTBHjMEx/z589W10L9z5MiRZR5ewTXXXKMmbIBYInScXbt2rd9z5+/6BfM7KO++WDkn9UuDU32JtHglKPEBBYmbCZ5GZs6c6VV4mP2KsHjk5OR43FRiBqTbVq3tf0zVOky3JVFHzq6cMhYPswDJLcxV4+wE94QXX3xRTS6YrPAEjhs6AkXNQXV6zAfuK3j6xIQ6aNAg9+cQF75STOE7x0MNrBm+gEDBzV5H/35YY420a9fOI/sGnHHGGR7vETiLyQHxDlaARQIiDE+1mKCnTZtWJjA/EDg2c4wAShvo+4/zA8szhAUsEADZRcC8HoSC8VwYwXnBBGk8ZlxHBAzr4DqVlJTI2LFjpW/fvuqYYKlYvHixe7vBHDNifIDxOyAS8TsxY7YKIfg2UFyKv+sXzO+gvPti5ZwcK71W/n7L4nTxcemll6pgUPzY77//fpVOi9fnn3/uHoMbDQKioJBxI4HowL//9re/SWziq84HIdHL7sLdto4LFj17BZMjntQBgiD9WSmGDh2q7iV4stYnB31dMwhAxYRpnkCM6BOxjp7Ka66rgPfmlulHjhzxeA8RhO3pKau6NcWI2c00ffp0JayuuuoqWb58uZqI4EqxCqwACAD1B0QC3C9whQA8yeNYzE/REHYY5w39vHg7Zh0IHd0VZgx2xevTTz8N+ph9faddNS/8Xb9gfgd2EOicbNu2TV3Hk046SZxEYrB534gARt64sYiYngYG3nnnHeUDQ1ouxAduHojkteKeiTrQt+Ww60boE3zO/i4kyqifWt/WcVYxP3EDuF/gSsWTsz/gMsB9Qp/gYZ5GnIK3uI5HH31UTc7mCsv+QC0M3KuM6f8QDEuXLpWzzz67TFyF2aKLp2n9KR+mcuMEj+1AEOnoWQ64V8IFNWfOHCWU5s2bp5Yj1sFbhosRZPqsXLnSY5l5HbgQEKeACQ3g/HXr1k0WLVrkHoPzjlgFWK39nRfjMeMh0+hOgFUA7pEPPiib/WfM6PB3zObvrFatmjr3OgcPHgzoTrGKv+sXzO8gWMzX1co5WbVqlbKMQIA7iaCKjCFwxgo4uf6eSGIGBpySGCWzQaakp6ar4FJvcR8JkqA+xzg7efrpp9XTHXzjrVq1UmZuPIwgoNAYTGoMOMWNHxPo5MmT5eqrr1aTEsDDC0zWWIb4EEzGiHN4+eWX1WQCUaI/kVsBT7yoqwDXAf4NvzvSdSGYEHNiZOPGjXLnnXeqWIBvv/1WFdqaNWuW+/PevXurY0WAIepp4HOjlQJproihgNUXQa84PrwefPBB9Xnr1q3V5AdxheNFbIv5WPCkjNgLuK0xOYEuXbqogMazzjpLnVN8ByYtpB7rIM4FQgNF0nr27KncWzhv2JY38LQ/evRoZcWG8INrAeLOaPnA+YIVC+Ng9cF2YaFCICUs4bg+gY7Z/J2I8cB3YsJG/AS2j9+LN6tSsPi7fsH8DoLFfF0RRIrUcH/nZO7cuWo/nUbQvV0chSngFKF5OVVSZHdSktQvLpbMI0clKZjAVEIiRFJikozrOE5ltUBoGAUI3oOxHceqcXYCdyyeshGDAP82Jhb48fEEjwkT4OkTwgQZBnjBHI4nQxTGQkaF2WQNdy/iR1577TU1WWDCh0XVn5kagsBblgwy62CJgXsYlhYEcEIsGZ86EaAIlxEmVwQqwr2D8Yi30MFymPYxScOqg8/gaoIlGMD6gAkG+484AkyuiJNDnIR+nmC1gJDAJI+eWOZ4ArhQ8Dm2AQEBECj55JNPqvd4eoflB+cN323MbIEww7lFICnONZ7qEQPjC1i0ca2eeOIJdS6wn5gsjUW2cG0wUcN9BgsI3DgYhywRK8dsLtoFoaKLGv07IQCwH8ZrYQ6exXYDlZvH9YEVyNf1s/I7MO+vlX3xdl39nZM1a9YoEe5E8ZGAfFuJMmDyw48Aaj0cmS+WQSbAM21FCrZLdmpVmVK3tuw0BG2lFxXJuLx9ktXnZZEzLq+4/SRxByY2pPJhAjXejIMF6bTIejEGn2akZijhkdXMusuCVAwoZIWsDWSGVFSPrHCBrBxjXA5iLvB7hwXMWzEyq8A6FO7y+HYxYsQIOe+881Srkni5P1mdvx1l+Sgu0WT15r2y68ARaVCjinRsUUeSEv2Y+PBUeOlkyV5wh4xuUK+M8XpXUpJaPu3LByTr9H7s80KiDgiM7k27q6wWBJcixgOuFrstHiQ84Ok8mCDVWAKBqnCfDRw4UMVG6FYwY8ZTvPOCTX1zYhHHiI9FG3bIIx9vlB35J6KgG9asIhP6tZFebRv6XK84tbayeCjhYfJFagkJMB3J1KqadP/ja0k6+cJwHgIhIQGh0SEjQEE9QiIMUk9Rp+X//u//lNUDwZ6IDyqvtdube4REH8lOER53zsopY7nIzT+ilk+/PtOnAMnZscrD1WIGAiQ3OVmN60DxQQghlkBgKTIo8bKTt99+m1cgBojB/NfgXS2weHgLbNGX4XOM88bug9ZK3lodRwghhDiduBcfiPEwulrMQHLgc4zzRq1Ua0FeVscRQgghTifuxQeCS8sz7lfN2vpWxxESDFGYjEYIcTiaDfeluBcfyGopz7htlayFxVgdR4gV9IJTgeoZEEJIpNHvS8EU+TMT9zMm0mmR1YLgUm9aDfkrGTVdabfeaFpkreun1XGEWAEphygchSqWekVIOyo/EkJIeSweEB64L+H+ZO6fFAxxLz5QxwPptMhq8QU+91Xv4+q01vJPTRPVlcLbzV/TlPkI4wixE71api5ACCEkGjBW8w2VuBcfAGm0t3VrITOWbRZjUgv0xq1dW/it81E5rbHclF8gr9dMU0LDQ4CU+r3wOcYRYiewdKAcM0o8B2pCRgghkQCulvJYPBxX5+NfSzeXcbtAO2D52SfV9i1AmnWW0cXVRPIPyBs1a7gsIKXA4nFT/gEZXVxdjSMkHOA/uh3/2QkhJFpIdnqdD9gx8HmPNhneXS8oQ91rqoyefaOM2Jcv76dVk63JydK0qEiuLjgklTFm8PMsrU4IIYRYJO7FRzB1Ps4/pa73QW36iwx+UyovGis3FBiKiVWtI3LeHSKt+4RhzwkhhJD4JO5Tbctb58NDgIzaIHLRAyJVS9tXH94rsuRxV+fbjfNt2FtCCCEk/ol78VHeOh8e/PSpyJLJIof3ey4v2CEy+0YKEEIIIcQCiU6p8+GrQgKWN/RT58NNSbHIorGGjjBGSpctGucaRwghhBDnig+9zgcwC5AEC3U+3Gz5RsQY71EGTaRgm2scIYQQQpwrPgDSaKdfn6kqmRrBeyz3V+fDzcGd1r7M6jhCCCHEocR9tosOBMbFrdPlrRV/yJa9hdKsTqrccH5zqZxsUX9VT7d3HCGEEOJQHCM+UGgM9TyMabf//nqzcrlYsnygiFhaI/+uF1Q5ZbExQgghxC+JThEe6O1irveBZnNYjs8DgmJjba/0P6btIBYbI4QQQpwuPgJVOAX4HOP8giyWDR/4H7NhDrNdCCGEEKeLj2AqnJYv2wX1PpjtQgghhIjTxYdtFU6Z7UIIIYTYQtyLD9sqnFrNYsnbZG0cIYQQ4lASnV7hFNRKrRS4wimyWGpYyIpZ9TLjPgghhBAniw+9wqm/cNL9hcdl8cbcwNkumTcF/kI0m/vj66D3kxBCCHEKcS8+QI82Gcq64YuEYDJerLB5WZB7SAghhDgHR4gPZLLAulHujJcA7V+CHkcIIYQ4EEeID9syXpp1sfaFVscRQgghDsQR4sO2jJcWXUWq1vY/pnJ11zhCCCGEOFd8BMp4wXJ8HjDjBUGn/Z7zP+bYQZGfPg15XwkhhJB4xxHiQ894AWYBor/H5xgXkNZ9Alg/EkQWjWO6LSGEEOJk8QHQuXb69ZmSUdPTtVK7WiV58bqzrXW21cusH97nZ4DGMuuEEEKIHxwjPgAExsN92kidapXdy/YeOi6TPv3RWmdbwDLrhBBCSLlwlPiAwBj+To7sPXTMY3lu/hG5c1aONQFitcy61XGEEEKIw3CM+EABMRQS81ZGTF9mqdAYyqynNfKIHkHpsTVVUmRBtVT1tzghUeRQnr0HQAghhMQJyeIQUEAMhcSsFBo7/5S6/jNeek0VmX2jepudWlWm1K0tO5NPnMr0oiIZt+BOyUpMFGnT394DIYQQQmIcx1g+bCs0BiAorpop2dVSZXSDerIzKclzG0lJMrpBXcn+glkvhBBCiGPFh22FxkoprlpbptSp5XLZJHim6Gql76dW1aSYTeYIIYQQZ4oP2wqNlZKTu8blajEJD6MAyU1OVuMIIYQQ4kDxYWuhMRHZnZRo6zhCCCHEKThqZvRVaAzvsdxyoTERqd+0s63jCCGEEKfgmGwXHQiMHm0yVFYLgksR4wFXi1WLh05mRgdJr5Qmu47lu2M8jCRomqSn1FLjCCGEEOJQy4cOhAbSaQe0b6z+Bis8XNtIknEXPKJiPiA0jKj3CQkytvNENY4QQgghIVo+iouLZe7cufLVV1/J8ePHpUOHDnLTTTdJpUqVPMZt3bpVpk+fLlu2bJGWLVvKiBEjpF69ehJvZDXLkmkXPS1TVk+RnYU73cvTK9eUsS2vlqym3St0/wghhJBoJEHTTI/tfujUqZM0a9ZMunbtKklJSfLss89KnTp1ZMmSJVK5sqtfyh9//KFESZcuXaR3797y3nvvyaZNm+S7776TunX9FO8yUFBQIDVr1pT8/HxJS0uTaKe4pFhyVj8vu9e8IvUP5UnmkaOi7B01Gopc9gQLjRFCCHEEBRbn76DEx19//SVNmjRxv//zzz+VGPnwww/l8ssvV8tuueUWycnJUWIjMTFRjhw5IqeeeqrcfPPN8uijj9q681HDxvkis2/w/fngtyhACCGExD0FFufvoGI+jMIDZGRkKJfLvn0nWswvWLBABg4cqIQHqFKlivTr108tj0tKikU+vtv/mI9HusYRQgghpHwBpy+//LL62727K7bh8OHDkpubKyeddJLHOLz//ffffW7n6NGjSi0ZX5EATeRWbMqTed9vU38DNpXzxuZlIodPiC+vHN7rGkcIIYSQ0FNtEXR63333yRNPPCHNmzd3iwiQmprqMbZ69erK/eKLyZMnyyOPPBLRy7Foww7VxdbYbA4VTlFoLJh6H7Lla+vjTrkohD0lhBBC4ouQLB/ffPONcqXcf//9MmrUKA+RgUBUoxsG5OXlSa1atXxub/z48co/pL+QLRNu4XHnrJwyXW5z84+o5fjcMlaNJSEYVQghhJB4JGjxsWLFCunVq5fcddddMmnSJI/PkpOT5fTTT5d169Z5LMf7du3a+dxmSkqKCkwxvsIFXCuweHjTAvoyfG7ZBdOiq7VxrPdBCCGEBC8+Vq1apYTH3XffLY899pjXMddff73Mnj1bZcaAH374QRYuXKiWRwOobGq2eBiB5MDnGGeJ5l1EqtYOPC7nTQadEkIIIcEWGYOrpaSkRH766Se58sor3csHDx6sXuCee+5RbpmzzjpL2rdvL2vWrJFrr71WbrjBTypqBEFJdTvHKYvGeXeKLHnc/7gD20W2fGPdUkIIIYTEKUGJj1deeUVVOTXTpo2rWyxAsbF58+bJ//73P1UHBDU+4IqJFtDLxc5xirqnWBv38wKKD0IIIY4nKPFxxRVXWB4Lywde0QaayCGrBcGlvqI6aqVWUuMsUz3d2rh1s0UufZTxH4QQQhyN4xrLoYkc0mn9hZPuLzwuizfmWt9os84iqRZKxxfucbleCCGEEAfjOPEBerTJUNYNfwSV8YK4j3ZXWxt78EQDOkIIIcSJOFJ8IJMF1g1/BJXxAlr1ttdFQwghhMQpjhQfufmHbR3ndr2kNUKvPt9jqtZxjSOEEEIcjCPFx95Dx2wd53a99Jrqv5Qperz89Kn1bRJCCCFxiCPFR53qKbaOc9O6T4CCYwkii8ax2BghhBBH40jxkZFWxdZxbpDJ4rfDrSZSsI0ZL4QQQhyNI8WHXuvDH/g8qFofwWSyMOOFEEKIg3Gk+NBrfSR4CQ/Vl+FzjAsKq5kseZuC2y4hhBASRzhSfIBebRvK9OszJcNkAcF7LMfnQWMl4wUsmSyycX7w2yeEEELigARN0yxW0oocBQUFUrNmTcnPz5e0tLSwfhcKiaGeBxrJoZ8LXC1BWzyMQFTMDtREL8ElUkatZ6l1QgghcYPV+duxlg8dCI3zT6krA9o3Vn/LJTxAm/4iFz0QYBADTwkhhDgXx4uPsGC1yy0DTwkhhDgQio9wYDXwlKXWCSGEOBCKj3AQMPAUMR+NWWqdEEKII6H4CMtZ1Uuti49kXqTbTGGwKSGEEEdC8WHKfFmxKU/mfb9N/cX7cgWeDn5TJM2UsguLCJbjc0IIIcSBJFf0DkQLizbskEc+3ig78o94VDlFsbGQan4ACAz0e0HZdQSXVqsvgszmwj0im5e53C6wkhBCCCEOwvF1PnThceesnDL9aHWHSchFx8z1PxaNFSnY7mkFgXuGVhBCCCFxAOt8WASuFVg8vDlY9GX4vFwuGFV47EZP4aGu0g7XclY7JYQQ4iAcH/OB6qZGV4sZSA58jnEhUVLssnj4kzeLxrnGEUIIIQ7A8eIDZdXtHFcGxHuYLR4esNopIYQQZ+F48YF+LnaOC7mK6c8LQts+IYQQEmM4XnygkRyyWvyUA1OfY1xIWK1iuvIlxn4QQghxBI4XH2gkh3RaP+XA1OchN5wLWO3UAGM/CCGEOADHiw+ANFqk02bU9HSt1KxaSUZltZQebTJsqHZqIVumYJsrRoQQQgiJYyg+DALk67EXyz1Zp0mtqpXUsv2Hj8vT2b9Kl6lfqFogIYM6Hp2GWRvLTreEEELiHIoPA4s35soz2b8o0WEEqbYoQlYuAdKqt7Vx7HRLCCEkzqH4sFBsDGjlLTbmjv0IQGFeaNsnhBBCYgSKD4vFxqS8xcYQ+3Hp5DKLUVpsTZUUWVAtVf0tXjSeBccIIYTENWwsV0pu/mFbx3mlWl2Pt9mpVWVK3dqyM/nEZUgvKpJxi+6WrN4vhv49hBBCSBRDy0cpew8ds3VcoGBSCI/RDerJziTPrra7kpJk9K6vJPvrslYSQgghJB6g+CilTvUUW8f5CyaFqwUWDxU9kuBZ/0MrfT/1l7eluKgcQocQQgiJUig+SslIs1Y+/c+8wtDPdmnQaU6VFJerxSQ8jAIkNylBcta/Ffp3EUIIIVEKxYepzHogkIobcsptacGx3SZXiy92//FVaN9DCCGERDEUH17KrAeiXCm3bfpL/TOvsTS0/h8rmPlCCCEk7qD4MFU5vSerpd8TppU35VZEMrOmSnpxiSRo3gUMlmcUFUlm/i6WWyeEEBJ3UHyYaF6vmqUTt+uA/5og/khKrizj6nRQ/zYLEP392Lx9opwzLLdOCCEkzqD4MNGgRhVbx/ki66xbZNquPdKgGLkvJ0gvLlbLswpL64ns+bVc30MIIYREGywy5iPwNDf/iNdS68hPQfdbjCsXzTpLVlIt6b51u8p+QRBq/eJiyTxy1GXx0Fn6hEj900XaXl6+7yOEEEKiBFo+/ASeekuEhSB58LLWalz5znySyDlDlNDocOSo9D5UqP6WyYPRSkQ+uElk4/zyfR8hhBASJVB8+Ag8nX59prJweOPu97+XyQs2lv/s1z3F+thF45j5QgghJC6g+PAjQPq2a+j1M2TZvrJ0c/kFSGnFU0sUbGPmCyGEkLiA4sMHx4pK5NWvN/s9eTOWbVbjylvx1DLMfCGEEBIHUHz44K0VfygLhz/wOcaFfvZdFU8tk7cp9O8ihBBCogSKDx9s2Vto6ziftOkvctUbIgkWLsWSxxl4SgghJOah+PBBszqpto7zyxmXiwx63drYj0cy8JQQQoizxMf69etl2LBh0qlTJ/nss8/KfD5z5kz1mfHVo0cPiTVuOL+5WMmmbWCxG25AUMfjogcCjzu8V2TpP+35TkIIISTai4xNnz5dXnrpJbn99tvVv3fv3l1mzF9//SUHDx6Uf//73ye+BO3jY4zKyYlyS5fmMmOZ/5iOxxf8KL3PbFj+uh/BpN6uelmk272umBFCCCEkni0f1113nbJ8jBgxwu+46tWre1g+zj33XIlFLm6dEXBMeZvMhZR6C+vHlm/s+U5CCCEkmsVHzZo1LY3btGmTcrX069dPJk2apCwhsYjV5nHlaTJXJvW2ai1rY39eYM93EkIIIbEecFqpUiVlIRk9erT6O2fOHDnnnHPk0KFDPtc5evSoFBQUeLyc1GTODdwo5w2zNnbdbAaeEkIIiUlsFx+jRo2SZ599Vi677DK59tprJTs7W3bu3KliRHwxefJkZVXRX02bNpVoajLnK5oDyxva0WTOCGI5UmoEHle4h64XQgghMYnt4iMlJcXjfb169aR9+/byv//9z+c648ePl/z8fPdr69atEu1N5vT3+NyWYFOj9ePsG6yNPbDDvu8lhBBC4qnOR25urlSrVs2vYElLS/N4RXuTObzHcnxuO616W282x263hBBCYgzbc2CnTp0qw4cPVxkvmqbJU089JT///LNft0u0A4HRo02GympBcCliPOBqsdXi4a3nS8F2/+MK80Rm3ygy+E1XpVRCCCEk3sTHmjVr5K677nK/nzhxorzwwgsyYMAA5ToBVapUkZYtW0qdOnVk7969kpCQIG+//bZ0795dYhkIjfNPqRuZL9N7vkBYKLTAFpDWfVj3gxBCSEyQoME8YRFkoWzcWLaNfIMGDeTkk092vy8uLpZff/1VUlNTpUmTJpKYGJx3B9+DwFPEf0STCybiwKXyyT2u4NJA3PSJSIuukdgrQgghpFzzd1CWD2wIRcMCkZSUJK1btw5m08QbcKUUHRGZe2vg88PgU0IIITECG8tFOzUsBrQuGs/gU0IIITEBxUe0owef+qw2Ygo+ZfYLIYSQKIfiI9rRg08Dop0IPi0pDvdeEUIIISFD8VFOiks0WbEpT+Z9v039xfuwxH4gnTY1ULaNJlKwjZVPCSGERDWx1+s+ili0YYc88vFG1dlWB+XWUfXU9uJjwQSfoukcM18IIYREKbR8lEN43Dkrx0N4ALy/Y1aO+rzCgk9XvsTYD0IIIVELxUcIwLUCi4c/B8u4uevtd8G4g08DkcDYD0IIIVELxUcIoMy62eJhZn/hcXnhi9+kwoJPGftBCCEkSqH4CAH0d7HCa8t/t9/6gdiPTsOsjf39K2a+EEIIiTooPkIAjeWskH+4yH7rRzBdb5c9KfJMW8Z/EEIIiSooPkIAHW1rVa1kaezT2b/YH3xqtfAYKNjB4mOEEEKiCoqPEDvcDrmgueXxCE611f3iEfsRSICw+BghhJDoguIjREZc3FJqpVqzfiA4FUGqYSk8lmYl/ZYBqIQQQqIHio9yWD+mDDzT8vjFG3PFdiBARm0Q6XaftfEoPkYIIYRUMBQf5QBVTO/Jamlp7Oxv/wpP6XW4YFpcaL342IaP7N8HQgghJAgoPmxwv9S24H45eDRMmS/BBqDOGSLyAwUIIYSQioPiwwb3y8CzG1sa+/o3m8Nn/VABqBa2rZWI/Ocmpt8SQgipMCg+bCCrTYalcah6anvgqYXiY8UisqZKiiyolqr+4r18PJIFyAghhFQIFB8RrvthtTqqXcXHslOrSs+mjWRow3QZ26Ce+ov32QmHRZb+M3z7QgghhPiA4iPCdT+sVke1o/EchMfoBvVkZ1KSx7BdSUlqeXbOdFo/CCGERByKjwjV/UAoaMOaVZSVJGwYio/BtTKlbm1XFEiCZyCqVvp+as2qUrxkSvj2hxBCCPECxYfNdT+85Zvoyyb0a6PGhRXEflz5huRUqSI7k5PLCA+jAMlNTpac1c8x/ZYQQkhEofiwue7H9OszlYXDSEbNKmo5Po8IbS+X3W0vtzR0d1KiyAfMfiGEEBI5kiP4XY4AAqNHmwyV1YLgUsR4wNUSdouHifqZQ0QWfxN4XLHKfRFZNE6kdR+X64YQQggJIxQfYQBC4/xT6nosQ32PSAqSzIwOkp6UKruKDrljPIwkaJqkFxdL5pGjrgUF20S2fCPSomvY9okQQggBFB8RYNGGHTJx/g+SW3D0hCsmLUUm9j8jbK6YpMQkGddlkoxeMloJDaMAwXswNm+fJJl7v1B8EEIICTOM+YiA8LhjVo6H8AB4j+X4PFxkNb9Upp36N2mgu1ZKgcVj2q49klV4uGzvlyVTmX5LCCEkrCRoWuljcBRRUFAgNWvWlPz8fElLS5NYBa6Wcx5drCqb+gLpud891COsLpjiDXMl59NhsjsxQcV4wNXiN7IDtUKQsovMGUIIIcTm+ZuWjzCyclOeX+EB8DnGhZOktgOlQ5+XpfehQukQSHiAgu0is29k/xdCCCFhgeIjjKz4fY+t48oFUm999H7xCTJgSjxdNoQQQkh5ofgIK1ZdKRFKw/XS+8U32okMGEIIIcRGKD7CiDndtrzj7Ov9EoTYWT2D1g9CCCG2QvERRjqdXNdvvxdQO7WSGhcRDL1fLPPjPJFn2jL+gxBCiG1QfESg34s/Jg88M7LVT5HBMvhNkRpB1BdRAag3UIAQQgixBYqPMIMiYi9fnykZaZ79XtD/5eVI9nsxC5B7fhBpMyC49T4eSRcMIYSQcsMKpw7q91LGBdPhVpGN86yvc3ivyFdPiHQfH849I4QQEudQfFRgv5eK6PniNQAVbhWrfDVVpH4rkbYDw7lnhBBC4hiKjwoEpdUf+Xij7Mg/4uGOmdCvTWTcMXoAKgqKIbXWEprIB0NEtq8VuXRSmHeQEEJIPMKYjwoUHnfOyvEQHiA3/4haHs6eL+UOQAXfPCey4aNw7RUhhJA4huKjAoCrBRYPb7YGfRk+x7iIBqCe9bfg1lswhgGohBBCgobiowJAjIfZ4mEEkgOfY1zEgAtmwPMiVetYX6dwDyugEkIICRqKjwoAwaV2jrNVgPR7Nrh1fpwvsnkZLSCEEEIsQ/FRASCrxc5xtrtgrnxDJMHiT2P1v0Te6MsqqIQQQixD8VEBIJ0WWS2+EmqxHJ9jXIWADriDXrU0FD1v11RJkQXF+2XN/L9L8Ya5Yd89QgghsQ3FRwWAOh5IpwVmAZJQGvNxTYem8sm67bJiU17kAk+NoI5H57v9DslOrSo9mzaSoQ3TZWyDeupvz1UPSvayxyK2m4QQQmKPBE3TKmBm809BQYHUrFlT8vPzJS0tTZxU50NvRLe/8HjF1P4wg3TaBaNFCvPKCI/RDeq5snMSTkiohNKf07RT/yZZXVgJlRBCnESBxfmb4qOCMVY4/WNPoTyT/UuZFFx9ap9eUb1gSopdWS0ILl39L+VqgcVjZ1KSh/AwCpD0Ek0WXf+tJFWuGvn9JYQQEtXiIyi3S3FxsXz88cfSu3dvycjIkDlz5ngdt3DhQunWrZs0a9ZMsrKy5Jtvvgn+CBxWdr1vu0by3po/o6f2hzkLpkVXkdP7q7c5VVJkZ3KyV+EBtIQEyU1KlJwX27ETLiGEkPKJj2effVZeeeUVGTZsmOzcuVMOHz5cZszy5culf//+0rdvX1m0aJFkZmYqAfLTTz8F81WOIyprf/joBbMbFg8L7C466CrdvnF+2HeNEEJInIqPUaNGySeffKKEhS8mT56sxMb9998vp59+ujzxxBNy6qmnylNPPWXH/sYtUVv7w0svmPrFcLwExjVOE/nkHpGiY2HfPUIIIXEoPhITAw9ftmyZ9OjRw2NZz549ZenSpcHvnYOI6tofRtr0l8y+L0t6UZE7uNQMlmcUFUnmkaMnKqFOa00LCCGEEPtTbQ8cOKCCTdLT0z2W4/327b7bth89elStZ3w5jaiv/WEgqe1AGXfa9a79MgkQ/f3YvH3i4ZxBtszsG0SWTGU1VEIIcTi2ig89azcZwYgG8B7Bqv5cNYiO1V9NmzYVpxGo9gd4uM/pKuZj3vfbKq7+RylZXR9U6bQNTPuQXlws03btkazCsvFAiiWPizx9Bq0ghBDiYDxVQjmpXr26VKlSRfbs2eOxfPfu3dKgQQOf640fP15Gjx7tfg/LhxMFCNJokU5rrv2RUbOK9D+roUz69EeP5RVa/wMCpMt46d5xlMpqQXApYjzgagkYjnpghysQdfCbrnLuhBBCHIWt4gMxIeeee658/fXXMnz4cPdyxHt07NjR53opKSnqRVwCpEebDHftD8R47Dt0TIa/k1MmDTc3/4jcOSun4up/wGJTuap06PmUS0x4TRT2hSby8d0iVWqKNO/iCmYlhBDiCGwvrz5ixAj58MMP5fPPP1dumHfffVfV+TCKEWKt9seA9o1VjMekTzf6rP+B18T5P1SoC0ZZL2DFSK0X3HqH94m82Z9N6QghxGEEJT6QyYLiYnjpQgP/HjNmjHvM1VdfLZMmTZLBgwdLamqqjBw5UmbMmCEXXnih/XvvAALV/wC5BUflhS9+kwoFAmT0jyKpdYNft2C7KxgVpdwJIYTEPUGVVz927Jjs3Vu2yBVEhrmMKgJM9TKrVlJ0ndjbxQoILh353veWxr5cge4XNygoFrQLppSERJErXxc54/Jw7BkhhJBYLK9euXJlt+XD+PL2BUlJSVK7du2ghQcJva7HPe9/X7HuF6MLJq1R8OtqJSL/uYmZMIQQEudQGcRI/Q8rHD5eIiPfXSsVDgTIqA0iFz0Q2vqsiEoIIXENxUcM1f+wwifrd8iCdTukwkH2ykVjRQa/FbwVhBVRCSEkrqH4iAEQx3FZW1eQrxUenreh4t0vZivIDfNEqta2vp6qiMqmdIQQEo9QfMQI13dqZnls3qFjMnP55ugRILCCnHKRSL/nQnPBrJstsnkZy7ITQogTs10iBbNdygIh0e6Rz+TQUWsdZaOhAqpXkE47Z4gruDRYUEekz1PMhiGEECdlu5CKjf14clC7oNbRK6Au2hAFMSA6bS93pdOGAmJBkA3z+cN27xUhhJAIQvERQ/Ru10hu7drc8njdpIVeMVHjggGo44FA1CArosLms6ZKiiz4379lzYqnpbjEuhWIEEJI9EDxEWM82OcMubVrC8vjITlQIRWVUqOKICuiZqdWlZ5NG8nQhukytkE9GfrLa9JzTk/J3pId9l0lhBBiLxQfMciDfdrIS9dlSp1qlS2vgyZ1UUdyZZG+zyD0qPTlW3iMblBPdiZ5Np/bVbhTRi+5R7KXTmJAKiGExBAUHzFK73YNZc2DWfJwn9Ntr5RaMRVRvQfFwrEypW5tlwspwVOgqGWaJlN/fUeK3+jLBnWEEBIjUHzEeBDqzRe0UFktvu0GInWqVZLcgiOyYlNedMV+mGuB3PSJyMAZIiknIqRzqqTIzuTkMsJDR0tIkNzkZDVOCnawNgghhMQAFB9xVAHVlwDZe+i46vty7YyV0mXqF9GV/WKsBdKiq0i7wSL9n3cv3m1ytfjCNa5UWC0ax5oghBASxVB8xAGo4zH9+kzJsNADJirTb71lw3S+W/2zfrG1jJYT4zSRgm0iW74J4w4SQggpD8nlWptElQDp0SZDZbXk5h+WSZ/+KHsPHSszTiu1kCD9FuNhOYlKLp0k0ugcyVwwWtKLimRXUpJysZhJ0DRJLy6WzCNHPT84sMMVhHpwp0j1dJFmnV3WFUIIIRUOxUccASFx/il1VWyHN+HhLf0W46OWtpdLUpt+Mu67F2X0xhlKNGkm4QHG5u2TMrJi0XhXUTIdNLfrNdUVX0IIIaRCodslDrGaVhuV6bdmEpMkq8PdMu2ip6VBarrHR7B4TNu1R7IKD5ddzyg8QMF2kdk3uEQJ+8QQQkiFQstHHGI1rTZq02+9kNUsS7o37S45u3Jk96+LpP7yF5SrJcnDFmKBlS+5XrSEEEJIhUHLRxzSsUUdv+m3WI7Pz2lWW7lo5n2/LXrTcA0kJSZJh4wO0rvrw9Kh/wxJMtcGCaZcO9NyCSGkwmBX2zgF2SzIahFznETp39u6tZD5/9uhYj+iuguuP9DbBVktelApgkzn3hrEBhJcFpBR6xmMSgghNsCutg7HV/ot3kN4/GvpZg/hAfD+jlk5smDddokJ9NogZ17p+lsjWNFUmpa76mWR9R8wFoQQQiIELR9xDlwpyGpBcCliPOBqufDJL8sIDyPIvn3h2kxVwj2mgCXkmbYul0qwsSA6jAUhhJCwWz4oPhwGYjtQ6dQK92S1lBEXt4zeWiDe2DjfVWJdEYoAKT3Wi8aL1D2FNUIIISQIKD6IVxBcOvK97y2fnYy0KjKxfwzFgegCZNFYV3qtHSCQFWXfW/VmsTJCCPEDYz6ILem1aEgX9eXY/TWq6zSsdGE5rDeFe6R45Uuy5v1BsuClM2XNymekGC4eQgghIcFUW4em4QYLyrFHeyqu12DUXpNFBr8lYk7LDYLs1KrSs2kjGdowXcbWSJKhP78qPd/rJtlbsm3dZUIIcQqM+XAgsGIgqyVY3r21U3SXY7ealovXZw9YFh6jG9RzRY8Yesuo0u4JCTKtza2SVaURY0MIIUQY80ECgHTaEe+ulWCMGTee30wua9tQWU9iKgg1xKwYOFZg8diZlOQhPMxN7RZt3e7qLcNMGUKIwymwmO1Ct4tD6d2ukUqnDYY3V2xRmTIdHlscO7VAfLlk0GRO4VtE5VRJkZ3JyV6FB0CX3dzkZDVOwf4xhBBiCYoPB4M6Hi+jEFla6eRpkb2Hjsuwd9bK5AUbJWZBUOrgN/3GguyGxcMCZcahd8wbfV3WFWTeEEII8YDiw+EghXb5uEvknqzTgl73laWbZcG6GMqC8ZcVM+hVkYv0OBCXpaN+sbWMFp/j2D+GEEK8wq62RMVvjMxqKa0yqqusFn/VT808PG+D9GybEbsxIHpWjE6D0901QtA1N72oSHYlJSkXi6+YD4zzDuJJEkQWjRNJSVMpu6oHTbPO7CVDCHE0zHYhXsuxL9ywQ8V4xH0WjL/MmJ8XSPbPH8jomi63lGbOdhGRabv2SFbh4eC2j8DUzJtZQZUQ4tiAU1o+iAewYOhCwqr4QN+YuEK3hrToKlmXPirTvntRpvz6vuw8XuAeAovH2Lx9wQsPPTB1yeMn3jNLhhDiMCg+iFeQTlunWiUVXGp31dSYIjFJsjrcLd3PGS45u3Jkd+Fuqb/rF8lcNNGVXmsHemwIAmBb9zlRj4QuGkJInEK3C/EJ0mmR1eIPVEv9euzFsRvzES39YxAbUrW2SHKKyAFDEC+tIoSQGIKN5YgtIJ0WWS3egNyYfn2mR9M5PWYErhhYRGK+IJnF2BCVXqvOiN0l6BNUsbOcXhNld4PTpH5qfclskClJcA0RQkiUQfFBbAPptA/N2yB7Dx3zsHhM6OfZ7RZl2yfO/0FyC05kf9ROrSSPXd5WFTWLa2y3hJwo7z6lbm1XsbNS0ivXknGNsySrUj2RavVFajRkBg0hJCqg+CC2EsiiEahfzO3dWsj43m3i+6oY+8dAFHx4u8iB3JCtIX77ypgzbeieIYREARQfJKLC5JxHF8v+Qv/BqS9dl6mqqjoGWEMQSKoIToAE3VfGtVTkovGuFF6IH4gU1hYhhEQQptqSiLFyU15A4QEeivWCZKGWcA/BHePuK+MDY1+ZDu4iZ5pnCq8RWkYIIVEEy6uTcrPi9z2WxiFmBK4bR+GthDtiNIykNXZluhia3IXcV8YXLPVOCIkiWOeD2IB1S0Zu/mFZsSnPGdkwvkq4d7u3bC2Pnz4tddG4MmbK3VfGX6l31BIxZssYY1VYW4QQEgEoPki5QUXUF778zdLYSZ/+GDBrJu4xixEvLpry95XxhiZSsM0lNPTv95alQxcNISTM0O1Cyk2nk+tKzarWdKxReIDc/CNy56wclS3jeAwumqRBr8q4dneKJCRKgsmypGe7oLx7SNU+YOEwBsSa41F8uGiKi47JmrWvyoKvJqi/eE8IIaHACqfEFgKl2vr9EYpIhlMrpQYge0u2TFk9RXYWlgoGnKuiotD7ygDEn8DV80xbP4GwCS4LyKj1ylKT/fVkmfLL27Iz6cT1SS/WZFzLayWrSTe6bAghCqbakgoRIOPmrJf9hz0zX6qnJMnBo8XO645rE8Ulxa6+Mr8ukvpr35XMvdtC7CtjEBRwvbzRN/AqN30i2du+ltG/vR1cvRH2qCHEkRRUVFdbTdOk2BQEl5CQIElWo/JJzIK4jR5tMlTqrSsDxtUhd9eBo3LP+99b7o7rqBLtFkAp9Q4ZHUTwuuABl3BA/5dDu131PPZuLk2x9VfevfT89ZriijnRXS8BKC7YpiweGhy0ptgTxKJAgEytW1u6Fx52CSLlsrnBlb1zeN+JwYwjIYSEU3w89thj8vDDD3uIjVq1asmePdbSMUlsA5FwQct66qWD7BYrQGjAevLIxxtlR75LiAB01310gANKtIcarAoanO6/noia/Ke4AlsBsloskLNnnYerJXC9kVLxYxQe5s69+j4Y8Zdxw2wcQuKOsGS7nHfeebJy5cpwbJrEILBcIKsFwaWan5iPfYeOyfB3csqM2XvouOque/tf++O/RHuoYEI3ujoCVTjFewgSiAJfVyWtkexOSrap3oifVF9/GTfA/FlqXSm+7J+SU7+p7C7czWZ7hMQgTLUlEbGGIJ0WWS1mx4D+TP1wn9Nl0qcb/RYhR3fdMxvVkr7taQEJyiriaywmd0NtkTJXpdcUqX90t431Rnyk+qp9MF153X3jhWwplCmrH/RstpeaLuM6jpOsZlleLSWqMzDiZsxihVYVQuJHfHz//fdStWpVqVKlinTs2FGmTp0q7du3D8dXkRiKB5l+fWYZlwosHhAmNatW9ljui7vfXyuJiQnO6hET6fLvBhdNZtExSV/7tOxKdLlYbKk3osebYOLHd3uVnFrgZnsGdhXuktFLRsu0U66VrNVvehxPdr3GMqVObdl5vMBTrDS8pMxYj9gUO4VJtG6LkHhJtX3ttdekRo0a0qtXL9m/f7888MADMm/ePFm/fr00a9bM6zpHjx5VL2O0bNOmTQNGy5LYw1cw6bzvt8nI9wIHpeq8fH2mswqThZMAkxnSbJHtArRA2S5WU31h+di8zFrGjdVmexAVRUUezfZ8dgZWB6N52ffSMZ3vEtnwgT3F1+ws5MaicCTKiZpU2+PHj0uLFi1kyJAhMmnSJK9jJk6cKI888kiZ5RQfzgFBqdfOsB4nhBgS1gWJHN7qfARfb8Szdois/0Bkzi2W92FNlRQZ2jBwoOxrO3aq4NfQOgMH2H/gK2jWG77cSrZuqxT0DULpflpBSAyIj7BXOK1UqZKcdtpp8ttvvstvjx8/Xu2o/tq6dWu4d4tEGbCAIKvFKnDR6E3qYE2BeIH1BH/xnthLVpfx8tmNOfJau1EytflA9XfReY9JVjIa4hmoWqf0Hwn+U32DyLgJtdmeuzOwF+FhztSxRunvCkGzsBYFwopbyZZtuaxCa1Y+JQteOlPWrHxG1YYhxNEBp8eOHZOffvpJMjMzfY5JSUlRL+Jc4HpBOi2yWqwC14231FxH9ouJAEnJlaXD2SZLRZsB3pvk+YkjsZ5x40mwzfZs7wzsK2jWFxjjs4KsfduCa2lK3donAnB/flXSfntHbmhysdxaJ1OS0EWZcSEk3sXHwIEDZcSIEXL22Wer2h4PPfSQMsPcdtttdn8ViTNQxwPptMhqscIfewrlmexfykxber8YBLhSgFRQkzwrFU4DZtx4Xtlgm+3Z3xnYgJUibRYLuZVnW74CcAuKD8uLWz6Vt36fLzfkH5CTKtWQ+ufeJpkd73Jl+RBSwdjudhk7dqw89dRT0rJlS7nkkktUtdPVq1cr1wshgUAdjxeuOduXpfxEXZC0FHl39Z9+8yRgEaELpoJFyZlXuv76mvD0jJs0k5UKFpHBb4l0vtu9CFsYl7fPI9hVR2++NzZvvzt+Qxcr5rHudTRNxa0E1xlYrLuMrLqVQtwWJBMsHuZgWiMFiYnyYp1aMrZGkgz9+VXpPqujPJF9t6zZvpKuGVKhsLEciUoWrNshw94p26hOv8WOymopT2f/GnA77BcTBxk3P3wk8ukYV8E03c1Qr55n8GtqhoxteLFkZU/1kKC6ZUAt8ZrtkidZhYVB7KgpaDbQManmff4LuYW6LasBuL5IT0qVcSf1laymF9EtQ2K/twshdoA6Hi8n+q4LcrSoxNJ29H4xJIYLpJ1xucjp/dziJKt6unRvep7k7Plf2aJhdc7wiDdBJs60gyVe6nyUipW9b4qIQXykNRZpO0jkm+dLF3gvvmYpo8RiIbfgtxVKrEpZdhYdktG/vy/TVr7oChwOJfWXkBCh5YPEZF0Qq6m53iwfbFwX59hR4dRrPY3GZYNmrWD3thberxoLltfy4ZluvMPlrrpovEjdU1i8jMR+nY9w7jxxLhAQXaZ+EbBfjLkWiLfsmFpVK8mQC5rLiItbOrp7LomRqqTY1tJ/SvGSx/3WMQkGvTaKB8iSOWcIxUicUVxS7F2I2wTFB4l7ICSQ1SI++sWYs1308b7Udq3USjJl4JnMkCGxwcb5kv3FOLmnemneQDkEyNRde6T3oQCxL4hP6TlZNfZjaffYJHtLtkxZPUV2Fu703hfJBig+iCOwWudDt5QE6h+D2zdTdEnMUFIs2d+9KBN/niX5xUGUuLdi+bBCqGXiSYUID/Q/0sR7pti0i6bZIkAoPohjsBLDEUz5doiXr+7rLt9t2ed3m4REkyl9xvoZMmvjLMk/ln/iA3jV/VhEgi8xX2YLrj9Il7ZS24VU2O+j55yeHhYPswCBBWTRoEXldsFQfBBiINjGdXWqVZa9h46537NqKokpf/6hnfLn1uXywdZs2Vni3doXcmPAslsSqVpbpFIVz6BaxoxEDWty18jQz4YGHPdaz9ekQ0aHcn0XU20JMQDrRTAYhYexauqL150ttaul0CJCohI8tbonj1P6yq0lj0rO6ufly+9nyCeVNdlnSM+FxSO4xoC+0EQO7xUxb+bADpElj3t30XgLwAW0nIQFBJfaOc4OWOeDOAK4TWC9CBTz4QvdSzri3bVi7FtHiwiJejHSaZR06HiXjPnja8nJXSO79/0m9Tctlcy9obpaQgQF0lCnpPNdIhs+8LSSqIaEEDGuCrZe40n8ZQwFm01kZ/ZRDFA/tb6t4+yAqbbEUcGpd5Rmx9iFr8waQqIa4+Sbt6nUQlG2n07FYognAV6bFU71/5m3QFivdVfiO3C2uDTmY1fhrjIBp4AxH6WwzgcJpwAZN3e97C88bts2zTVFWMSMxBzeJuSooDSeBG4db5/5FEsG4WIUFDhOVSVWszY+DrNdgFGAMNvFAMUHCScQBy988Zu8vnyz7D98QoTUqVZJ9h4KXZSgmmr+4WOWUn8JiXprSM7MKBQjwWDqnePuj+PrmILotRNHdT4y0Gqg41jW+QAUHyQSmC0U5zSrLRc++aXPqqmBGHpBc3l9+R++nqnomiGxhTku4lCeyOfjTRktjUSKjpTGakSTy8bATZ+4+gZtXibyRl/r4+OU4iipcMqAU+JY4CIx932BhQJZLaF4vz/6frvXdbAM24NF5OLW6ap+SG7+YZVRU6d6imSksY4IiZFmf236lQ3U/OlTH83zogTsq/Gv1fFOyIiqQCg+CDEA1wiCR82uE9QXM2a5GMEtt7Zy2Xim5xrBqthep8n/9TqOrhkSu4KkvytWIipjRsQlkox/rY4nYYXZLoRYcMnsO3RMhr/ju48MXC6vLv+jfP8Z6Zoh8Rwzgk6+bQeJfPN86YJAVpLS/11Va4kc3h+CVcVXzMcOH9uK/5iPSEC3CyE2u2SmJ5a1iCDLBa6amlUrl1t8AGy/R5sMlnInsW8V6Xav91oaTTqUtZL4rPMxxfVvr24d43tvn8GUOeWEkMBfpNP63JZpPAkrtHwQEgS+0mj1xnWhBquas2bMwoeQuCLYCqdea3M0PiFOfH1muc6Hn/EkKNjbhZAKqCGCYFVQHgHy7DXtZUD7xkGJIQawkriHFU5jArpdCImSYNVg64dY7UMDsWP+Lh0GsBJHBLta+cyO8cR26HYhJEL1QwL1lTFXSrViZfFnYWEAKyEkWi0fiRHdK0IcFKwK1wn+Vk5OVEGp/uWEC4wLJDwgbmDxsOLawTiMJ4SQaILig5AIumTgDvEGllttTgeripXuvHptEYz3BkTJik15Mu/7beovRQohJFKwyBghEQLCAmm05Q0QhTsnGLyN9xYvwjgRQkikoPggpILrhwSL1YBUX+N9xYsgTRjLrVpgCCEkVOh2ISTGgJXEl/vGCOwoGIfxVuJFtABxInTTEELsgpYPQmLQeqI3wNOCDGANFC9ijBMxWmjopiGE2AktH4Q4KIDVaryIcZzupjGLFt1Ng88JISQYaPkgxEEBrFbjRfRxgdw0+Ab2oyGEBAvFByEOCmDV40V89aDRC53pcSJW3TQzl2+WejVSPPrdEEKILyg+CHFovIiPvp4ecSJW3TSTPv3R/W+m7BJCAsGYD0IcGi8CC4eRDC9xIsGm9QLGghBCAkHLByEOjxfRe9B4c5cEctN4g7EghJBA0PJBiEMx96DxFqehu2lAMFEcgUq7E0KcDcUHISQkN40VrMSMsHgZIc6DbhdCSNBumj0HjnoEmfoiUMyIt+JldapVkkcHtJXe7RrxyhASp1B8EEKCTuuFteLfX2+2nLLrDV89ZvYeOi7D3lkrt/+1X8b3drl8CCHxBd0uhJCg8RcL4i1l14y/4mU6ryzdLAvWsXoqIfEIxQchJOwpu2YCFS/TeXjeBja5IyQOoduFEBL2lF0zVouX5R06FnKTO1hXgt0vQkhkoPgghES0xHuwxcu8NbnTfBQ20y0u7MJLSHRDtwshJOLACoGsFjub3AF8jjgRduElJLqh+CCEVIi1BOm0gWgYQpO7h+ZtCChQIGR8wbojhIQful0IIRUC6nggnRZZLd5ICLHJ3d5DxyxVXvXmKvLnrgkltsUXjEchTofigxBSYaCOx1lNaitrhVE0eAsgDaXJnS+8CRl/8SR3zMqRWqmVZH/hcb/7aAXGoxAikqBpmtV+URGjoKBAatasKfn5+ZKWllbRu0MICTNWLAEY02XqF34Lm9WuVkkVKQvEu7d28rB86Nu2kv5r/D4QKK3YisAJZVuERCNW529aPgghMZExoxc2w+SNyVrzMnkjjgRl34OtvGq17kh5uvcGCpgNtC2jQKtXLUWtsOfgUVvSiPVt5+YfVhaoOtVTJCON6ckkfIRNfBw7dkzy8vKkfv36kpxMjUMIsa+wmTkuI8PgAklMTPArULxVXrUaTxJsDIkRqwGz3rblzVVjJFQXUKBtl2e7hEQ822XSpElSp04dad26tRIf06dPD8fXEEIcCCbCr8derFwnz17TXv3Fe32CDKXyannjSayIF6sCxzxOd9X4Ey56nROMDYZA294R4nb9wWwiAmw3Sbz99tsyefJkWbRokXTr1k3mzp0rgwcPltNOO00uueQSnnVCSNjdNMFWXsVneMr35a4JhBXxYlXgGMdZ6YETigsomG1LkNv1B4NtSdgsHy+99JIMHDhQCQ+Af3fp0oXWD0JIhQiUAe0bq7/+Jk5/jfL8kWCqReIPXeAkBLGtYGJRjG4bK1jddrDbDdbKEqrVhsQ2toqPkpIS+e6776Rz584eyyE+1qxZY+dXEUKIrfhy1yDFNtTuveXtBBxKLEqo7h27xxuxWp3WX/E3El/Y6nY5cOCAHD16VOrW9TSH1qtXT3bv3u1zPayDlzFVhxBCIo0vd83ijbl+g1ztDJgtbyxKKO6dcIy3K9iWxCe2io/ERJchpaioyGP58ePHJSkpyed6iBF55JFH7NwVQgixLZ4k1O693ghmW8HEovhKI/aFvu1ArpdgtxtJawyJXWx1u9SoUUMVF8nNzfVYjveNGzf2ud748eNVQRL9tXXrVjt3ixBCIhpDYte2rMaiBOsCMm7byuhgthtJawyJXWwPOL3wwgvls88+81iGzBcs90VKSoqqhGZ8EUII8R2LYjWN2Mq2YQHxBpbbUXU1lGBbEt/YXl591apV0rVrV3nooYekX79+MnPmTHn11Vdl7dq10rJlS0vbYHl1QgiJrwqneraL+Cj+xtLy8YHV+TssvV2++OILFcfx559/KsExYcIE6dChg+X1KT4IIST+YJ2P+KegIsVHeaH4IIQQ5zYRJLELG8sRQgiJySaCJP4JS28XQgghhBBfUHwQQgghJKJQfBBCCCEkolB8EEIIISSiUHwQQgghJKJQfBBCCCEkolB8EEIIIYTigxBCCCHxS7JEIXrRVVRKI4QQQkhsoM/bgYqnR6X4OHDggPrbtGnTit4VQgghhIQwj6PHS0z1dikpKZHt27dLjRo1JCEhIaYUHwTT1q1b/TbUiTeceNxOPGanHrcTj9mpx+3EY7b7uCEpIDwaNWokiYmJsWX5wA43adJEYhVcPCf9cJ183E48ZqcetxOP2anH7cRjtvO4/Vk8dJjtQgghhJCIQvFBCCGEkIhC8WEjKSkpMmHCBPXXSTjxuJ14zE49bices1OP24nHXFHHHZUBp4QQQgiJX2j5IIQQQkhEofgghBBCSESh+CCEEEJIRInKOh+xxvr16+WHH37wWFa1alUZMGCAxBs4Thxv165dpXHjxmU+RwjRt99+K3/99Zecfvrp0rp1a4l1Dh8+LJ9//rm6ppdeeqnHZ0VFRfLBBx+UWee8886TFi1aSKyyb98+ycnJUUX+zjrrLKlbt67Xa7169WrZsWOHtGnTRk477TSJZXA8GzZskD/++EOaNWsm7dq18/j82LFjMnfu3DLrde7cWU466SSJVQoLC+W7776TgwcPqv+zzZs391r4cdWqVbJz505p27atnHrqqRLrbNq0STZu3Kh+2x06dJBKlSp5/J+fN29emXV83fdija1bt8ry5cvV/1vz77y4uFhd6127dqnPTj755LDsA8WHDbz//vvyr3/9Sy6++GL3slq1asWV+MAPdfz48ZKbmyu//vqrfPjhh2X+E+Im1q9fPyVQMGGtWLFCrr/+ennppZckFsENd8yYMer6JiUlSXp6ehnxceTIEbn22mslKyvLY4Ju2LBhzIqPu+++W+bMmaNuTEePHlUi5J///Kfccccd7jGoYNinTx/57bff1GT0zTffyK233ipPP/20xOrve9iwYUqAYPJds2aNun4ff/yx+7qiCiSudc+ePdX/bx0IlVgVH6+++qo8+uijaoJJTk6Wr7/+Wq6++mr597//7a5OmZ+fL7169ZI///xT/SZwre+66y6ZMmWKxCKonj1kyBB1PBDMP/30kxw6dEj95vHQAPLy8tS17t27t6q0rQPRFevi49ixYzJw4EBZt26d3HPPPR7iA8eN3zdEJh4cca3vvfdeeeSRR+zfEWS7kPLx4IMPahdeeGFcn8aFCxdqS5Ys0Q4cOIDsKO3DDz/0eh6aNGmi7dq1S73PycnRkpOTtQ8++ECLRY4dO6ZNmzZNy8vL00aOHKmdc845Zcbo52PFihVavDB9+nTtyJEj7vevvfaalpiYqP3yyy/uZWPGjNFatGihzg1YtWqVGvPJJ59oscjixYu19evXu98XFBRorVu31oYMGeJetnv3bnWt165dq8ULc+bMUceqs27dOnWM8+fPdy8bPny4dtppp2n79+9X75ctW6bGZGdna7EIfsfLly/3WDZgwACtY8eO7vdbt25Vx/jjjz9q8cY999yj3XLLLVqrVq20sWPHenyG5WeccYa6r+n/L3AecM3thjEfNgGT5SeffCJffPGFUo/xBp58LrzwQr9jZs2aJTfccIPUr19fvT/77LOVNQjLYxGYYfFkUKdOnYBjYR2AmRZm+1jPXoeFw5jvP2jQIGUFgrtNB9f05ptvdp+bjh07SpcuXWL2WsNyBQuODp52e/ToId9//32ZsXArzp8/X5nsYx08ARuf7HULiN7cE7/lt99+W2655RZ3yWxcZ1zvWL3WLVu2VK4yI7CA6MdsBO4HXOsff/xR4oEFCxaoeeqZZ54p8xlcyO+9957cdtttUr16dff/C1hG8BuwG7pdbALm5xdeeEEJD7gdHn/8cRk1apQ4BfzH3bJli8cNHJx55pny0UcfSbwDtxtcLbhZnXHGGcpVg8ZK8UB2drb6i+MCu3fvdvv+zdd6yZIlEg/A741jad++vcdyxMBMnz5dGjRoICtXrlSf44YNl1ysAlcqjhVuJQgKuBquvPJK9Rlit/bv3+/1WnsTZrEE4rjwW4aIfPfdd5WryXytcU+H2w0uZLhkMM5b/FOsuJv+/ve/K5e5Li6MbN68WbmfvF1r44OHXdDyYZNVAAE8ixYtUr5iTESjR4+WpUuXilOAXxiYrQT4j4qbV7wC68h///tfdSNeuHChEqE4F4h/iAcw+QwfPlyGDh0qrVq1csy1fuihh9TNGH91qlSpIl999ZUKzsS1/uWXX1SwLWJFYhkISTwgzJ49W37++WcVfAnrR7xfa1xLxHngQQEWH2PcTrVq1VS8A+7nuK/jWiNAFfFQsUhJSYmKv7vzzjvdcS1mIn2tKT5sAGZIo+kSFxmmPZi3nIJupof7yQje46Ydz8dtDDTGf1y4avBUhWDNWJ+U4HqA2dUYNBzv1/qpp55SZmlkthgzePC0iGwHHbgXR44cqUzZsJTEKggOh/UGFq4vv/xSBZIiEDXer/Vjjz2mrjGEBcQHAqj161i7dm3p1KmTeywsWyNGjIjZe/qbb76pAkxxnLjWeMFaDXcSRGdFXGuKjzCBtsQw6TmFevXqqWNGBLkRuGLClaoVreA8wH8ay0+GSLODqGrSpIl6KjbGgMCdhJtRPF5riA5YO2CahvCycq2R8WS+YccqyGY555xzZNmyZeo9rAGwgsTjtdZBVs91112nLF3btm3ze63hmorFh4p69eqp+A1kb+H/M14QH8hc1FOKkd0FV1OkrjXFhw3A9GoEpnf4yGC+dAr40V522WWq5oUecIkfN8zTeKKIV+AvN4OnKdy0YzUOAKIZwgMiA8F2qG9iBGnHSMf7z3/+414GoQVrTyxf6+eee06lk+P6wZUa6P85wFhYOfVgzFgCT/l79uzxWAYRBUtA06ZN1XuIzksuucTjWmMdBNbH6rX2dh3hSoOgxiTt71oj/iEWm8717dvXbfHQX/j/3b9/f3cwKSx73bp187jWOA8IHwjHtWbAqU0X9vzzz1fmSzwxIkgJ0eDwk8cLeCLA05Cu+lEPAE98iANAVgv4xz/+ofyJV111lbphvfXWWyowL5Z94vD3YmLFDRmFt/CfFiAgD0+EEFevv/66+g2g9gNM8IsXL3aPizVgscETEkQVapwYCy3hiRgTrW6yxm8etRDgipg5c6aqd4FI+VjknXfeUS4U1H+A71u/fsZigXpcBAIy8RSMp0jEDXgrMhcLHD9+XE02OB5YPPA7f+ONN9QkZIxtgBsGrmVksuGaIzATxciQ7RSLYLLF/1HU7MH/WWQv4f8wjjM1NVWNQRwIXCwQoTgfuPYIOo334PknnnhCZTVi7sL/91deeUUyMzPlb3/7m+3fxa62NoAJGRMtgpPwQ4WvEJMTrAHxAv6DotCUGdy4brzxRg8THX6wECsoUgPhEYtPhTr33XefCiY2g5u0/gSENFtMQLAYwDyJ8xGrhYjwW77pppu8foYbkrHIGgLwZsyYoZ6OkAmDYDZj7FOs+cQhHM1gcnr55Zfd7/F/HE/AePpHwSmcq4yMDIlVkN2A3/LatWtVkCUeJCAoK1eu7DEO4huiA3FAePrHtcb4WAVCAsIa1xFWysGDB5epxqyLDTx0QHTjWuNhKl4YM2aMnHvuuep6G0EcCGJ+cD9DNhdS783WTzug+CCEEEJIRGHMByGEEEIiCsUHIYQQQiIKxQchhBBCIgrFByGEEEIiCsUHIYQQQiIKxQchhBBCIgrFByGEEEIiCsUHIYQQQiIKxQchhBBCIgrFByGEEEIiCsUHIYQQQiIKxQchhBBCJJL8P1HH0DU2RIuWAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from sb_arch_opt.algo.arch_sbo.api import get_arch_sbo_gp\n", + "\n", + "# Create the algorithm (with an initial database size of 20)\n", + "n_init = 20\n", + "sbo = get_arch_sbo_gp(problem, n_parallel=4, init_size=n_init)\n", + "\n", + "# Run the optimization: 30 infill points (20 + 30 = 50 design points)\n", + "# NOTE: remove \"seed\" argument when using in production!\n", + "n_infill = 30\n", + "result_sbo = minimize(problem, sbo, termination=('n_eval', n_init+n_infill), seed=42)\n", + "\n", + "# Compare both robust Pareto fronts to the deterministic one\n", + "plot(pf_deterministic, result_nsga2.opt.get('F'), result_sbo.opt.get('F'),\n", + " labels=['Deterministic Pareto front', 'NSGA2 robust (2000 design points)',\n", + " 'SBO robust (50 design points)'])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "SBO reaches the same robust Pareto front as NSGA2 using 40 times fewer design points. Given that for each design points 12 evaluations are performed to propagate uncertainty, the computational budget savings are tremendous compared with NSGA-2 (overall 480 less evaluations)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.14" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/mkdocs.yml b/mkdocs.yml index 1a08fc5..d98247d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -26,11 +26,14 @@ nav: - 'HEBO': 'algo/hebo.md' - 'SEGOMOE': 'algo/segomoe.md' - 'SMARTy': 'algo/smarty.md' + - 'Stochastic Optimization': 'algo/stochastic.md' - Tutorials: - SBArchOpt: tutorial.ipynb + - Stochastic Optimization: tutorial_stochastic.ipynb - 'Test Problems': 'test_problems.md' - API Reference: - Problem Definition: 'api/problem.md' + - Stochastic Optimization: 'api/robust.md' - pymoo: 'api/pymoo.md' - ArchSBO: 'api/arch_sbo.md' - 'BoTorch (Ax)': 'api/botorch.md' diff --git a/sb_arch_opt/stochastic_problem.py b/sb_arch_opt/stochastic_problem.py index e8c46df..2de8136 100644 --- a/sb_arch_opt/stochastic_problem.py +++ b/sb_arch_opt/stochastic_problem.py @@ -36,15 +36,54 @@ class StochasticArchOptProblem(ArchOptProblemBase): """ - Base class for a stochastic (robust) architecture optimization problem. + Base class for a stochastic (robust) architecture optimization problem. It extends the ArchOptProblemBase class + with support for stochastic optimization problems. + + Stochastic problem evaluates design vector for many realizations of the uncertain parameters and + reduces the resulting distribution of each response to the single value the optimizer sees. + Three things are therefore needed on top of the design variables: + + - `param_space`: a `StochasticParameterSpace`, the joint distribution of the uncertain parameters. These are + quantities that influence the evaluation but are not chosen by the optimizer, so they are NOT design + variables. + - `uq_method`: a `UQMethod` (`MonteCarlo`, `PolynomialChaos`) that decides which realizations to evaluate and + how to turn the responses into statistics. It is given a budget of `n_evaluations` expensive evaluations per + design point. + - a `Scalarization` per response, below. + + Requires specifying a type of optimization problem for each objective and constraint by providing a child + instance of Scalarization object: + + - Mean: Minimize the expectation of the objective or constraint function + --> for example min(E[F(x)]) + - Margin: Gaussian output distribution expected. Minimize for the objective or constraint function for a given + confidence interval. + --> min(E[F(x)] + k*sigma[F(x)]); The default k=1.645 is the one-sided 95% interval of a normal distribution. + - Quantile: Makes no distributional assumption. Minimize the q quantile of the objective or constraint + function --> min(F_q(x)), with q=0.95 by default. + + When type of optimization problem is not provided, the response defaults to `Mean()`. + + Implement `_arch_evaluate_sample`: it is called once per realization. All design points in a batch see the same + realizations (common random numbers), so they stay comparable to each other and to a surrogate fitted through + them. + + A failed evaluation (NaN) in any realization fails the whole design point: `scalarize` returns NaN, + which is how SBArchOpt treats hidden-constraint violations elsewhere. + + After each evaluation the fitted response distributions are stored in the pymoo output dictionary as + `StochasticOutput` objects, in `out['f_stochastic']`, `out['g_stochastic']` and `out['h_stochastic']. + For polynomial chaos each output also carries the fitted expansion (`method_results`), from which Sobol + indices can be obtained. If the output turns out to be deterministic after evaluation, a float is stored instead of a + 'StochasticOutput' object. """ def __init__(self, des_vars: Union[List[Variable], ArchDesignSpace], param_space: StochasticParameterSpace, uq_method: UQMethod, n_obj=1, n_ieq_constr=0, n_eq_constr=0, - obj_scalar: List[Scalarization] = None, - ieq_constr_scalar: List[Scalarization] = None, - eq_constr_scalar: List[Scalarization] = None, + obj_scalar: Optional[List[Scalarization]] = None, + ieq_constr_scalar: Optional[List[Scalarization]] = None, + eq_constr_scalar: Optional[List[Scalarization]] = None, **kwargs): self.obj_scalar = self.check_scalars(obj_scalar, n_obj) @@ -62,8 +101,6 @@ def __init__(self, des_vars: Union[List[Variable], ArchDesignSpace], self.param_space = param_space self.uq_method = uq_method - # List for storing stochastic results object for each design point - self.stochastic_results: List[StochasticResults] = [] super().__init__(des_vars, n_obj=n_obj, n_ieq_constr=n_ieq_constr, n_eq_constr=n_eq_constr, **kwargs) @@ -81,16 +118,31 @@ def check_scalars(scalars: Optional[List[Scalarization]], n: int) -> List[Scalar raise ValueError(f'scalars should contain Scalarization instances, got: {scalar!r}') return list(scalars) + def _print_extra_stats(self): + print(f'stochastic : True') + print(f'n_params : {self.param_space.n_parameters}') + print(f'uq_method : {self.uq_method}') + print(f'n_evaluations: {self.uq_method.n_evaluations}') + print(f'obj : {self.obj_scalar}') + print(f'ieq_constr : {self.ieq_constr_scalar}') + print(f'eq_constr : {self.eq_constr_scalar}') + + def _evaluate(self, x, out, *args, **kwargs): - # The pymoo outputs are processed by the parent method + n = x.shape[0] + f_stoch = np.empty((n, self.n_obj), dtype=object) + g_stoch = np.empty((n, self.n_ieq_constr), dtype=object) + h_stoch = np.empty((n, self.n_eq_constr), dtype=object) + kwargs.update(f_stoch_out=f_stoch, g_stoch_out=g_stoch, h_stoch_out=h_stoch) + super()._evaluate(x, out, *args, **kwargs) + out['f_stochastic'] = f_stoch + out['g_stochastic'] = g_stoch + out['h_stochastic'] = h_stoch - # Add stochastic result to pymoo out dictionary. - if len(self.stochastic_results) == len(out['X']): - out['stochastic'] = list(self.stochastic_results) def _arch_evaluate(self, x: np.ndarray, is_active_out: np.ndarray, f_out: np.ndarray, g_out: np.ndarray, - h_out: np.ndarray, *args, **kwargs): + h_out: np.ndarray, *args, f_stoch_out: np.ndarray=None, g_stoch_out: np.ndarray=None, h_stoch_out: np.ndarray=None, **kwargs): """ Evaluate architecture for the provided design vectors and samples. Implement _arch_evaluate_sample to evaluate architecture for single samples realizations. @@ -101,7 +153,6 @@ def _arch_evaluate(self, x: np.ndarray, is_active_out: np.ndarray, f_out: np.nda # Get samples and include deterministic parameter values for evaluation samples = self.uq_method.get_samples(self.param_space) - # parameter_values = self.uq_method.param_space.include_deterministic_values(samples) n_x, n_s = x.shape[0], samples.shape[0] @@ -111,23 +162,26 @@ def _arch_evaluate(self, x: np.ndarray, is_active_out: np.ndarray, f_out: np.nda # Evaluate all design vectors for each realization of the uncertain parameters for i in range(n_s): - self._arch_evaluate_sample( - x, is_active_out, f_s[:, i, :], g_s[:, i, :], h_s[:, i, :], samples[i, :],*args, **kwargs) + self._arch_evaluate_sample(x, is_active_out, f_s[:, i, :], g_s[:, i, :], h_s[:, i, :], samples[i, :],*args, **kwargs) # Evaluate the stochastic result for all the evaluated design vectors and samples - self.stochastic_results = [] for x_i in range(n_x): - results = self.uq_method.process_results(np.concatenate([f_s[x_i], g_s[x_i], h_s[x_i]], axis=1), self.param_space) - self.stochastic_results.append(results) + outputs = self.uq_method.process_results(np.concatenate([f_s[x_i], g_s[x_i], h_s[x_i]], axis=1), self.param_space) # Reduce the sampled responses of each design point to the values the optimizer sees n_f, n_g = self.n_obj, self.n_ieq_constr - for f_i, output in enumerate(results.outputs[:n_f]): - f_out[x_i, f_i] = output.reduce(self.obj_scalar[f_i]) - for g_i, output in enumerate(results.outputs[n_f:n_f+n_g]): - g_out[x_i, g_i] = output.reduce(self.ieq_constr_scalar[g_i]) - for h_i, output in enumerate(results.outputs[n_f+n_g:]): - h_out[x_i, h_i] = output.reduce(self.eq_constr_scalar[h_i]) + for f_i, output in enumerate(outputs[:n_f]): + f_stoch_out[x_i, f_i] = output + obj_scalar = self.obj_scalar[f_i] + f_out[x_i, f_i] = obj_scalar.scalarize(output) if not isinstance(output, float) else output + for g_i, output in enumerate(outputs[n_f:n_f+n_g]): + g_stoch_out[x_i, g_i] = output + ieq_constr_scalar = self.ieq_constr_scalar[g_i] + g_out[x_i, g_i] = ieq_constr_scalar.scalarize(output) if not isinstance(output, float) else output + for h_i, output in enumerate(outputs[n_f+n_g:]): + h_stoch_out[x_i, h_i] = output + eq_constr_scalar = self.eq_constr_scalar[h_i] + h_out[x_i, h_i] = eq_constr_scalar.scalarize(output) if not isinstance(output, float) else output def _arch_evaluate_sample(self, x: np.ndarray, is_active: np.ndarray, f_out: np.ndarray, g_out: np.ndarray, h_out: np.ndarray, parameters: np.ndarray, *args, **kwargs): diff --git a/sb_arch_opt/tests/conftest.py b/sb_arch_opt/tests/conftest.py index 270139f..b131fea 100644 --- a/sb_arch_opt/tests/conftest.py +++ b/sb_arch_opt/tests/conftest.py @@ -1,15 +1,25 @@ import pytest import itertools import numpy as np -import openturns as ot from typing import Optional, Tuple from sb_arch_opt.sampling import * -from sb_arch_opt.uncertainty import * from sb_arch_opt.problems.problems_base import * -from sb_arch_opt.stochastic_problem import StochasticArchOptProblem from pymoo.core.variable import Real, Integer, Choice from pymoo.problems.multi.zdt import ZDT1 +try: + import openturns as ot + from sb_arch_opt.uncertainty import * + from sb_arch_opt.stochastic_problem import StochasticArchOptProblem + + HAS_UNCERTAINTY = True + +except ImportError: + HAS_UNCERTAINTY = False + + class StochasticArchOptProblem: + pass + class DummyProblem(ArchOptTestProblemBase): @@ -103,7 +113,7 @@ def failing_problem(): return DummyProblem(fail=True) -def make_space(*distributions) -> StochasticParameterSpace: +def make_space(*distributions) -> 'StochasticParameterSpace': return StochasticParameterSpace([StochasticParameter(f'u{i}', dist) for i, dist in enumerate(distributions)]) @@ -180,6 +190,24 @@ def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, * h_out[:, 0] = parameters[0] * x[:, 0] +class DeterministicResponseProblem(StochasticArchOptProblem): + """Two objectives, of which the second does not depend on the stochastic parameters at all""" + + def __init__(self, uq_method=None, **kwargs): + super().__init__([Real(bounds=(0., 1.))], param_space=make_space(ot.Normal(1., .2)), + uq_method=uq_method or MonteCarlo(50, seed=3), n_obj=2, **kwargs) + + def _is_conditionally_active(self): + return [False] + + def _correct_x(self, x, is_active): + pass + + def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, *args, **kwargs): + f_out[:, 0] = parameters[0] * x[:, 0] + f_out[:, 1] = 2. * x[:, 0] + + @pytest.fixture def stochastic_problem(): return VectorizedProblem() @@ -195,6 +223,11 @@ def all_response_kinds_problem(): return AllResponseKindsProblem() +@pytest.fixture +def deterministic_response_problem(): + return DeterministicResponseProblem() + + def pytest_sessionstart(session): from sb_arch_opt.util import _prevent_capture print('PREVENT CAPTURE') diff --git a/sb_arch_opt/tests/test_stochastic_problem.py b/sb_arch_opt/tests/test_stochastic_problem.py index 19d7a66..0ca1ae2 100644 --- a/sb_arch_opt/tests/test_stochastic_problem.py +++ b/sb_arch_opt/tests/test_stochastic_problem.py @@ -1,23 +1,20 @@ import pytest import numpy as np -import openturns as ot -from sb_arch_opt.uncertainty import * -from sb_arch_opt.stochastic_problem import StochasticArchOptProblem -from sb_arch_opt.problems.robust_optimization.rosenbrock import StochasticRosenbrock -from sb_arch_opt.tests.conftest import VectorizedProblem, HierarchicalProblem, make_space -from pymoo.core.variable import Real +from sb_arch_opt.tests.conftest import (HAS_UNCERTAINTY, VectorizedProblem, HierarchicalProblem, + DeterministicResponseProblem, make_space) +pytestmark = pytest.mark.skipif(not HAS_UNCERTAINTY, reason='OpenTURNS dependency not installed: ' + 'pip install sb-arch-opt[uncertainty]') -def _output(values): - return StochasticOutput(ot.Sample(np.array(values, dtype=float).reshape((-1, 1)))) +if HAS_UNCERTAINTY: + import openturns as ot + from sb_arch_opt.uncertainty import * + from sb_arch_opt.stochastic_problem import StochasticArchOptProblem + from sb_arch_opt.problems.robust_optimization.rosenbrock import StochasticRosenbrock -def _n_occupied_strata(space, samples, i_param): - # An LHS puts exactly one point in each of the n equiprobable strata of every marginal - n = samples.shape[0] - marginal = space.joint_dist.getMarginal(i_param) - cdf = np.array([marginal.computeCDF(v) for v in samples[:, i_param]]) - return len(np.unique(np.floor(cdf*n).astype(int))) +def _output(values): + return StochasticOutput.from_output_samples(ot.Sample(np.array(values, dtype=float).reshape((-1, 1)))) def test_parameter_space(): @@ -34,27 +31,23 @@ def test_parameter_space(): assert samples.shape == (20, 2) assert np.all(np.isfinite(samples)) - # The two draws differ in how evenly they cover each marginal - assert all(_n_occupied_strata(space, space.get_lhs_samples(40), i) == 40 for i in range(2)) - assert any(_n_occupied_strata(space, space.get_random_samples(40), i) < 40 for i in range(2)) - def test_scalars(): out = _output([1., 2., 3., 4., 5.]) mean, std = out.mean, out.std - assert out.reduce(Mean()) == pytest.approx(mean) - assert out.reduce(Quantile(q=.9)) == pytest.approx(out.quantile(.9)) - assert out.reduce(Margin(k=2.)) == pytest.approx(out.margin(2.)) + assert Mean().scalarize(out) == pytest.approx(mean) + assert Quantile(q=.9).scalarize(out) == pytest.approx(out.quantile(.9)) + assert Margin(k=2.).scalarize(out) == pytest.approx(out.margin(2.)) # The margin is the conservative value, so which tail that is depends on the direction - assert out.reduce(Margin(k=2.)) == pytest.approx(mean + 2.*std) - assert out.reduce(Margin(k=2., direction=-1)) == pytest.approx(mean + 2.*std) - assert out.reduce(Margin(k=2., direction=1)) == pytest.approx(mean - 2.*std) + assert Margin(k=2.).scalarize(out) == pytest.approx(mean + 2. * std) + assert Margin(k=2., direction=-1).scalarize(out) == pytest.approx(mean + 2. * std) + assert Margin(k=2., direction=1).scalarize(out) == pytest.approx(mean - 2. * std) # Each instance carries its own parameters - assert out.reduce(Quantile(q=.5)) != out.reduce(Quantile(q=.95)) - assert out.reduce(Margin(k=1.)) != out.reduce(Margin(k=3.)) + assert Quantile(q=.5).scalarize(out) != Quantile(q=.95).scalarize(out) + assert Margin(k=1.).scalarize(out) != Margin(k=3.).scalarize(out) with pytest.raises(ValueError): Quantile(q=2.) @@ -64,20 +57,13 @@ def test_scalars_penalize_spread(): wide, narrow = _output([1., 3., 5.]), _output([2.5, 3., 3.5]) assert wide.mean == pytest.approx(narrow.mean) - assert narrow.reduce(Margin(k=2.)) < wide.reduce(Margin(k=2.)) # minimized: lower is better - assert narrow.reduce(Margin(k=2., direction=1)) > wide.reduce(Margin(k=2., direction=1)) # maximized - + assert Margin(k=2.).scalarize(narrow) < Margin(k=2.).scalarize(wide) # minimized: lower is better + assert Margin(k=2., direction=1).scalarize(narrow) > Margin(k=2., direction=1).scalarize(wide) # maximized -def test_custom_scalar(): - class WorstCase(Scalarization): - def reduce(self, samples): - return float(samples.getMax()[0]) - assert _output([1., 5., 3.]).reduce(WorstCase()) == pytest.approx(5.) - - -@pytest.mark.parametrize('method_class', [MonteCarlo, PolynomialChaos]) -def test_uq_method_samples(method_class): +@pytest.mark.parametrize('method_name', ['MonteCarlo', 'PolynomialChaos']) +def test_uq_method_samples(method_name): + method_class = {'MonteCarlo': MonteCarlo, 'PolynomialChaos': PolynomialChaos}[method_name] space = make_space(ot.Normal(0., 1.), ot.Normal(0., 1.)) method = method_class(n_evaluations=40, seed=42) @@ -87,9 +73,6 @@ def test_uq_method_samples(method_class): # Drawn once and reused, so every design point sees the same realizations (common random numbers) assert np.all(method.get_samples(space) == samples) - # Both methods use an LHS rather than a plain random draw - assert all(_n_occupied_strata(space, samples, i) == 40 for i in range(2)) - method.resample() assert method.get_samples(space).shape == (40, 2) @@ -97,11 +80,6 @@ def test_uq_method_samples(method_class): method.get_samples(None) -def test_uq_method_requires_n_evaluations(): - with pytest.raises(ValueError): - MonteCarlo(n_evaluations=None) - - def test_parameter_realization(): space = StochasticParameterSpace([ StochasticParameter('a', ot.Normal(0., 1.)), @@ -122,12 +100,11 @@ def test_process_results(): space = make_space(ot.Normal(0., 1.)) method = MonteCarlo(n_evaluations=3, seed=42) - result = method.process_results(np.array([[1., 10., 20.], [2., 11., 21.], [3., 12., 22.]]), space) + outputs = method.process_results(np.array([[1., 10., 20.], [2., 11., 21.], [3., 12., 22.]]), space) - assert isinstance(result, StochasticResults) - assert len(result.outputs) == 3 # one output per response column - assert [output.mean for output in result.outputs] == pytest.approx([2., 11., 21.]) - assert result.method_result is None # Monte Carlo has nothing beyond the samples + assert len(outputs) == 3 # one output per response column + assert [output.mean for output in outputs] == pytest.approx([2., 11., 21.]) + assert all(output.method_results is None for output in outputs) # Monte Carlo has nothing beyond the samples def test_user_only_implements_arch_evaluate_sample(stochastic_problem): out = stochastic_problem.evaluate(np.array([[1., 0.], [0., 0.]]), return_as_dictionary=True) @@ -139,17 +116,6 @@ def test_user_only_implements_arch_evaluate_sample(stochastic_problem): assert out['F'][1, 0] == pytest.approx(1. + .05**2, abs=2e-2) -def test_evaluate_sample_receives_one_realization_per_sample(): - problem = VectorizedProblem(n=25) - problem.evaluate(np.array([[1., 0.], [0., 0.]]), return_as_dictionary=True) - - seen = problem.seen_parameters - assert len(seen) == 25 # once per sample, not once per design point - assert all(realization.shape == (1,) for realization in seen) # one row, not the whole design - assert len({tuple(realization) for realization in seen}) == 25 # a different row each time - assert np.allclose(np.array(seen), problem.uq_method.get_samples(problem.param_space)) - - def test_multiple_design_points_at_once(stochastic_problem): x = np.random.RandomState(0).uniform(-2., 2., size=(7, 2)) out = stochastic_problem.evaluate(x, return_as_dictionary=True) @@ -179,17 +145,6 @@ def test_resample_draws_new_samples(): assert not np.allclose(f_a, problem.evaluate(x, return_as_dictionary=True)['F']) -def test_correction_runs_for_implicit_design_space(hierarchical_problem): - # Correction only happens up front for explicit design spaces, so the loop has to do it: otherwise the - # evaluation function sees an all-True activeness matrix and un-imputed design vectors - x = np.array([[0, .5, .3], [1, .5, .3]]) - out = hierarchical_problem.evaluate(x, return_as_dictionary=True) - - assert np.all(out['is_active'] == [[True, True, False], [True, True, True]]) - assert out['X'][0, 2] == pytest.approx(.25) # imputed to the middle of its bounds - assert np.all(np.isfinite(out['F'])) - - def test_hierarchical_problem_with_constraint(hierarchical_problem): x = np.array([[0, .5, .3], [1, .9, .35]]) out = hierarchical_problem.evaluate(x, return_as_dictionary=True) @@ -203,15 +158,14 @@ def test_hierarchical_problem_with_constraint(hierarchical_problem): def test_response_kinds_use_their_own_scalar(all_response_kinds_problem): problem = all_response_kinds_problem out = problem.evaluate(np.array([[.5]]), return_as_dictionary=True) - result = out['stochastic'][0] - assert len(result.outputs) == 3 - assert out['F'][0, 0] == pytest.approx(result.outputs[0].reduce(Mean())) - assert out['G'][0, 0] == pytest.approx(result.outputs[1].reduce(Mean())) - assert out['H'][0, 0] == pytest.approx(result.outputs[2].reduce(Margin(k=3.))) + # Each response kind is published in its own object array, laid out like F, G and H + assert out['F'][0, 0] == pytest.approx(Mean().scalarize(out['f_stochastic'][0, 0])) + assert out['G'][0, 0] == pytest.approx(Mean().scalarize(out['g_stochastic'][0, 0])) + assert out['H'][0, 0] == pytest.approx(Margin(k=3.).scalarize(out['h_stochastic'][0, 0])) # The equality constraint uses a margin, so it is above its own mean - assert out['H'][0, 0] > result.outputs[2].mean + assert out['H'][0, 0] > out['h_stochastic'][0, 0].mean def test_scalar_counts_checked_per_response_kind(): @@ -225,42 +179,24 @@ def test_scalar_counts_checked_per_response_kind(): HierarchicalProblem(ieq_constr_scalar=[Mean()]) # a matching count is accepted -def test_problem_configuration_checks(): - space = make_space(ot.Normal(0., 1.)) - - with pytest.raises(ValueError): - StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=space, uq_method=None, n_obj=1) - with pytest.raises(ValueError): - StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=space, uq_method='monte carlo', n_obj=1) - with pytest.raises(ValueError): - StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=None, uq_method=MonteCarlo(10), n_obj=1) - with pytest.raises(ValueError): - StochasticArchOptProblem([Real(bounds=(0., 1.))], param_space=StochasticParameterSpace([]), - uq_method=MonteCarlo(10), n_obj=1) - - def test_statistics_available_per_design_point(stochastic_problem): out = stochastic_problem.evaluate(np.array([[1., 0.], [0., 0.]]), return_as_dictionary=True) - assert len(out['stochastic']) == 2 - for result in out['stochastic']: - assert isinstance(result, StochasticResults) - assert len(result.outputs) == 1 - assert len(result.outputs[0].to_numpy()) == 100 + assert out['f_stochastic'].shape == (2, 1) + assert all(isinstance(output, StochasticOutput) for output in out['f_stochastic'].ravel()) + assert out['g_stochastic'].shape == (2, 0) # The realizations really reach the model, so the response scatters - values = out['stochastic'][0].outputs[0].to_numpy() - assert len(set(values.tolist())) == 100 - assert values.std() > 0. + for output in out['f_stochastic'][:, 0]: + assert output.std > 0. def test_reported_statistics_reproduce_the_reduced_value(): problem = VectorizedProblem(n=200, obj_scalar=[Margin(k=2.)]) out = problem.evaluate(np.array([[1., 0.], [.5, .5]]), return_as_dictionary=True) - for i, result in enumerate(out['stochastic']): - output = result.outputs[0] - assert out['F'][i, 0] == pytest.approx(output.reduce(Margin(k=2.))) + for i, output in enumerate(out['f_stochastic'][:, 0]): + assert out['F'][i, 0] == pytest.approx(Margin(k=2.).scalarize(output)) assert out['F'][i, 0] == pytest.approx(output.mean + 2.*output.std) @@ -268,11 +204,32 @@ def test_stochastic_output_statistics(): out = _output(np.linspace(0., 10., 101)) assert out.mean == pytest.approx(5.) - assert out.var == pytest.approx(out.std**2) + # The kernel widens the fitted distribution slightly compared to the samples it was built from + assert out.std == pytest.approx(np.std(np.linspace(0., 10., 101)), rel=.1) assert out.quantile(.5) == pytest.approx(5., abs=.1) - assert out.prob_exceeds(5.) == pytest.approx(.5, abs=.02) - assert len(out.to_numpy()) == 101 - assert out.to_distribution.computeCDF(5.) == pytest.approx(.5, abs=.05) + assert out.dist.computeCDF(5.) == pytest.approx(.5, abs=.05) + assert str(out) == f'(mean = {out.mean:.4g}, sigma = {out.std:.4g})' + + +@pytest.mark.parametrize('method_name', ['MonteCarlo', 'PolynomialChaos']) +def test_a_response_that_does_not_depend_on_the_parameters(method_name): + uq_method = {'MonteCarlo': MonteCarlo, 'PolynomialChaos': PolynomialChaos}[method_name](50, seed=3) + # A response that is constant over the realizations has no distribution to fit: it is reported as the value + # itself, so no scalarization can add a margin to it + problem = DeterministicResponseProblem(uq_method=uq_method, obj_scalar=[Mean(), Margin(k=2.)]) + out = problem.evaluate(np.array([[.5], [1.]]), return_as_dictionary=True) + + assert np.all(np.isfinite(out['F'])) + assert not problem.get_failed_points(out).any() + + for i, x_i in enumerate([.5, 1.]): + varying, constant = out['f_stochastic'][i, :] + assert isinstance(varying, StochasticOutput) + assert varying.std > 0. + + assert isinstance(constant, float) + assert constant == pytest.approx(2.*x_i) + assert out['F'][i, 1] == pytest.approx(2.*x_i) def test_a_failed_sample_fails_the_design_point(): @@ -281,8 +238,8 @@ def test_a_failed_sample_fails_the_design_point(): problem = VectorizedProblem(n=50, fail=True) out = problem.evaluate(np.array([[.5, .5]]), return_as_dictionary=True) - values = out['stochastic'][0].outputs[0].to_numpy() - assert np.any(~np.isfinite(values)) + # There is no distribution to fit, so the response comes back as NaN rather than as an output + assert not np.isfinite(out['f_stochastic'][0, 0]) assert not np.isfinite(out['F'][0, 0]) assert problem.get_failed_points(out)[0] @@ -310,8 +267,8 @@ def test_pce_through_the_problem(): assert np.all(np.isfinite(out['F'])) assert np.allclose(out['F'], problem.evaluate(x, return_as_dictionary=True)['F']) - # Statistics come from the cheap metamodel, not from the 50 expensive evaluations - assert len(out['stochastic'][0].outputs[0].to_numpy()) == problem.uq_method.n_metamodel_samples + # Statistics come from the fitted expansion, not from the 50 expensive evaluations + assert isinstance(out['f_stochastic'][0, 0].method_results, ot.FunctionalChaosResult) def test_pce_terms_and_sample_requirement(): @@ -328,20 +285,19 @@ def test_pce_provides_chaos_result_for_sensitivity_analysis(): problem = VectorizedProblem(uq_method=PolynomialChaos(50, seed=42), obj_scalar=[Mean()]) out = problem.evaluate(np.array([[.5, .5]]), return_as_dictionary=True) - chaos_results = out['stochastic'][0].method_result - assert len(chaos_results) == 1 - assert isinstance(chaos_results[0], ot.FunctionalChaosResult) + chaos_result = out['f_stochastic'][0, 0].method_results + assert isinstance(chaos_result, ot.FunctionalChaosResult) - sobol = ot.FunctionalChaosSobolIndices(chaos_results[0]) + sobol = ot.FunctionalChaosSobolIndices(chaos_result) assert sobol.getSobolIndex(0) == pytest.approx(1., abs=1e-6) # a single parameter explains everything -def test_pce_falls_back_to_raw_samples_when_evaluations_fail(): +def test_pce_fails_the_design_point_when_evaluations_fail(): # A response with failed evaluations cannot be fitted; the design point then fails as usual problem = VectorizedProblem(uq_method=PolynomialChaos(50, seed=42), obj_scalar=[Mean()], fail=True) out = problem.evaluate(np.array([[.5, .5]]), return_as_dictionary=True) - assert out['stochastic'][0].method_result[0] is None + assert not np.isfinite(out['f_stochastic'][0, 0]) assert not np.isfinite(out['F'][0, 0]) diff --git a/sb_arch_opt/uncertainty.py b/sb_arch_opt/uncertainty.py index 5c5b6ca..b40cbff 100644 --- a/sb_arch_opt/uncertainty.py +++ b/sb_arch_opt/uncertainty.py @@ -22,64 +22,40 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from typing import List, Optional, Dict +from typing import List, Optional, Union import openturns as ot import numpy as np __all__ = ['Scalarization', 'Mean', 'Margin', 'Quantile', 'StochasticParameter', 'StochasticParameterSpace', - 'StochasticOutput', 'StochasticResults', 'UQMethod', 'MonteCarlo', 'PolynomialChaos'] + 'StochasticOutput', 'UQMethod', 'MonteCarlo', 'PolynomialChaos', 'EvaluationOutput'] +EvaluationOutput = Union['StochasticOutput', float] +"""Output either distribution or numeric value.""" - -class Scalarization: - """ - Parent class for all robust scalars etc. Margin, Mean, Quantile +class StochasticParameter: """ + A stochastic parameter used to evaluate architecture objectives or constraints. - def reduce(self, samples: ot.Sample) -> float: - """Reduce an (n_samples x 1) sample of one response to a single value""" - raise NotImplementedError - - -class Mean(Scalarization): - """The expected value of the response""" - - def reduce(self, samples: ot.Sample) -> float: - return float(samples.computeMean()[0]) - -class Margin(Scalarization): - """Assumes the response is Gaussian""" - - def __init__(self, k: float = 1.645, direction: int =-1): - self.k = k - self.direction = direction - - def reduce(self, samples: ot.Sample) -> float: - return float(samples.computeMean()[0] - self.k * np.sign(self.direction) * samples.computeStandardDeviation()[0]) - -class Quantile(Scalarization): - """The q quantile of the response for arbitrary distribution""" - - def __init__(self, q: float = 0.95): - if not 0. <= q <= 1.: - raise ValueError(f'Quantile should be between 0 and 1: {q}') - self.q = q - - def reduce(self, samples: ot.Sample) -> float: - return float(samples.computeQuantile(self.q)[0]) - + The parameter carries two distinct things. `dist` defines the parameter distribution defined by OpenTurns object. + `sample` is one realization of it. -class StochasticParameter: - """A stochastic parameter used to evaluate architecture objectives or constraints.""" + :param name: identifies the parameter + :param dist: OpenTurns distribution object + :param ref: Optional reference object used for matching with the related class + """ - def __init__(self, name, value: ot.DistributionImplementation): + def __init__(self, name, dist: ot.DistributionImplementation, ref=None): self.name = name - self.value = value + self.dist = dist + self.ref = ref self._sample = None @property def sample(self) -> float: + """ + The parameter realization of it, written by `StochasticParameterSpace.param_realization` + """ return self._sample @sample.setter @@ -87,127 +63,198 @@ def sample(self, value: float): self._sample = value - - class StochasticParameterSpace: - """The joint distribution of all stochastic parameters of a problem.""" + """ + The joint distribution of all stochastic parameters of a problem, which are assumed to be independent. + The space does not store the sampled parameters as its attribute. The samples are processed by the chosen UQ method. + + :param parameters: the stochastic parameters + """ def __init__(self, parameters: List[StochasticParameter]): self._parameters = parameters @property def n_parameters(self) -> int: + """Number of stochastic parameters""" return len(self._parameters) @property def parameter_names(self) -> List[str]: + """List of stochastic parameter names""" return [parameter.name for parameter in self._parameters] @property def joint_dist(self) -> ot.JointDistribution: - """ Joint distribution of independent variables is chosen """ - return ot.JointDistribution([parameter.value for parameter in self._parameters], + """Joint distribution of independent parameters""" + return ot.JointDistribution([parameter.dist for parameter in self._parameters], ot.IndependentCopula(self.n_parameters)) def param_realization(self, samples: np.ndarray, i_realization: int) -> List[StochasticParameter]: + """ + Return `i_realization` of `samples` (n_samples x n_parameters) as a list of stochastic parameters. + """ for j, param in enumerate(self._parameters): param.sample = samples[i_realization, j] return self._parameters def get_random_samples(self, n_samples: int) -> np.ndarray: - """ Draw n samples of the stochastic parameters; returns an n x n_parameters matrix """ + """Draw n samples of the stochastic parameters; returns an n x n_parameters matrix""" result = self.joint_dist.getSample(n_samples) return np.array(result) def get_lhs_samples(self, n_samples: int) -> np.ndarray: - """ Draw n samples of the stochastic parameters; returns an n x n_parameters matrix built with LHS""" + """Draw n samples of the stochastic parameters; returns an n x n_parameters matrix built with LHS""" lhs = ot.LHSExperiment(self.joint_dist, n_samples) result = lhs.generate() return np.array(result) - class StochasticOutput: - """The sampled values of one response (objective or constraint) for one design point.""" + """ + The object representing the stochastic output of a single objective or constraint for a given design point. + + :param dist: the output distribution for the specific objective or constraint. + """ - def __init__(self, output_samples: ot.Sample): - self.output_samples = output_samples # ot.Sample of shape (n, 1) + def __init__(self, dist: ot.Distribution, method_results=None): + self.dist = dist + self.method_results = method_results + + @staticmethod + def build_distribution(output_samples: ot.Sample) -> ot.Distribution | float: + values = np.asarray(output_samples, dtype=float).ravel() + + if values.size == 0 or not np.all(np.isfinite(values)): + return np.nan + + mean, std = float(np.mean(values)), float(np.std(values)) + if std <= 1e-12*max(abs(mean), 1.): + return mean + else: + return ot.KernelSmoothing().build(output_samples) @classmethod - def from_results(cls, results: ot.Sample, index: int) -> 'StochasticOutput': - """Extract one response's column from the full (n_samples, n_outputs) results of a single design point.""" - return cls(output_samples=results.getMarginal(index)) + def from_output_samples(cls, output_samples: ot.Sample, method_results=None) -> EvaluationOutput: + """ + Construct the object from evaluation output samples of a single objective or constraint. + If the variance of the provided samples is statistically insignificant, + a float value is returned to represent deterministic output. If any + sample is nan, np.nan is returned. + """ + dist = cls.build_distribution(output_samples) + if isinstance(dist, float): + return dist + return cls(dist=dist, method_results=method_results) @property def mean(self) -> float: - return self.output_samples.computeMean()[0] + return self.dist.getMean()[0] @property def std(self) -> float: - return self.output_samples.computeStandardDeviation()[0] - - @property - def var(self) -> float: - return self.output_samples.computeVariance()[0] + return self.dist.getStandardDeviation()[0] def quantile(self, q: float) -> float: - return self.output_samples.computeQuantile(q)[0] - - def prob_exceeds(self, threshold: float) -> float: - arr = self.to_numpy() - return float(np.mean(arr > threshold)) + return self.dist.computeQuantile(q)[0] def margin(self, k: float = 1.645, direction: int =-1) -> float: - """mean + k*sigma - the Gaussian-assumption margin formulation.""" return self.mean - np.sign(direction) * k * self.std - @property - def to_distribution(self) -> ot.Distribution: - """Fit a continuous distribution if you need PDF/CDF rather than raw samples.""" - return ot.KernelSmoothing().build(self.output_samples) + def __repr__(self): + return f"(mean = {self.mean:.4g}, sigma = {self.std:.4g})" - def to_numpy(self) -> np.ndarray: - return np.array(self.output_samples).flatten() + def __str__(self): + return repr(self) - def reduce(self, scalar: Scalarization) -> float: - """ - Reduce the sampled values to the single value the optimizer sees, by applying the given scalar. - """ - values = self.to_numpy() - if values.size == 0 or not np.all(np.isfinite(values)): - return np.nan - samples = self.output_samples +class Scalarization: + """ + Parent class for all types of stochastic optimization problem. Do not instantiate this class directly, use its subclasses: - return scalar.reduce(samples) + - Mean + - Margin + - Quantile + """ + def scalarize(self, output: StochasticOutput) -> float: + """Reduce an (n_samples x 1) sample of one response to a single value that the optimizer sees based on the optimization problem type.""" + raise NotImplementedError + + def __repr__(self): + return self.__class__.__name__ def __str__(self): - if self.std / self.mean < 1e-6: - return f'{self.mean:.4g}' - return f"(mean = {self.mean:.4g}, sigma = {self.std:.4g})" + return repr(self) + +class Mean(Scalarization): + """Minimize the expectation of the objective or constraint function for example min(E[F(x)])""" + def scalarize(self, output: StochasticOutput) -> float: + return float(output.mean) -class StochasticResults: + +class Margin(Scalarization): """ - All sampled responses of a single design point. + Gaussian output distribution expected. Minimize for the objective or constraint function for a given + confidence interval. + --> min(E[F(x)] + k*sigma[F(x)]); The default k=1.645 is the one-sided 95% interval of a normal distribution. + + Requires: - `method_result` optionally carries whatever the UQ method produced beyond the samples themselves; for polynomial - chaos that is the list of `ot.FunctionalChaosResult`s, from which for example Sobol indices can be obtained. + - :param k: coverage factor, i.e. the number of standard deviations added to the mean (default 1.645 covers 95% of a normal response on one side; 1, 2 and 3 give 84.1%, 97.7% and 99.9%). + - :param direction: -1 for minimizing and +1 for maximizing. """ - def __init__(self, outputs: List[StochasticOutput], method_result=None): - self.outputs = outputs - self.method_result = method_result + def __init__(self, k: float = 1.645, direction: int =-1): + self.k = k + self.direction = direction + + def scalarize(self, output: StochasticOutput) -> float: + return float(output.margin(self.k, self.direction)) + + def __repr__(self): + return f"Margin with k = {self.k}" + + +class Quantile(Scalarization): + """ + Makes no distributional assumption. Minimize the q quantile of the objective or constraint function --> min(F_q(x)), with q=0.95 by default. + + Requires: + + :param q: quantile level in [0, 1], i.e. the probability that the response falls below the returned value (default 0.95); + unlike Margin this assumes nothing about the distribution shape, since the quantile is taken from the samples directly. + + """ + + def __init__(self, q: float = 0.95): + if not 0. <= q <= 1.: + raise ValueError(f'Quantile should be between 0 and 1: {q}') + self.q = q + + def scalarize(self, output: StochasticOutput) -> float: + return float(output.quantile(self.q)) + + def __repr__(self): + return f"Quantile with q = {self.q}" class UQMethod: """ - Base class for an uncertainty propagation method. + Base class for an uncertainty propagation methods. + + It is responsible for sampling the provided 'param_space' based on the provided number of evaluations and + for performing uncertainty quantification study to return a list of StochasticOutputs or floats for a given design vector. + + Subclass by overriding `_draw_samples` (sampling method) and `process_results` (statistics computation). + + :param n_evaluations: number of expensive evaluations spent per design point + :param seed: optional seed the OpenTURNS generator before drawing, so the design is reproducible """ def __init__(self, n_evaluations: int, seed: int = None): - if n_evaluations is None: + if n_evaluations is None or n_evaluations <= 0: raise ValueError('n_evaluations must be specified: it is the number of expensive evaluations ' 'per design point') self.n_evaluations = n_evaluations @@ -216,13 +263,18 @@ def __init__(self, n_evaluations: int, seed: int = None): self._samples: Optional[np.ndarray] = None def get_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: + """ + Get samples from the joint distribution provided by the param_space. + """ if param_space is None: raise ValueError('No parameter space to sample') - if self._samples is None: + + samples = self._samples + if samples is None: if self.seed is not None: ot.RandomGenerator.SetSeed(self.seed) - self._samples = self._draw_samples(param_space) - return self._samples + samples = self._draw_samples(param_space) + return samples def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: """Draw the design of experiments in the parameter space; override to use a different design""" @@ -232,13 +284,19 @@ def resample(self): """Draw a new design on the next evaluation""" self._samples = None - def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace) -> StochasticResults: + def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace) -> list[EvaluationOutput]: """ - Turn the responses of ONE design point (an n_samples x (n_obj+n_ieq_constr+n_eq_constr) matrix) into the - stochastic result object that contains list of stochastic outputs. + Turn the responses of one design point (an n_samples x (n_obj+n_ieq_constr+n_eq_constr) matrix) into the + list of EvaluationOutput objects. """ raise NotImplementedError + def __repr__(self) -> str: + return self.__class__.__name__ + + def __str__(self): + return repr(self) + class MonteCarlo(UQMethod): """ @@ -249,32 +307,32 @@ def __init__(self, n_evaluations: int = 100, seed: int = 42): super().__init__(n_evaluations, seed) def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: - # Draw samples with LHS method + """Draw samples with LHS method""" samples = param_space.get_lhs_samples(self.n_evaluations) return samples - def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace = None) -> StochasticResults: + def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace = None) -> list[EvaluationOutput]: results = np.asarray(results, dtype=float) sample = ot.Sample(results) - outputs = [StochasticOutput.from_results(sample, i) for i in range(results.shape[1])] - return StochasticResults(outputs) + outputs = [StochasticOutput.from_output_samples(sample[:, i]) for i in range(results.shape[1])] + return outputs + + def __repr__(self) -> str: + return 'Monte Carlo' class PolynomialChaos(UQMethod): """ - Polynomial chaos expansion (PCE): a surrogate of each response as a function of the uncertain parameters is - fitted from n_evaluations evaluations of the expensive model, and the statistics are then taken from that - surrogate. + Polynomial chaos expansion (PCE): a surrogate model is fitted for each response as a function of the uncertain parameters. + The trained model is then sampled with Monte Carlo to compute output statistics. - The expensive model is evaluated `n_evaluations` times per design point, exactly like Monte Carlo, but the - statistics come from `n_metamodel_samples` evaluations of the cheap fitted expansion instead of from the - expensive ones. For a response that the expansion represents well this gives far more accurate quantiles and - tail statistics for the same number of expensive evaluations. + The fitted `ot.FunctionalChaosResult` of each response is kept on the `StochasticOutput` (`method_result`) with + Sobol sensitivity indices available through `ot.FunctionalChaosSobolIndices`. - The fitted `ot.FunctionalChaosResult` of each response is kept on the `StochasticResult` (`method_result`), so - Sobol sensitivity indices are available for free through `ot.FunctionalChaosSobolIndices`. - """ + :param degree: total degree of the expansion + :param n_metamodel_samples: number of cheap evaluations of the trained model used to obtain the statistics + """ def __init__(self, n_evaluations: int, seed: int = None, degree: int = 3, n_metamodel_samples: int = 10000): self.degree = degree @@ -288,6 +346,7 @@ def n_terms(self, param_space: StochasticParameterSpace) -> int: return int(enumerate_function.getStrataCumulatedCardinal(self.degree)) def _validate(self, param_space: StochasticParameterSpace): + """Check the expansion can be fitted from the budget""" n_terms = self.n_terms(param_space) if self.n_evaluations < n_terms: raise ValueError(f'A degree-{self.degree} expansion in {param_space.n_parameters} ' @@ -295,17 +354,19 @@ def _validate(self, param_space: StochasticParameterSpace): f'n_evaluations = {self.n_evaluations}') def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: + """Draw samples with LHS method""" self._validate(param_space) - # Draw samples with LHS method samples = param_space.get_lhs_samples(self.n_evaluations) return samples def _get_metamodel_input(self, param_space: StochasticParameterSpace) -> ot.Sample: + """Get samples used for evaluating the surrogate model""" if self._metamodel_input is None: self._metamodel_input = param_space.joint_dist.getSample(self.n_metamodel_samples) return self._metamodel_input def _build_algorithm(self, param_space: StochasticParameterSpace, input_sample: ot.Sample, output_sample: ot.Sample) -> ot.FunctionalChaosAlgorithm: + """Train surrogate model with the least-squares method""" distribution = param_space.joint_dist dimension = distribution.getDimension() @@ -319,13 +380,13 @@ def _build_algorithm(self, param_space: StochasticParameterSpace, input_sample: return ot.FunctionalChaosAlgorithm(input_sample, output_sample, distribution, adaptive_strategy, projection_strategy) - def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace) -> StochasticResults: + def process_results(self, results: np.ndarray, param_space: StochasticParameterSpace) -> list[EvaluationOutput]: results = np.asarray(results, dtype=float) input_sample = ot.Sample(self.get_samples(param_space)) metamodel_input = self._get_metamodel_input(param_space) - outputs, chaos_results = [], [] + outputs = [] for i_out in range(results.shape[1]): values = results[:, i_out] @@ -333,14 +394,16 @@ def process_results(self, results: np.ndarray, param_space: StochasticParameterS # the problem's nan_policy decides what happens to this design point if not np.all(np.isfinite(values)): samples = ot.Sample(values.reshape((-1, 1))) - chaos_results.append(None) + chaos_result = None else: algorithm = self._build_algorithm(param_space, input_sample, ot.Sample(values.reshape((-1, 1)))) algorithm.run() chaos_result = algorithm.getResult() - chaos_results.append(chaos_result) samples = chaos_result.getMetaModel()(metamodel_input) - outputs.append(StochasticOutput(samples)) - return StochasticResults(outputs, method_result=chaos_results) \ No newline at end of file + outputs.append(StochasticOutput.from_output_samples(samples, chaos_result)) + return outputs + + def __repr__(self) -> str: + return 'Polynomial Chaos Expansion' \ No newline at end of file diff --git a/setup.py b/setup.py index 63caeb8..ad28c3c 100644 --- a/setup.py +++ b/setup.py @@ -67,6 +67,9 @@ def _get_readme(): 'numba', 'scikit-learn', ], + 'uncertainty': [ # Stochastic (robust) problems: sb_arch_opt.uncertainty, sb_arch_opt.stochastic_problem + 'openturns', + ], # 'ota': [ # pip install -r requirements-ota.txt # 'open_turb_arch @ git+https://github.com/jbussemaker/OpenTurbofanArchitecting@pymoo_optional#egg=open_turb_arch', # ], From 12a42335e4910dc5e2a7afb6ba4c04d57d1c9102 Mon Sep 17 00:00:00 2001 From: bial_fr Date: Thu, 24 Sep 2026 16:06:37 +0200 Subject: [PATCH 3/5] print_extra_stats update --- docs/tutorial_stochastic.ipynb | 4 +--- sb_arch_opt/stochastic_problem.py | 14 +++++++------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/tutorial_stochastic.ipynb b/docs/tutorial_stochastic.ipynb index 76a3f20..07d91b2 100644 --- a/docs/tutorial_stochastic.ipynb +++ b/docs/tutorial_stochastic.ipynb @@ -222,9 +222,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": [ - "The robust Pareto front lies above the deterministic one, which clealry shows that the perturbation in design vector has effect on the objectives. The deterministic front is therefore not achievable in practice, and optimizing towards it would select designs that are more sensitive to the perturbations." - ] + "source": "The robust Pareto front lies above the deterministic one, which clearly shows that the perturbation in design vector has effect on the objectives. The deterministic front is therefore not achievable in practice, and optimizing towards it would select designs that are more sensitive to the perturbations." }, { "cell_type": "markdown", diff --git a/sb_arch_opt/stochastic_problem.py b/sb_arch_opt/stochastic_problem.py index 2de8136..e78a363 100644 --- a/sb_arch_opt/stochastic_problem.py +++ b/sb_arch_opt/stochastic_problem.py @@ -119,13 +119,13 @@ def check_scalars(scalars: Optional[List[Scalarization]], n: int) -> List[Scalar return list(scalars) def _print_extra_stats(self): - print(f'stochastic : True') - print(f'n_params : {self.param_space.n_parameters}') - print(f'uq_method : {self.uq_method}') - print(f'n_evaluations: {self.uq_method.n_evaluations}') - print(f'obj : {self.obj_scalar}') - print(f'ieq_constr : {self.ieq_constr_scalar}') - print(f'eq_constr : {self.eq_constr_scalar}') + print(f'stochastic : True') + print(f'n_stochastic_params : {self.param_space.n_parameters}') + print(f'uq_method : {self.uq_method}') + print(f'n_uq_samples : {self.uq_method.n_evaluations}') + print(f'obj_scalarize : {self.obj_scalar}') + print(f'ieq_constr_scalarize : {self.ieq_constr_scalar}') + print(f'eq_constr_scalarize : {self.eq_constr_scalar}') def _evaluate(self, x, out, *args, **kwargs): From f8f76e6f34043f4427fa77ce62f3775773e38792 Mon Sep 17 00:00:00 2001 From: bial_fr Date: Thu, 24 Sep 2026 16:06:37 +0200 Subject: [PATCH 4/5] print_extra_stats update --- docs/tutorial_stochastic.ipynb | 88 +++++++++++++++---------------- sb_arch_opt/stochastic_problem.py | 14 ++--- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/docs/tutorial_stochastic.ipynb b/docs/tutorial_stochastic.ipynb index 76a3f20..aeccb0c 100644 --- a/docs/tutorial_stochastic.ipynb +++ b/docs/tutorial_stochastic.ipynb @@ -26,10 +26,10 @@ "start_time": "2026-09-23T13:01:12.092456200Z" }, "execution": { - "iopub.execute_input": "2026-09-24T12:05:33.295694Z", - "iopub.status.busy": "2026-09-24T12:05:33.294696Z", - "iopub.status.idle": "2026-09-24T12:05:33.307190Z", - "shell.execute_reply": "2026-09-24T12:05:33.304172Z" + "iopub.execute_input": "2026-09-24T14:08:48.217956Z", + "iopub.status.busy": "2026-09-24T14:08:48.216925Z", + "iopub.status.idle": "2026-09-24T14:08:48.227395Z", + "shell.execute_reply": "2026-09-24T14:08:48.226377Z" } }, "outputs": [], @@ -66,10 +66,10 @@ "start_time": "2026-09-24T09:25:09.763161600Z" }, "execution": { - "iopub.execute_input": "2026-09-24T12:05:33.312192Z", - "iopub.status.busy": "2026-09-24T12:05:33.312192Z", - "iopub.status.idle": "2026-09-24T12:05:34.996078Z", - "shell.execute_reply": "2026-09-24T12:05:34.996078Z" + "iopub.execute_input": "2026-09-24T14:08:48.230905Z", + "iopub.status.busy": "2026-09-24T14:08:48.230905Z", + "iopub.status.idle": "2026-09-24T14:09:13.455124Z", + "shell.execute_reply": "2026-09-24T14:09:13.455124Z" } }, "outputs": [], @@ -120,10 +120,10 @@ "start_time": "2026-09-24T09:25:12.171654800Z" }, "execution": { - "iopub.execute_input": "2026-09-24T12:05:34.998092Z", - "iopub.status.busy": "2026-09-24T12:05:34.998092Z", - "iopub.status.idle": "2026-09-24T12:05:35.001389Z", - "shell.execute_reply": "2026-09-24T12:05:35.001389Z" + "iopub.execute_input": "2026-09-24T14:09:13.457652Z", + "iopub.status.busy": "2026-09-24T14:09:13.457652Z", + "iopub.status.idle": "2026-09-24T14:09:13.462157Z", + "shell.execute_reply": "2026-09-24T14:09:13.461652Z" } }, "outputs": [ @@ -142,13 +142,13 @@ "n_valid_discr: 10\n", "imp_ratio : 1.00 (discr.: 1.00; cont.: 1.00)\n", "corr_ratio : 1.00 (discr.: 1.00; cont.: 1.00; fraction of imp_ratio: 0.0%)\n", - "stochastic : True\n", - "n_params : 1\n", - "uq_method : Polynomial Chaos Expansion\n", - "n_evaluations: 12\n", - "obj : [Mean, Mean]\n", - "ieq_constr : []\n", - "eq_constr : []\n" + "stochastic : True\n", + "n_stochastic_params : 1\n", + "uq_method : Polynomial Chaos Expansion\n", + "n_uq_samples : 12\n", + "obj_scalarize : [Mean, Mean]\n", + "ieq_constr_scalarize : []\n", + "eq_constr_scalarize : []\n" ] } ], @@ -184,10 +184,10 @@ "start_time": "2026-09-24T09:25:16.824124100Z" }, "execution": { - "iopub.execute_input": "2026-09-24T12:05:35.031443Z", - "iopub.status.busy": "2026-09-24T12:05:35.031443Z", - "iopub.status.idle": "2026-09-24T12:06:07.456866Z", - "shell.execute_reply": "2026-09-24T12:06:07.456866Z" + "iopub.execute_input": "2026-09-24T14:09:13.489596Z", + "iopub.status.busy": "2026-09-24T14:09:13.489596Z", + "iopub.status.idle": "2026-09-24T14:09:39.215685Z", + "shell.execute_reply": "2026-09-24T14:09:39.214671Z" } }, "outputs": [ @@ -223,7 +223,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The robust Pareto front lies above the deterministic one, which clealry shows that the perturbation in design vector has effect on the objectives. The deterministic front is therefore not achievable in practice, and optimizing towards it would select designs that are more sensitive to the perturbations." + "The robust Pareto front lies above the deterministic one, which clearly shows that the perturbation in design vector has effect on the objectives. The deterministic front is therefore not achievable in practice, and optimizing towards it would select designs that are more sensitive to the perturbations." ] }, { @@ -244,10 +244,10 @@ "start_time": "2026-09-23T13:01:31.942026500Z" }, "execution": { - "iopub.execute_input": "2026-09-24T12:06:07.458881Z", - "iopub.status.busy": "2026-09-24T12:06:07.458881Z", - "iopub.status.idle": "2026-09-24T12:06:18.583246Z", - "shell.execute_reply": "2026-09-24T12:06:18.583246Z" + "iopub.execute_input": "2026-09-24T14:09:39.219685Z", + "iopub.status.busy": "2026-09-24T14:09:39.219685Z", + "iopub.status.idle": "2026-09-24T14:09:52.920934Z", + "shell.execute_reply": "2026-09-24T14:09:52.920934Z" } }, "outputs": [ @@ -255,112 +255,112 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:09,799 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (20 real unique, 20 eval)\n" + "INFO 2026-09-24 16:09:45,150 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (20 real unique, 20 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:09,896 sb_arch_opt.sbo : Infill: 4 new (eval 20 real unique, 20 eval)\n" + "INFO 2026-09-24 16:09:45,265 sb_arch_opt.sbo : Infill: 4 new (eval 20 real unique, 20 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:10,904 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (24 real unique, 24 eval)\n" + "INFO 2026-09-24 16:09:46,212 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (24 real unique, 24 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:11,024 sb_arch_opt.sbo : Infill: 4 new (eval 24 real unique, 24 eval)\n" + "INFO 2026-09-24 16:09:46,317 sb_arch_opt.sbo : Infill: 4 new (eval 24 real unique, 24 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:11,901 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (28 real unique, 28 eval)\n" + "INFO 2026-09-24 16:09:47,199 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (28 real unique, 28 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:12,037 sb_arch_opt.sbo : Infill: 4 new (eval 28 real unique, 28 eval)\n" + "INFO 2026-09-24 16:09:47,358 sb_arch_opt.sbo : Infill: 4 new (eval 28 real unique, 28 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:12,977 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (32 real unique, 32 eval)\n" + "INFO 2026-09-24 16:09:48,234 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (32 real unique, 32 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:13,694 sb_arch_opt.sbo : Infill: 4 new (eval 32 real unique, 32 eval)\n" + "INFO 2026-09-24 16:09:48,362 sb_arch_opt.sbo : Infill: 4 new (eval 32 real unique, 32 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:14,763 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (36 real unique, 36 eval)\n" + "INFO 2026-09-24 16:09:49,250 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (36 real unique, 36 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:15,028 sb_arch_opt.sbo : Infill: 4 new (eval 36 real unique, 36 eval)\n" + "INFO 2026-09-24 16:09:49,507 sb_arch_opt.sbo : Infill: 4 new (eval 36 real unique, 36 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:15,909 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (40 real unique, 40 eval)\n" + "INFO 2026-09-24 16:09:50,378 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (40 real unique, 40 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:16,001 sb_arch_opt.sbo : Infill: 4 new (eval 40 real unique, 40 eval)\n" + "INFO 2026-09-24 16:09:50,472 sb_arch_opt.sbo : Infill: 4 new (eval 40 real unique, 40 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:16,977 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (44 real unique, 44 eval)\n" + "INFO 2026-09-24 16:09:51,385 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (44 real unique, 44 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:17,086 sb_arch_opt.sbo : Infill: 4 new (eval 44 real unique, 44 eval)\n" + "INFO 2026-09-24 16:09:51,494 sb_arch_opt.sbo : Infill: 4 new (eval 44 real unique, 44 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:18,011 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (48 real unique, 48 eval)\n" + "INFO 2026-09-24 16:09:52,397 sb_arch_opt.sbo : Surrogate infill gen 10 @ 1000 points evaluated (48 real unique, 48 eval)\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO 2026-09-24 14:06:18,408 sb_arch_opt.sbo : Infill: 4 new (eval 48 real unique, 48 eval)\n" + "INFO 2026-09-24 16:09:52,761 sb_arch_opt.sbo : Infill: 4 new (eval 48 real unique, 48 eval)\n" ] }, { diff --git a/sb_arch_opt/stochastic_problem.py b/sb_arch_opt/stochastic_problem.py index 2de8136..e78a363 100644 --- a/sb_arch_opt/stochastic_problem.py +++ b/sb_arch_opt/stochastic_problem.py @@ -119,13 +119,13 @@ def check_scalars(scalars: Optional[List[Scalarization]], n: int) -> List[Scalar return list(scalars) def _print_extra_stats(self): - print(f'stochastic : True') - print(f'n_params : {self.param_space.n_parameters}') - print(f'uq_method : {self.uq_method}') - print(f'n_evaluations: {self.uq_method.n_evaluations}') - print(f'obj : {self.obj_scalar}') - print(f'ieq_constr : {self.ieq_constr_scalar}') - print(f'eq_constr : {self.eq_constr_scalar}') + print(f'stochastic : True') + print(f'n_stochastic_params : {self.param_space.n_parameters}') + print(f'uq_method : {self.uq_method}') + print(f'n_uq_samples : {self.uq_method.n_evaluations}') + print(f'obj_scalarize : {self.obj_scalar}') + print(f'ieq_constr_scalarize : {self.ieq_constr_scalar}') + print(f'eq_constr_scalarize : {self.eq_constr_scalar}') def _evaluate(self, x, out, *args, **kwargs): From f064c2a5607aafd25e4ac8627b87025827e7fc5f Mon Sep 17 00:00:00 2001 From: bial_fr Date: Fri, 25 Sep 2026 15:30:34 +0200 Subject: [PATCH 5/5] Robust optimization 1.1.1 --- docs/tutorial_stochastic.ipynb | 58 +++++------ sb_arch_opt/__init__.py | 2 +- .../robust_optimization/rosenbrock.py | 99 ------------------- sb_arch_opt/stochastic_problem.py | 20 +++- sb_arch_opt/tests/conftest.py | 13 ++- sb_arch_opt/tests/test_stochastic_problem.py | 41 +++----- sb_arch_opt/uncertainty.py | 61 +++++++++--- 7 files changed, 111 insertions(+), 183 deletions(-) delete mode 100644 sb_arch_opt/problems/robust_optimization/rosenbrock.py diff --git a/docs/tutorial_stochastic.ipynb b/docs/tutorial_stochastic.ipynb index 3730df6..397cccd 100644 --- a/docs/tutorial_stochastic.ipynb +++ b/docs/tutorial_stochastic.ipynb @@ -15,7 +15,8 @@ "```\n", "pip install sb-arch-opt[uncertainty,arch_sbo]\n", "```" - ] + ], + "id": "cb2dfcb356ad626b" }, { "metadata": {}, @@ -25,7 +26,8 @@ "source": [ "import warnings\n", "warnings.filterwarnings('ignore')" - ] + ], + "id": "853186ccc64b311b" }, { "metadata": {}, @@ -44,7 +46,8 @@ "Evaluation is implemented in `_arch_evaluate_sample` instead of `_arch_evaluate`. This function is called once per realization of the uncertain parameters, receives it in `parameters`, and writes the responses for that realization only.\n", "\n", "Below, `MDMOHimmelblau` is wrapped in such a class. Its continuous design variable is realized with a normally-distributed deviation from its nominal value, and the underlying problem is then evaluated at the realized design vector. Since the design space of the underlying problem is reused, the design variables and the correction of design vectors are inherited as-is and `_correct_x` does not need to be implemented." - ] + ], + "id": "56d6dab3b1e5f7b" }, { "metadata": {}, @@ -87,7 +90,8 @@ "\n", " def __repr__(self):\n", " return f'{self.__class__.__name__}(std={self.std}, n={self.uq_method.n_evaluations})'" - ] + ], + "id": "7c735da49e170b2b" }, { "metadata": {}, @@ -100,12 +104,14 @@ "# P = (n+d)! / (n! d!) is the number of polynomial terms for n parameters.\n", "problem = StochasticMDMOHimmelblau(std=.3, n=12)\n", "problem.print_stats()" - ] + ], + "id": "2db9a8d421e3f270" }, { "metadata": {}, "cell_type": "markdown", - "source": "The resulting problem has the same design variables and objectives as the deterministic one, only the evaluation has changed. Note that one design point now costs 12 evaluations of the underlying problem to train the PCE surrogate. Then the cheap PCE model is evaluated with 1000 samples to construct the distribution. Note that the uncertainty propagation requires overall more model evaluation which makes Surrogate-Based Optimization (SBO) especially attractive for stochastic problems." + "source": "The resulting problem has the same design variables and objectives as the deterministic one, only the evaluation has changed. Note that one design point now costs 12 evaluations of the underlying problem to train the PCE surrogate. Then the cheap PCE model is evaluated with 1000 samples to construct the distribution. Note that the uncertainty propagation requires overall more model evaluation which makes Surrogate-Based Optimization (SBO) especially attractive for stochastic problems.", + "id": "d0ecd1d63f400dc5" }, { "metadata": {}, @@ -114,7 +120,8 @@ "## Solving with NSGA2\n", "\n", "The stochastic problem is an `ArchOptProblemBase` subclass, so the algorithms in SBArchOpt can be used without modification. The code below runs the pre-configured NSGA2 and compares the resulting robust Pareto front to the known deterministic Pareto front of `MDMOHimmelblau`." - ] + ], + "id": "d58823d06acbc6ed" }, { "metadata": {}, @@ -136,12 +143,14 @@ "pf_deterministic = MDMOHimmelblau().pareto_front()\n", "plot(pf_deterministic, result_nsga2.opt.get('F'),\n", " labels=['Deterministic Pareto front', 'NSGA2 robust (2000 design points)'])" - ] + ], + "id": "762c1b2ad0127509" }, { "metadata": {}, "cell_type": "markdown", - "source": "The robust Pareto front lies above the deterministic one, which clearly shows that the perturbation in design vector has effect on the objectives. The deterministic front is therefore not achievable in practice, and optimizing towards it would select designs that are more sensitive to the perturbations." + "source": "The robust Pareto front lies above the deterministic one, which clearly shows that the perturbation in design vector has effect on the objectives. The deterministic front is therefore not achievable in practice, and optimizing towards it would select designs that are more sensitive to the perturbations.", + "id": "a4f99e6120985213" }, { "metadata": {}, @@ -150,7 +159,8 @@ "## Solving with SBO\n", "\n", "Because every design point requires 12 evaluations for each sample realization, the number of design points for evaluation should be kept low. Surrogate-Based Optimization (SBO) in general requires fewer evaluations than genetic algorithms to find the Pareto front, therefore they are well suited for the stochastic optimization problems." - ] + ], + "id": "efe12d81968cda69" }, { "metadata": {}, @@ -173,33 +183,17 @@ "plot(pf_deterministic, result_nsga2.opt.get('F'), result_sbo.opt.get('F'),\n", " labels=['Deterministic Pareto front', 'NSGA2 robust (2000 design points)',\n", " 'SBO robust (50 design points)'])" - ] + ], + "id": "68b1694f096c0818" }, { "metadata": {}, "cell_type": "markdown", - "source": "SBO reaches the same robust Pareto front as NSGA2 using 40 times fewer design points. Given that for each design points 12 evaluations are performed to propagate uncertainty, the computational budget savings are tremendous compared with NSGA-2 (overall 480 less evaluations)." + "source": "SBO reaches the same robust Pareto front as NSGA2 using 40 times fewer design points. Given that for each design points 12 evaluations are performed to propagate uncertainty, the computational budget savings are tremendous compared with NSGA-2 (overall 480 less evaluations).", + "id": "d4f0ff16942cc3c4" } ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.14" - } - }, + "metadata": {}, "nbformat": 4, - "nbformat_minor": 4 + "nbformat_minor": 5 } diff --git a/sb_arch_opt/__init__.py b/sb_arch_opt/__init__.py index 4574cc8..218f431 100644 --- a/sb_arch_opt/__init__.py +++ b/sb_arch_opt/__init__.py @@ -1 +1 @@ -__version__ = '1.6.3' +__version__ = '1.7' diff --git a/sb_arch_opt/problems/robust_optimization/rosenbrock.py b/sb_arch_opt/problems/robust_optimization/rosenbrock.py deleted file mode 100644 index 73a15d6..0000000 --- a/sb_arch_opt/problems/robust_optimization/rosenbrock.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -MIT License - -Copyright: (c) 2023, Deutsches Zentrum fuer Luft- und Raumfahrt e.V. -Contact: jasper.bussemaker@dlr.de - -Stochastic version of the Rosenbrock problem: a robust optimization test problem. -""" -from typing import List - -import numpy as np -import openturns as ot -from pymoo.core.variable import Real - -from sb_arch_opt.stochastic_problem import StochasticArchOptProblem -from sb_arch_opt.uncertainty import * - -__all__ = ['StochasticRosenbrock'] - - -class StochasticRosenbrock(StochasticArchOptProblem): - """ - The Rosenbrock function where the location of the valley is uncertain: - - f(x, u) = sum_i [ 100*(x_{i+1} - x_i^2)^2 + (u_i - x_i)^2 ] - - with one uncertain parameter u_i ~ N(mean, std) per term. - - With the default MEAN reduction the expected objective is minimized; since E[(u-x)^2] = (E[u]-x)^2 + Var[u], - the optimum stays at x_i = mean and the optimal value is the sum of the parameter variances rather than 0. - - Note that only `_arch_evaluate_sample` is implemented, and that it is vectorized over all design points: the - loop over uncertain-parameter samples is owned by `StochasticArchOptProblem`. - - A UQ method is configured independently and receives the parameter space when drawing samples / processing results: - - ```python - method = PolynomialChaos(n_evaluations=40, degree=3, seed=42) - problem = StochasticRosenbrock(n_var=2, uq_method=method) - ``` - """ - - def __init__(self, n_var=2, mean=1., std=.05, n=100, seed=42, - uq_method: UQMethod = None, obj_scalar: List[Scalarization] = None): - if n_var < 2: - raise ValueError('Need at least 2 design variables') - self.mean = mean - self.std = std - self._n_param = n_var - 1 - - # The problem and its UQ method must be defined over the same parameter space; when a method is supplied - # it already carries the space it was built over, so take that one rather than building a second - if uq_method is None: - uq_method = MonteCarlo(n_evaluations=n, seed=seed) - - super().__init__( - [Real(bounds=(-2.048, 2.048)) for _ in range(n_var)], - param_space=self.get_parameter_space(n_var, mean, std), - uq_method=uq_method, - n_obj=1, obj_scalar=obj_scalar - ) - - @staticmethod - def get_parameter_space(n_var=2, mean=1., std=.05) -> StochasticParameterSpace: - """The uncertain valley location: one parameter per Rosenbrock term. Build this first if you want to - construct the UQ method yourself, since a method is defined over a parameter space.""" - parameters = [] - for i in range(n_var-1): - parameters.append(StochasticParameter(f'u{i}', ot.Normal(mean, std))) - param_space = StochasticParameterSpace(parameters) - return param_space - - def _is_conditionally_active(self) -> List[bool]: - return [False]*self.n_var - - def _correct_x(self, x: np.ndarray, is_active: np.ndarray): - pass - - def _get_n_valid_discrete(self) -> int: - return 1 - - def might_have_hidden_constraints(self): - return False - - def _arch_evaluate_sample(self, x, is_active, f_out, g_out, h_out, parameters, *args, **kwargs): - f = np.zeros((x.shape[0],)) - for i in range(self._n_param): - f += 100 * (x[:, i+1] - x[:, i]**2) ** 2 + (parameters[i] - x[:, i]) ** 2 - f_out[:, 0] = f - - def _calc_pareto_front(self, *args, **kwargs): - # E[(u-x)^2] is minimized at x = E[u], leaving the parameter variance - return np.array([[self._n_param * self.std**2]]) - - def _calc_pareto_set(self, *args, **kwargs): - return np.full((1, self.n_var), self.mean) - - def __repr__(self): - return f'{self.__class__.__name__}(n_var={self.n_var}, std={self.std}, n={self.uq_method.n_evaluations})' \ No newline at end of file diff --git a/sb_arch_opt/stochastic_problem.py b/sb_arch_opt/stochastic_problem.py index e78a363..dc31adb 100644 --- a/sb_arch_opt/stochastic_problem.py +++ b/sb_arch_opt/stochastic_problem.py @@ -22,16 +22,25 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from typing import Union, List, Optional - import numpy as np +from typing import Union, List, Optional from pymoo.core.variable import Variable - from sb_arch_opt.design_space import ArchDesignSpace from sb_arch_opt.problem import ArchOptProblemBase -from sb_arch_opt.uncertainty import * -__all__ = ['StochasticArchOptProblem'] +try: + from sb_arch_opt.uncertainty import StochasticParameterSpace, UQMethod, Scalarization, Mean + HAS_UNCERTAINTY = True + +except ImportError: + HAS_UNCERTAINTY = False + +__all__ = ['StochasticArchOptProblem', 'HAS_UNCERTAINTY', 'check_dependency'] + + +def check_dependency(): + if not HAS_UNCERTAINTY: + raise ImportError('Looks like SBArchOpt uncertainty package is not installed! Run: pip install sb-arch-opt[uncertainty]') class StochasticArchOptProblem(ArchOptProblemBase): @@ -86,6 +95,7 @@ def __init__(self, des_vars: Union[List[Variable], ArchDesignSpace], eq_constr_scalar: Optional[List[Scalarization]] = None, **kwargs): + check_dependency() self.obj_scalar = self.check_scalars(obj_scalar, n_obj) self.ieq_constr_scalar = self.check_scalars(ieq_constr_scalar, n_ieq_constr) self.eq_constr_scalar = self.check_scalars(eq_constr_scalar, n_eq_constr) diff --git a/sb_arch_opt/tests/conftest.py b/sb_arch_opt/tests/conftest.py index b131fea..357f016 100644 --- a/sb_arch_opt/tests/conftest.py +++ b/sb_arch_opt/tests/conftest.py @@ -10,10 +10,8 @@ try: import openturns as ot from sb_arch_opt.uncertainty import * - from sb_arch_opt.stochastic_problem import StochasticArchOptProblem - + from sb_arch_opt.stochastic_problem import * HAS_UNCERTAINTY = True - except ImportError: HAS_UNCERTAINTY = False @@ -21,6 +19,12 @@ class StochasticArchOptProblem: pass +def check_dependency(): + if not HAS_UNCERTAINTY: + raise ImportError( + 'Looks like SBArchOpt uncertainty package is not installed! Run: pip install sb-arch-opt[uncertainty]') + + class DummyProblem(ArchOptTestProblemBase): def __init__(self, only_discrete=False, fail=False): @@ -121,6 +125,7 @@ class VectorizedProblem(StochasticArchOptProblem): """f = (u - x0)^2 + x1^2, evaluated for all design points at once""" def __init__(self, n=100, seed=42, uq_method=None, fail=False, **kwargs): + check_dependency() self.fail = fail self.seen_parameters = [] super().__init__([Real(bounds=(-2., 2.)), Real(bounds=(-2., 2.))], @@ -145,6 +150,7 @@ class HierarchicalProblem(StochasticArchOptProblem): """Implicit (hierarchical) design space, a constraint, and the per-row evaluation pattern""" def __init__(self, n=50, seed=1, **kwargs): + check_dependency() super().__init__([Choice(options=['electric', 'hybrid']), Real(bounds=(.2, 1.)), Real(bounds=(.1, .4))], param_space=StochasticParameterSpace([ StochasticParameter('payload', ot.Normal(2., .3)), @@ -171,6 +177,7 @@ class AllResponseKindsProblem(StochasticArchOptProblem): """One objective, one inequality and one equality constraint, each with its own scalar""" def __init__(self, **kwargs): + check_dependency() kwargs.setdefault('obj_scalar', [Mean()]) kwargs.setdefault('ieq_constr_scalar', [Mean()]) kwargs.setdefault('eq_constr_scalar', [Margin(k=3.)]) diff --git a/sb_arch_opt/tests/test_stochastic_problem.py b/sb_arch_opt/tests/test_stochastic_problem.py index 0ca1ae2..0917e76 100644 --- a/sb_arch_opt/tests/test_stochastic_problem.py +++ b/sb_arch_opt/tests/test_stochastic_problem.py @@ -1,16 +1,18 @@ import pytest import numpy as np -from sb_arch_opt.tests.conftest import (HAS_UNCERTAINTY, VectorizedProblem, HierarchicalProblem, - DeterministicResponseProblem, make_space) -pytestmark = pytest.mark.skipif(not HAS_UNCERTAINTY, reason='OpenTURNS dependency not installed: ' - 'pip install sb-arch-opt[uncertainty]') - -if HAS_UNCERTAINTY: +try: import openturns as ot from sb_arch_opt.uncertainty import * from sb_arch_opt.stochastic_problem import StochasticArchOptProblem - from sb_arch_opt.problems.robust_optimization.rosenbrock import StochasticRosenbrock + from sb_arch_opt.tests.conftest import (VectorizedProblem, HierarchicalProblem, + DeterministicResponseProblem, make_space) + +except ImportError: + pytest.skip( + "SBArchOpt uncertainty package is not installed! Run: pip install sb-arch-opt[uncertainty]", + allow_module_level=True, + ) def _output(values): @@ -26,6 +28,7 @@ def test_parameter_space(): assert space.n_parameters == 2 assert space.parameter_names == ['a', 'b'] assert space.joint_dist.getDimension() == 2 + assert [parameter.name for parameter in space.parameters] == ['a', 'b'] for samples in [space.get_random_samples(20), space.get_lhs_samples(20)]: assert samples.shape == (20, 2) @@ -298,26 +301,4 @@ def test_pce_fails_the_design_point_when_evaluations_fail(): out = problem.evaluate(np.array([[.5, .5]]), return_as_dictionary=True) assert not np.isfinite(out['f_stochastic'][0, 0]) - assert not np.isfinite(out['F'][0, 0]) - - -def test_stochastic_rosenbrock(): - problem = StochasticRosenbrock(n_var=3, std=.05, n=500) - assert problem.n_var == 3 - assert problem.param_space.n_parameters == 2 - assert repr(problem) - - # At the optimum the expected objective equals the sum of the parameter variances - out = problem.evaluate(problem.pareto_set(), return_as_dictionary=True) - assert out['F'][0, 0] == pytest.approx(problem.pareto_front()[0, 0], abs=1e-3) - - # Moving away from the optimum makes it worse - f_off = problem.evaluate(np.array([[.5, .5, .5]]), return_as_dictionary=True)['F'] - assert f_off[0, 0] > out['F'][0, 0] - - -def test_stochastic_rosenbrock_with_pce(): - problem = StochasticRosenbrock(n_var=3, uq_method=PolynomialChaos(40, seed=42, degree=2)) - out = problem.evaluate(problem.pareto_set(), return_as_dictionary=True) - - assert out['F'][0, 0] == pytest.approx(problem.pareto_front()[0, 0], abs=1e-3) \ No newline at end of file + assert not np.isfinite(out['F'][0, 0]) \ No newline at end of file diff --git a/sb_arch_opt/uncertainty.py b/sb_arch_opt/uncertainty.py index b40cbff..aaa18e0 100644 --- a/sb_arch_opt/uncertainty.py +++ b/sb_arch_opt/uncertainty.py @@ -22,13 +22,28 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ -from typing import List, Optional, Union +from __future__ import annotations + +import copy -import openturns as ot import numpy as np +from typing import List, Optional, Union + +try: + import openturns as ot + HAS_UNCERTAINTY = True +except ImportError: + HAS_UNCERTAINTY = False + + +def check_dependency(): + if not HAS_UNCERTAINTY: + raise ImportError( + 'Looks like SBArchOpt uncertainty package is not installed! Run: pip install sb-arch-opt[uncertainty]') + __all__ = ['Scalarization', 'Mean', 'Margin', 'Quantile', 'StochasticParameter', 'StochasticParameterSpace', - 'StochasticOutput', 'UQMethod', 'MonteCarlo', 'PolynomialChaos', 'EvaluationOutput'] + 'StochasticOutput', 'UQMethod', 'MonteCarlo', 'PolynomialChaos', 'EvaluationOutput', 'HAS_UNCERTAINTY', 'check_dependency'] EvaluationOutput = Union['StochasticOutput', float] """Output either distribution or numeric value.""" @@ -46,6 +61,7 @@ class StochasticParameter: """ def __init__(self, name, dist: ot.DistributionImplementation, ref=None): + check_dependency() self.name = name self.dist = dist self.ref = ref @@ -72,6 +88,7 @@ class StochasticParameterSpace: """ def __init__(self, parameters: List[StochasticParameter]): + check_dependency() self._parameters = parameters @property @@ -79,6 +96,11 @@ def n_parameters(self) -> int: """Number of stochastic parameters""" return len(self._parameters) + @property + def parameters(self) -> List[StochasticParameter]: + """The stochastic parameters, in the order of the sample columns""" + return list(self._parameters) + @property def parameter_names(self) -> List[str]: """List of stochastic parameter names""" @@ -94,9 +116,12 @@ def param_realization(self, samples: np.ndarray, i_realization: int) -> List[Sto """ Return `i_realization` of `samples` (n_samples x n_parameters) as a list of stochastic parameters. """ + realization = [] for j, param in enumerate(self._parameters): - param.sample = samples[i_realization, j] - return self._parameters + realized = copy.copy(param) + realized.sample = float(samples[i_realization, j]) + realization.append(realized) + return realization def get_random_samples(self, n_samples: int) -> np.ndarray: """Draw n samples of the stochastic parameters; returns an n x n_parameters matrix""" @@ -118,11 +143,12 @@ class StochasticOutput: """ def __init__(self, dist: ot.Distribution, method_results=None): + check_dependency() self.dist = dist self.method_results = method_results @staticmethod - def build_distribution(output_samples: ot.Sample) -> ot.Distribution | float: + def build_distribution(output_samples: ot.Sample) -> Union[ot.Distribution, float]: values = np.asarray(output_samples, dtype=float).ravel() if values.size == 0 or not np.all(np.isfinite(values)): @@ -176,6 +202,10 @@ class Scalarization: - Margin - Quantile """ + + def __init__(self): + check_dependency() + def scalarize(self, output: StochasticOutput) -> float: """Reduce an (n_samples x 1) sample of one response to a single value that the optimizer sees based on the optimization problem type.""" raise NotImplementedError @@ -207,6 +237,7 @@ class Margin(Scalarization): """ def __init__(self, k: float = 1.645, direction: int =-1): + super().__init__() self.k = k self.direction = direction @@ -229,6 +260,7 @@ class Quantile(Scalarization): """ def __init__(self, q: float = 0.95): + super().__init__() if not 0. <= q <= 1.: raise ValueError(f'Quantile should be between 0 and 1: {q}') self.q = q @@ -254,6 +286,7 @@ class UQMethod: """ def __init__(self, n_evaluations: int, seed: int = None): + check_dependency() if n_evaluations is None or n_evaluations <= 0: raise ValueError('n_evaluations must be specified: it is the number of expensive evaluations ' 'per design point') @@ -261,20 +294,22 @@ def __init__(self, n_evaluations: int, seed: int = None): self.seed = seed self._samples: Optional[np.ndarray] = None + self._samples_space: Optional[StochasticParameterSpace] = None def get_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: """ - Get samples from the joint distribution provided by the param_space. + Get samples from the joint distribution provided by the param_space. The samples are drawn once and reused + for every design point (common random numbers), until `resample` is called or another space is given. """ if param_space is None: raise ValueError('No parameter space to sample') - samples = self._samples - if samples is None: + if self._samples is None or self._samples_space is not param_space: if self.seed is not None: ot.RandomGenerator.SetSeed(self.seed) - samples = self._draw_samples(param_space) - return samples + self._samples = self._draw_samples(param_space) + self._samples_space = param_space + return self._samples def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: """Draw the design of experiments in the parameter space; override to use a different design""" @@ -357,12 +392,12 @@ def _draw_samples(self, param_space: StochasticParameterSpace) -> np.ndarray: """Draw samples with LHS method""" self._validate(param_space) samples = param_space.get_lhs_samples(self.n_evaluations) + self._metamodel_input = param_space.joint_dist.getSample(self.n_metamodel_samples) return samples def _get_metamodel_input(self, param_space: StochasticParameterSpace) -> ot.Sample: """Get samples used for evaluating the surrogate model""" - if self._metamodel_input is None: - self._metamodel_input = param_space.joint_dist.getSample(self.n_metamodel_samples) + self.get_samples(param_space) return self._metamodel_input def _build_algorithm(self, param_space: StochasticParameterSpace, input_sample: ot.Sample, output_sample: ot.Sample) -> ot.FunctionalChaosAlgorithm: